所需脚本网址
jqop https://blog.csdn.net/qq_51903852/article/details/121120429
xsync https://blog.csdn.net/qq_51903852/article/details/121141166
zkop https://blog.csdn.net/qq_51903852/article/details/121120516
[root@mihaoyu152 install]# tar -zxvf hadoop-2.6.0-cdh5.14.2.tar.gz -C ../soft [root@mihaoyu152 soft]# mv hadoop-2.6.0-cdh5.14.2/ hadoop260 [root@mihaoyu152 hadoop]# vi ./hadoop-env.sh 25 export JAVA_HOME=/opt/soft/jdk180 [root@mihaoyu152 hadoop]# vi ./yarn-env.sh 23 export JAVA_HOME=/opt/soft/jdk180 [root@mihaoyu152 hadoop]# vi ./mapred-env.sh 16 export JAVA_HOME=/opt/soft/jdk180 [root@mihaoyu152 hadoop]# vi ./core-site.xml第一次启动集群服务[root@mihaoyu152 hadoop]# vi ./hdfs-site.xml fs.defaultFS hdfs://mycluster/ hadoop.tmp.dir /opt/soft/hadoop260/hadooptmp/ ha.zookeeper.quorum mihaoyu153:2181,mihaoyu154:2181,mihaoyu155:2181 hadoop.proxyuser.bigdata.hosts * hadoop.proxyuser.bigdata.groups * [root@mihaoyu152 hadoop]# cp mapred-site.xml.template mapred-site.xml [root@mihaoyu152 hadoop]# vi ./mapred-site.xml dfs.replication 3 dfs.nameservices mycluster dfs.ha.namenodes.mycluster nn1,nn2 dfs.namenode.rpc-address.mycluster.nn1 mihaoyu152:9000 dfs.namenode.http-address.mycluster.nn1 mihaoyu152:50070 dfs.namenode.rpc-address.mycluster.nn2 mihaoyu153:9000 dfs.namenode.http-address.mycluster.nn2 mihaoyu153:50070 dfs.journalnode.edits.dir /opt/soft/hadoop260/journaldata dfs.namenode.shared.edits.dir qjournal://mihaoyu152:8485;mihaoyu153:8485;mihaoyu154:8485/mycluster dfs.ha.automatic-failover.enabled true dfs.client.failover.proxy.provider.mycluster org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider dfs.ha.fencing.methods sshfence shell(/bin/true) dfs.ha.fencing.ssh.private-key-files /root/.ssh/id_rsa dfs.ha.fencing.ssh.connect-timeout 30000 dfs.webhdfs.enabled true [root@mihaoyu152 hadoop]# vi ./yarn-site.xml mapreduce.framework.name yarn mapreduce.jobhistory.address mihaoyu155:10020 mapreduce.jobhistory.webapp.address mihaoyu155:19888 [root@mihaoyu152 hadoop]# vi ./slaves mihaoyu152 mihaoyu153 mihaoyu154 mihaoyu155 [root@mihaoyu152 hadoop]# vi /etc/profile #hadoop export HADOOP_HOME=/opt/soft/hadoop260 export HADOOP_MAPRED_HOME=$HADOOP_HOME export HADOOP_COMMON_HOME=$HADOOP_HOME export HADOOP_HDFS_HOME=$HADOOP_HOME export YARN_HOME=$HADOOP_HOME export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib" export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin [root@mihaoyu152 soft]# source /etc/profile [root@mihaoyu152 soft]# xsync hadoop260 [root@mihaoyu152 soft]# xsync /etc/profile yarn.resourcemanager.ha.enabled true yarn.resourcemanager.cluster-id yrc yarn.resourcemanager.ha.rm-ids rm1,rm2 yarn.resourcemanager.hostname.rm1 mihaoyu152 yarn.resourcemanager.hostname.rm2 mihaoyu153 yarn.resourcemanager.zk-address mihaoyu153:2181,mihaoyu154:2181,mihaoyu155:2181 yarn.nodemanager.aux-services mapreduce_shuffle yarn.log-aggregation-enable true yarn.log-aggregation.retain-seconds 86400 yarn.resourcemanager.recovery.enabled true yarn.resourcemanager.store.class org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
1、先将zookeeper启动 [root@mihaoyu152 soft]# zkop start --------------- mihaoyu153 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED --------------- mihaoyu154 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED --------------- mihaoyu155 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED [root@mihaoyu152 soft]# zkop status --------------- mihaoyu153 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: follower --------------- mihaoyu154 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: leader --------------- mihaoyu155 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: follower 2、启动journalnode [root@mihaoyu152 soft]# hadoop-daemon.sh start journalnode starting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu152.out [root@mihaoyu152 soft]# ssh mihaoyu153 "source /etc/profile; hadoop-daemon.sh start journalnode" starting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu153.out [root@mihaoyu152 soft]# ssh mihaoyu154 "source /etc/profile; hadoop-daemon.sh start journalnode" starting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu154.out [root@mihaoyu152 soft]# jqop jps --------------- mihaoyu152 指令信息 --------------- jps 5986 Jps 5894 JournalNode --------------- mihaoyu153 指令信息 --------------- jps 4676 QuorumPeerMain 4807 Jps 4748 JournalNode --------------- mihaoyu154 指令信息 --------------- jps 3570 JournalNode 3493 QuorumPeerMain 3629 Jps --------------- mihaoyu155 指令信息 --------------- jps 3378 Jps 3318 QuorumPeerMain 3、格式化 namenode mihaoyu152 mihaoyu153 [root@mihaoyu152 soft]# hadoop namenode -format [root@mihaoyu152 soft]# cd ./hadoop260/ 将mihaoyu152格式化后的hadooptmp文件同步到mihaoyu153 [root@mihaoyu152 hadoop260]# scp -r ./hadooptmp/ root@mihaoyu153:/opt/soft/hadoop260/ VERSION 100% 205 235.0KB/s 00:00 seen_txid 100% 2 2.4KB/s 00:00 fsimage_0000000000000000000.md5 100% 62 106.3KB/s 00:00 fsimage_0000000000000000000 100% 320 565.5KB/s 00:00 4、初始化zookeeper [root@mihaoyu152 hadoop260]# hdfs zkfc -formatZK [root@mihaoyu153 hadoop260]# zkCli.sh 5、启动HDFS [root@mihaoyu152 hadoop260]# start-dfs.sh 21/11/04 21:40:35 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting namenodes on [mihaoyu152 mihaoyu153] mihaoyu153: starting namenode, logging to /opt/soft/hadoop260/logs/hadoop-root-namenode-mihaoyu153.out mihaoyu152: starting namenode, logging to /opt/soft/hadoop260/logs/hadoop-root-namenode-mihaoyu152.out mihaoyu153: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu153.out mihaoyu155: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu155.out mihaoyu154: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu154.out mihaoyu152: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu152.out Starting journal nodes [mihaoyu152 mihaoyu153 mihaoyu154] mihaoyu153: journalnode running as process 4748. Stop it first. mihaoyu154: journalnode running as process 3570. Stop it first. mihaoyu152: journalnode running as process 5894. Stop it first. 21/11/04 21:40:47 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting ZK Failover Controllers on NN hosts [mihaoyu152 mihaoyu153] mihaoyu153: starting zkfc, logging to /opt/soft/hadoop260/logs/hadoop-root-zkfc-mihaoyu153.out mihaoyu152: starting zkfc, logging to /opt/soft/hadoop260/logs/hadoop-root-zkfc-mihaoyu152.out [root@mihaoyu152 hadoop260]# jqop jps --------------- mihaoyu152 指令信息 --------------- jps 6581 DFSZKFailoverController 5894 JournalNode 6312 DataNode 6185 NameNode 6666 Jps --------------- mihaoyu153 指令信息 --------------- jps 4883 NameNode 4676 QuorumPeerMain 5147 Jps 4748 JournalNode 4972 DataNode 5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps 3570 JournalNode 3493 QuorumPeerMain 3767 Jps 3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps 3318 QuorumPeerMain 3495 Jps 3407 DataNode 6、启动yarn [root@mihaoyu152 hadoop260]# start-yarn.sh starting yarn daemons starting resourcemanager, logging to /opt/soft/hadoop260/logs/yarn-root-resourcemanager-mihaoyu152.out mihaoyu154: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu154.out mihaoyu153: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu153.out mihaoyu155: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu155.out mihaoyu152: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu152.out [root@mihaoyu152 hadoop260]# jqop jps --------------- mihaoyu152 指令信息 --------------- jps 7139 Jps 6581 DFSZKFailoverController 6821 NodeManager 5894 JournalNode 6312 DataNode 6185 NameNode 6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps 4883 NameNode 4676 QuorumPeerMain 5188 NodeManager 4748 JournalNode 4972 DataNode 5085 DFSZKFailoverController 5310 Jps --------------- mihaoyu154 指令信息 --------------- jps 3570 JournalNode 3922 Jps 3493 QuorumPeerMain 3800 NodeManager 3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps 3650 Jps 3318 QuorumPeerMain 3528 NodeManager 3407 DataNode [root@mihaoyu152 hadoop260]# ssh mihaoyu153 "source /etc/profile; yarn-daemon.sh start resourcemanager" starting resourcemanager, logging to /opt/soft/hadoop260/logs/yarn-root-resourcemanager-mihaoyu153.out [root@mihaoyu152 hadoop260]# jqop jps --------------- mihaoyu152 指令信息 --------------- jps 6581 DFSZKFailoverController 6821 NodeManager 5894 JournalNode 6312 DataNode 6185 NameNode 7179 Jps 6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps 4883 NameNode 4676 QuorumPeerMain 5188 NodeManager 5352 ResourceManager 5418 Jps 4748 JournalNode 4972 DataNode 5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps 3570 JournalNode 3493 QuorumPeerMain 3800 NodeManager 3659 DataNode 3948 Jps --------------- mihaoyu155 指令信息 --------------- jps 3318 QuorumPeerMain 3528 NodeManager 3676 Jps 3407 DataNode 7、启动historyserver [root@mihaoyu152 hadoop260]# ssh mihaoyu155 "source /etc/profile; mr-jobhistory-daemon.sh start historyserver" starting historyserver, logging to /opt/soft/hadoop260/logs/mapred-root-historyserver-mihaoyu155.out [root@mihaoyu152 hadoop260]# jqop jps --------------- mihaoyu152 指令信息 --------------- jps 7219 Jps 6581 DFSZKFailoverController 6821 NodeManager 5894 JournalNode 6312 DataNode 6185 NameNode 6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps 5458 Jps 4883 NameNode 4676 QuorumPeerMain 5188 NodeManager 5352 ResourceManager 4748 JournalNode 4972 DataNode 5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps 3570 JournalNode 3972 Jps 3493 QuorumPeerMain 3800 NodeManager 3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps 3797 Jps 3318 QuorumPeerMain 3528 NodeManager 3720 JobHistoryServer 3407 DataNode
