0) My environment
macOS Big Sur Version 11.6
JDK jdk1.8
1) Download and install
## download hadoop-2.8.0.tar.gz from http://archive.apache.org/dist/
## unpack to ~/work/hadoop
## set environment variables
$ cd ~/work/hadoop
$ export HADOOP_HOME=~/work/hadoop
$ export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
2) configure and start
## modify configuration for pseudo-dist
$ cd ~/work/hadoop/etc/hadoop
$ diff hadoop-env.sh.orig hadoop-env.sh
25c25
< export JAVA_HOME=${JAVA_HOME}
---
> export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_321.jdk/Contents/Home
$ diff core-site.xml.orig core-site.xml
>
>
>
>
>
>
>
>
$ diff hdfs-site.xml.orig hdfs-site.xml
>
>
>
>
>
>
>
>
## start hadoop as pseudo-dist
$ cd ~/work/hadoop
$ mr.sh start
## modify for pseudo-dist with yarn
$ cd ~/work/hadoop/etc/hadoop
$ diff yarn-site.xml.orig yarn-site.xml
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
$ diff mapred-site.xml.template mapred-site.xml
>
>
>
>
## start hadoop as pseudo-dist yarn
$ cd ~/work/hadoop
$ mr.sh start yarn
$ jps
26976 NameNode
27378 NodeManager
27427 JobHistoryServer
27061 DataNode
27159 SecondaryNameNode
27295 ResourceManager
3) run a mapreduce case
$ mr.sh pseudo-dist
## check from a browser
## hadoop-hdfs - http://localhost:50070/explorer.html#/
## hadoop-mapreduce - http://localhost:8088/cluster
## mr.sh
#!/bin/sh
usage() {
echo "Usage: $0 standalone"
echo " $0 start [yarn]"
echo " $0 stop [yarn]"
echo " $0 pseudo_dist"
}
standalone() {
cd $HADOOP_HOME/etc/hadoop
ln -sf core-site.xml.orig core-site.xml
ln -sf hdfs-site.xml.orig hdfs-site.xml
ln -sf yarn-site.xml.orig yarn-site.xml
rm -f mapred-site.xml
cd $HADOOP_HOME
mkdir -p input
echo test apache hadoop hadoop sqoop hue mapreduce sqoop oozie http > input/in.txt
rm -rf output
hadoop jar $jarfile wordcount input output
cat output/*
}
start() {
cd $HADOOP_HOME/etc/hadoop
ln -sf core-site.xml.pseudo core-site.xml
ln -sf hdfs-site.xml.pseudo hdfs-site.xml
if [ $# -eq 0 ]; then
ln -sf yarn-site.xml.orig yarn-site.xml
rm -f mapred-site.xml
else
ln -sf yarn-site.xml.pseudo yarn-site.xml
ln -sf mapred-site.xml.pseudo mapred-site.xml
fi
cd $HADOOP_HOME
# hdfs namenode -format
if [ "x`jps | grep -w "NameNode"`" = "x" ]; then
start-dfs.sh
fi
hdfs dfsadmin -safeoper leave
if [ $# -eq 1 -a "x`jps | grep -w "NodeManager"`" = "x" ]; then
start-yarn.sh
mr-jobhistory-daemon.sh start historyserver
fi
}
stop() {
if [ $# -eq 1 ]; then
mr-jobhistory-daemon.sh stop historyserver
stop-yarn.sh
fi
stop-dfs.sh
}
pseudo_dist() {
hdfs dfs -rm -f -r input
hdfs dfs -mkdir input
hdfs dfs -put $HADOOP_HOME/input/in.txt input
hdfs dfs -rm -f -r output
hadoop jar $jarfile wordcount input output
hdfs dfs -cat output/*
}
oper=$1
yarn=$2
jarfile=$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.0.jar
if [ ( x$oper = "xstandalone" -o x$oper = "xstart" -o x$oper = "xstop" -o x$oper = "xpseudo_dist" )
-a ( x$yarn = "x" -o x$yarn = "xyarn" ) ]; then
echo "$oper $yarn"
$oper $yarn
else
usage
exit 1
fi
reference: https://hadoop.apache.org/docs/r2.8.0/



