栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

搭建Hadoop pseudo-distributed 环境

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

搭建Hadoop pseudo-distributed 环境

0) My environment
macOS Big Sur Version 11.6
JDK jdk1.8

1) Download and install
## download hadoop-2.8.0.tar.gz from http://archive.apache.org/dist/
## unpack to ~/work/hadoop
## set environment variables
$ cd ~/work/hadoop
$ export HADOOP_HOME=~/work/hadoop
$ export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

2) configure and start
## modify configuration for pseudo-dist
$ cd ~/work/hadoop/etc/hadoop
$ diff hadoop-env.sh.orig hadoop-env.sh
25c25
< export JAVA_HOME=${JAVA_HOME}
---
> export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_321.jdk/Contents/Home
$ diff core-site.xml.orig core-site.xml
>    
>         fs.defaultFS
>         hdfs://localhost:9000
>    

>    
>         hadoop.tmp.dir
>         /Users/sun_xo/work/hadoop/data/tmp
>    

$ diff hdfs-site.xml.orig hdfs-site.xml
>    
>         dfs.replication
>         1
>    

>    
>         dfs.user.home.dir.prefix
>         /Users
>    

## start hadoop as pseudo-dist
$ cd ~/work/hadoop
$ mr.sh start

## modify for pseudo-dist with yarn
$ cd ~/work/hadoop/etc/hadoop
$ diff yarn-site.xml.orig yarn-site.xml
>    
>         yarn.resourceManager.hostname
>         localhost
>    

>    
>         yarn.nodemanager.aux-services
>         mapreduce_shuffle
>    

>    
>         yarn.log-aggregation-enable
>         true
>    

>    
>         yarn.log-aggregation.retain-seconds
>         604800
>    

$ diff mapred-site.xml.template mapred-site.xml
>    
>         mapreduce.framework.name
>         yarn
>    

## start hadoop as pseudo-dist yarn
$ cd ~/work/hadoop
$ mr.sh start yarn
$ jps
26976 NameNode
27378 NodeManager
27427 JobHistoryServer
27061 DataNode
27159 SecondaryNameNode
27295 ResourceManager

3) run a mapreduce case
$ mr.sh pseudo-dist
## check from a browser
## hadoop-hdfs - http://localhost:50070/explorer.html#/
## hadoop-mapreduce  - http://localhost:8088/cluster

## mr.sh

#!/bin/sh

usage() {
    echo "Usage: $0 standalone"
    echo "       $0 start [yarn]"
    echo "       $0 stop [yarn]"
    echo "       $0 pseudo_dist"
}

standalone() {
    cd $HADOOP_HOME/etc/hadoop
    ln -sf core-site.xml.orig core-site.xml
    ln -sf hdfs-site.xml.orig hdfs-site.xml
    ln -sf yarn-site.xml.orig yarn-site.xml
    rm -f mapred-site.xml
    cd $HADOOP_HOME

    mkdir -p input
    echo test apache hadoop hadoop sqoop hue mapreduce sqoop oozie http > input/in.txt
    rm -rf output
    hadoop jar $jarfile wordcount input output
    cat output/*
}

start() {
    cd $HADOOP_HOME/etc/hadoop
    ln -sf core-site.xml.pseudo core-site.xml
    ln -sf hdfs-site.xml.pseudo hdfs-site.xml
    if [ $# -eq 0 ]; then
        ln -sf yarn-site.xml.orig yarn-site.xml
        rm -f mapred-site.xml
    else
        ln -sf yarn-site.xml.pseudo yarn-site.xml
        ln -sf mapred-site.xml.pseudo mapred-site.xml
    fi

    cd $HADOOP_HOME
    # hdfs namenode -format
    if [ "x`jps | grep -w "NameNode"`" = "x" ]; then
        start-dfs.sh
    fi
    hdfs dfsadmin -safeoper leave
    if [ $# -eq 1 -a "x`jps | grep -w "NodeManager"`" = "x" ]; then
        start-yarn.sh
        mr-jobhistory-daemon.sh start historyserver
    fi
}

stop() {
    if [ $# -eq 1 ]; then
        mr-jobhistory-daemon.sh stop historyserver
        stop-yarn.sh
    fi
    stop-dfs.sh
}

pseudo_dist() {
    hdfs dfs -rm -f -r input
    hdfs dfs -mkdir input
    hdfs dfs -put $HADOOP_HOME/input/in.txt input
    hdfs dfs -rm -f -r output
    hadoop jar $jarfile wordcount input output
    hdfs dfs -cat output/*
}

oper=$1
yarn=$2
jarfile=$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.0.jar

if [ ( x$oper = "xstandalone" -o x$oper = "xstart" -o x$oper = "xstop" -o x$oper = "xpseudo_dist" ) 
        -a ( x$yarn = "x" -o x$yarn = "xyarn" ) ]; then
    echo "$oper $yarn"
    $oper $yarn
else
    usage
    exit 1
fi

reference: https://hadoop.apache.org/docs/r2.8.0/

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/943395.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号