伪分布hadoop涉及的配置文件

core-site.xml

        
                        fs.defaultFS
                        hdfs://localhost:9000
                        默认的HDFS端口，用于NameNode与DataNode之间到的通讯，IP为NameNode的地址
        


        
                hadoop.security.authorization
                false
                
         

hdfs-site.xml

 
 dfs.replication
        1



   dfs.namenode.name.dir
   /app/hdfs/namenode



   dfs.datanode.data.dir
   /app/hdfs/datanode



    dfs.permissions
    false

mapred-site.xml

        
                mapreduce.framework.name
                yarn
        


        
                 mapreduce.application.classpath
                 
                        $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*,
                
                 设置MapReduce资源调度类路径，如果不设置可能会报错
        `

yarn-site.xml

        
        yarn.nodemanager.aux-services
        mapreduce_shuffle
        NodeManager上运行的附属服务。需配置成mapreduce_shuffle，才可运行MapReduce程序
    

        
        yarn.nodemanager.env-whitelist
        JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
        
    

hadoop-env.sh

export HADOOP JAVA_HOME=/usr/lib/jvm/java
export HADOOP_MAPRED_HOME=/app/hadoop
export HADOOP_NAMENODE_USER=root
export HADOOP_DATANODE_USER=root
export HADOOP_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

ps:5步搞定 Windows 10 docker WSL2的镜像存储位置的修改

ps2:docker-compose.yarm

version: '3'
services:
  master1:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D
    ports:
      - "9000:9000"
      - "9870:9870"
      - "8088:8088"
 
  master2:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

    
  worker1:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

  worker2:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D

  worker3:
    image: hadoop-distributed:v1
    stdin_open: true
    tty: true
    command: /sbin/sshd -D