1. 构建centos版本的镜像
1.1 创建目录，并在目录下创建dockerfile文件
mkdir centos-ssh-root
cd centos-ssh-root
vim dockerfile
# 选择已有的OS镜像作为基础
FROM centos:7.5.1804
# 镜像作者
MAINTAINER ljs
ENV TZ "Asia/Shanghai"
RUN yum install -y curl wget tar unzip vim
RUN yum install -y gcc gcc-c++ git make
# 安装openssh-server和sudo软件包,将sshd的UsePAM参数设置成no
RUN yum install -y openssh-server sudo
RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
# 安装open-clients
RUN yum install -y openssh-clients
# 添加测试用户root,密码root,将此用户添加至sudoers内
RUN echo "root:root" | chpasswd
RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
# 这两句比较特殊,必须要有,否则创建出来的容器sshd无法登录
RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key
RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
# 启动sshd服务并爆露22端口
RUN mkdir /var/run/sshd
EXPOSE 22
CMD ["/usr/sbin/sshd", "-D"]
1.2 在所创建目录下执行以下命令生成镜像
sudo docker build -t="ljs/centos-ssh-root" .;
2. 构建JDK的镜像
2.1 创建目录并创建dockerfile文件
mkdir centos-ssh-root-jdk
cd centos-ssh-root-jdk
vim dockerfile
FROM liangsw/centos-ssh-root
ADD jdk-8u261-linux-x64.tar.gz /usr/local/
RUN mv /usr/local/jdk1.8.0_261 /usr/local/jdk-8u261
ENV JAVA_HOME /usr/local/jdk-8u261
ENV PATH $JAVA_HOME/bin:$PATH
- 将 jdk-8u261-linux-x64.tar.gz拷贝到当前目录
2.2 在所创建目录下执行以下命令生成镜像
sudo docker build -t="ljs/centos-ssh-root-jdk" .;
3. 构建Hadoop的镜像
3.1 创建目录并创建dockerfile文件
mkdir centos-root-hadoop
cd centos-root-hadoop
vim dockerfile
FROM ljs/centos-ssh-root-jdk
ADD hadoop-3.2.1.tar.gz /usr/local/
RUN mv /usr/local/hadoop-3.2.1 /usr/local/hadoop
ENV HADOOP_HOME /usr/local/hadoop
ENV PATH $HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
- 将 hadoop-3.2.1.tar.gz拷贝当前目录
3.2 在所创建目录下执行以下命令生成镜像
sudo docker build -t="ljs/centos-root-hadoop" .;
4. 构建容器
sudo docker run --name hadoop-master --hostname master -d -P -p 50070:50070 -p 8088:8088 ljs/centos-root-hadoop
sudo docker run --name hadoop-slave1 --hostname slave1 -d -P ljs/centos-root-hadoop
sudo docker run --name hadoop-slave2 --hostname slave2 -d -P ljs/centos-root-hadoop
sudo docker run --name hadoop-slave3 --hostname slave3 -d -P ljs/centos-root-hadoop
5. 进入每个容器，为每个容器配置ssh登录
5. 1 开四个终端，每个终端进入一个容器内部
sudo docker exec -it hadoop-master /bin/bash
sudo docker exec -it hadoop-slave1 /bin/bash
sudo docker exec -it hadoop-slave2 /bin/bash
sudo docker exec -it hadoop-slave3 /bin/bash
5.2 在容器内部查看和添加各个容器的IP
vim /etc/hosts
172.17.0.12 master
172.17.0.13 slave1
172.17.0.14 slave2
172.17.0.15 slave3
5.3 在每个容器内输入以下命令生成秘钥
ssh-keygen -t rsa
5.4 在每个容器下输入以下命令设置ssh免密登录
ssh-copy-id -i localhost
ssh-copy-id -i master
ssh-copy-id -i slave1
ssh-copy-id -i slave2
ssh-copy-id -i slave3
ssh master
exit
ssh slave1
exit
ssh slave2
exit
ssh slave3
exit
6. 在master节点上进行以下配置
6.1 在master容器下执行
cd /usr/local/hadoop/etc/hadoop
6.2 vim hadoop-env.sh
export JAVA_HOME=/usr/local/jdk-8u261
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
6.3 vim cor-site.xml
fs.defaultFS
hdfs://master:9000
hadoop.tmp.dir
/usr/local/hadoop/tmp
fs.trash.interval
1440
6.4 vim hdfs-site.xml
dfs.replication
3
dfs.permissions
false
6.5 vim yarn-site.xml
yarn.resourcemanager.address
master:8032
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler
yarn.resourcemanager.scheduler.address
master:8030
yarn.resourcemanager.resource-tracker.address
master:8031
yarn.resourcemanager.admin.address
master:8033
yarn.resourcemanager.webapp.address
master:8089
yarn.log-aggregation-enable
true
6.6 vim mapred-site.xml
mapreduce.framework.name
yarn
mapreduce.jobhistory.address
0.0.0.0:10020
mapreduce.jobhistory.webapp.address
0.0.0.0:19888
6.7 格式化NameNode
hadoop namenode -format
6.8 配置启用用户信息（在master容器下）
cd /usr/local/hadoop-3.2.1
vi ./sbin/start-dfs.sh 和 ./sbin/stop-dfs.sh
HDFS_NAMENODE_USER=root
HDFS_DATANODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
vi ./sbin/start-yarn.sh 和 ./sbin/stop-yarn.sh
YARN_RESOURCEMANAGER_USER=root
YARN_NODEMANAGER_USER=root
# 启动hadoop集群(nna容器)
start-all.sh
# nna容器查看进程
jps
