spark3.x 集群部署

# 配置hosts解析
cat > /etc/hosts < /etc/profile.d/zookeeper.sh <<'EOF'
export ZK_HOME=/opt/zookeeper
export PATH=$ZK_HOME/bin:$PATH
EOF

source /etc/profile


# 复制zookeeper配置文件
cp /opt/zookeeper/conf/{zoo_sample.cfg,zoo.cfg}

# 修改zookeeper配置文件
cat > /opt/zookeeper/conf/zoo.cfg </opt/zookeeper/data/myid
[root@spark02 ~]# echo '2' >/opt/zookeeper/data/myid
[root@spark03 ~]# echo '3' >/opt/zookeeper/data/myid


# 创建zookeeper用户
useradd -r -s /bin/bash zookeeper
chown -R zookeeper:zookeeper /opt/zookeeper


# 使用systemd管理zookeeper服务
cat > /usr/lib/systemd/system/zookeeper.service << EOF
[Unit]
Description=Zookeeper Service
After=network.target

[Service]
Type=forking
User=zookeeper
Group=zookeeper
Environment=JAVA_HOME=/usr/local/java
ExecStart=/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg
ExecStop=/opt/zookeeper/bin/zkServer.sh stop
PIDFile=/opt/zookeeper/data/zookeeper_server.pid
Restart=always
TimeoutSec=20
SuccessExitStatus=130 143


[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload
systemctl start zookeeper
systemctl stop zookeeper
systemctl restart zookeeper
# 启动zookeeper服务并配置为开机启动
systemctl enable --now zookeeper

# 查看zookeeper服务运行状态
zkServer.sh status


# 只有leader节点在监听2888端口
ss -antlp | grep 888

在集群任意一个节点使用zkCli.sh即可连接当前节点的2181端口；连接以后会有一个zk的控制台
zkCli.sh -server spark01:2181

zkCli.sh -server spark01:2181,spark01:2181,spark01:2181
# 查看集群成员配置信息
config

# 配置spark环境变量
cat > /etc/profile.d/spark.sh <<'EOF'
export SPARK_HOME=/opt/spark
export PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
EOF

source /etc/profile

# 编辑spark-env.sh文件
cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh

cat > $SPARK_HOME/conf/spark-env.sh < $SPARK_HOME/conf/workers < 
zookeeper
 参考:https://blog.csdn.net/networken/article/details/116207969
 参考:https://www.codenong.com/js4c594b949b91/