（1）基础配置
- mysql5.7系列
- JDK1.8+
- ZooKeeper3.4.6+
- hadoop2.6+
（2）tar安装包下载
https://www.apache.org/dyn/closer.lua/dolphinscheduler/2.0.0/apache-dolphinscheduler-2.0.0-bin.tar.gz
（3）安装包上传
集群模式：上传到每台节点
配置文件：master节点配置即可
2、解压安装包tar -xzvf apache-dolphinscheduler-2.0.0-bin.tar.gz -C /opt/module/ apache-dolphinscheduler-2.0.0-bin mv /opt/module/apache-dolphinscheduler-2.0.0-bin/ /opt/module/apache-dolphinscheduler-bin/3、创建部署用户
- 单机版本：在单个节点创建
# 创建用户 useradd dolphinscheduler; # 添加密码 echo "dolphinscheduler" | passwd --stdin dolphinscheduler10.4、配置免密登录
# 配置sudo免密 sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers5、赋予用户目录操作权限
#部署用户对dolphinscheduler-bin目录有操作权限 chown -R dolphinscheduler:dolphinscheduler dolphinscheduler-bin6、ssh免密配置
- 切换到部署用户并配置ssh本机免密登录
su dolphinscheduler ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys chmod 600 ~/.ssh/authorized_keys7、mysql数据库初始化
（1）将mysql的驱动包放到/lib驱动目录下
cp mysql-connector-java-5.1.27.jar /opt/module/apache-dolphinscheduler-bin/lib/
（2）在mysql执行以下命令
CREATE DATAbase dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%' IDENTIFIED BY '123456'; GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost' IDENTIFIED BY '123456'; flush privileges;
（3）修改配置文件datasource.properties
vim datasource.properties
- 将PG数据库相关注释掉，修改mysql相关配置
sudo ln -s /usr/java/jdk1.8.0_181-cloudera /usr/bin/java9、修改ds运行环境配置文件dolphinscheduler_env.sh
vim dolphinscheduler_env.sh export HADOOP_HOME=/opt/cloudera/parcels/CDH/lib/hadoop export HADOOP_CONF_DIR=/opt/cloudera/parcels/CDH/lib/hadoop/etc/hadoop export SPARK_HOME1=/opt/cloudera/parcels/CDH/lib/spark export SPARK_HOME2=/opt/cloudera/parcels/CDH/lib/spark export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera export HIVE_HOME=/opt/cloudera/parcels/CDH/lib/hive #export Flink_HOME=/opt/soft/flink export DATAX_HOME=/opt/module/datax export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$DATAX_HOME/bin:$PATH10、修改一键部署配置文件conf/config/install_config.conf中的各参数
vim install_config.conf #修改以下参数 # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR ConDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # --------------------------------------------------------- # INSTALL MACHINE # --------------------------------------------------------- # A comma separated list of machine hostname or IP would be installed DolphinScheduler, # including master, worker, api, alert. If you want to deploy in pseudo-distributed # mode, just write a pseudo-distributed hostname # Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" ips="localhost" # Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine # modify it if you use different ssh port sshPort="22" # A comma separated list of machine hostname or IP would be installed Master server, it # must be a subset of configuration `ips`. # Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" masters="localhost" # A comma separated list of machine11、运行建表脚本: or : .All hostname or IP must be a # subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts # Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" workers="localhost:default" # A comma separated list of machine hostname or IP would be installed alert server, it # must be a subset of configuration `ips`. # Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" alertServer="localhost" # A comma separated list of machine hostname or IP would be installed API server, it # must be a subset of configuration `ips`. # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" apiServers="localhost" # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. # Do not set this configuration same as the current path (pwd) installPath="/opt/module/dolphinscheduler" # The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` # script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs # to be created by this user deployUser="dolphinscheduler" # The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory. databasedirPath="/tmp/dolphinscheduler" # --------------------------------------------------------- # DolphinScheduler ENV # --------------------------------------------------------- # JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler # and this configuration only support one parameter so far. javaHome="/usr/java/jdk1.8.0_181-cloudera" # DolphinScheduler API service port, also this is your DolphinScheduler UI component's URL port, default value is 12345 apiServerPort="123456" # --------------------------------------------------------- # Database # NOTICE: If database value has special characters, such as `.*[]^${}+?|()@#&`, Please add prefix `` for escaping. # --------------------------------------------------------- # The type for the metadata database # Supported values: ``postgresql``, ``mysql``. dbtype="mysql" # The : connection pair DolphinScheduler connect to the metadata database dbhost="localhost:3306" # The username DolphinScheduler connect to the metadata database username="dolphinscheduler" # The password DolphinScheduler connect to the metadata database password="123456" # The database DolphinScheduler connect to the metadata database dbname="dolphinscheduler" # --------------------------------------------------------- # Registry Server # --------------------------------------------------------- # Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir. # For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`. # If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html registryPluginDir="lib/plugin/registry/zookeeper" # Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency registryPluginName="zookeeper" # Registry Server address. registryServers="hadoop106:2181" # The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. zkRoot="/dolphinscheduler" # --------------------------------------------------------- # alert Server # --------------------------------------------------------- # alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir. alertPluginDir="lib/plugin/alert" # --------------------------------------------------------- # Worker Task Server # --------------------------------------------------------- # Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir. taskPluginDir="lib/plugin/task" # resource storage type: HDFS, S3, NONE resourceStorageType="HDFS" # resource store on HDFS/S3 path, resource file will store to this hdfs path, self configuration, please make sure the directory exists on hdfs and has read write permissions. "/dolphinscheduler" is recommended resourceUploadPath="/data/dolphinscheduler" # if resourceStorageType is HDFS，defaultFS write namenode address，HA, you need to put core-site.xml and hdfs-site.xml in the conf directory. # if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， # Note，S3 be sure to create the root directory /dolphinscheduler defaultFS="hdfs://hadoop106:8020" # if resourceStorageType is S3, the following three configuration is required, otherwise please ignore #s3Endpoint="http://192.168.xx.xx:9010" #s3AccessKey="xxxxxxxxxx" #s3SecretKey="xxxxxxxxxx" # resourcemanager port, the default value is 8088 if not specified resourceManagerHttpAddressPort="8088" # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single node, keep this value empty #yarnHaIps="" # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single node, you only need to replace 'yarnIp1' to actual resourcemanager hostname singleYarnIp="hadoop106" # who has permission to create directory under HDFS/S3 root path # Note: if kerberos is enabled, please config hdfsRootUser= hdfsRootUser="hdfs" # kerberos config # whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore kerberosStartUp="false" # kdc krb5 config file path krb5ConfPath="$installPath/conf/krb5.conf" # keytab username,watch out the @ sign should followd by \ keytabUserName="hdfs-mycluster\@ESZ.COM" # username keytab path keytabPath="$installPath/conf/hdfs.headless.keytab" # kerberos expire time, the unit is hour kerberosExpireTime="2" # use sudo or not sudoEnable="true" # worker tenant auto create workerTenantAutoCreate="false"
sh script/create-dolphinscheduler.sh12、运行一键部署脚本
- 资源中心，运行以下命令
sudo mkdir -p /data/dolphinscheduler sudo chown -R dolphinscheduler:dolphinscheduler /data/dolphinscheduler
- 运行一键部署脚本
sh install.sh
- jps查看相关服务
http://hadoop106:12345/dolphinscheduler 账号:admin 密码:dolphinscheduler123附件：资源下载
链接：https://pan.baidu.com/s/1LHCbMwrDsV3VB_P3p7o5kw 提取码：ntjj
