spark任务调度脚本 同一任务只能有一个在队列总执行

脚本会判断相同任务是否正在执行
如果正在执行中，则忽略
如果没有执行，根据用户传递参数执行任务

#!/bin/bash
. /etc/profile.d/hadoop.sh
#runing=`ps aux | grep 'edu.xd.spark.FenXiShiShiZaiXiao' | grep -v grep | grep -v "bigdata.sh" | wc -l`
runing=`ps aux | grep "$1" | grep -v grep | grep -v "bigdata.sh"  | wc -l`
echo $runing

if [ $runing -lt 1 ];then
        echo "can runing"
        param=$@
        echo $param
        echo param-sub:
        #替换第一个空格前的参数为空
        pnew=`echo $param | sed 's/[^ ]* //'`
        echo $pnew
        dir=$(dirname `which spark-submit`)
        cmd="$dir/$pnew > ./bigdata.log 2>&1  "
        echo "cmd:"$cmd >>  ./bigdata.log  
        eval $cmd
else
        echo "You cannot run two identical tasks at the same time"
fi

运行脚本

/bin/sh ./bigdata.sh  edu.xd.spark.FenXiShiShiZaiXiao   spark-submit --class edu.xd.spark.FenXiShiShiZaiXiao --master yarn /data/app/spark-libs/xdSpark-1.0-SNAPSHOT.jar >> bigdata.log