脚本test_数据挖掘与分析

基本环境脚本

#!bin/bash
#基础部分
ntp_path=/etc/ntp.conf
java_version=/usr/package277/jdk-8u221-linux-x64.tar.gz
hadoop_path=/usr/package277/hadoop-2.7.7.tar.gz

echo "==============正在关闭防火墙============="
for host in master slave1 slave2
do
  echo "================$host=================="
  ssh $host systemctl stop firewalld
done
mkdir /usr/java
echo server 127.127.1.0  >> $ntp_path
echo fudge 127.127.1.0 stratum 10 >> $ntp_path 
echo '正在开启ntp'
systemctl restart ntpd
#java部分
tar -xzvf $java_version -C  /usr/java
scp -r /usr/java root@slave1:/usr
scp -r /usr/java root@slave2:/usr

环境变量追加脚本

#!bin/bash
path=/etc/profile
echo export script_HOME=/usr/script >> $path
echo =====正在添加java环境变量=====
echo #JAVA_HOME >> $path
echo export JAVA_HOME=/usr/java/jdk1.8.0_221 >> $path
echo export JRE_HOME=$JAVA_HOME/jre >> $path
echo export CLASSPATH=$JAVA_HOME/lib/:$JRE_HOME/lib >> $path
echo export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin  >> $path
echo export PATH JAVA_HOME CLASSPATH  JRE_HOME >> $path
echo ======Java添加完成=======

echo ======添加Hadoop环境变量 
echo #Hadoop PATH  >> $path
echo export HADOOP_HOME=/usr/hadoop/hadoop-2.7.7 >> $path
echo export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib >> $path
echo export PATH=$PATH:$HADOOP_HOME/bin >> $path
echo export PATH=$PATH:$HADOOP_HOME/sbin >> $path
echo ==========Hadoop添加完成===========

echo ==========添加zookeeper环境变量===========
echo #zookeeprt PATH >> $path
echo export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.14 >> $path
echo PATH=$PATH:$ZOOKEEPER_HOME/bin >> $path
echo ========zookeeper添加完成========

echo ========添加hive环境变量=========
echo #HIVE_HOME >> $path
echo export HIVE_HOME=/usr/hive/apache-hive-2.3.4-bin >> $path
echo export PATH=$PATH:$HIVE_HOME/bin >> $path
echo ===========hive添加完成===========
sh /usr/script/xsync /etc/profile

export JAVA_HOME=/usr/java/jdk1.8.0_171 export CLASSPATH= J A V A H O M E / l i b / e x p o r t P A T H = JAVA_HOME/lib/ export PATH= JAVAHOME/lib/exportPATH=PATH:$JAVA_HOME/bin export PATH JAVA_HOME CLASSPATH

关闭防火墙

#!bin/bash
echo "==============正在关闭防火墙============="
for host in master slave1 slave2
do
  echo "================$host=================="
  ssh $host systemctl stop firewalld
done

systemctl stop firewalld

hostnamectl set-hostname master/slave1/slave2

分发脚本

#1. 判断参数个数
if [ $# -lt 1 ]
then
 echo Not Enough Arguement!
 exit;
fi
#2. 遍历集群所有机器
for host in master slave1 slave2
do
 echo ==================== $host ====================
 #3. 遍历所有目录，挨个发送
 for file in $@
 do
 #4. 判断文件是否存在
 if [ -e $file ]
 then
 #5. 获取父目录
 pdir=$(cd -P $(dirname $file); pwd)
 #6. 获取当前文件的名称
 fname=$(basename $file)
 ssh $host "mkdir -p $pdir"
 rsync -av $pdir/$fname $host:$pdir
 else
 echo $file does not exists!
 fi
 done
done

Hadoop脚本

#!bin/bash
path=/etc/profile
hadoop_path=/usr/package277/hadoop-2.7.7.tar.gz
mkdir /usr/hadoop
tar -xzvf $hadoop_path -C  /usr/hadoop
cd /usr/hadoop/hadoop-2.7.7/etc/hadoop

echo #Hadoop PATH  >> $path
echo export HADOOP_HOME=/usr/hadoop/hadoop-2.7.7 >> $path
echo export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib >> $path

echo 'export JAVA_HOME=/usr/java/jdk1.8.0_221'>>hadoop-env.sh
echo 'export JAVA_HOME=/usr/java/jdk1.8.0_221'>>yarn-env.sh
echo ======hadoop-env.sh添加完成====

echo '



    fs.default.name
    hdfs://master:9000


    hadoop.tmp.dir
    /root/hadoopData/tmp
A base for other temporary directories.


    io.file.buffer.size
    131072


    fs.checkpoint.period
    60


    fs.checkpoint.size
    67108864


 
 hadoop.http.staticuser.user
 root
 
'>core-site.xml
echo ======core-site.xml添加完成====

echo '


    yarn.resourcemanager.address
    master:18040


    yarn.resourcemanager.scheduler.address
    master:18030


    yarn.resourcemanager.webapp.address
    master:18088


    yarn.resourcemanager.resource-tracker.address
    master:18025


    yarn.resourcemanager.admin.address
    master:18141


    yarn.nodemanager.aux-services
    mapreduce_shuffle


    yarn.nodemanager.auxservices.mapreduce.shuffle.class
    org.apache.hadoop.mapred.ShuffleHandler


'>yarn-site.xml
echo ====yarn-site.xml添加完成====

echo 'slave1
slave2
slave3'>slaves
echo ======slaves添加成功=====

echo 'master'>master
echo ======master添加成功=====

echo '



    dfs.replication
    2


    dfs.namenode.name.dir
    file:/root/hadoopData/name
    true


    dfs.datanode.data.dir
    file:/root/hadoopData/data
    true


    dfs.namenode.secondary.http-address
    master:9001


    dfs.webhdfs.enabled
    true


    dfs.permissions
    false


        dfs.datanode.use.datanode.hostname
        true



 dfs.namenode.http-address
 master:50070
 
'>hdfs-site.xml
echo ====hdfs-site.xml添加完成====

cp mapred-site.xml.template mapred-site.xml
echo '



    mapreduce.framework.name
    yarn

'>mapred-site.xml
echo ====mapred-site.xml添加完成====
scp -r /usr/hadoop root@slave2:/usr
scp -r /usr/hadoop root@slave1:/usr
scp -r /usr/java root@slave3:/usr
scp -r /usr/hadoop root@slave3:/usr

刷新变量脚本

#!bin/bash
for host in master slave1 slave2
 do
 ssh $host source /etc/profile
done

hive

#!bin/bash
mkdir /usr/hive
tar -xzvf /usr/package277/apache-hive-2.3.4-bin.tar.gz -C /usr/hive
cp /usr/hive/apache-hive-2.3.4-bin/lib/jline-2.12.jar /usr/hadoop/hadoop-2.7.7/share/hadoop/yarn/lib/

cp /usr/hive/apache-hive-2.3.4-bin/conf/hive-env.sh.template hive-env.sh
touch hive-site.xml

echo 'HADOOP_HOME=/usr/hadoop/hadoop-2.7.7
export HIVE_CONF_DIR=/usr/hive/apache-hive-2.3.4-bin/conf
export HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.3.4-bin/lib' >> hive-env.sh

touch hive-site.xml

cd /usr/hive/apache-hive-2.3.4-bin/conf
cp /usr/hive/apache-hive-2.3.4-bin/lib/jline-2.12.jar /usr/hadoop/hadoop-2.7.7/share/hadoop/yarn/lib/
touch hive-site.xml
echo '

hive.metastore.uris
thrift://slave1:9083



hive.metastore.warehouse.dir
/user/hive_remote/warehouse



javax.jdo.option.ConnectionURL 
jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false



javax.jdo.option.ConnectionDriverName
com.mysql.jdbc.Driver



javax.jdo.option.ConnectionUserName
root



javax.jdo.option.ConnectionPassword
123456


hive.metastore.schema.verification
false


datanucleus.schema.autoCreateAll
true

 ' > hive-site.xml
cp /usr/package277/mysql-connector-java-5.1.47-bin.jar /usr/hive/apache-hive-2.3.4-bin/lib

#!bin/bash
mkdir /usr/hive
tar -xzvf /usr/package277/apache-hive-2.3.4-bin.tar.gz -C /usr/hive
cp /usr/hive/apache-hive-2.3.4-bin/lib/jline-2.12.jar /usr/hadoop/hadoop-2.7.7/share/hadoop/yarn/lib/

cd /usr/hive/apache-hive-2.3.4-bin/conf
cp hive-env.sh.template hive-env.sh
touch hive-site.xml

echo 'HADOOP_HOME=/usr/hadoop/hadoop-2.7.7
export HIVE_CONF_DIR=/usr/hive/apache-hive-2.3.4-bin/conf
export HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.3.4-bin/lib' >> hive-env.sh

touch hive-site.xml
echo '


hive.metastore.warehouse.dir
/user/hive_remote/warehouse



hive.metastore.local
false



hive.metastore.uris
thrift://slave1:9083


' > hive-site.xml

zookeeper

#!bin/bash
cd /usr
mkdir zookeeper
tar /usr/package277/zookeeper-3.4.14.tar.gz -C /usr/zookeeper
cd /usr/zookeeper/zookeeper-3.4.14/
mkdir zkdata
mkdir zkdatalog
cd /usr/zookeeper/zookeeper-3.4.14/conf/
mv zoo_sample.cfg zoo.cfg
echo 'tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/zookeeper/zookeeper-3.4.14/zkdata
clientPort=2181
dataLogDir=/usr/zookeeper/zookeeper-3.4.14/zkdatalog
server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888'>zoo.cfg

cd /usr/zookeeper/zookeeper-3.4.14/zkdata
echo '1'>myid

建表

hadoop dfs -mkdir /college
hadoop dfs -put /root/college/fraud.txt /college/

create table fraud(
id string,
state string,
class string,
loss bigint,
degree string,
mode string,
source string,
uppertime string,
lowertime string,
place string,
accept_unit string,
accept_time string,
take_time string,
police string,
break_time string
) ROW FORMAT DELIMITED FIELDS TERMINATED BY 't';

load data local inpath '/root/college/fraud.txt' into table fraud;

insert overwrite local directory 
'/root/fraud01'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select sum(loss) from fraud where take_time like '2017-03%';

insert overwrite local directory 
'/root/fraud02'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select class,sum(loss) from fraud group by class limit 1,3;
select class,sum(loss) from fraud group by class having class='d' ;

select sum(loss) from fraud
join (select class from fraud order by loss limit 1) as tmp
on (tmp.class=fraud.class);

insert overwrite local directory 
'/root/fraud03'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select sum(loss) from fraud where like take_time ='2016-03%';

insert overwrite local directory 
'/root/fraud04'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select count(*) from fraud where degree ='特别巨大' group by class limit 1;

insert overwrite local directory 
'/root/fraud05'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select place,sum(loss) from fraud group by place limit;

insert overwrite local directory 
'/root/fraud06'
ROW FORMAT DELIMITED FIELDS TERMINATED BY 't'
select sum(loss) from fraud where take_time regexp '2016-0[1-3]%'

语句 &坑

hive --service metastore

set ff=unix

脚本test

数据挖掘与分析相关栏目本月热门文章