[root@master ~]# cd /usr/local/soft/hadoop-2.7.6/ [root@master hadoop-2.7.6]# ls bin include libexec logs README.txt share etc lib LICENSE.txt NOTICE.txt sbin tmp [root@master hadoop-2.7.6]# cd etc/ [root@master etc]# ls hadoop [root@master etc]# cd hadoop/ [root@master hadoop]# ls capacity-scheduler.xml kms-env.sh configuration.xsl kms-log4j.properties container-executor.cfg kms-site.xml core-site.xml log4j.properties core-site.xml.0 mapred-env.cmd hadoop-env.cmd mapred-env.sh hadoop-env.sh mapred-queues.xml.template hadoop-env.sh.0 mapred-site.xml hadoop-metrics2.properties mapred-site.xml.template hadoop-metrics.properties slaves hadoop-policy.xml slaves.0 hdfs-site.xml ssl-client.xml.example hdfs-site.xml.0 ssl-server.xml.example httpfs-env.sh yarn-env.cmd httpfs-log4j.properties yarn-env.sh httpfs-signature.secret yarn-site httpfs-site.xml yarn-site.xml kms-acls.xml yarn-site.xml.0 [root@master hadoop]# pwd /usr/local/soft/hadoop-2.7.6/etc/hadoop
修改hdfs-site.xml
dfs.permissions
true
[root@master hadoop]# vim hdfs-site.xml [root@master hadoop]# scp hdfs-site.xml node1:`pwd` hdfs-site.xml 100% 960 2.1MB/s 00:00 [root@master hadoop]# scp hdfs-site.xml node2:`pwd` hdfs-site.xml 100% 960 1.4MB/s 00:00 [root@master hadoop]# stop-dfs.sh Stopping namenodes on [master] master: stopping namenode node1: stopping datanode node2: stopping datanode Stopping secondary namenodes [0.0.0.0] 0.0.0.0: stopping secondarynamenode [root@master hadoop]# jps 82486 Jps 81785 ResourceManager [root@master hadoop]# start-dfs.sh Starting namenodes on [master] master: starting namenode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-namenode-master.out node2: starting datanode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-datanode-node2.out node1: starting datanode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-datanode-node1.out Starting secondary namenodes [0.0.0.0] 0.0.0.0: starting secondarynamenode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-secondarynamenode-master.out [root@master hadoop]# jps 82614 NameNode 81785 ResourceManager 82825 SecondaryNameNode 82957 Jps
[root@master hadoop]# hdfs dfs -chmod 764 /student [root@master hadoop]# useradd shujia [root@master hadoop]# passwd shujia 更改用户 shujia 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。 [root@master hadoop]# su shujia [shujia@master hadoop]$ cd ~ [shujia@master ~]$ pwd /home/shujia [shujia@master ~]$ hdfs dfs -ls / Found 8 items drwxr-xr-x - lenovo supergroup 0 2022-03-23 10:04 /TestAPI drwxr-xr-x - root supergroup 0 2022-05-17 20:53 /data drwxr-xr-x - root supergroup 0 2022-04-16 09:43 /hbase drwxr-xr-x - root supergroup 0 2022-04-18 19:47 /sqoop drwxrw-r-- - root supergroup 0 2022-04-06 19:33 /student drwx------ - root supergroup 0 2022-04-02 16:34 /tmp drwx------ - root supergroup 0 2022-05-15 20:25 /user drwxr-xr-x - root supergroup 0 2022-04-28 17:53 /zzj [shujia@master ~]$ hdfs dfs -ls /student ls: Permission denied: user=shujia, access=READ_EXECUTE, inode="/student":root:supergroup:drwxrw-r--
[root@master ~]# hdfs dfs -chmod o+x /student [shujia@master ~]$ hdfs dfs -ls / Found 8 items drwxr-xr-x - lenovo supergroup 0 2022-03-23 10:04 /TestAPI drwxr-xr-x - root supergroup 0 2022-05-17 20:53 /data drwxr-xr-x - root supergroup 0 2022-04-16 09:43 /hbase drwxr-xr-x - root supergroup 0 2022-04-18 19:47 /sqoop drwxrw-r-x - root supergroup 0 2022-04-06 19:33 /student drwx------ - root supergroup 0 2022-04-02 16:34 /tmp drwx------ - root supergroup 0 2022-05-15 20:25 /user drwxr-xr-x - root supergroup 0 2022-04-28 17:53 /zzj [shujia@master ~]$ hdfs dfs -ls /student Found 1 items drwxr-xr-x - root supergroup 0 2022-04-06 19:43 /student/input1
[shujia@master ~]$ ls [shujia@master ~]$ vim words.txt [shujia@master ~]$ hdfs dfs -put words.txt /student/input1/ put: Permission denied: user=shujia, access=WRITE, inode="/student/input1":root:supergroup:drwxr-xr-x [shujia@master ~]$ hdfs dfs -chmod o+w /student/input1 chmod: changing permissions of '/student/input1': Permission denied. user=shujia is not the owner of inode=/student/input1 [root@master ~]# hdfs dfs -chmod o+w /student/input1 [shujia@master ~]$ hdfs dfs -put words.txt /student/input1/ [shujia@master ~]$ hdfs dfs -ls /student/input1/ Found 6 items drwxr-xr-x - root supergroup 0 2022-04-06 19:43 /student/input1/pt=20220401 drwxr-xr-x - root supergroup 0 2022-04-06 19:43 /student/input1/pt=20220402 drwxr-xr-x - root supergroup 0 2022-04-06 19:43 /student/input1/pt=20220403 drwxr-xr-x - root supergroup 0 2022-04-06 19:43 /student/input1/pt=20220404 drwxr-xr-x - root supergroup 0 2022-04-06 19:38 /student/input1/pt=20220405 -rw-r--r-- 1 shujia supergroup 25 2022-05-30 16:03 /student/input1/words.txt
修改hdfs-site.xml
dfs.namenode.acls.enabled
true
[root@master ~]# cd /usr/local/soft/hadoop-2.7.6/etc/hadoop/ [root@master hadoop]# ls capacity-scheduler.xml kms-env.sh configuration.xsl kms-log4j.properties container-executor.cfg kms-site.xml core-site.xml log4j.properties core-site.xml.0 mapred-env.cmd hadoop-env.cmd mapred-env.sh hadoop-env.sh mapred-queues.xml.template hadoop-env.sh.0 mapred-site.xml hadoop-metrics2.properties mapred-site.xml.template hadoop-metrics.properties slaves hadoop-policy.xml slaves.0 hdfs-site.xml ssl-client.xml.example hdfs-site.xml.0 ssl-server.xml.example httpfs-env.sh yarn-env.cmd httpfs-log4j.properties yarn-env.sh httpfs-signature.secret yarn-site httpfs-site.xml yarn-site.xml kms-acls.xml yarn-site.xml.0 [root@master hadoop]# vim hdfs-site.xml [root@master hadoop]# scp hdfs-site.xml node1:`pwd` hdfs-site.xml 100% 1066 1.7MB/s 00:00 [root@master hadoop]# scp hdfs-site.xml node2:`pwd` hdfs-site.xml 100% 1066 1.9MB/s 00:00 [root@master hadoop]# stop-dfs.sh Stopping namenodes on [master] master: stopping namenode node2: stopping datanode node1: stopping datanode Stopping secondary namenodes [0.0.0.0] 0.0.0.0: stopping secondarynamenode [root@master hadoop]# jps 86052 ResourceManager 86936 Jps [root@master hadoop]# start-dfs.sh Starting namenodes on [master] master: starting namenode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-namenode-master.out node1: starting datanode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-datanode-node1.out node2: starting datanode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-datanode-node2.out Starting secondary namenodes [0.0.0.0] 0.0.0.0: starting secondarynamenode, logging to /usr/local/soft/hadoop-2.7.6/logs/hadoop-root-secondarynamenode-master.out [root@master hadoop]# jps 87281 SecondaryNameNode 87072 NameNode 87411 Jps 86052 ResourceManager
[root@master hadoop]# useradd ods [root@master hadoop]# passwd ods 更改用户 ods 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。 [root@master hadoop]# useradd dwi [root@master hadoop]# passwd dwi 更改用户 dwi 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。 [root@master hadoop]# useradd dws [root@master hadoop]# passwd dws 更改用户 dws 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。 [root@master hadoop]# useradd dim [root@master hadoop]# passwd dim 更改用户 dim 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。 [root@master hadoop]# useradd dal [root@master hadoop]# passwd dal 更改用户 dal 的密码 。 新的 密码： 无效的密码： 密码少于 8 个字符 重新输入新的 密码： passwd：所有的身份验证令牌已经成功更新。
[root@master hadoop]# cd .. [root@master etc]# cd .. [root@master hadoop-2.7.6]# cd .. [root@master soft]# ls 0?? hive-1.2.1 redis show A?? jars redis-6.2.6 spark-2.4.5 data jdk1.8.0_171 scripts sqoop-1.4.7 hadoop-2.7.6 packages shell test.txt hbase-1.4.6 phoenix-4.15.0 shell01 zookeeper-3.4.6 [root@master soft]# cd hive-1.2.1/ [root@master hive-1.2.1]# ls bin examples lib NOTICE RELEASE_NOTES.txt conf hcatalog LICENSE README.txt scripts [root@master hive-1.2.1]# cd conf/ [root@master conf]# ls beeline-log4j.properties.template hive-default.xml.template hive-env.sh hive-env.sh.template hive-exec-log4j.properties.template hive-log4j.properties.template hive-site.xml ivysettings.xml [root@master conf]# vim hive-site.xml [root@master conf]# hive --service metastore Starting Hive Metastore Server [root@master conf]# hive Logging initialized using configuration in jar:file:/usr/local/soft/hive-1.2.1/lib/hive-common-1.2.1.jar!/hive-log4j.properties hive> show databases; OK default stu stu_spark test1 testsqoop Time taken: 0.183 seconds, Fetched: 5 row(s) hive> drop database stu cascade; OK Time taken: 1.239 seconds hive> drop database stu_spark cascade; OK Time taken: 0.414 seconds hive> drop database test1 cascade; OK Time taken: 0.375 seconds hive> drop database testsqoop cascade; OK Time taken: 0.193 seconds hive> use default; OK Time taken: 0.013 seconds hive> show tables; OK ods_yiqing_data student students_hbase Time taken: 0.013 seconds, Fetched: 3 row(s) hive> drop table ods_yiqing_data; OK Time taken: 0.104 seconds hive> drop table student; OK Time taken: 0.066 seconds hive> drop table students_hbase; OK Time taken: 0.081 seconds hive> show databases; OK default Time taken: 0.008 seconds, Fetched: 1 row(s) hive> create database ods; OK Time taken: 0.138 seconds hive> create database dwi; OK Time taken: 0.128 seconds hive> create database dws; OK Time taken: 0.141 seconds hive> create database dim; OK Time taken: 0.122 seconds hive> create database dal; OK Time taken: 0.025 seconds
[root@master conf]# hdfs dfs -mkdir -p /daas/motl/ods [root@master conf]# hdfs dfs -mkdir -p /daas/motl/dwi [root@master conf]# hdfs dfs -mkdir -p /daas/motl/dws [root@master conf]# hdfs dfs -mkdir -p /daas/motl/dim [root@master conf]# hdfs dfs -mkdir -p /daas/motl/dal [root@master conf]# hdfs dfs -chmod 750 /daas/motl/ods [root@master conf]# hdfs dfs -chmod 750 /daas/motl/dwi [root@master conf]# hdfs dfs -chmod 750 /daas/motl/dws [root@master conf]# hdfs dfs -chmod 750 /daas/motl/dim [root@master conf]# hdfs dfs -chmod 750 /daas/motl/dal [root@master conf]# hdfs dfs -chown ods:ods /daas/motl/ods [root@master conf]# hdfs dfs -chown dwi:dwi /daas/motl/dwi [root@master conf]# hdfs dfs -chown dws:dws /daas/motl/dws [root@master conf]# hdfs dfs -chown dim:dim /daas/motl/dim [root@master conf]# hdfs dfs -chown dal:dal /daas/motl/dal
[root@master ~]# exit 登出 Connection closed. Disconnected from remote host(master) at 15:07:02. Type `help' to learn how to use Xshell prompt. [C:~]$ ssh ods@master Host 'master' resolved to fe80::20c:29ff:fe5c:f5e7. Connecting to fe80::20c:29ff:fe5c:f5e7:22... Connection established. To escape to local shell, press Ctrl+Alt+]. /usr/bin/xauth: file /home/ods/.Xauthority does not exist [ods@master ~]$
[root@master soft]# chmod 777 -R /usr/local/soft/hive-1.2.1
[root@master soft]# ll
总用量 28
-rw-r--r--. 1 root root 0 3月 21 16:52 0??
-rw-r--r--. 1 root root 0 3月 21 16:52 A??
drwxr-xr-x. 3 root root 4096 5月 17 21:06 data
drwxr-xr-x. 11 20415 101 172 3月 21 17:48 hadoop-2.7.6
drwxr-xr-x. 8 root root 172 4月 11 20:32 hbase-1.4.6
drwxrwxrwx. 8 root root 159 5月 31 14:58 hive-1.2.1
drwxr-xr-x. 3 root root 262 5月 18 09:44 jars
drwxr-xr-x. 8 10 143 255 3月 29 2018 jdk1.8.0_171
drwxr-xr-x. 2 root root 286 5月 12 14:54 packages
drwxr-xr-x. 5 1411715874 wheel 4096 12月 17 2019 phoenix-4.15.0
drwxr-xr-x. 3 root root 68 3月 20 20:51 redis
drwxrwxr-x. 7 root root 4096 10月 4 2021 redis-6.2.6
drwxr-xr-x. 2 root root 61 4月 8 15:56 scripts
drwxr-xr-x. 2 root root 4096 2月 28 20:58 shell
drwxr-xr-x. 2 root root 37 3月 2 15:48 shell01
drwxr-xr-x. 2 root root 72 2月 26 11:13 show
drwxr-xr-x. 14 test test 223 5月 12 15:20 spark-2.4.5
drwxr-xr-x. 10 test test 4096 4月 18 19:44 sqoop-1.4.7
-rw-r--r--. 1 root root 55 3月 2 15:33 test.txt
drwxr-xr-x. 11 test test 4096 3月 24 14:18 zookeeper-3.4.6
[root@master ~]# hdfs dfs -chmod 777 /tmp
[root@master ~]# hive --service metastore
Starting Hive Metastore Server
[ods@master ~]$ hive
Logging initialized using configuration in jar:file:/usr/local/soft/hive-1.2.1/lib/hive-common-1.2.1.jar!/hive-log4j.properties
hive> show databases;
OK
dal
default
dim
dwi
dws
ods
Time taken: 0.535 seconds, Fetched: 6 row(s)
hive> use ods;
OK
Time taken: 0.025 seconds
hive> create table ods.student
> (
> id string,
> name string,
> age int,
> gender string,
> clazz string
> )
> ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
> STORED AS textfile
> location '/daas/motl/ods/student/';
OK
Time taken: 0.86 seconds
[ods@master ~]$ hdfs dfs -ls /daas/motl/ods/
Found 1 items
drwxr-x--- - ods ods 0 2022-05-31 15:22 /daas/motl/ods/student
[ods@master ~]$ ll
总用量 44
-rw-rw-r--. 1 ods ods 41998 5月 31 15:26 students.txt
[ods@master ~]$ hdfs dfs -put students.txt /daas/motl/ods/student
[ods@master ~]$ hdfs dfs -ls /daas/motl/ods/student/
Found 1 items
-rw-r--r-- 1 ods ods 41998 2022-05-31 15:26 /daas/motl/ods/student/students.txt
hive> select * from student limit 10;
OK
1500100001 施笑槐 22 女 文科六班
1500100002 吕金鹏 24 男 文科六班
1500100003 单乐蕊 22 女 理科六班
1500100004 葛德曜 24 男 理科三班
1500100005 宣谷芹 22 女 理科五班
1500100006 边昂雄 21 男 理科二班
1500100007 尚孤风 23 女 文科六班
1500100008 符半双 22 女 理科六班
1500100009 沈德昌 21 男 理科一班
1500100010 羿彦昌 23 男 理科六班
Time taken: 0.05 seconds, Fetched: 10 row(s)
[root@master ~]# chmod 777 -R /usr/local/soft/hive-1.2.1 [dwi@master ~]$ hive Logging initialized using configuration in jar:file:/usr/local/soft/hive-1.2.1/lib/hive-common-1.2.1.jar!/hive-log4j.properties hive> use ods; OK Time taken: 0.553 seconds hive> show tables; OK student Time taken: 0.145 seconds, Fetched: 1 row(s) [root@master ~]# hdfs dfs -setfacl -R -m user:dwi:r-x /daas/motl/ods hive> select * from student limit 10; OK 1500100001 施笑槐 22 女 文科六班 1500100002 吕金鹏 24 男 文科六班 1500100003 单乐蕊 22 女 理科六班 1500100004 葛德曜 24 男 理科三班 1500100005 宣谷芹 22 女 理科五班 1500100006 边昂雄 21 男 理科二班 1500100007 尚孤风 23 女 文科六班 1500100008 符半双 22 女 理科六班 1500100009 沈德昌 21 男 理科一班 1500100010 羿彦昌 23 男 理科六班 Time taken: 0.184 seconds, Fetched: 10 row(s)
[root@master ~]# chmod 777 -R /usr/local/soft/hive-1.2.1 [dim@master ~]$ hive Logging initialized using configuration in jar:file:/usr/local/soft/hive-1.2.1/lib/hive-common-1.2.1.jar!/hive-log4j.properties hive> use ods; OK Time taken: 0.514 seconds hive> show tables; OK student Time taken: 0.129 seconds, Fetched: 1 row(s) [root@master ~]# hdfs dfs -ls /daas/motl/ods Found 1 items drwxr-x---+ - ods ods 0 2022-05-31 15:26 /daas/motl/ods/student
