Windos HDFS客户端操作

HDFS客户端环境准备

1．根据自己电脑的操作系统拷贝对应的编译后的hadoop jar包到非中文路径

2．配置HADOOP_HOME环境变量

 3.重新启动电脑

4.创建一个Maven工程hdfs

5．pom.xml中导入相应的依赖坐标+日志添加

    4.0.0

    com.9343
    hdfs
    1.0-SNAPSHOT

    
        junit
        junit
        4.12
    
    
        org.apache.logging.log4j
        log4j-slf4j-impl
        2.12.0
    
    
        org.apache.hadoop
        hadoop-client-api
        3.1.3
    
    
        org.apache.hadoop
        hadoop-client-runtime
        3.1.3
    

6.在项目的src/main/resources目录下，新建一个文件，命名为“log4j2.xml”，在文件中在文件中填入

7.创建包名创建HdfsClient类

package hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;

public class HDFSClient {
    FileSystem fileSystem;

    @Test
    public void put() throws IOException, InterruptedException {

        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "2"); //设置副本数量
        configuration.set("dfs.blocksize", "67108864"); //设置磁盘块的大小
        //1.新建HDFS对象
        fileSystem = FileSystem.get(URI.create("hdfs://hadoop101:8020"), configuration, "lisi");//整个HDFS集群
        //2.操作集群
        fileSystem.copyFromLocalFile(new Path("D:\JavaWeb.md"), new Path("/"));
        //3.关闭资源
        fileSystem.close();
    }

    @Before
    public void before() throws IOException, InterruptedException {
        fileSystem = FileSystem.get(URI.create("hdfs://hadoop101:8020"), new Configuration(), "lisi");
    }

    @After
    public void After() throws IOException {
        fileSystem.close();
    }

    
    @Test
    public void get() throws IOException {
        fileSystem.copyToLocalFile(new Path("/JavaWeb.md"), new Path("D:\"));
    }

    
    @Test
    public void append() throws IOException {
        FSDataOutputStream append = fileSystem.append(new Path("/README.txt"));//拿到输出流
        append.write("TestTest".getBytes());//往流入写入
        IOUtils.closeStream(append);//关闭流
    }

    
    @Test
    public void delete() throws IOException {
        fileSystem.delete(new Path("/1_出了问题看日志.mp4"), true);
    }

    
    @Test
    public void ls() throws IOException {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            System.out.println(fileStatus.getPath());//获取文件名
            System.out.println(fileStatus.getOwner());//文件拥有者
            System.out.println("========================");
        }
    }

    
    @Test
    public void lf() throws IOException {
        RemoteIterator locatedFileStatusRemoteIterator =
                fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus fileStatus = locatedFileStatusRemoteIterator.next();
            System.out.println(fileStatus.getPath());
            System.out.println(fileStatus.getOwner());
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (int i = 0; i < blockLocations.length; i++)  {
                System.out.println("第"+i+"块");
                String[] hosts = blockLocations[i].getHosts();
                for (String host : hosts) {
                    System.out.print("host  " + host);
                }
                System.out.println();
            }
            System.out.println("=====================================================");
        }
    }

    
    @Test
    public void mv() throws IOException {
        fileSystem.rename(new Path("/JavaWeb.md"),new Path("/logs/JavaWeb.md"));
    }
}