hdfs的ide远程管理

windows下载hadoop2.x版本配置环境变量

 配置pom.xml

    4.0.0
    org.sw.hadoop.api
    hadoop-api-m
    0.0.1-SNAPSHOT

    
        2.7.6
    

    
        
            jdk.tools
            jdk.tools
            1.8
            system
            E:/ProgramFiles/Java/jdk1.8.0_291/lib/tools.jar
        
        
        
            org.apache.hadoop
            hadoop-common
            ${hadoop.version}
        
        
        
            org.apache.hadoop
            hadoop-hdfs
            ${hadoop.version}
        
        
        
        
            org.apache.hadoop
            hadoop-mapreduce-client-core
            ${hadoop.version}
        
        
        
            org.apache.hadoop
            hadoop-yarn-common
            ${hadoop.version}
        
        
        
            org.apache.hadoop
            hadoop-mapreduce-client-common
            ${hadoop.version}
        
        
            junit
            junit
            4.8.2
            compile
        
    


    
        
            
            
                org.apache.maven.plugins
                maven-compiler-plugin
                
                    1.8
                    1.8
                    UTF-8
                
            
        
    

 文件系统对象

#    将代码提取出来
FileSystem fileSystem=null;
Configuration configuration=null;
@Before
    public void before() throws IOException {
    System.setProperty("HADOOP_USER_NAME","root");
    configuration = new Configuration();
    configuration.set("fs.defaultFS","hdfs://192.168.10.101:8020");
    fileSystem = FileSystem.get(configuration);
    }
#    关闭流
@After
    public void testFileSystem() throws IOException {
        fileSystem.close();
    }

 api之上传文件

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\input\file.txt");
        final Path path1 = new Path("/input/file");
        fileSystem.copyFromLocalFile(path,path1);
    }

-rw-r--r--

root

supergroup

2.32 KB

2021/12/7 下午7:11:03

3

128 MB

file

 api之下载文件

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\input");
        final Path path1 = new Path("/input/file");
        //fileSystem.copyFromLocalFile(path,path1);
        fileSystem.copyToLocalFile(path1,path);
    }

 api之创建目录

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\input");
        final Path path1 = new Path("/input/mkdir.txt");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        fileSystem.mkdirs(path1);
    }

drwxr-xr-x

root

supergroup

0 B

2021/12/7 下午7:21:53

0

0 B

mkdir.txt

 api之删除目录

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\input");
        final Path path1 = new Path("/input/mkdir.txt");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        fileSystem.delete(path1,true);
    }
#    true表示可以递归删除目录
#    false只能删除空文件

 api之重命名

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("/input/mkdir.txt");
        final Path path1 = new Path("/input/mkdir");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        //fileSystem.delete(path1,true);
        fileSystem.rename(path,path1);
    }

drwxr-xr-x

root

supergroup

0 B

2021/12/7 下午7:35:04

0

0 B

mkdir

 IOUtil上传文件

@Test
    public void putFile() throws IOException {
    FileInputStream fileInputStream = new FileInputStream(new File("D://input//file.txt"));
    FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/input/file.txt"));
    org.apache.hadoop.io.IOUtils.copyBytes(fileInputStream,fsDataOutputStream,configuration);
    org.apache.hadoop.io.IOUtils.closeStream(fileInputStream);
    org.apache.hadoop.io.IOUtils.closeStream(fsDataOutputStream);
    }

-rw-r--r--

root

supergroup

2.32 KB

2021/12/7 下午8:01:08

3

128 MB

file.txt

IOUtil下载文件

@Test
      public void putFile() throws IOException, URISyntaxException {
          FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.10.101:8020"), configuration);
          FSDataInputStream open = fileSystem.open(new Path("/input/file.txt"));
          FileOutputStream fileOutputStream = new FileOutputStream(new File("D:\output\file.txt"));
          IOUtils.copyBytes(open,fileOutputStream,configuration);
          IOUtils.closeStream(open);
          IOUtils.closeStream(fileOutputStream);
      }
#    本地存储需要指定文件名

 查看文件信息

@Test
        public void statusFile() throws IOException {
            final Path path = new Path("/input/file.txt");
            //获取文件状态
            final RemoteIterator locatedFileStatusRemoteIterator = fileSystem.listLocatedStatus(path);
            while(locatedFileStatusRemoteIterator.hasNext()){
                //取出对象
                final LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
                System.out.println("name:"+next.getPath());
                //获取位置
                final BlockLocation[] blockLocations = next.getBlockLocations();
                for (BlockLocation blockLocation : blockLocations) {
                    System.out.println("当前块的副本位置:"+ Arrays.toString(blockLocation.getHosts()));
                    System.out.println("当前块大小:"+blockLocation.getLength());
                    System.out.println("当前块的副本的IP地址信息:"+Arrays.toString(blockLocation.getNames()));
                    System.out.println("系统块大小:"+next.getBlockSize());
                    System.out.println("文件总长度:"+next.getLen());
                }
            }
        }

 log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
name:hdfs://192.168.10.101:8020/input/file.txt
当前块的副本位置:[qianfeng02, qianfeng01, qianfeng03]
当前块大小:2373
当前块的副本的IP地址信息:[192.168.10.102:50010, 192.168.10.101:50010, 192.168.10.103:50010]
系统块大小:134217728
文件总长度:2373