学大数据小胖的第四十四天

package com.shijia.HDFS;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;

public class Demo01TestAPI {
    public static void main(String[] args) throws IOException {
        //创建一个配置文件
        Configuration conf = new Configuration();
        //设置NameNode的地址
         conf.set("fs.defaultFS","hdfs://master:9000");
         //与NameNode建立连接
        FileSystem fs = FileSystem.get(conf);

        System.out.println(fs.exists(new Path("/data")));
        System.out.println(fs.exists(new Path("/data1")));
        //关闭连接
        fs.close();
    } 
}

package com.shijia.HDFS;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;

public class Demo02HDFSAPI {
    FileSystem fs;

   @Before
   public  void init() throws IOException {
       //创建一个配置文件
       Configuration conf = new Configuration();
       //设置NameNode的地址
       conf.set("fs.defaultFS", "hdfs://master:9000");
       //设置副本数量
       conf.set("dfs.replication","1");
       //与NameNode建立连接
       fs = FileSystem.get(conf);
   }

   @Test
   //判断路径存在不存在
   public void exists() throws IOException {
       Path path = new Path("/data");
       System.out.println(fs.exists(path));
   }

   @Test
   //创建一个TestAPI/test1目录
   public void mkdirs() throws IOException {
       fs.mkdirs(new Path("/TestAPI/test1"));
       fs.mkdirs(new Path("/TestAPI/test2"));
   }

   @Test
   public void delete() throws IOException {
       fs.delete(new Path("/TestAPI/test2"),true);
   }

   @Test
   //上传文件到TestAPI/test1
   public void copyFromLocalFile() throws IOException {
       fs.copyFromLocalFile(new Path("data/students.txt"),new Path("/TestAPI/test1"));
   }

   @Test
   //上传文件到/TestAPI/test1,并移除本地文件
   public void moveFromLocalFile() throws IOException {
       fs.moveFromLocalFile(new Path("data/students.txt"), new Path("/TestAPI/test1/students_new.txt"));
   }

   @Test
   //将HDFS上的文件下载到本地，并移除HdFS的文件
  public void moveToLocalFile() throws IOException {
       fs.moveToLocalFile(new Path("/TestAPI/test1/students.txt"),new Path("data/"));
   }

   @Test
   //读取HDFS文件
   public void readHDFSFile() throws IOException {
       FSDataInputStream fsDataInputStream = fs.open(new Path("/TestAPI/test1/students_new.txt"));
       BufferedReader br = new BufferedReader(new InputStreamReader(fsDataInputStream));
       String line;
       while((line=br.readLine())!=null){
           System.out.println(line);
       }
       br.close();
       fsDataInputStream.close();
   }

   @Test
   //将数据写入HDFS
   public void writeHDFSFlie() throws IOException {
       FSDataOutputStream fsDataOutputStream;
       //如果文件存在，追加
       //如果文件不存在，则创建
       Path path = new Path("/TestAPI/test1/newFile.txt");
       if(fs.exists(path)){
           System.out.println("文件已存在，进行追加");
            fsDataOutputStream= fs.append(path);
       }else{
           System.out.println("创建文件");
           fsDataOutputStream=fs.create(path);
       }
       //写入文件
       BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream));
       bw.write("hjm,2018110710");
       bw.newline();
       bw.write("dxb,2018110702");
       bw.newline();
       bw.write("zzj,2018110755");
       bw.newline();
       bw.write("wyc,2018110734");
       bw.newline();
       bw.write("yzw,2018110743");
       bw.newline();
       bw.flush();
       bw.close();
       fsDataOutputStream.close();
   }

   @Test
   //查看/TestAPI/test1目录下的文件
   public void listStatus() throws IOException {
       FileStatus[] fileStatuses = fs.listStatus(new Path("/TestAPI/test1"));
       for (FileStatus fileStatus : fileStatuses) {
           System.out.println(fileStatus.getPath());
       }
   }

   @After
    public  void close() throws IOException {
       fs.close();
   }
}