HDFS与Java
1.引入jar包
2.7.1
org.apache.hadoop
hadoop-client
${hadoop.version}
org.apache.hadoop
hadoop-common
${hadoop.version}
org.apache.hadoop
hadoop-hdfs
${hadoop.version}
2.连接Hadoop客户端,去下面的API工具类修改对象位置
1.application.yml
hdfs:
path: hdfs://node1:9000
username: root
2.Java API
@Value("${hdfs.path}")
private String path;
@Value("${hdfs.username}")
private String username;
//获取HDFS配置信息
public Configuration getConf() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", path);
return conf;
}
1.application.yml
hdfs:
path: hdfs://yc #hdfs://node1:9000
username: root
nameservices: yc
namenodes: nn1,nn2
node1: node1:8020
node2: node2:8020
provider: org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
2.Java API
@Value("${hdfs.path}")
private String path;
@Value("${hdfs.username}")
private String username;
@Value("${hdfs.nameservices}")
private String nameservices;
@Value("${hdfs.namenodes}")
private String namenodes;
@Value("${hdfs.node1}")
private String node1;
@Value("${hdfs.node2}")
private String node2;
@Value("${hdfs.provider}")
private String provider;
//获取HDFS配置信息
public Configuration getConf() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", path);
conf.set("dfs.nameservices", nameservices);
conf.set("dfs.ha.namenodes.yc", namenodes);
conf.set("dfs.namenode.rpc-address.yc.nn1", node1);
conf.set("dfs.namenode.rpc-address.yc.nn2", node2);
conf.set("dfs.client.failover.proxy.provider.yc", provider);
return conf;
}
3.API工具类
package com.yc.T1_API.service;
import org.apache.hadoop.fs.BlockLocation;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.ResponseEntity;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
public interface HdfsService {
//获取配置文件,以map形式返回
public Map getConfInfoAsMap() throws InterruptedException, IOException, URISyntaxException;
//在HDFS新建文件夹
public boolean mkdir(String path) throws InterruptedException, IOException, URISyntaxException;
//判断HDFS文件是否存在
public boolean existFile(String path) throws InterruptedException, IOException, URISyntaxException;
//读取HDFS目录信息
public List
package com.yc.service;
import com.yc.util.MyFileUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.*;
import org.apache.zookeeper.common.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
public class HdfsServiceImpl implements HdfsService {
@Value("${hdfs.path}")
private String path;
@Value("${hdfs.username}")
private String username;
@Value("${hdfs.nameservices}")
private String nameservices;
@Value("${hdfs.namenodes}")
private String namenodes;
@Value("${hdfs.node1}")
private String node1;
@Value("${hdfs.node2}")
private String node2;
@Value("${hdfs.provider}")
private String provider;
private final int bufferSize = 1024 * 1024 * 64;
//获取HDFS配置信息
public Configuration getConf() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", path);
conf.set("dfs.nameservices", nameservices);
conf.set("dfs.ha.namenodes.yc", namenodes);
conf.set("dfs.namenode.rpc-address.yc.nn1", node1);
conf.set("dfs.namenode.rpc-address.yc.nn2", node2);
conf.set("dfs.client.failover.proxy.provider.yc", provider);
return conf;
}
//获取HDFS文件系统对象
public FileSystem getFileSystem() throws URISyntaxException, IOException, InterruptedException {
//客户端操作hdfs需要一个用户身份,默认hdfs客户端API会从JVM中获取一个参数作为自己的用户身份
//1.在Windows环境变量中添加一个变量 HADOOP_USER_NAME=root
//2.创建fs对象时,通过参数传递
FileSystem fs = FileSystem.get(new URI(path), getConf(), username);
return fs;
}
//获取配置文件,以map形式返回
@Override
public Map getConfInfoAsMap() throws InterruptedException, IOException, URISyntaxException {
FileSystem fs = getFileSystem();
Configuration conf = getConf();
Iterator> ite = conf.iterator();
Map map = new HashMap<>();
while (ite.hasNext()) {
Map.Entry entry = ite.next();
map.put(entry.getKey(), entry.getValue());
}
return map;
}
//判断HDFS文件是否存在
@Override
public boolean existFile(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return false;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
return fs.exists(srcpath);
}
//新建HDFS目录
@Override
public boolean mkdir(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return false;
}
if (existFile(path)) {//文件已经存在,返回true
return true;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
return fs.mkdirs(srcpath);
}
//读取HDFS目录信息
@Override
public List> readPathInfo(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
FileStatus[] statusList = fs.listStatus(srcpath);//读取出来的状态信息
List> res = new ArrayList<>();
if (statusList != null && statusList.length > 0) {
for (FileStatus status : statusList) {
Map map = new HashMap<>();
map.put("filePath", status.getPath());
map.put("fileStatus", status.toString());
res.add(map);
}
return res;
}
return null;
}
//创建HDFS文件
@Override
public void createFile(String path, MultipartFile file) throws IOException, URISyntaxException, InterruptedException {
if (StringUtils.isEmpty(path) || file.getBytes() == null) {
return;
}
FileSystem fs = getFileSystem();
String fileName = file.getOriginalFilename();
Path newpath = null;
if ("/".equals(path)) {
newpath = new Path(path + fileName);
} else {
newpath = new Path(path + "/" + fileName);
}
//打开一个输出流
FSDataOutputStream outputStream = fs.create(newpath);
outputStream.write(file.getBytes());
outputStream.close();
fs.close();
}
//读取HDFS文件内容
@Override
public String readFile(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
//打开一个输入流
FSDataInputStream inputStream = fs.open(srcpath);
//防止中文乱码
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String lineTxt = "";
StringBuffer sb = new StringBuffer();
while ((lineTxt = reader.readLine()) != null) {
sb.append(lineTxt);
}
return sb.toString();
}
//读取HDFS文件列表
@Override
public List> listFile(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
//递归找到所有文件
RemoteIterator filesList = fs.listFiles(srcpath, true);
List> resList = new ArrayList<>();
while (filesList.hasNext()) {
LocatedFileStatus next = filesList.next();
Map map = new HashMap<>();
map.put("fileName", next.getPath().getName());
map.put("filePath", next.getPath().toString());
resList.add(map);
}
fs.close();
return resList;
}
//获取某个文件的信息
@Override
public Map getFileInfo(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
FileStatus fileStatus = fs.getFileStatus(srcpath);
return fileStatusToMap(fileStatus);
}
//HDFS重命名文件
@Override
public boolean renameFile(String oldName, String newName) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(oldName) || StringUtils.isEmpty(newName)) {
return false;
}
FileSystem fs = getFileSystem();
Path oldpath = new Path(oldName);
Path newpath = new Path(newName);
return fs.rename(oldpath, newpath);
}
//删除HDFS文件
@Override
public boolean deleteFile(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return false;
}
if (!existFile(path)) {
return false;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
return fs.delete(srcpath, true);
}
//上传HDFS文件
@Override
public void uploadFile(String path, String uploadpath) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path) || StringUtils.isEmpty(uploadpath)) {
return;
}
FileSystem fs = getFileSystem();
Path oldpath = new Path(path);
Path newpath = new Path(uploadpath);
fs.copyFromLocalFile(false, oldpath, newpath);
fs.close();
}
//下载HDFS文件
@Override
public ResponseEntity downloadFile(String path, String fileName) throws InterruptedException, IOException, URISyntaxException {
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
FSDataInputStream in = fs.open(srcpath);
return MyFileUtil.downloadFile(in,fileName);
}
//将fileStatus转为map
public Map fileStatusToMap(FileStatus fileStatus) {
Map map = new HashMap<>();
Path path = fileStatus.getPath();
map.put("fileName", path.getName());//a
map.put("filePath", path.toUri().toString());// hdfs://node1:9000/idea/a.txt
map.put("relativePath", path.toUri().toString().substring(this.path.length()));// /idea/a.txt
map.put("parenPath", path.getParent().toUri().toString().substring(this.path.length())); // /idea
map.put("owner", fileStatus.getOwner());
map.put("group", fileStatus.getGroup());
map.put("isFile", fileStatus.isFile() + "");
map.put("duplicates", fileStatus.getReplication() + "");
map.put("size", MyFileUtil.formatFileSize(fileStatus.getLen()));
map.put("rights", fileStatus.getPermission().toString());
map.put("modifyTime", MyFileUtil.formatTime(fileStatus.getModificationTime()));
return map;
}
public void search(Path srcpath, List> list, Pattern pattern) throws InterruptedException, IOException, URISyntaxException {
FileSystem fs = getFileSystem();
FileStatus[] fileStatuses = fs.listStatus(srcpath);
if (fileStatuses != null && fileStatuses.length > 0) {
for (FileStatus fileStatus : fileStatuses) {
boolean res = fileStatus.isFile();
if (!res) {//是目录则递归
search(fileStatus.getPath(), list, pattern);
} else {
boolean bool = pattern.matcher(fileStatus.getPath().getName()).find();
if (bool) {
Map map = this.fileStatusToMap(fileStatus);
list.add(map);
}
}
}
}
}
//在整个磁盘搜索所有的某类型的文件
@Override
public List> listStatus(int type) throws InterruptedException, IOException, URISyntaxException {
String path = "/";//查找全盘所有文件
Path srcpath = new Path(path);//目标路径
List> resList = new ArrayList<>();
String reg = null;
if (type == 1) {
reg = ".+(.jpeg|.jpg|.png|.bmp|.gif)$";
} else if (type == 2) {
reg = ".+(.txt|.rtf|.doc|.docx|.xls|.xlsx|.html|.htm|.xml)$";
} else if (type == 3) {
reg = ".+(.mp4|.avi|.wmv)$";
} else if (type == 4) {
reg = ".+(.mp3|.wav)$";
} else if (type == 5) {
reg = "^\S+\.*$";
}
Pattern pattern = Pattern.compile(reg, Pattern.CASE_INSENSITIVE);
search(srcpath, resList, pattern);
return resList;
}
//在给定路径下搜索文件status,并统计在list
@Override
public List> listStatus(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
Path srcpath = new Path(path);
FileStatus[] fileStatuses = fs.listStatus(srcpath);
if (fileStatuses == null || fileStatuses.length <= 0) {
return null;
}
List> resList = new ArrayList<>();
for (FileStatus fileStatus : fileStatuses) {
Map map = fileStatusToMap(fileStatus);
resList.add(map);
}
fs.close();
return resList;
}
public void compress(String baseDir, ZipOutputStream zipOutputStream, FileSystem fs) {
try {
FileStatus[] fileStatuses = fs.listStatus(new Path(baseDir));
String[] strs = baseDir.split("/");
//lastName代表路径最后的单词
String lastName = strs[strs.length - 1];
for (int i = 0; i < fileStatuses.length; i++) {//循环baseDir路径下每一个文件
String name = fileStatuses[i].getPath().toString();//文件path
name = name.substring(name.indexOf("/" + lastName));//文件名
if (fileStatuses[i].isFile()) {
Path path = fileStatuses[i].getPath();
FSDataInputStream in = fs.open(path);
zipOutputStream.putNextEntry(new ZipEntry(name.substring(1)));
IOUtils.copyBytes(in, zipOutputStream, this.bufferSize);
in.close();
} else {
zipOutputStream.putNextEntry(new ZipEntry(fileStatuses[i].getPath().getName() + "/"));
compress(fileStatuses[i].getPath().toString(), zipOutputStream, fs);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
//压缩打包目录
@Override
public ResponseEntity downloadDirectory(String path, String fileName) throws IOException {
//1.获取对象
ByteArrayOutputStream out = null;
try {
FileSystem fs = this.getFileSystem();
out = new ByteArrayOutputStream();
ZipOutputStream zos = new ZipOutputStream(out);
compress(path, zos, fs);
zos.close();
} catch (Exception e) {
e.printStackTrace();
}
byte[] bs = out.toByteArray();
out.close();
return null;
//todo return YcFileUtil.downloadDirectory(bs, YvFileUtil.getFileName());
}
//HDFS文件复制
@Override
public void copyFile(String sourcePath, String targetPath) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) {
return;
}
FileSystem fs = getFileSystem();
Path oldpath = new Path(sourcePath);
Path newpath = new Path(targetPath);
FSDataInputStream in = null;
FSDataOutputStream out = null;
try {
in = fs.open(oldpath);
out = fs.create(newpath);
IOUtils.copyBytes(in, out, bufferSize, false);
} finally {
in.close();
out.close();
fs.close();
}
}
//打开HDFS上的文件并返回byte数组
@Override
public byte[] openFileToBytes(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
//目标路径
Path srcpath = new Path(path);
byte[] res = null;
try {
FSDataInputStream in = fs.open(srcpath);
InputStream iss = in.getWrappedStream();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] bs = new byte[10 * 1024];
int len = 0;
while ((len = iss.read(bs, 0, bs.length)) != -1) {
baos.write(bs, 0, len);
}
baos.flush();
res = baos.toByteArray();
} finally {
fs.close();
}
return res;
}
//获取某个文件在HDFS集群的位置
@Override
public BlockLocation[] getFileBlockLocations(String path) throws InterruptedException, IOException, URISyntaxException {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
//目标路径
Path srcpath = new Path(path);
FileStatus fileStatus = fs.getFileStatus(srcpath);
return fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
}
}
package com.yc.T1_API.service;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
public class MyFileUtil {
public static String formatTime(long time) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:sss");
Date date = new Date(time);
return simpleDateFormat.format(date);
}
public static String formatFileSize(long fileS) {
DecimalFormat df = new DecimalFormat("#.00");
String fileSizeString = "";
String wrongSize = "0B";
if (fileS == 0) {
return wrongSize;
}
if (fileS < 1024) {
fileSizeString = df.format((double) fileS) + "B";
} else if (fileS < 1048576) {
fileSizeString = df.format((double) fileS / 1024) + "KB";
} else if (fileS < 1073741824) {
fileSizeString = df.format((double) fileS / 1048576) + "MB";
} else {
fileSizeString = df.format((double) fileS / 1073741824) + "GB";
}
return fileSizeString;
}
public static ResponseEntity downloadFile(InputStream in, String fileName) {
try {
byte[] bytes = new byte[in.available()];
//http响应协议
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.add("Cache-Control", "no-cache, no-store, must-revalidate");
//下载文件的方式
httpHeaders.add("Content-Disposition", String.format("attachment; filename="%s"", fileName));
httpHeaders.add("Pragma", "no-cache");
httpHeaders.add("Expires", "8");
httpHeaders.add("Content-Language", "UTF-8");
//文件内容以流的形式输出
return ResponseEntity.ok().headers(httpHeaders).contentLength(bytes.length)
.contentType(MediaType.parseMediaType("application/octet-stream")).body(new InputStreamResource(in));
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}