JAVA+VUE3.0+MINIO 大文件上传（极速上传，分片上传）

• • 定义分片大小
  • 急速上传
  • • 文件MD5
  • 分片上传
  • • 创建分片上传任务
    • • 前端计算文件分片数量
      • 自定义MINIO CLIENT
      • 调用后台接口创建上传任务
      • 创建单文件上传任务
      • 创建分块上传任务
      • 校验桶是否存在（不存在则创建）
    • 分片文件合并
    • • 获取指定 uploadId 下已上传的分块信息
      • MINIO 文件合并

大体思路如下：
1、数据库中存放文件路径，所有文件保存在 MINIO 中，文件名即是文件的 MD5。
2、当用户上传文件时，首先判断该文件信息是否存在在数据库中，如果存在则直接显示上传成功（急速上传），若不存在则执行上传操作。
3、文件在真正上传之前先判断文件大小，太小的不需要创建分片上传任务，一次性上传即可。
4、后台调用 MINIO 的 API 创建分片上传任务（得到一个任务 ID ），并为该任务生成分片上传链接（上传地址列表）后返回给前端，前端将对应分片按照到对应的连接传递到 MINIO 中。
5、分片上传成功后更新进度信息。
6、所有分片上传结束后，调用 MINIO 的 API 将当前任务的分片全部合并形成整个文件。

const chunkSize = 5 * 1024 * 1024; // 切片大小为5M

前端使用 SparkMD5 获取文件的 MD5 信息，当该 MD5 信息已经存在在数据库中时，即上传完成（急速上传）
下面是获取文件 MD5 的方法

import SparkMD5 from 'spark-md5';

 
//获取文件的MD5信息 分片获取
const ReadFileMD5 = (param) => {
  return new Promise((resolve, reject) => {
    const file = param.file;
    const fileReader = new FileReader();
    const md5 = new SparkMD5();
    let index = 0;
    const loadFile = () => {
      const slice = file.slice(index, index + chunkSize);
      fileReader.readAsBinaryString(slice);
    };
    loadFile();
    fileReader.onload = (e) => {
      md5.appendBinary(e.target.result);
      if (index < file.size) {
        index += chunkSize;
        loadFile();
      } else {
        // md5.end() 就是文件md5码
        var md5Str = md5.end();
        return resolve(md5Str);
      }
    };
    fileReader.onerror = () => {
      reject('文件MD5获取失败');
    };
  });
};

当确认该文件的 MD5 在数据库中不存在时，开始触发我们的上传操作

let chunks = Math.ceil(file.file.size / chunkSize);

我们需要重新写一个 MINIO 客户端来实现我们的分片上传。

public class CustomMinioClient extends MinioClient {
 
    protected CustomMinioClient(MinioClient client) {
        super(client);
    }
 	//创建分块上传任务
    public String initMultiPartUpload(String bucket, String region, String object, Multimap headers, Multimap extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
        CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams);
 
        return response.result().uploadId();
    }
    //合并指定上传任务的分块文件
    public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap extraHeaders, Multimap extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
 
        return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);
    }
 	//获取指定上传任务内的已上传的分块信息
    public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap extraHeaders, Multimap extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException {
        return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);
    }
}

...
if (partCount == 1) {
   //只有一个分片的情况下 直接返回上传地址
    String uploadObjectUrl = MinioUtils.getUploadObjectUrl(MinioUtils.FILE_WAREHOUSE,objectName);
    result.setUploadUrl(new ArrayList(){{add(uploadObjectUrl);}});
}else {
    Map initRsl = MinioUtils.initMultiPartUpload(MinioUtils.FILE_WAREHOUSE, objectName, partCount, contentType);
	result.setFinished(false);
    result.setUploadId(initRsl.get("uploadId").toString());
    result.setUploadUrl((List)initRsl.get("uploadUrls"));
}
...

其中比较重要的创建 MINIO 上传任务的方法如下

public static String getUploadObjectUrl(String bucketName, String objectName) {
    try {
    	//创建 MINIO 连接
        CustomMinioClient customMinioClient = new CustomMinioClient(MinioClient.builder()
                .endpoint(properties.getUrl())//MINIO 服务地址
                .credentials(properties.getAccessKey(), properties.getSecureKey())//用户名和密码
                .build());
        return customMinioClient.getPresignedObjectUrl(
                GetPresignedObjectUrlArgs.builder()
                        .method(Method.PUT)//GET方式请求
                        .bucket(bucketName)//存储桶的名字
                        .object(objectName)//文件的名字
                        .expiry(1, TimeUnit.DAYS)//上传地址有效时长
                        .build()
        );
    } catch (Exception e) {
        return null;
    }
}

public static Map initMultiPartUpload(String bucketName,String objectName, int partCount,String contentType) {
    Map result = new HashMap<>();
    try {
        //如果类型使用默认流会导致无法预览
        contentType = "application/octet-stream";

        HashMultimap headers = HashMultimap.create();
        headers.put("Content-Type", contentType);
        CustomMinioClient customMinioClient = new CustomMinioClient(MinioClient.builder()
                .endpoint(properties.getUrl())
                .credentials(properties.getAccessKey(), properties.getSecureKey())
                .build());
        checkBucket(customMinioClient,false,bucketName);
        //初始化分块上传任务
        String uploadId = customMinioClient.initMultiPartUpload(bucketName, null, objectName, headers, null);

        result.put("uploadId", uploadId);
        List partList = new ArrayList<>();

        Map reqParams = new HashMap<>();
        reqParams.put("uploadId", uploadId);
        for (int i = 1; i <= partCount; i++) {
            reqParams.put("partNumber", String.valueOf(i))
            //返回带签名URL
            String uploadUrl = customMinioClient.getPresignedObjectUrl(
                    GetPresignedObjectUrlArgs.builder()
                            .method(Method.PUT)//GET方式请求
	                        .bucket(bucketName)//存储桶的名字
	                        .object(objectName)//文件的名字
	                        .expiry(1, TimeUnit.DAYS)//上传地址有效时长
                            .extraQueryParams(reqParams)//指定任务ID和当前是第几个分块，生成上传链接
                            .build());
            partList.add(uploadUrl);
        }
        //返回任务ID 和 分块任务列表
        result.put("uploadUrls", partList);
    } catch (Exception e) {
        return null;
    }

    return result;
}

private static void checkBucket(MinioClient minioClient ,boolean versioning, String bucket) throws Exception {

    boolean exists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucket).build());
    if (!exists) {
        minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucket).build());
        //设置Procy属性 默认所有文件都能读取 遵循 AmazonS3 规则
        String config = "{ " +
                "    "Id": "Policy1", " +
                "    "Version": "2012-10-17", " +
                "    "Statement": [ " +
                "        { " +
                "            "Sid": "Statement1", " +
                "            "Effect": "Allow", " +
                "            "Action": [ " +
                "                "s3:ListBucket", " +
                "                "s3:GetObject" " +
                "            ], " +
                "            "Resource": [ " +
                "                "arn:aws:s3:::"+bucket+"", " +
                "                "arn:aws:s3:::"+bucket+"
        customMinioClient = new CustomMinioClient(MinioClient.builder()
                .endpoint(properties.getUrl())
                .credentials(properties.getAccessKey(), properties.getSecureKey())
                .build());
        ListPartsResponse partResult = customMinioClient.listMultipart(bucketName, null, objectName, 1024, 0, uploadId, null, null);

        for (Part part : partResult.result().partList()) {
            parts.add(part.etag());
        }
        //合并分片
    } catch (Exception e) {
        //
        log.error("查询任务分片错误");
    }

    return parts;
}

public static boolean mergeMultipartUpload(String bucketName, String objectName, String uploadId) {
   try {
       Part[] parts = new Part[1000];
       
       CustomMinioClient customMinioClient = new CustomMinioClient(MinioClient.builder()
               .endpoint(properties.getUrl())
               .credentials(properties.getAccessKey(), properties.getSecureKey())
               .build());
       ListPartsResponse partResult = customMinioClient.listMultipart(bucketName, null, objectName, 1000, 0, uploadId, null, null);
       int partNumber = 1;
       for (Part part : partResult.result().partList()) {
           parts[partNumber - 1] = new Part(partNumber, part.etag());
           partNumber++;
       }
       //合并分片
       customMinioClient.mergeMultipartUpload(bucketName, null, objectName, uploadId, parts, null, null);
   } catch (Exception e) {
       return false;
   }
   return true;
}