当前位置：首页 > 卖家故事 > Java大文件分片上传(minio版)，超详细

Java大文件分片上传(minio版)，超详细

时间：2024-03-26 14:36:00 来源：网络cs 作者：亙句栏目：卖家故事阅读：

标签：详细文件

阅读本书更多章节>>>>

本文使用spring boot 结合minio文件服务做的大文件分片上传，思路：①：初始化文件调用后端接口，后端再调用minio把文件分片成几份，生成每个分片的minio上传url②：把提起分片好的文件依次调用上一把返回的url ③：调用合并分片文件接口，完成上传 PS：文件并未经过后端服务器，而是直接上传到minio文件服务的

文章目录

第一步：pom引入minio第二步：配置minio第三步：自定义minio客户端与工具类1. 自定义客户端2.封装minio配置3.上传工具类4.service5.service实现6.接口控制器第四步：文件切片第五步：示例截图

第一步：pom引入minio

 <!--minio大文件上传-->        <dependency>            <groupId>io.minio</groupId>            <artifactId>minio</artifactId>            <version>8.0.3</version>        </dependency>

第二步：配置minio

# minio 配置（大文件分片上传使用）minio:  endpoint: ${MINIO_ENDPOINT:http://minio服务IP:端口号}  accesskey: ${MINIO_ASSESSKEY:密钥}  secretkey: ${MINIO_SECRETKEY:密匙}  bucket: ${MINIO_BUCKET:桶名称}

在这里插入图片描述

第三步：自定义minio客户端与工具类

1. 自定义客户端

package com.gcddd.visual.uav.largeFileUpload.util;import com.google.common.collect.Multimap;import io.minio.CreateMultipartUploadResponse;import io.minio.ListPartsResponse;import io.minio.MinioClient;import io.minio.ObjectWriteResponse;import io.minio.errors.*;import io.minio.messages.Part;import java.io.IOException;import java.security.InvalidKeyException;import java.security.NoSuchAlgorithmException;/** * 自定义minio * @date 2023-09-15 */public class CustomMinioClient extends MinioClient {    public CustomMinioClient(MinioClient client) {        super(client);    }    public String initMultiPartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {        CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams);        return response.result().uploadId();    }    public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {        return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);    }    public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException {        return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);    }}

2.封装minio配置

package com.gcddd.visual.uav.largeFileUpload.configurer;import lombok.Getter;import lombok.Setter;import org.springframework.boot.context.properties.ConfigurationProperties;/** * @Description: MinioProperties * minio配置类 */@ConfigurationProperties(prefix = "minio")@Getter@Setterpublic class MinioProperties {    private String endpoint;    private String accesskey;    private String secretkey;    private String bucket;}

3.上传工具类

package com.gcddd.visual.uav.largeFileUpload.configurer;import cn.hutool.core.util.StrUtil;import com.gcddd.visual.uav.largeFileUpload.util.CustomMinioClient;import com.google.common.collect.HashMultimap;import io.minio.*;import io.minio.http.Method;import io.minio.messages.Part;import org.springframework.boot.context.properties.EnableConfigurationProperties;import org.springframework.context.annotation.Configuration;import org.springframework.stereotype.Component;import javax.annotation.PostConstruct;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.concurrent.TimeUnit;/*** @Description: MinIoUtils * 工具类 */@Component@Configuration@EnableConfigurationProperties({MinioProperties.class})public class MinIoUtils {    private final MinioProperties minioProperties;    private CustomMinioClient customMinioClient;    public MinIoUtils(MinioProperties minioProperties) {        this.minioProperties = minioProperties;    }    @PostConstruct    public void init() {        MinioClient minioClient = MinioClient.builder()                .endpoint(minioProperties.getEndpoint())                .credentials(minioProperties.getAccesskey(), minioProperties.getSecretkey())                .build();        customMinioClient = new CustomMinioClient(minioClient);    }    /**     * 单文件签名上传     *     * @param objectName 文件全路径名称     * @return /     */    public String getUploadObjectUrl(String objectName) {        try {            return customMinioClient.getPresignedObjectUrl(                    GetPresignedObjectUrlArgs.builder()                            .method(Method.PUT)                            .bucket(minioProperties.getBucket())                            .object(objectName)                            .expiry(1, TimeUnit.DAYS)                            //.extraHeaders(headers)                            .build()            );        } catch (Exception e) {            e.printStackTrace();            return null;        }    }    /**     *  初始化分片上传     *     * @param objectName 文件全路径名称     * @param partCount 分片数量     * @param contentType 类型，如果类型使用默认流会导致无法预览     * @return /     */    public Map<String, Object> initMultiPartUpload(String objectName, int partCount, String contentType) {        Map<String, Object> result = new HashMap<>();        try {            if (StrUtil.isBlank(contentType)) {                contentType = "application/octet-stream";            }            HashMultimap<String, String> headers = HashMultimap.create();            headers.put("Content-Type", contentType);            String uploadId = customMinioClient.initMultiPartUpload(minioProperties.getBucket(), null, objectName, headers, null);            result.put("uploadId", uploadId);            List<String> partList = new ArrayList<>();            Map<String, String> reqParams = new HashMap<>();            //reqParams.put("response-content-type", "application/json");            reqParams.put("uploadId", uploadId);            for (int i = 1; i <= partCount; i++) {                reqParams.put("partNumber", String.valueOf(i));                String uploadUrl = customMinioClient.getPresignedObjectUrl(                        GetPresignedObjectUrlArgs.builder()                                .method(Method.PUT)                                .bucket(minioProperties.getBucket())                                .object(objectName)                                .expiry(1, TimeUnit.DAYS)                                .extraQueryParams(reqParams)                                .build());                partList.add(uploadUrl);            }            result.put("uploadUrls", partList);        } catch (Exception e) {            e.printStackTrace();            return null;        }        return result;    }    /**     * 分片上传完后合并     *     * @param objectName 文件全路径名称     * @param uploadId 返回的uploadId     * @return /     */    public boolean mergeMultipartUpload(String objectName, String uploadId) {        try {            //TODO::目前仅做了最大1000分片            Part[] parts = new Part[1000];            ListPartsResponse partResult = customMinioClient.listMultipart(minioProperties.getBucket(), null, objectName, 1000, 0, uploadId, null, null);            int partNumber = 1;            for (Part part : partResult.result().partList()) {                parts[partNumber - 1] = new Part(partNumber, part.etag());                partNumber++;            }            customMinioClient.mergeMultipartUpload(minioProperties.getBucket(), null, objectName, uploadId, parts, null, null);        } catch (Exception e) {            e.printStackTrace();            return false;        }        return true;    }}

4.service

package com.gcddd.visual.uav.largeFileUpload.service;import java.util.Map;/** * @date 2023-09-13 */public interface UploadService {    /**     * 分片上传初始化     *     * @param path        路径     * @param filename    文件名     * @param partCount   分片数量     * @param contentType /     * @return /     */    Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType);    /**     * 完成分片上传     *     * @param objectName 文件名     * @param uploadId 标识     * @return /     */     boolean mergeMultipartUpload(String objectName, String uploadId);}

5.service实现

package com.gcddd.visual.uav.largeFileUpload.service.impl;import com.gcddd.visual.uav.largeFileUpload.configurer.MinIoUtils;import com.gcddd.visual.uav.largeFileUpload.service.UploadService;import com.google.common.collect.ImmutableList;import com.google.common.collect.ImmutableMap;import lombok.RequiredArgsConstructor;import org.springframework.stereotype.Service;import java.util.Map;/** * @date 2023-09-13 */@Service@RequiredArgsConstructorpublic class UploadServiceImpl implements UploadService {    private final MinIoUtils minIoUtils;    @Override    public Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType) {        path = path.replaceAll("/+", "/");        if (path.indexOf("/") == 0) {            path = path.substring(1);        }        String filePath = path + "/" + filename;        Map<String, Object> result;        // 单文件，直接上传        if (partCount == 1) {            String uploadObjectUrl = minIoUtils.getUploadObjectUrl(filePath);            result = ImmutableMap.of("uploadUrls", ImmutableList.of(uploadObjectUrl));        } else {//多文件，分片上传            result = minIoUtils.initMultiPartUpload(filePath, partCount, contentType);        }        return result;    }    @Override    public boolean mergeMultipartUpload(String objectName, String uploadId) {        return minIoUtils.mergeMultipartUpload(objectName, uploadId);    }}

6.接口控制器

import cn.hutool.core.lang.Assert;import cn.hutool.json.JSONObject;import com.gcddd.visual.uav.largeFileUpload.service.UploadService;import com.gcddd.visual.uav.util.DateUtils;import com.google.common.collect.ImmutableMap;import io.swagger.v3.oas.annotations.security.SecurityRequirement;import io.swagger.v3.oas.annotations.tags.Tag;import lombok.RequiredArgsConstructor;import org.springframework.http.HttpHeaders;import org.springframework.http.HttpStatus;import org.springframework.http.ResponseEntity;import org.springframework.web.bind.annotation.*;import java.util.Map;/** * 大文件上传控制器 */@RestController@RequiredArgsConstructor@RequestMapping("/largeFileUpload" )@Tag(description = "largeFileUpload" , name = "大文件分片上传" )@SecurityRequirement(name = HttpHeaders.AUTHORIZATION)public class uploadController {    private final UploadService uploadService;//    1.用户调用初始化接口，后端调用minio初始化，得到uploadId，生成每个分片的minio上传url//初始化 示例： $ curl --location --request POST 'http://127.0.0.1:9998/admin/largeFileUpload/multipart/init' \//            --header 'Content-Type: application/json' \//            --data-raw '{    //            "filename": "result.png",    //            "partCount": 2//              }'    //    返回示例：//{//    "uploadId": "eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3",//        "uploadUrls": [//    "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6",//            "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3"//    ]//}//    2.用户调用对应分片的上传地址，多次上传会覆盖//    示例：//    curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6' \//            --header 'Content-Type: application/octet-stream' \//            --data-binary '@/D:/jpgone'//    curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3' \//            --header 'Content-Type: application/octet-stream' \//            --data-binary '@/D:/jpgtwo'//    3.调用完成接口，后端查询所有上传的分片并合并，uploadId为第一步返回的uploadId//    示例：//    curl --location --request PUT '127.0.0.1:8006/multipart/complete' \//            --header 'Content-Type: application/json' \//            --data-raw '{                    //    "objectName":"20230913/result.png",                    //    "uploadId":"eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3"//                      }'    /**     * 分片初始化     *     * @param requestParam 请求参数     * @return /     */    @PostMapping("/multipart/init")    public ResponseEntity<Object> initMultiPartUpload(@RequestBody JSONObject requestParam) {        // 路径（存入minio的路径）        String pathText = DateUtils.getTodays("yyyyMMdd");//当前日期        String path = requestParam.getStr("path", pathText);        // 文件名        String filename = requestParam.getStr("filename", "test.obj");        // content-type        String contentType = requestParam.getStr("contentType", "application/octet-stream");        // md5-可进行秒传判断        String md5 = requestParam.getStr("md5", "");        // 分片数量        Integer partCount = requestParam.getInt("partCount", 100);        //TODO::业务判断+秒传判断        Map<String, Object> result = uploadService.initMultiPartUpload(path, filename, partCount, contentType);        return new ResponseEntity<>(result, HttpStatus.OK);    }    /**     * 合并分片文件，完成上传     *     * @param requestParam 用户参数     * @return /     */    @PutMapping("/multipart/complete")    public ResponseEntity<Object> completeMultiPartUpload(            @RequestBody JSONObject requestParam    ) {        // 文件名完整路径        String objectName = requestParam.getStr("objectName");        //初始化返回的uploadId        String uploadId = requestParam.getStr("uploadId");        Assert.notNull(objectName, "objectName must not be null");        Assert.notNull(uploadId, "uploadId must not be null");        boolean result = uploadService.mergeMultipartUpload(objectName, uploadId);        return new ResponseEntity<>(ImmutableMap.of("success", result), HttpStatus.OK);    }}