opensource/RuoYi-Vue-Plus
2
0
Fork 1
mirror of https://gitee.com/dromara/RuoYi-Vue-Plus.git synced 2026-09-19 01:48:47 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add 新增预签名式分片上传

Browse Source
This commit is contained in:
AprilWind 2024-12-02 18:05:47 +08:00
parent eda67dd572
commit d049877fb8
9 changed files with 547 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
ruoyi-common/ruoyi-common-core/src/main/java/org/dromara/common/core/constant/GlobalConstants.java
View File

@ -31,4 +31,15 @@ public interface GlobalConstants {
* 三方认证 redis key
*/
String SOCIAL_AUTH_CODE_KEY = GLOBAL_REDIS_KEY + "social_auth_codes:";
/**
* OSS断点续传
*/
String OSS_CONTINUATION = GLOBAL_REDIS_KEY + "oss_continuation:";
/**
* OSS分片上传
*/
String OSS_MULTIPART = GLOBAL_REDIS_KEY + "oss_multipart:";
}

124
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/core/OssClient.java
View File

@ -1,5 +1,6 @@
package org.dromara.common.oss.core;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import org.dromara.common.core.constant.Constants;
@ -7,6 +8,7 @@ import org.dromara.common.core.utils.DateUtils;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.core.utils.file.FileUtils;
import org.dromara.common.oss.constant.OssConstant;
import org.dromara.common.oss.entity.PartUploadResult;
import org.dromara.common.oss.entity.UploadResult;
import org.dromara.common.oss.enumd.AccessPolicyType;
import org.dromara.common.oss.exception.OssException;
@ -20,7 +22,9 @@ import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.Part;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.*;
@ -32,6 +36,8 @@ import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.stream.Collectors;
/**
* S3 存储协议 所有兼容S3协议的云厂商均支持
@ -300,6 +306,124 @@ public class OssClient {
return url.toString();
}
/**
* 创建分片上传任务
*
* @param suffix 对象键的后缀
* @return 包含上传后的文件信息
*/
public UploadResult initiateMultipart(String suffix) {
return initiateMultipartUpload(getPath(properties.getPrefix(), suffix));
}
/**
* 创建分片上传任务
*
* @param key 在 Amazon S3 中的对象键
* @return 包含上传后的文件信息
*/
public UploadResult initiateMultipartUpload(String key) {
try {
String uploadId = client.createMultipartUpload(
x -> x.bucket(properties.getBucketName())
.key(key)
.build()
).join().uploadId();
return UploadResult.builder().filename(key).uploadId(uploadId).build();
} catch (Exception e) {
// 捕获异常并抛出自定义异常
throw new OssException("创建分片上传任务失败，请检查配置信息:[" + e.getMessage() + "]");
}
}
/**
* 生成预签名的分片上传 URL
*
* @param key 在 Amazon S3 中的对象键
* @param uploadId 分片上传任务的 Upload ID
* @param partNumber 分片编号（从1开始递增）
* @param second 签名持续时间（秒）
* @return 预签名 URL 字符串
*/
public String uploadPartFutures(String key, String uploadId, Integer partNumber, Integer second) {
URL url = presigner.presignUploadPart(
x -> x.signatureDuration(Duration.ofSeconds(second))
.uploadPartRequest(
y -> y.bucket(properties.getBucketName())
.key(key)
.uploadId(uploadId)
.partNumber(partNumber)
.build()
).build()
).url();
return url.toString();
}
/**
* 获取指定对象分片上传的分片列表
*
* @param key 在 Amazon S3 中的对象键
* @param uploadId 分片上传任务的 Upload ID
* @param maxParts 最大返回的分片数（默认为1000）
* @param partNumberMarker 分片编号的标记，用于分页查询（默认为0，表示从第一个分片开始查询）
* @return 包含分片上传结果信息的 PartUploadResult 对象列表
*/
public List<PartUploadResult> listParts(String key, String uploadId, Integer maxParts, Integer partNumberMarker) {
try {
List<Part> parts = client.listParts(
x -> x.bucket(properties.getBucketName())
.key(key)
.uploadId(uploadId)
.maxParts(maxParts != null ? maxParts : 1000)
.partNumberMarker(partNumberMarker != null ? partNumberMarker : 0)
.build()).join().parts();
return parts.stream()
.map(x -> PartUploadResult.builder()
.partNumber(x.partNumber())
.eTag(x.eTag())
.build())
.collect(Collectors.toList());
} catch (Exception e) {
// 捕获异常并抛出自定义异常
throw new OssException("获取分片列表失败，请检查配置信息:[" + e.getMessage() + "]");
}
}
/**
* 完成分片上传任务
*
* @param key 在 Amazon S3 中的对象键
* @param uploadId 分片上传任务的 Upload ID
* @param partUploadResults 已完成的分片列表（必须是唯一且按照递增顺序排列，严格检查是否漏传）
* @return 包含上传后的文件信息
*/
public UploadResult completeMultipartUpload(String key, String uploadId, List<PartUploadResult> partUploadResults) {
if (CollUtil.isEmpty(partUploadResults)) {
throw new OssException("分片列表不能为空");
}
List<CompletedPart> completedParts = partUploadResults.stream()
.map(x -> CompletedPart.builder()
.partNumber(x.getPartNumber())
.eTag(x.getETag())
.build())
.collect(Collectors.toList());
try {
String eTag = client.completeMultipartUpload(
x -> x.bucket(properties.getBucketName())
.key(key)
.uploadId(uploadId)
.multipartUpload(y -> y.parts(completedParts)
.build())
.build()
).join().eTag();
// 提取上传结果中的 ETag，并构建一个自定义的 UploadResult 对象
return UploadResult.builder().url(getUrl() + StringUtils.SLASH + key).filename(key).eTag(eTag).build();
} catch (Exception e) {
// 捕获异常并抛出自定义异常
throw new OssException("合并文件失败，请检查配置信息:[" + e.getMessage() + "]");
}
}
/**
* 上传 byte[] 数据到 Amazon S3，使用指定的后缀构造对象键。
*

25
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadResult.java Normal file
View File

@ -0,0 +1,25 @@
package org.dromara.common.oss.entity;
import lombok.Builder;
import lombok.Data;
/**
* 上传部件返回体
*
* @author AprilWind
*/
@Data
@Builder
public class PartUploadResult {
/**
* 分片编号（从1开始递增）
*/
private Integer partNumber;
/**
* 从上传部分的内容生成的实体标签
*/
private String eTag;
}

12
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/UploadResult.java
View File

@ -27,4 +27,16 @@ public class UploadResult {
*/
private String eTag;
/**
* 用于分片上传任务的 Upload ID
* 在初始化分片上传时获取，并在后续的分片上传和完成上传过程中使用
*/
private String uploadId;
/**
* 用于私有预签名 URL
* 用于上传文件或分片的预签名 URL，确保文件或分片在上传时具有私有访问权限
*/
private String privateUrl;
}

36
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/controller/system/SysOssController.java
View File

@ -4,12 +4,17 @@ package org.dromara.system.controller.system;
import cn.dev33.satoken.annotation.SaCheckPermission;
import cn.hutool.core.util.ObjectUtil;
import org.dromara.common.core.domain.R;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.core.utils.ValidatorUtils;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.common.core.validate.QueryGroup;
import org.dromara.common.web.core.BaseController;
import org.dromara.common.log.annotation.Log;
import org.dromara.common.log.enums.BusinessType;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.page.TableDataInfo;
import org.dromara.system.domain.bo.MultipartBo;
import org.dromara.system.domain.bo.SysOssBo;
import org.dromara.system.domain.vo.SysOssUploadVo;
import org.dromara.system.domain.vo.SysOssVo;
@ -105,4 +110,35 @@ public class SysOssController extends BaseController {
return toAjax(ossService.deleteWithValidByIds(List.of(ossIds), true));
}
/**
* 分片上传
*/
@SaCheckPermission("system:oss:multipart")
@PostMapping(value = "/multipart")
public R<?> multipart(@RequestBody MultipartBo multipartBo) {
return switch (multipartBo.getOssStatus()) {
case "initiate" -> {
if (StringUtils.isNotEmpty(multipartBo.getOriginalName()) && StringUtils.isNotEmpty(multipartBo.getMd5Digest())) {
ValidatorUtils.validate(multipartBo);
yield R.ok(ossService.initiateMultipart(multipartBo));
} else {
yield R.fail("Original name and MD5 digest cannot be empty");
}
}
case "upload" -> {
ValidatorUtils.validate(multipartBo, AddGroup.class);
yield R.ok(ossService.uploadPart(multipartBo));
}
case "query" -> {
ValidatorUtils.validate(multipartBo, QueryGroup.class);
yield R.ok(ossService.uploadPartList(multipartBo));
}
case "complete" -> {
ValidatorUtils.validate(multipartBo, EditGroup.class);
yield R.ok(ossService.completeMultipartUpload(multipartBo));
}
default -> R.fail("Invalid OSS status");
};
}
}

106
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/MultipartBo.java Normal file
View File

@ -0,0 +1,106 @@
package org.dromara.system.domain.bo;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.common.core.validate.QueryGroup;
import org.hibernate.validator.constraints.Range;
import java.io.Serial;
import java.io.Serializable;
import java.util.List;
/**
* 分片上传
*
* @author AprilWind
*/
@Data
public class MultipartBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 分片类型（必传）
*/
private String ossStatus;
/**
* 文件原名（分片初始化的时候使用）
*/
@Size(min = 2, max = 255, message = "文件原名长度必须在2到255之间")
private String originalName;
/**
* 用于分片上传任务的 Upload ID
* 在初始化分片上传时获取，并在后续的分片上传和完成上传过程中使用
*/
@NotBlank(message = "上传任务ID不能为空", groups = {QueryGroup.class, AddGroup.class, EditGroup.class})
private String uploadId;
/**
* 分片编号（从1开始递增）
*/
@NotNull(message = "分片编号不能为空", groups = AddGroup.class)
@Range(min = 1, max = 10000, message = "分片编号必须介于1和10,000之间", groups = AddGroup.class)
private Integer partNumber;
/**
* 内容的 MD5 摘要
* initiate初始化需要第一片的md5值（或者直接计算整体的md5）用来判断断点续传，以及秒传
*/
@Size(max = 255, message = "MD5 摘要不能超过255")
private String md5Digest;
/**
* 最大返回的分片数（默认为1000，最大值1000）
* 最多分片一万，一次性返回会造成前端性能问题，需要前端多次校验
*/
@NotNull(message = "最大返回的分片数不能为空", groups = QueryGroup.class)
@Range(max = 1000, message = "最大返回的分片数不能超过1000", groups = QueryGroup.class)
private Integer maxParts;
/**
* 分片编号的标记，用于分页查询（默认为0，表示从第一个分片开始查询）
*/
@NotNull(message = "分片编号的标记不能为空", groups = QueryGroup.class)
@Range(min = 0, max = 10000, message = "分片编号的标记长度必须在0到10000之间", groups = QueryGroup.class)
private Integer partNumberMarker;
/**
* 已上传列表（最大长度一万）
* 必须是唯一且按照递增顺序排列，严格检查是否漏传
*/
@Valid
@NotNull(message = "已上传列表不能为空", groups = EditGroup.class)
@Size(min = 2, max = 10000, message = "已上传列表长度必须在2到10000之间", groups = EditGroup.class)
private List<PartUploadResult> partUploadList;
@Data
public static class PartUploadResult implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 分片编号（从1开始递增）
*/
@NotNull(message = "分片编号不能为空", groups = EditGroup.class)
@Range(min = 1, max = 10000, message = "分片编号必须介于1和10,000之间", groups = EditGroup.class)
private Integer partNumber;
/**
* 从上传部分的内容生成的实体标签
*/
@NotBlank(message = "实体标签不能为空", groups = EditGroup.class)
@JsonProperty("eTag")
private String eTag;
}
}

83
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/MultipartVo.java Normal file
View File

@ -0,0 +1,83 @@
package org.dromara.system.domain.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.io.Serial;
import java.io.Serializable;
import java.util.List;
/**
* 分片上传
*
* @author AprilWind
*/
@Data
public class MultipartVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 文件名
*/
private String filename;
/**
* 文件原名（分片初始化的时候使用）
*/
private String originalName;
/**
* 分片第一片的md5值
*/
private String md5Digest;
/**
* 用于分片上传任务的 Upload ID
* 在初始化分片上传时获取，并在后续的分片上传和完成上传过程中使用
*/
private String uploadId;
/**
* 文件后缀
*/
private String suffix;
/**
* 分片编号（从1开始递增）
*/
private Integer partNumber;
/**
* 用于私有预签名 URL
* 用于上传文件或分片的预签名 URL，确保文件或分片在上传时具有私有访问权限
*/
private String privateUrl;
/**
* 已上传列表（最大长度一千）
* 必须是唯一且按照递增顺序排列，严格检查是否漏传
* ‘initiate’初始化状态时如果返回值，说明是断点续传，如果前端分片大于一千，需要请求‘query’获取
*/
private List<PartUploadResult> partUploadList;
@Data
@AllArgsConstructor
public static class PartUploadResult implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 分片编号（从1开始递增）
*/
private Integer partNumber;
/**
* 从上传部分的内容生成的实体标签
*/
private String entryTag;
}
}

34
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/ISysOssService.java
View File

@ -2,7 +2,9 @@ package org.dromara.system.service;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.page.TableDataInfo;
import org.dromara.system.domain.bo.MultipartBo;
import org.dromara.system.domain.bo.SysOssBo;
import org.dromara.system.domain.vo.MultipartVo;
import org.dromara.system.domain.vo.SysOssVo;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.web.multipart.MultipartFile;
@ -77,4 +79,36 @@ public interface ISysOssService {
*/
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
/**
* 初始化分片上传任务
*
* @param multipartBo 初始化分片的参数对象
* @return 分片上传对象信息
*/
MultipartVo initiateMultipart(MultipartBo multipartBo);
/**
* 上传文件的分段（分片上传）
*
* @param multipartBo 分段上传的参数对象
* @return 分片上传成功后的对象信息
*/
MultipartVo uploadPart(MultipartBo multipartBo);
/**
* 获取上传分段进度
*
* @param multipartBo 分片上传对象信息
* @return 分片上传对象信息
*/
MultipartVo uploadPartList(MultipartBo multipartBo);
/**
* 合并分段
*
* @param multipartBo 分片上传对象信息
* @return OSS对象存储视图对象
*/
SysOssVo completeMultipartUpload(MultipartBo multipartBo);
}

116
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java
View File

@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.constant.CacheNames;
import org.dromara.common.core.constant.GlobalConstants;
import org.dromara.common.core.domain.dto.OssDTO;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.service.OssService;
@ -20,11 +21,16 @@ import org.dromara.common.core.utils.file.FileUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.page.TableDataInfo;
import org.dromara.common.oss.core.OssClient;
import org.dromara.common.oss.entity.PartUploadResult;
import org.dromara.common.oss.entity.UploadResult;
import org.dromara.common.oss.enumd.AccessPolicyType;
import org.dromara.common.oss.factory.OssFactory;
import org.dromara.common.redis.utils.RedisUtils;
import org.dromara.common.satoken.utils.LoginHelper;
import org.dromara.system.domain.SysOss;
import org.dromara.system.domain.bo.MultipartBo;
import org.dromara.system.domain.bo.SysOssBo;
import org.dromara.system.domain.vo.MultipartVo;
import org.dromara.system.domain.vo.SysOssVo;
import org.dromara.system.mapper.SysOssMapper;
import org.dromara.system.service.ISysOssService;
@ -36,10 +42,12 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 文件上传 服务层实现
@ -252,6 +260,114 @@ public class SysOssServiceImpl implements ISysOssService, OssService {
return baseMapper.deleteByIds(ids) > 0;
}
/**
* 初始化分片上传任务
*
* @param multipartBo 初始化分片的参数对象
* @return 分片上传对象信息
*/
@Override
public MultipartVo initiateMultipart(MultipartBo multipartBo) {
OssClient storage = OssFactory.instance();
String md5Digest = multipartBo.getMd5Digest();
String osskey = GlobalConstants.OSS_CONTINUATION + LoginHelper.getUserId() + md5Digest;
MultipartVo multipartVo = new MultipartVo();
// 检查是否存在缓存，如果存在且超时时间在2小时内，则从缓存中获取上传信息
if (RedisUtils.getTimeToLive(osskey) > 60 * 60 * 2 * 1000) {
multipartVo = RedisUtils.getCacheObject(osskey);
// 获取上传分段进度
List<PartUploadResult> listParts = storage.listParts(multipartVo.getFilename(), multipartVo.getUploadId(), null, null);
multipartVo.setPartUploadList(listParts.stream()
.map(x -> new MultipartVo.PartUploadResult(x.getPartNumber(), x.getETag()))
.collect(Collectors.toList()));
} else {
String originalName = multipartBo.getOriginalName();
String suffix = StringUtils.substring(originalName, originalName.lastIndexOf("."), originalName.length());
UploadResult uploadResult = storage.initiateMultipart(suffix);
multipartVo.setFilename(uploadResult.getFilename());
multipartVo.setUploadId(uploadResult.getUploadId());
multipartVo.setMd5Digest(md5Digest);
multipartVo.setOriginalName(originalName);
multipartVo.setSuffix(suffix);
RedisUtils.setCacheObject(osskey, multipartVo, Duration.ofMillis(60 * 60 * 72));
RedisUtils.setCacheObject(GlobalConstants.OSS_MULTIPART + multipartVo.getUploadId(), multipartVo, Duration.ofMillis(60 * 60 * 72));
}
return multipartVo;
}
/**
* 上传文件的分段（分片上传）
*
* @param multipartBo 分段上传的参数对象
* @return 分片上传成功后的对象信息
*/
@Override
public MultipartVo uploadPart(MultipartBo multipartBo) {
String uploadId = multipartBo.getUploadId();
Integer partNumber = multipartBo.getPartNumber();
MultipartVo multipartVo = RedisUtils.getCacheObject(GlobalConstants.OSS_MULTIPART + uploadId);
if (ObjectUtil.isNull(multipartVo)) {
throw new ServiceException("该分片任务不存在!");
}
OssClient storage = OssFactory.instance();
String privateUrl = storage.uploadPartFutures(multipartVo.getFilename(), uploadId, partNumber, 60 * 60 * 72);
multipartVo.setPrivateUrl(privateUrl);
multipartVo.setPartNumber(partNumber);
return multipartVo;
}
/**
* 获取上传分段进度
*
* @param multipartBo 分片上传对象信息
* @return 分片上传对象信息
*/
@Override
public MultipartVo uploadPartList(MultipartBo multipartBo) {
String uploadId = multipartBo.getUploadId();
MultipartVo multipartVo = RedisUtils.getCacheObject(GlobalConstants.OSS_MULTIPART + uploadId);
if (ObjectUtil.isNull(multipartVo)) {
throw new ServiceException("该分片任务不存在!");
}
OssClient storage = OssFactory.instance();
List<PartUploadResult> listParts = storage.listParts(multipartVo.getFilename(), uploadId, multipartBo.getMaxParts(), multipartBo.getPartNumberMarker());
multipartVo.setPartUploadList(listParts.stream()
.map(x -> new MultipartVo.PartUploadResult(x.getPartNumber(), x.getETag()))
.collect(Collectors.toList()));
return multipartVo;
}
/**
* 合并分段
*
* @param multipartBo 分片上传对象信息
* @return OSS对象存储视图对象
*/
@Override
public SysOssVo completeMultipartUpload(MultipartBo multipartBo) {
String uploadId = multipartBo.getUploadId();
String uploadIdKey = GlobalConstants.OSS_MULTIPART + uploadId;
MultipartVo multipartVo = RedisUtils.getCacheObject(uploadIdKey);
if (ObjectUtil.isNull(multipartVo)) {
throw new ServiceException("该分片任务不存在!");
}
List<PartUploadResult> listParts = multipartBo.getPartUploadList().stream()
.map(x -> PartUploadResult.builder()
.partNumber(x.getPartNumber())
.eTag(x.getETag())
.build())
.collect(Collectors.toList());
OssClient storage = OssFactory.instance();
UploadResult uploadResult = storage.completeMultipartUpload(multipartVo.getFilename(), uploadId, listParts);
// 保存文件信息
SysOssVo sysOssVo = buildResultEntity(multipartVo.getOriginalName(), multipartVo.getSuffix(), storage.getConfigKey(), uploadResult);
RedisUtils.deleteObject(uploadIdKey);
RedisUtils.deleteObject(GlobalConstants.OSS_CONTINUATION + LoginHelper.getUserId() + multipartVo.getMd5Digest());
return sysOssVo;
}
/**
* 桶类型为 private 的URL 修改为临时URL时长为120s
*