opensource/RuoYi-Vue-Plus
2
0
Fork 1
mirror of https://gitee.com/dromara/RuoYi-Vue-Plus.git synced 2026-09-20 02:08:15 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

update 更新分片上传逻辑

Browse Source
This commit is contained in:
L 2024-03-31 01:16:42 +08:00
parent 43a8b503ff
commit 0fb0555bc4
6 changed files with 228 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartInfo.java
View File

@ -20,7 +20,13 @@ public class PartInfo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 分片序号
*/
private Integer partNumber;
/**
* 已上传分片的实体标记（用来校验文件）
*/
private String eTag;
}

11
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadInfo.java
View File

@ -1,6 +1,9 @@
package org.dromara.common.oss.entity;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serial;
import java.io.Serializable;
@ -8,17 +11,21 @@ import java.util.List;
/**
* 分片上传信息对象
* TODO 用于创建分片上传任务时，数据落库之前，存放分片上传信息数据到Redis
*
* @author SunnyDeer0911
*/
@NoArgsConstructor
@AllArgsConstructor
@Data
@Builder
public class PartUploadInfo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 文件上传ID（分片上传扩展字段，文件分片上传S3 OSS返回的唯一标识）
* 文件上传ID（创建分片上传任务时，S3 OSS返回的唯一标识）
*/
private String uploadId;
@ -28,7 +35,7 @@ public class PartUploadInfo implements Serializable {
private String fileName;
/**
* 原名
* 原文件名
*/
private String originalName;

101
ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/utils/PartUploadCacheHelper.java Normal file
View File

@ -0,0 +1,101 @@
package org.dromara.common.oss.utils;
import cn.hutool.core.collection.CollUtil;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.oss.constant.OssConstant;
import org.dromara.common.oss.core.OssClient;
import org.dromara.common.oss.entity.PartUploadInfo;
import org.dromara.common.oss.factory.OssFactory;
import org.dromara.common.redis.utils.RedisUtils;
import java.util.Collection;
/**
* 分片上传缓存助手
*
* @author SunnyDeer0911
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class PartUploadCacheHelper {
/**
* 获取分片上传信息cacheKey
* @param uploadId 文件分片上传ID
* @return 分片上传信息cacheKey
*/
public static String getPartUploadInfoCacheKey(String uploadId){
if (StringUtils.isBlank(uploadId)) {
return null;
}
return OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId;
}
/**
* 从缓存中获取分片上传信息对象
* @param uploadId 文件分片上传ID
* @return 获取分片上传信息对象
*/
public static PartUploadInfo getCache(String uploadId){
String cacheKey = getPartUploadInfoCacheKey(uploadId);
if (cacheKey==null) {
return null;
}
return RedisUtils.getCacheObject(cacheKey);
}
/**
* 将分片上传信息对象放入缓存
* @param partUploadInfo 分片上传信息对象
*/
public static void putCache(PartUploadInfo partUploadInfo){
String cacheKey = getPartUploadInfoCacheKey(partUploadInfo.getUploadId());
if (cacheKey==null) {
return;
}
RedisUtils.setCacheObject(cacheKey, partUploadInfo);
}
/**
* 移除分片上传信息对象缓存
* @param uploadId 文件分片上传ID
*/
public static void removeCache(String uploadId){
String cacheKey = getPartUploadInfoCacheKey(uploadId);
if (cacheKey==null) {
return;
}
removeCacheByKey(cacheKey);
}
/**
* 移除分片上传信息对象缓存
* @param uploadId 文件分片上传ID
*/
private static void removeCacheByKey(String cacheKey){
PartUploadInfo cache = RedisUtils.getCacheObject(cacheKey);
if (cache==null) {
return;
}
// 如果未合并，啧调用OssClient终止分片上传任务的方法，让OSS删除已上传的分片
if (!cache.isNeedMerge() && CollUtil.isNotEmpty(cache.getPartInfoList())) {
OssClient instance = OssFactory.instance(cache.getService());
// 终止分片上传任务
instance.abortPartUpload(cache.getUploadId(),cache.getFileName());
}
RedisUtils.deleteObject(cacheKey);
}
/**
* 移除所有分片上传信息对象缓存
*/
public static void removeAllCache(){
Collection<String> removeKeys = RedisUtils.keys(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + "*");
if (CollUtil.isEmpty(removeKeys)) {
return;
}
removeKeys.forEach(PartUploadCacheHelper::removeCacheByKey);
}
}

30
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/SysOssPartUploadBo.java
View File

@ -1,5 +1,6 @@
package org.dromara.system.domain.bo;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
@ -18,45 +19,34 @@ public class SysOssPartUploadBo implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 文件上传ID（分片上传扩展字段，文件分片上传S3 OSS返回的唯一标识）
* TODO 上传第一片分片时允许为空，后续需要从第一次分片上传的响应中获取回
* 文件上传ID（创建分片上传任务时，S3 OSS返回的唯一标识）
* TODO 上传第一片分片时允许为空，后续需要从第一次分片上传的响应中获取回填
*/
private String uploadId;
/**
* 文件名
* TODO 允许为空，如果为空，则从上传的文件中获取
* TODO 非必填，如果为空则从上传的文件中获取
*/
// @NotBlank(message = "文件名不能为空")
private String fileName;
/**
* 文件大小
* TODO 文件的总大小，上传第一片分片时不允许为空，后续请求用不到，可传可不传
* 文件大小（文件的总大小）
*/
@NotNull(message = "文件大小不能为空")
private Long fileSize;
/**
* 分片序号
* TODO 分片序号不能为空，从1开始
* 分片序号*（分片序号从1开始）
*/
@NotNull(message = "分片序号不能为空")
@Min(value = 1, message = "分片序号不能小于1")
private Integer partNumber;
/**
* 分片大小
*/
private Long partSize;
/**
* 分片数量
*/
private Long totalParts;
/**
* 完成状态
* TODO 允许为空，为空默认
* 是否需要合并
*/
@NotNull(message = "是否需要合并不能为空")
private Boolean needMerge;
}

3
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/SysOssPartUploadVo.java
View File

@ -1,9 +1,7 @@
package org.dromara.system.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.common.oss.entity.PartInfo;
import org.dromara.common.oss.entity.PartUploadInfo;
import java.io.Serial;
import java.io.Serializable;
@ -15,7 +13,6 @@ import java.util.List;
* @author SunnyDeer0911
*/
@Data
@AutoMapper(target = PartUploadInfo.class)
public class SysOssPartUploadVo implements Serializable {
@Serial

164
ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java
View File

@ -18,12 +18,11 @@ import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.core.utils.file.FileUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.page.TableDataInfo;
import org.dromara.common.oss.constant.OssConstant;
import org.dromara.common.oss.core.OssClient;
import org.dromara.common.oss.entity.*;
import org.dromara.common.oss.enumd.AccessPolicyType;
import org.dromara.common.oss.factory.OssFactory;
import org.dromara.common.redis.utils.RedisUtils;
import org.dromara.common.oss.utils.PartUploadCacheHelper;
import org.dromara.system.domain.SysOss;
import org.dromara.system.domain.bo.SysOssBo;
import org.dromara.system.domain.bo.SysOssPartUploadBo;
@ -243,73 +242,114 @@ public class SysOssServiceImpl implements ISysOssService, OssService {
@Override
public SysOssPartUploadVo partUpload(MultipartFile file, SysOssPartUploadBo bo) {
OssClient storage;
PartUploadInfo partUploadInfo;
String uploadId = bo.getUploadId();
// 如果上传ID为空，则说明是创建任务此时
if (StringUtils.isBlank(uploadId)) {
storage = OssFactory.instance();
String originalFileName = bo.getFileName();
// 如果文件名为空，则使用从上传的文件中获取
if (StringUtils.isBlank(originalFileName)) {
originalFileName = file.getOriginalFilename();
}
String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length());
CreatePartUploadResult partUpload = storage.createPartUploadSuffix(suffix);
uploadId = partUpload.getUploadId();
partUploadInfo = new PartUploadInfo();
partUploadInfo.setUploadId(uploadId);
partUploadInfo.setFileName(partUpload.getFilename());
partUploadInfo.setOriginalName(originalFileName);
partUploadInfo.setFileSuffix(suffix);
partUploadInfo.setUrl(partUpload.getUrl());
partUploadInfo.setFileSize(bo.getFileSize());
partUploadInfo.setService(storage.getConfigKey());
partUploadInfo.setPartSize(bo.getPartSize());
partUploadInfo.setTotalParts(bo.getTotalParts());
partUploadInfo.setPartInfoList(new ArrayList<>());
partUploadInfo.setNeedMerge(bo.getNeedMerge());
} else {
// uploadId不为空，且文件不为空，则进行文件上传
partUploadInfo = RedisUtils.getCacheObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId);
partUploadInfo.setNeedMerge(bo.getNeedMerge());
storage = OssFactory.instance(partUploadInfo.getService());
}
return partUpload(storage, file, partUploadInfo, bo.getPartNumber());
}
private SysOssPartUploadVo partUpload(OssClient storage, MultipartFile file, PartUploadInfo partUploadInfo, Integer partNumber) {
try {
String uploadId = partUploadInfo.getUploadId();
PartUploadResult partUploadResult = storage.partUpload(file.getInputStream(), partUploadInfo.getFileName(), partUploadInfo.getUploadId(), partNumber, file.getSize());
// 将完成上传的分片信息放入集合中
partUploadInfo.getPartInfoList().add(new PartInfo(partUploadResult.getPartNumber(), partUploadResult.getETag()));
// 分片上传信息放入redis
RedisUtils.setCacheObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + partUploadInfo.getUploadId(), partUploadInfo);
// 检查是否需要合并 - 已经完成所有上传时操作
if (partUploadInfo.isNeedMerge()) {
// 如果已经完成最后一片的上传，则进行合并
UploadResult uploadResult = storage.completePartUpload(uploadId, partUploadInfo.getFileName(), partUploadInfo.getPartInfoList());
// 不报错即合并成功，删除redis中的分片上传信息
RedisUtils.deleteObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId);
// 数据落库
SysOss oss = new SysOss();
oss.setFileSuffix(partUploadInfo.getFileSuffix());
oss.setOriginalName(partUploadInfo.getOriginalName());
oss.setService(storage.getConfigKey());
oss.setFileName(partUploadInfo.getFileName());
oss.setUrl(partUploadInfo.getUrl());
baseMapper.insert(oss);
String uploadId = bo.getUploadId();
// 如果 uploadId 为空，则说明是新建上传分片
if (StringUtils.isBlank(uploadId)) {
return createPartUpload(file, bo);
}
SysOssPartUploadVo sysOssPartUploadVo = MapstructUtils.convert(partUploadInfo, SysOssPartUploadVo.class);
sysOssPartUploadVo.setMergeCompleted(partUploadInfo.isNeedMerge());
return sysOssPartUploadVo;
// uploadId不为空，从缓存中获取分片上传信息
PartUploadInfo partUploadInfo = PartUploadCacheHelper.getCache(uploadId);
if (ObjectUtil.isNull(partUploadInfo)) {
throw new ServiceException("未找到分片上传信息！");
}
// TODO 是否需要合并分片
partUploadInfo.setNeedMerge(bo.getNeedMerge());
OssClient storage = OssFactory.instance(partUploadInfo.getService());
// 上传分片并返回结果
return partUpload(storage, partUploadInfo, file, bo.getPartNumber());
} catch (Exception e) {
throw new ServiceException(e.getMessage());
}
}
/**
* 创建并上传分片
*
* @param file 需要进行上传的分片文件
* @param bo OSS分片上传业务对象
* @return 分片上传对象信息VO
*/
private SysOssPartUploadVo createPartUpload(MultipartFile file, SysOssPartUploadBo bo) throws IOException {
// 文件大小
Long fileSize = bo.getFileSize();
// 获取第一片文件的大小作为分片大小
long partSize = file.getSize();
// 计算总片数
long totalParts = fileSize / partSize;
// 当除以分数有余数时每分片总数+1
if (fileSize % partSize > 0) {
totalParts += 1;
}
// 获取文件名，如果文件名为空，则使用从上传的文件中获取
String originalFileName = bo.getFileName();
if (StringUtils.isBlank(originalFileName)) {
originalFileName = file.getOriginalFilename();
}
String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length());
OssClient storage = OssFactory.instance();
CreatePartUploadResult partUpload = storage.createPartUploadSuffix(suffix);
// 构建分片上传信息对象
PartUploadInfo partUploadInfo = new PartUploadInfo();
partUploadInfo.setUploadId(partUpload.getUploadId());
partUploadInfo.setFileName(partUpload.getFilename());
partUploadInfo.setOriginalName(originalFileName);
partUploadInfo.setFileSuffix(suffix);
partUploadInfo.setUrl(partUpload.getUrl());
partUploadInfo.setFileSize(fileSize);
partUploadInfo.setService(storage.getConfigKey());
partUploadInfo.setPartSize(partSize);
partUploadInfo.setTotalParts(totalParts);
partUploadInfo.setPartInfoList(new ArrayList<>());
// TODO 是否需要合并分片
partUploadInfo.setNeedMerge(bo.getNeedMerge());
// 上传分片并返回结果
return partUpload(storage, partUploadInfo, file, bo.getPartNumber());
}
/**
* 上传分片
*
* @param storage OSS客户端
* @param partUploadInfo 分片上传信息对象
* @param file 需要进行上传的分片文件
* @param partNumber 分片序号
* @return 分片上传对象信息VO
*/
private SysOssPartUploadVo partUpload(OssClient storage, PartUploadInfo partUploadInfo, MultipartFile file, Integer partNumber) throws IOException {
String uploadId = partUploadInfo.getUploadId();
// 上传分片
PartUploadResult partUploadResult = storage.partUpload(file.getInputStream(), partUploadInfo.getFileName(), uploadId, partNumber, file.getSize());
// 将完成上传的分片信息放入集合中
partUploadInfo.getPartInfoList().add(new PartInfo(partUploadResult.getPartNumber(), partUploadResult.getETag()));
// 分片上传信息放入缓存
PartUploadCacheHelper.putCache(partUploadInfo);
// 检查是否需要合并 - 已经完成所有上传时操作
if (partUploadInfo.isNeedMerge()) {
// 如果已经完成最后一片的上传，则进行合并
UploadResult uploadResult = storage.completePartUpload(uploadId, partUploadInfo.getFileName(), partUploadInfo.getPartInfoList());
// 不报错即合并成功，删除缓存中的分片上传信息
PartUploadCacheHelper.removeCache(uploadId);
// 数据落库
SysOss oss = new SysOss();
oss.setFileSuffix(partUploadInfo.getFileSuffix());
oss.setOriginalName(partUploadInfo.getOriginalName());
oss.setService(storage.getConfigKey());
oss.setFileName(partUploadInfo.getFileName());
oss.setUrl(partUploadInfo.getUrl());
baseMapper.insert(oss);
}
// 构建分片上传对象信息VO
SysOssPartUploadVo sysOssPartUploadVo = new SysOssPartUploadVo();
sysOssPartUploadVo.setUploadId(uploadId);
sysOssPartUploadVo.setUrl(partUploadInfo.getUrl());
sysOssPartUploadVo.setPartInfoList(partUploadInfo.getPartInfoList());
sysOssPartUploadVo.setMergeCompleted(partUploadInfo.isNeedMerge());
return sysOssPartUploadVo;
}
/**
* 桶类型为 private 的URL 修改为临时URL时长为120s
*