From 0fb0555bc48555859fdf6da2a71bcceeccc96390 Mon Sep 17 00:00:00 2001 From: L <545073804@qq.com> Date: Sun, 31 Mar 2024 01:16:42 +0800 Subject: [PATCH] =?UTF-8?q?update=20=E6=9B=B4=E6=96=B0=E5=88=86=E7=89=87?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dromara/common/oss/entity/PartInfo.java | 6 + .../common/oss/entity/PartUploadInfo.java | 11 +- .../oss/utils/PartUploadCacheHelper.java | 101 +++++++++++ .../system/domain/bo/SysOssPartUploadBo.java | 30 ++-- .../system/domain/vo/SysOssPartUploadVo.java | 3 - .../service/impl/SysOssServiceImpl.java | 164 +++++++++++------- 6 files changed, 228 insertions(+), 87 deletions(-) create mode 100644 ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/utils/PartUploadCacheHelper.java diff --git a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartInfo.java b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartInfo.java index 8d89ac7e5..75c583439 100644 --- a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartInfo.java +++ b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartInfo.java @@ -20,7 +20,13 @@ public class PartInfo implements Serializable { @Serial private static final long serialVersionUID = 1L; + /** + * 分片序号 + */ private Integer partNumber; + /** + * 已上传分片的实体标记（用来校验文件） + */ private String eTag; } diff --git a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadInfo.java b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadInfo.java index d0a5b5fb9..d3230a241 100644 --- a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadInfo.java +++ b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/entity/PartUploadInfo.java @@ -1,6 +1,9 @@ package org.dromara.common.oss.entity; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.io.Serial; import java.io.Serializable; @@ -8,17 +11,21 @@ import java.util.List; /** * 分片上传信息对象 + * TODO 用于创建分片上传任务时，数据落库之前，存放分片上传信息数据到Redis * * @author SunnyDeer0911 */ +@NoArgsConstructor +@AllArgsConstructor @Data +@Builder public class PartUploadInfo implements Serializable { @Serial private static final long serialVersionUID = 1L; /** - * 文件上传ID（分片上传扩展字段，文件分片上传S3 OSS返回的唯一标识） + * 文件上传ID（创建分片上传任务时，S3 OSS返回的唯一标识） */ private String uploadId; @@ -28,7 +35,7 @@ public class PartUploadInfo implements Serializable { private String fileName; /** - * 原名 + * 原文件名 */ private String originalName; diff --git a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/utils/PartUploadCacheHelper.java b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/utils/PartUploadCacheHelper.java new file mode 100644 index 000000000..d4f3284b1 --- /dev/null +++ b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/utils/PartUploadCacheHelper.java @@ -0,0 +1,101 @@ +package org.dromara.common.oss.utils; + +import cn.hutool.core.collection.CollUtil; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.oss.constant.OssConstant; +import org.dromara.common.oss.core.OssClient; +import org.dromara.common.oss.entity.PartUploadInfo; +import org.dromara.common.oss.factory.OssFactory; +import org.dromara.common.redis.utils.RedisUtils; + +import java.util.Collection; + +/** + * 分片上传缓存助手 + * + * @author SunnyDeer0911 + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class PartUploadCacheHelper { + + /** + * 获取分片上传信息cacheKey + * @param uploadId 文件分片上传ID + * @return 分片上传信息cacheKey + */ + public static String getPartUploadInfoCacheKey(String uploadId){ + if (StringUtils.isBlank(uploadId)) { + return null; + } + return OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId; + } + + /** + * 从缓存中获取分片上传信息对象 + * @param uploadId 文件分片上传ID + * @return 获取分片上传信息对象 + */ + public static PartUploadInfo getCache(String uploadId){ + String cacheKey = getPartUploadInfoCacheKey(uploadId); + if (cacheKey==null) { + return null; + } + return RedisUtils.getCacheObject(cacheKey); + } + + /** + * 将分片上传信息对象放入缓存 + * @param partUploadInfo 分片上传信息对象 + */ + public static void putCache(PartUploadInfo partUploadInfo){ + String cacheKey = getPartUploadInfoCacheKey(partUploadInfo.getUploadId()); + if (cacheKey==null) { + return; + } + RedisUtils.setCacheObject(cacheKey, partUploadInfo); + } + + /** + * 移除分片上传信息对象缓存 + * @param uploadId 文件分片上传ID + */ + public static void removeCache(String uploadId){ + String cacheKey = getPartUploadInfoCacheKey(uploadId); + if (cacheKey==null) { + return; + } + removeCacheByKey(cacheKey); + } + + /** + * 移除分片上传信息对象缓存 + * @param uploadId 文件分片上传ID + */ + private static void removeCacheByKey(String cacheKey){ + PartUploadInfo cache = RedisUtils.getCacheObject(cacheKey); + if (cache==null) { + return; + } + // 如果未合并，啧调用OssClient终止分片上传任务的方法，让OSS删除已上传的分片 + if (!cache.isNeedMerge() && CollUtil.isNotEmpty(cache.getPartInfoList())) { + OssClient instance = OssFactory.instance(cache.getService()); + // 终止分片上传任务 + instance.abortPartUpload(cache.getUploadId(),cache.getFileName()); + } + RedisUtils.deleteObject(cacheKey); + } + + /** + * 移除所有分片上传信息对象缓存 + */ + public static void removeAllCache(){ + Collection removeKeys = RedisUtils.keys(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + "*"); + if (CollUtil.isEmpty(removeKeys)) { + return; + } + removeKeys.forEach(PartUploadCacheHelper::removeCacheByKey); + } + +} diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/SysOssPartUploadBo.java b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/SysOssPartUploadBo.java index 5c9aa930d..0291ce95f 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/SysOssPartUploadBo.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/bo/SysOssPartUploadBo.java @@ -1,5 +1,6 @@ package org.dromara.system.domain.bo; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; import lombok.Data; @@ -18,45 +19,34 @@ public class SysOssPartUploadBo implements Serializable { private static final long serialVersionUID = 1L; /** - * 文件上传ID（分片上传扩展字段，文件分片上传S3 OSS返回的唯一标识） - * TODO 上传第一片分片时允许为空，后续需要从第一次分片上传的响应中获取回 + * 文件上传ID（创建分片上传任务时，S3 OSS返回的唯一标识） + * TODO 上传第一片分片时允许为空，后续需要从第一次分片上传的响应中获取回填 */ private String uploadId; /** * 文件名 - * TODO 允许为空，如果为空，则从上传的文件中获取 + * TODO 非必填，如果为空则从上传的文件中获取 */ +// @NotBlank(message = "文件名不能为空") private String fileName; /** - * 文件大小 - * TODO 文件的总大小，上传第一片分片时不允许为空，后续请求用不到，可传可不传 + * 文件大小（文件的总大小） */ @NotNull(message = "文件大小不能为空") private Long fileSize; /** - * 分片序号 - * TODO 分片序号不能为空，从1开始 + * 分片序号*（分片序号从1开始） */ @NotNull(message = "分片序号不能为空") + @Min(value = 1, message = "分片序号不能小于1") private Integer partNumber; /** - * 分片大小 - */ - private Long partSize; - - /** - * 分片数量 - */ - private Long totalParts; - - /** - * 完成状态 - * TODO 允许为空，为空默认 + * 是否需要合并 */ + @NotNull(message = "是否需要合并不能为空") private Boolean needMerge; - } diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/SysOssPartUploadVo.java b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/SysOssPartUploadVo.java index 6165c1ee9..784f0af06 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/SysOssPartUploadVo.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/domain/vo/SysOssPartUploadVo.java @@ -1,9 +1,7 @@ package org.dromara.system.domain.vo; -import io.github.linpeilie.annotations.AutoMapper; import lombok.Data; import org.dromara.common.oss.entity.PartInfo; -import org.dromara.common.oss.entity.PartUploadInfo; import java.io.Serial; import java.io.Serializable; @@ -15,7 +13,6 @@ import java.util.List; * @author SunnyDeer0911 */ @Data -@AutoMapper(target = PartUploadInfo.class) public class SysOssPartUploadVo implements Serializable { @Serial diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java index fbe8a2a93..ec633ffd5 100644 --- a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java +++ b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java @@ -18,12 +18,11 @@ import org.dromara.common.core.utils.StringUtils; import org.dromara.common.core.utils.file.FileUtils; import org.dromara.common.mybatis.core.page.PageQuery; import org.dromara.common.mybatis.core.page.TableDataInfo; -import org.dromara.common.oss.constant.OssConstant; import org.dromara.common.oss.core.OssClient; import org.dromara.common.oss.entity.*; import org.dromara.common.oss.enumd.AccessPolicyType; import org.dromara.common.oss.factory.OssFactory; -import org.dromara.common.redis.utils.RedisUtils; +import org.dromara.common.oss.utils.PartUploadCacheHelper; import org.dromara.system.domain.SysOss; import org.dromara.system.domain.bo.SysOssBo; import org.dromara.system.domain.bo.SysOssPartUploadBo; @@ -243,73 +242,114 @@ public class SysOssServiceImpl implements ISysOssService, OssService { @Override public SysOssPartUploadVo partUpload(MultipartFile file, SysOssPartUploadBo bo) { - OssClient storage; - PartUploadInfo partUploadInfo; - String uploadId = bo.getUploadId(); - // 如果上传ID为空，则说明是创建任务此时 - if (StringUtils.isBlank(uploadId)) { - storage = OssFactory.instance(); - String originalFileName = bo.getFileName(); - // 如果文件名为空，则使用从上传的文件中获取 - if (StringUtils.isBlank(originalFileName)) { - originalFileName = file.getOriginalFilename(); - } - String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length()); - CreatePartUploadResult partUpload = storage.createPartUploadSuffix(suffix); - uploadId = partUpload.getUploadId(); - - partUploadInfo = new PartUploadInfo(); - partUploadInfo.setUploadId(uploadId); - partUploadInfo.setFileName(partUpload.getFilename()); - partUploadInfo.setOriginalName(originalFileName); - partUploadInfo.setFileSuffix(suffix); - partUploadInfo.setUrl(partUpload.getUrl()); - partUploadInfo.setFileSize(bo.getFileSize()); - partUploadInfo.setService(storage.getConfigKey()); - partUploadInfo.setPartSize(bo.getPartSize()); - partUploadInfo.setTotalParts(bo.getTotalParts()); - partUploadInfo.setPartInfoList(new ArrayList<>()); - partUploadInfo.setNeedMerge(bo.getNeedMerge()); - } else { - // uploadId不为空，且文件不为空，则进行文件上传 - partUploadInfo = RedisUtils.getCacheObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId); - partUploadInfo.setNeedMerge(bo.getNeedMerge()); - storage = OssFactory.instance(partUploadInfo.getService()); - } - return partUpload(storage, file, partUploadInfo, bo.getPartNumber()); - } - - private SysOssPartUploadVo partUpload(OssClient storage, MultipartFile file, PartUploadInfo partUploadInfo, Integer partNumber) { try { - String uploadId = partUploadInfo.getUploadId(); - PartUploadResult partUploadResult = storage.partUpload(file.getInputStream(), partUploadInfo.getFileName(), partUploadInfo.getUploadId(), partNumber, file.getSize()); - // 将完成上传的分片信息放入集合中 - partUploadInfo.getPartInfoList().add(new PartInfo(partUploadResult.getPartNumber(), partUploadResult.getETag())); - // 分片上传信息放入redis - RedisUtils.setCacheObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + partUploadInfo.getUploadId(), partUploadInfo); - // 检查是否需要合并 - 已经完成所有上传时操作 - if (partUploadInfo.isNeedMerge()) { - // 如果已经完成最后一片的上传，则进行合并 - UploadResult uploadResult = storage.completePartUpload(uploadId, partUploadInfo.getFileName(), partUploadInfo.getPartInfoList()); - // 不报错即合并成功，删除redis中的分片上传信息 - RedisUtils.deleteObject(OssConstant.PART_UPLOAD_INFO_CACHE_KEY + uploadId); - // 数据落库 - SysOss oss = new SysOss(); - oss.setFileSuffix(partUploadInfo.getFileSuffix()); - oss.setOriginalName(partUploadInfo.getOriginalName()); - oss.setService(storage.getConfigKey()); - oss.setFileName(partUploadInfo.getFileName()); - oss.setUrl(partUploadInfo.getUrl()); - baseMapper.insert(oss); + String uploadId = bo.getUploadId(); + + // 如果 uploadId 为空，则说明是新建上传分片 + if (StringUtils.isBlank(uploadId)) { + return createPartUpload(file, bo); } - SysOssPartUploadVo sysOssPartUploadVo = MapstructUtils.convert(partUploadInfo, SysOssPartUploadVo.class); - sysOssPartUploadVo.setMergeCompleted(partUploadInfo.isNeedMerge()); - return sysOssPartUploadVo; + + // uploadId不为空，从缓存中获取分片上传信息 + PartUploadInfo partUploadInfo = PartUploadCacheHelper.getCache(uploadId); + if (ObjectUtil.isNull(partUploadInfo)) { + throw new ServiceException("未找到分片上传信息！"); + } + // TODO 是否需要合并分片 + partUploadInfo.setNeedMerge(bo.getNeedMerge()); + OssClient storage = OssFactory.instance(partUploadInfo.getService()); + // 上传分片并返回结果 + return partUpload(storage, partUploadInfo, file, bo.getPartNumber()); } catch (Exception e) { throw new ServiceException(e.getMessage()); } } + /** + * 创建并上传分片 + * + * @param file 需要进行上传的分片文件 + * @param bo OSS分片上传业务对象 + * @return 分片上传对象信息VO + */ + private SysOssPartUploadVo createPartUpload(MultipartFile file, SysOssPartUploadBo bo) throws IOException { + // 文件大小 + Long fileSize = bo.getFileSize(); + // 获取第一片文件的大小作为分片大小 + long partSize = file.getSize(); + // 计算总片数 + long totalParts = fileSize / partSize; + // 当除以分数有余数时每分片总数+1 + if (fileSize % partSize > 0) { + totalParts += 1; + } + // 获取文件名，如果文件名为空，则使用从上传的文件中获取 + String originalFileName = bo.getFileName(); + if (StringUtils.isBlank(originalFileName)) { + originalFileName = file.getOriginalFilename(); + } + String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length()); + OssClient storage = OssFactory.instance(); + CreatePartUploadResult partUpload = storage.createPartUploadSuffix(suffix); + // 构建分片上传信息对象 + PartUploadInfo partUploadInfo = new PartUploadInfo(); + partUploadInfo.setUploadId(partUpload.getUploadId()); + partUploadInfo.setFileName(partUpload.getFilename()); + partUploadInfo.setOriginalName(originalFileName); + partUploadInfo.setFileSuffix(suffix); + partUploadInfo.setUrl(partUpload.getUrl()); + partUploadInfo.setFileSize(fileSize); + partUploadInfo.setService(storage.getConfigKey()); + partUploadInfo.setPartSize(partSize); + partUploadInfo.setTotalParts(totalParts); + partUploadInfo.setPartInfoList(new ArrayList<>()); + // TODO 是否需要合并分片 + partUploadInfo.setNeedMerge(bo.getNeedMerge()); + // 上传分片并返回结果 + return partUpload(storage, partUploadInfo, file, bo.getPartNumber()); + } + + /** + * 上传分片 + * + * @param storage OSS客户端 + * @param partUploadInfo 分片上传信息对象 + * @param file 需要进行上传的分片文件 + * @param partNumber 分片序号 + * @return 分片上传对象信息VO + */ + private SysOssPartUploadVo partUpload(OssClient storage, PartUploadInfo partUploadInfo, MultipartFile file, Integer partNumber) throws IOException { + String uploadId = partUploadInfo.getUploadId(); + // 上传分片 + PartUploadResult partUploadResult = storage.partUpload(file.getInputStream(), partUploadInfo.getFileName(), uploadId, partNumber, file.getSize()); + // 将完成上传的分片信息放入集合中 + partUploadInfo.getPartInfoList().add(new PartInfo(partUploadResult.getPartNumber(), partUploadResult.getETag())); + // 分片上传信息放入缓存 + PartUploadCacheHelper.putCache(partUploadInfo); + // 检查是否需要合并 - 已经完成所有上传时操作 + if (partUploadInfo.isNeedMerge()) { + // 如果已经完成最后一片的上传，则进行合并 + UploadResult uploadResult = storage.completePartUpload(uploadId, partUploadInfo.getFileName(), partUploadInfo.getPartInfoList()); + // 不报错即合并成功，删除缓存中的分片上传信息 + PartUploadCacheHelper.removeCache(uploadId); + // 数据落库 + SysOss oss = new SysOss(); + oss.setFileSuffix(partUploadInfo.getFileSuffix()); + oss.setOriginalName(partUploadInfo.getOriginalName()); + oss.setService(storage.getConfigKey()); + oss.setFileName(partUploadInfo.getFileName()); + oss.setUrl(partUploadInfo.getUrl()); + baseMapper.insert(oss); + } + // 构建分片上传对象信息VO + SysOssPartUploadVo sysOssPartUploadVo = new SysOssPartUploadVo(); + sysOssPartUploadVo.setUploadId(uploadId); + sysOssPartUploadVo.setUrl(partUploadInfo.getUrl()); + sysOssPartUploadVo.setPartInfoList(partUploadInfo.getPartInfoList()); + sysOssPartUploadVo.setMergeCompleted(partUploadInfo.isNeedMerge()); + return sysOssPartUploadVo; + } + /** * 桶类型为 private 的URL 修改为临时URL时长为120s *