Pre Merge pull request !478 from David Wei/feature/aws-s3_v2

2026-09-20 10:13:28 +08:00 · 2024-01-13 10:03:50 +00:00 · 2024-01-13 10:03:50 +00:00 · 4ef89dd6d6
commit 4ef89dd6d6
parent d4f8b93fe3 4b07b23aba
5 changed files with 262 additions and 109 deletions
--- a/pom.xml
+++ b/pom.xml
@ -44,7 +44,7 @@
        <ip2region.version>2.7.0</ip2region.version>

        <!-- OSS 配置 -->
-        <aws-java-sdk-s3.version>1.12.600</aws-java-sdk-s3.version>
+        <aws-java-sdk-s3.version>2.22.9</aws-java-sdk-s3.version>
        <!-- SMS 配置 -->
        <sms4j.version>2.2.0</sms4j.version>
        <!-- 限制框架中的fastjson版本 -->
@ -236,10 +236,13 @@
            </dependency>

            <dependency>
-                <groupId>com.amazonaws</groupId>
-                <artifactId>aws-java-sdk-s3</artifactId>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bom</artifactId>
                <version>${aws-java-sdk-s3.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
            </dependency>
+
            <!--短信sms4j-->
            <dependency>
                <groupId>org.dromara.sms4j</groupId>
--- a/ruoyi-common/ruoyi-common-oss/pom.xml
+++ b/ruoyi-common/ruoyi-common-oss/pom.xml
@ -27,9 +27,12 @@
        </dependency>

        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-s3</artifactId>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>netty-nio-client</artifactId>
        </dependency>
    </dependencies>
-
 </project>
--- a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/core/OssClient.java
+++ b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/core/OssClient.java
@ -2,18 +2,7 @@ package org.dromara.common.oss.core;

 import cn.hutool.core.io.IoUtil;
 import cn.hutool.core.util.IdUtil;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.HttpMethod;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.*;
+import io.netty.handler.ssl.SslProvider;
 import org.dromara.common.core.utils.DateUtils;
 import org.dromara.common.core.utils.StringUtils;
 import org.dromara.common.oss.constant.OssConstant;
@ -22,18 +11,32 @@ import org.dromara.common.oss.enumd.AccessPolicyType;
 import org.dromara.common.oss.enumd.PolicyType;
 import org.dromara.common.oss.exception.OssException;
 import org.dromara.common.oss.properties.OssProperties;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.async.AsyncRequestBody;
+import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.*;
+import software.amazon.awssdk.services.s3.model.*;
+
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;

 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.InputStream;
+import java.net.URI;
 import java.net.URL;
-import java.util.Date;
+import java.time.Duration;
+import java.util.concurrent.Executors;

 /**
- * S3 存储协议 所有兼容S3协议的云厂商均支持
+ * S3-v2 存储协议 所有兼容S3协议的云厂商均支持
 * 阿里云 腾讯云 七牛云 minio
 *
- * @author Lion Li
+ * @author Lion Li / David Wei
 */
 public class OssClient {

@ -41,33 +44,45 @@ public class OssClient {

    private final OssProperties properties;

-    private final AmazonS3 client;
+    private final S3AsyncClient client;
+
+    private final S3Presigner presigner;

    public OssClient(String configKey, OssProperties ossProperties) {
        this.configKey = configKey;
        this.properties = ossProperties;
        try {
-            AwsClientBuilder.EndpointConfiguration endpointConfig =
-                new AwsClientBuilder.EndpointConfiguration(properties.getEndpoint(), properties.getRegion());
+            StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(
+                AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()));
+
+            S3AsyncClientBuilder client = S3AsyncClient.builder()
+                .httpClient(
+                    NettyNioAsyncHttpClient.builder()
+                        .sslProvider(SslProvider.OPENSSL)
+                        .build()
+                )
+                .asyncConfiguration(
+                    b -> b.advancedOption(SdkAdvancedAsyncClientOption
+                            .FUTURE_COMPLETION_EXECUTOR,
+                        Executors.newFixedThreadPool(10)
+                    )
+                )
+                .credentialsProvider(credentialsProvider)
+                .endpointOverride(URI.create(getEndpoint()))
+                .region(Region.of(properties.getRegion()));
+
+            S3Presigner.Builder presigner = S3Presigner.builder()
+                .credentialsProvider(credentialsProvider)
+                .endpointOverride(URI.create(getPresignerEndpoint()))
+                .region(Region.of(properties.getRegion()));

-            AWSCredentials credentials = new BasicAWSCredentials(properties.getAccessKey(), properties.getSecretKey());
-            AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials);
-            ClientConfiguration clientConfig = new ClientConfiguration();
-            if (OssConstant.IS_HTTPS.equals(properties.getIsHttps())) {
-                clientConfig.setProtocol(Protocol.HTTPS);
-            } else {
-                clientConfig.setProtocol(Protocol.HTTP);
-            }
-            AmazonS3ClientBuilder build = AmazonS3Client.builder()
-                .withEndpointConfiguration(endpointConfig)
-                .withClientConfiguration(clientConfig)
-                .withCredentials(credentialsProvider)
-                .disableChunkedEncoding();
            if (!StringUtils.containsAny(properties.getEndpoint(), OssConstant.CLOUD_SERVICE)) {
                // minio 使用https限制使用域名访问 需要此配置 站点填域名
-                build.enablePathStyleAccess();
+                client.forcePathStyle(true);
            }
-            this.client = build.build();
+
+            this.client = client.build();
+            this.presigner = presigner.build();

            createBucket();
        } catch (Exception e) {
@ -81,14 +96,28 @@ public class OssClient {
    public void createBucket() {
        try {
            String bucketName = properties.getBucketName();
-            if (client.doesBucketExistV2(bucketName)) {
+            try {
+                client.headBucket(HeadBucketRequest.builder()
+                    .bucket(bucketName)
+                    .build())
+                    .get();
                return;
+            } catch (Exception e) {
+                // 桶不存在，捕获异常。
+                // 继续桶创建操作
            }
-            CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);
            AccessPolicyType accessPolicy = getAccessPolicy();
-            createBucketRequest.setCannedAcl(accessPolicy.getAcl());
-            client.createBucket(createBucketRequest);
-            client.setBucketPolicy(bucketName, getPolicy(bucketName, accessPolicy.getPolicyType()));
+            CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
+                .bucket(bucketName)
+                .acl(accessPolicy.getBucketCannedACL())
+                .build();
+
+            PutBucketPolicyRequest putBucketPolicyRequest = PutBucketPolicyRequest.builder()
+                .bucket(bucketName)
+                .policy(getPolicy(bucketName, accessPolicy.getPolicyType()))
+                .build();
+            client.createBucket(createBucketRequest).get();
+            client.putBucketPolicy(putBucketPolicyRequest).get();
        } catch (Exception e) {
            throw new OssException("创建Bucket失败, 请核对配置信息:[" + e.getMessage() + "]");
        }
@ -103,13 +132,17 @@ public class OssClient {
            inputStream = new ByteArrayInputStream(IoUtil.readBytes(inputStream));
        }
        try {
-            ObjectMetadata metadata = new ObjectMetadata();
-            metadata.setContentType(contentType);
-            metadata.setContentLength(inputStream.available());
-            PutObjectRequest putObjectRequest = new PutObjectRequest(properties.getBucketName(), path, inputStream, metadata);
-            // 设置上传对象的 Acl 为公共读
-            putObjectRequest.setCannedAcl(getAccessPolicy().getAcl());
-            client.putObject(putObjectRequest);
+            PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(properties.getBucketName())
+                .key(path)
+                .contentType(contentType)
+                .acl(getAccessPolicy().getObjectCannedACL()) // 设置上传对象的 Acl
+                .build();
+            client.putObject(putObjectRequest, AsyncRequestBody.fromInputStream(
+                inputStream,
+                (long) inputStream.available(),
+                Executors.newFixedThreadPool(1)
+            )).get();
        } catch (Exception e) {
            throw new OssException("上传文件失败，请检查配置信息:[" + e.getMessage() + "]");
        }
@ -118,10 +151,12 @@ public class OssClient {

    public UploadResult upload(File file, String path) {
        try {
-            PutObjectRequest putObjectRequest = new PutObjectRequest(properties.getBucketName(), path, file);
-            // 设置上传对象的 Acl 为公共读
-            putObjectRequest.setCannedAcl(getAccessPolicy().getAcl());
-            client.putObject(putObjectRequest);
+            PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(properties.getBucketName())
+                .key(path)
+                .acl(getAccessPolicy().getObjectCannedACL())// 设置上传对象的 Acl
+                .build();
+            client.putObject(putObjectRequest, AsyncRequestBody.fromFile(file)).get();
        } catch (Exception e) {
            throw new OssException("上传文件失败，请检查配置信息:[" + e.getMessage() + "]");
        }
@ -131,7 +166,11 @@ public class OssClient {
    public void delete(String path) {
        path = path.replace(getUrl() + "/", "");
        try {
-            client.deleteObject(properties.getBucketName(), path);
+            DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
+                .bucket(properties.getBucketName())
+                .key(path)
+                .build();
+            client.deleteObject(deleteObjectRequest).get();
        } catch (Exception e) {
            throw new OssException("删除文件失败，请检查配置信息:[" + e.getMessage() + "]");
        }
@ -154,16 +193,34 @@ public class OssClient {
     *
     * @param path 完整文件路径
     */
-    public ObjectMetadata getObjectMetadata(String path) {
+    public HeadObjectResponse getObjectMetadata(String path) {
        path = path.replace(getUrl() + "/", "");
-        S3Object object = client.getObject(properties.getBucketName(), path);
-        return object.getObjectMetadata();
+        HeadObjectRequest headObjectRequest = HeadObjectRequest.builder()
+            .bucket(properties.getBucketName())
+            .key(path)
+            .build();
+        return client.headObject(headObjectRequest).join();
    }

-    public InputStream getObjectContent(String path) {
-        path = path.replace(getUrl() + "/", "");
-        S3Object object = client.getObject(properties.getBucketName(), path);
-        return object.getObjectContent();
+    public String getEndpoint() {
+        String endpoint = properties.getEndpoint();
+        String header = OssConstant.IS_HTTPS.equals(properties.getIsHttps()) ? "https://" : "http://";
+        return header + endpoint;
+    }
+
+    /**
+     * 获取预签名地址的 Endpoint 地址
+     * 注意：此地址不能带bucket，「云服务商」使用非路径形式，生成的url会自动带上bucket前缀
+     * @return
+     */
+    public String getPresignerEndpoint() {
+        String domain = properties.getDomain();
+        String endpoint = properties.getEndpoint();
+        String header = OssConstant.IS_HTTPS.equals(properties.getIsHttps()) ? "https://" : "http://";
+        if (StringUtils.isNotBlank(domain)) {
+            return header + domain;
+        }
+        return header + endpoint;
    }

    public String getUrl() {
@ -195,7 +252,6 @@ public class OssClient {
        return path + suffix;
    }

-
    public String getConfigKey() {
        return configKey;
    }
@ -207,12 +263,24 @@ public class OssClient {
     * @param second    授权时间
     */
    public String getPrivateUrl(String objectKey, Integer second) {
-        GeneratePresignedUrlRequest generatePresignedUrlRequest =
-            new GeneratePresignedUrlRequest(properties.getBucketName(), objectKey)
-                .withMethod(HttpMethod.GET)
-                .withExpiration(new Date(System.currentTimeMillis() + 1000L * second));
-        URL url = client.generatePresignedUrl(generatePresignedUrlRequest);
-        return url.toString();
+        try {
+            GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                .bucket(properties.getBucketName())
+                .key(objectKey)
+                .build();
+            GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder()
+                .signatureDuration(Duration.ofSeconds(second))
+                .getObjectRequest(getObjectRequest)
+                .build();
+
+            PresignedGetObjectRequest presignedGetObjectRequest = presigner
+                .presignGetObject(getObjectPresignRequest);
+            URL url = presignedGetObjectRequest.url();
+
+            return url.toString();
+        } catch (Exception e) {
+            return null;
+        }
    }

    /**
@ -231,32 +299,108 @@ public class OssClient {
        return AccessPolicyType.getByType(properties.getAccessPolicy());
    }

+    /**
+     * 获取 minio 的 policy 定义
+     * @param bucketName
+     * @param policyType
+     * @return
+     */
    private static String getPolicy(String bucketName, PolicyType policyType) {
-        StringBuilder builder = new StringBuilder();
-        builder.append("{\n\"Statement\": [\n{\n\"Action\": [\n");
-        builder.append(switch (policyType) {
-            case WRITE -> "\"s3:GetBucketLocation\",\n\"s3:ListBucketMultipartUploads\"\n";
-            case READ_WRITE -> "\"s3:GetBucketLocation\",\n\"s3:ListBucket\",\n\"s3:ListBucketMultipartUploads\"\n";
-            default -> "\"s3:GetBucketLocation\"\n";
-        });
-        builder.append("],\n\"Effect\": \"Allow\",\n\"Principal\": \"*\",\n\"Resource\": \"arn:aws:s3:::");
-        builder.append(bucketName);
-        builder.append("\"\n},\n");
-        if (policyType == PolicyType.READ) {
-            builder.append("{\n\"Action\": [\n\"s3:ListBucket\"\n],\n\"Effect\": \"Deny\",\n\"Principal\": \"*\",\n\"Resource\": \"arn:aws:s3:::");
-            builder.append(bucketName);
-            builder.append("\"\n},\n");
-        }
-        builder.append("{\n\"Action\": ");
-        builder.append(switch (policyType) {
-            case WRITE -> "[\n\"s3:AbortMultipartUpload\",\n\"s3:DeleteObject\",\n\"s3:ListMultipartUploadParts\",\n\"s3:PutObject\"\n],\n";
-            case READ_WRITE -> "[\n\"s3:AbortMultipartUpload\",\n\"s3:DeleteObject\",\n\"s3:GetObject\",\n\"s3:ListMultipartUploadParts\",\n\"s3:PutObject\"\n],\n";
-            default -> "\"s3:GetObject\",\n";
-        });
-        builder.append("\"Effect\": \"Allow\",\n\"Principal\": \"*\",\n\"Resource\": \"arn:aws:s3:::");
-        builder.append(bucketName);
-        builder.append("/*\"\n}\n],\n\"Version\": \"2012-10-17\"\n}\n");
-        return builder.toString();
-    }
+        String policy = switch (policyType) {
+            case WRITE -> """
+                {
+                    "Version": "2012-10-17",
+                    "Statement": []
+                }
+                """;
+            case READ_WRITE -> """
+                {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Effect": "Allow",
+                            "Principal": {
+                                "AWS": [
+                                    "*"
+                                ]
+                            },
+                            "Action": [
+                                "s3:GetBucketLocation",
+                                "s3:ListBucket",
+                                "s3:ListBucketMultipartUploads"
+                            ],
+                            "Resource": [
+                                "arn:aws:s3:::bucketName"
+                            ]
+                        },
+                        {
+                            "Effect": "Allow",
+                            "Principal": {
+                                "AWS": [
+                                    "*"
+                                ]
+                            },
+                            "Action": [
+                                "s3:AbortMultipartUpload",
+                                "s3:DeleteObject",
+                                "s3:GetObject",
+                                "s3:ListMultipartUploadParts",
+                                "s3:PutObject"
+                            ],
+                            "Resource": [
+                                "arn:aws:s3:::bucketName/*"
+                            ]
+                        }
+                    ]
+                }
+                """;
+            case READ -> """
+                {
+                  "Version": "2012-10-17",
+                  "Statement": [
+                    {
+                      "Effect": "Allow",
+                      "Principal": {
+                        "AWS": [
+                          "*"
+                        ]
+                      },
+                      "Action": [
+                        "s3:GetBucketLocation"
+                      ],
+                      "Resource": [
+                        "arn:aws:s3:::bucketName"
+                      ]
+                    },
+                    {
+                      "Effect": "Deny",
+                      "Principal": "*",
+                      "Action": [
+                        "s3:ListBucket"
+                      ],
+                      "Resource": [
+                        "arn:aws:s3:::bucketName"
+                      ]
+                    },
+                    {
+                      "Effect": "Allow",
+                      "Principal": {
+                        "AWS": [
+                          "*"
+                        ]
+                      },
+                      "Action": [
+                        "s3:GetObject"
+                      ],
+                      "Resource": [
+                        "arn:aws:s3:::bucketName/*"
+                      ]
+                    }
+                  ]
+                }
+                """;
+        };

+        return policy.replaceAll("bucketName", bucketName);
+    }
 }
--- a/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/enumd/AccessPolicyType.java
+++ b/ruoyi-common/ruoyi-common-oss/src/main/java/org/dromara/common/oss/enumd/AccessPolicyType.java
@ -1,8 +1,9 @@
 package org.dromara.common.oss.enumd;

-import com.amazonaws.services.s3.model.CannedAccessControlList;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import software.amazon.awssdk.services.s3.model.BucketCannedACL;
+import software.amazon.awssdk.services.s3.model.ObjectCannedACL;

 /**
 * 桶访问策略配置
@ -16,27 +17,32 @@ public enum AccessPolicyType {
    /**
     * private
     */
-    PRIVATE("0", CannedAccessControlList.Private, PolicyType.WRITE),
+    PRIVATE("0", BucketCannedACL.PRIVATE, ObjectCannedACL.PRIVATE, PolicyType.WRITE),

    /**
     * public
     */
-    PUBLIC("1", CannedAccessControlList.PublicRead, PolicyType.READ),
+    PUBLIC("1", BucketCannedACL.PUBLIC_READ_WRITE, ObjectCannedACL.PUBLIC_READ_WRITE, PolicyType.READ_WRITE),

    /**
     * custom
     */
-    CUSTOM("2",CannedAccessControlList.PublicRead, PolicyType.READ);
+    CUSTOM("2", BucketCannedACL.PUBLIC_READ, ObjectCannedACL.PUBLIC_READ, PolicyType.READ);

    /**
-     * 桶 权限类型
+     * 桶 权限类型（数据库值）
     */
    private final String type;

+    /**
+     * 桶 权限类型
+     */
+    private final BucketCannedACL bucketCannedACL;
+
    /**
     * 文件对象 权限类型
     */
-    private final CannedAccessControlList acl;
+    private final ObjectCannedACL objectCannedACL;

    /**
     * 桶策略类型
--- a/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java
+++ b/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/system/service/impl/SysOssServiceImpl.java
@ -1,8 +1,8 @@
 package org.dromara.system.service.impl;

 import cn.hutool.core.convert.Convert;
-import cn.hutool.core.io.IoUtil;
 import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.http.HttpUtil;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@ -114,18 +114,15 @@ public class SysOssServiceImpl implements ISysOssService, OssService {
    }

    @Override
-    public void download(Long ossId, HttpServletResponse response) throws IOException {
+    public void download(Long ossId, HttpServletResponse response) {
        SysOssVo sysOss = SpringUtils.getAopProxy(this).getById(ossId);
        if (ObjectUtil.isNull(sysOss)) {
            throw new ServiceException("文件数据不存在!");
        }
        FileUtils.setAttachmentResponseHeader(response, sysOss.getOriginalName());
        response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE + "; charset=UTF-8");
-        OssClient storage = OssFactory.instance(sysOss.getService());
-        try(InputStream inputStream = storage.getObjectContent(sysOss.getUrl())) {
-            int available = inputStream.available();
-            IoUtil.copy(inputStream, response.getOutputStream(), available);
-            response.setContentLength(available);
+        try {
+            HttpUtil.download(this.matchingUrl(sysOss).getUrl(), response.getOutputStream(), true);
        } catch (Exception e) {
            throw new ServiceException(e.getMessage());
        }