feat: add verification function to the configuration of s3 object storage policy (#134)

```release-note 在 S3 存储策略配置中增加了验证配置的功能。 ``` fixes https://github.com/halo-dev/plugin-s3/issues/132
2026-01-28 02:00:45 +08:00 · 2024-04-22 12:26:10 +08:00
parent 68b1a88b14
commit 47b6a37d0a
7 changed files with 271 additions and 131 deletions
--- a/build.gradle
+++ b/build.gradle
@@ -16,7 +16,7 @@ repositories {
 }

 dependencies {
-    implementation platform('run.halo.tools.platform:plugin:2.13.0-SNAPSHOT')
+    implementation platform('run.halo.tools.platform:plugin:2.14.0-SNAPSHOT')
    compileOnly 'run.halo.app:api'

    implementation platform('software.amazon.awssdk:bom:2.19.8')
@@ -36,7 +36,7 @@ configurations.runtimeClasspath {


 halo {
-    version = '2.12.1'
+    version = '2.14.0'
 }

 haloPlugin {
--- a/src/main/java/run/halo/s3os/PolicyConfigValidationController.java
+++ b/src/main/java/run/halo/s3os/PolicyConfigValidationController.java
@@ -0,0 +1,123 @@
+package run.halo.s3os;
+
+import static run.halo.s3os.S3OsAttachmentHandler.MULTIPART_MIN_PART_SIZE;
+import static run.halo.s3os.S3OsAttachmentHandler.checkResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.buffer.DataBuffer;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.core.io.buffer.DefaultDataBufferFactory;
+import org.springframework.util.StringUtils;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import run.halo.app.infra.utils.PathUtils;
+import run.halo.app.plugin.ApiVersion;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
+
+@ApiVersion("s3os.halo.run/v1alpha1")
+@RestController
+@RequiredArgsConstructor
+@Slf4j
+public class PolicyConfigValidationController {
+    private final S3OsAttachmentHandler handler;
+
+    @PostMapping("/policies/s3/validation")
+    public Mono<Void> validatePolicyConfig(@RequestBody S3OsProperties properties) {
+        var filename = "halo-s3-plugin-test-file-" + System.currentTimeMillis() + ".jpg";
+        var content = readImage();
+        return Mono.using(() -> handler.buildS3Client(properties),
+                client -> {
+                    var uploadState =
+                        new S3OsAttachmentHandler.UploadState(properties, filename, false);
+
+                    return handler.checkFileExistsAndRename(uploadState, client)
+                        // init multipart upload
+                        .flatMap(state -> Mono.fromCallable(() -> client.createMultipartUpload(
+                            CreateMultipartUploadRequest.builder()
+                                .bucket(properties.getBucket())
+                                .contentType(state.contentType)
+                                .key(state.objectKey)
+                                .build())))
+                        .doOnNext((response) -> {
+                            checkResult(response, "createMultipartUpload");
+                            uploadState.uploadId = response.uploadId();
+                        })
+                        .thenMany(handler.reshape(content, MULTIPART_MIN_PART_SIZE))
+                        // buffer to part
+                        .windowUntil((buffer) -> {
+                            uploadState.buffered += buffer.readableByteCount();
+                            if (uploadState.buffered >= MULTIPART_MIN_PART_SIZE) {
+                                uploadState.buffered = 0;
+                                return true;
+                            } else {
+                                return false;
+                            }
+                        })
+                        // upload part
+                        .concatMap((window) -> window.collectList().flatMap((bufferList) -> {
+                            var buffer = S3OsAttachmentHandler.concatBuffers(bufferList);
+                            return handler.uploadPart(uploadState, buffer, client);
+                        }))
+                        .reduce(uploadState, (state, completedPart) -> {
+                            state.completedParts.put(completedPart.partNumber(), completedPart);
+                            return state;
+                        })
+                        // complete multipart upload
+                        .flatMap((state) -> Mono.just(client.completeMultipartUpload(
+                            CompleteMultipartUploadRequest
+                                .builder()
+                                .bucket(properties.getBucket())
+                                .uploadId(state.uploadId)
+                                .multipartUpload(CompletedMultipartUpload.builder()
+                                    .parts(state.completedParts.values())
+                                    .build())
+                                .key(state.objectKey)
+                                .build())
+                        ))
+                        // get object metadata
+                        .flatMap((response) -> {
+                            checkResult(response, "completeUpload");
+                            return Mono.just(client.headObject(
+                                HeadObjectRequest.builder()
+                                    .bucket(properties.getBucket())
+                                    .key(uploadState.objectKey)
+                                    .build()
+                            ));
+                        })
+                        // check object metadata
+                        .doOnNext((response) -> {
+                            checkResult(response, "headObject");
+                        })
+                        // delete object
+                        .flatMap((response) -> Mono.just(client.deleteObject(
+                            software.amazon.awssdk.services.s3.model.DeleteObjectRequest.builder()
+                                .bucket(properties.getBucket())
+                                .key(uploadState.objectKey)
+                                .build()
+                        )))
+                        .doOnNext((response) -> checkResult(response, "deleteObject"))
+                        .then();
+                },
+                SdkAutoCloseable::close)
+            .onErrorMap(S3ExceptionHandler::map);
+    }
+
+    private Flux<DataBuffer> readImage() {
+        DefaultResourceLoader resourceLoader = new DefaultResourceLoader(this.getClass()
+            .getClassLoader());
+        String path = PathUtils.combinePath("validation.jpg");
+        String simplifyPath = StringUtils.cleanPath(path);
+        Resource resource = resourceLoader.getResource(simplifyPath);
+        return DataBufferUtils.read(resource, new DefaultDataBufferFactory(), 1024);
+    }
+}
--- a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
+++ b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
@@ -390,8 +390,8 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
            SdkAutoCloseable::close);
    }

-    private Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
-        S3Client s3client) {
+    Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
+                                               S3Client s3client) {
        return Mono.defer(() -> {
                // deduplication of uploading files
                if (uploadingFile.put(uploadState.getUploadingMapKey(),
@@ -437,8 +437,8 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
    }


-    private Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
-        S3Client s3client) {
+    Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
+                                   S3Client s3client) {
        final int partNumber = ++uploadState.partCounter;
        return Mono.just(s3client.uploadPart(UploadPartRequest.builder()
                    .bucket(uploadState.properties.getBucket())
@@ -457,7 +457,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
            });
    }

-    private static void checkResult(SdkResponse result, String operation) {
+    static void checkResult(SdkResponse result, String operation) {
        log.info("operation: {}, result: {}", operation, result);
        if (result.sdkHttpResponse() == null || !result.sdkHttpResponse().isSuccessful()) {
            log.error("Failed to upload object, response: {}", result.sdkHttpResponse());
@@ -465,7 +465,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
        }
    }

-    private static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
+    static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
        int partSize = 0;
        for (DataBuffer b : buffers) {
            partSize += b.readableByteCount();
--- a/src/main/resources/extensions/policy-template-s3os.yaml
+++ b/src/main/resources/extensions/policy-template-s3os.yaml
@@ -14,130 +14,134 @@ spec:
  forms:
    - group: default
      formSchema:
-        - $formkit: text
-          name: bucket
-          label: Bucket 桶名称
-          validation: required
-        - $formkit: select
-          name: endpointProtocol
-          label: Endpoint 访问协议
-          options:
-            - label: HTTPS
-              value: https
-            - label: HTTP
-              value: http
-          validation: required
-        - $formkit: select
-          name: enablePathStyleAccess
-          label: Endpoint 访问风格
-          options:
-            - label: Virtual Hosted Style
-              value: false
-            - label: Path Style
-              value: true
-          value: false
-          validation: required
-        - $formkit: text
-          name: endpoint
-          label: EndPoint
-          placeholder: 请填写不带bucket-name的Endpoint
-          validation: required
-          help: 协议头请在上方设置，此处无需以"http://"或"https://"开头，系统会自动拼接
-        - $formkit: password
-          name: accessKey
-          label: Access Key ID
-          placeholder: 存储桶用户标识(用户名)
-          validation: required
-        - $formkit: password
-          name: accessSecret
-          label: Access Key Secret
-          placeholder: 存储桶密钥(密码)
-          validation: required
-        - $formkit: text
-          name: region
-          label: Region
-          placeholder: 如不填写，则默认为"Auto"
-          help: 若Region为Auto无法使用，才需要填写对应Region
-        - $formkit: text
-          name: location
-          label: 上传目录
-          placeholder: 如不填写，则默认上传到根目录
-          help: 支持的占位符请查阅：https://github.com/halo-dev/plugin-s3#上传目录
-        - $formkit: select
-          name: randomFilenameMode
-          label: 上传时重命名文件方式
-          options:
-            - label: 保留原文件名
-              value: none
-            - label: 自定义（请在下方输入自定义模板）
-              value: custom
-            - label: 使用UUID
-              value: uuid
-            - label: 使用毫秒时间戳
-              value: timestampMs
-            - label: 使用原文件名 + 随机字母
-              value: withString
-            - label: 使用日期 + 随机字母
-              value: dateWithString
-            - label: 使用日期时间 + 随机字母
-              value: datetimeWithString
-            - label: 使用随机字母
-              value: string
-          validation: required
-        - $formkit: number
-          name: randomStringLength
-          key: randomStringLength
-          label: 随机字母长度
-          min: 4
-          max: 16
-          if: "$randomFilenameMode == 'dateWithString' || $randomFilenameMode == 'datetimeWithString' || $randomFilenameMode == 'withString' || $randomFilenameMode == 'string'"
-          help: 支持4~16位, 默认为8位
-        - $formkit: text
-          name: customTemplate
-          key: customTemplate
-          label: 自定义文件名模板
-          if: "$randomFilenameMode == 'custom'"
-          value: "${origin-filename}"
-          help: 支持的占位符请查阅：https://github.com/halo-dev/plugin-s3#自定义文件名模板
-        - $formkit: select
-          name: duplicateFilenameHandling
-          label: 重复文件名处理方式
-          options:
-            - label: 加随机字母数字后缀
-              value: randomAlphanumeric
-            - label: 加随机字母后缀
-              value: randomAlphabetic
-            - label: 报错不上传
-              value: exception
-          validation: required
-        - $formkit: select
-          name: protocol
-          label: 绑定域名协议
-          options:
-            - label: HTTPS
-              value: https
-            - label: HTTP
-              value: http
-          validation: required
-        - $formkit: text
-          name: domain
-          label: 绑定域名（CDN域名）
-          placeholder: 如不设置，那么将使用 Bucket + EndPoint 作为域名
-          help: 协议头请在上方设置，此处无需以"http://"或"https://"开头，系统会自动拼接
-        - $formkit: repeater
-          name: urlSuffixes
-          label: 网址后缀
-          help: 用于对指定文件类型的网址添加后缀处理参数，优先级从上到下只取第一个匹配项
-          value: [ ]
-          min: 0
+        - $formkit: verificationForm
+          action: "/apis/s3os.halo.run/v1alpha1/policies/s3/validation"
+          label: 对象存储验证
          children:
            - $formkit: text
-              name: fileSuffix
-              label: 文件后缀
-              placeholder: 以半角逗号分隔，例如：jpg,jpeg,png,gif
+              name: bucket
+              label: Bucket 桶名称
+              validation: required
+            - $formkit: select
+              name: endpointProtocol
+              label: Endpoint 访问协议
+              options:
+                - label: HTTPS
+                  value: https
+                - label: HTTP
+                  value: http
+              validation: required
+            - $formkit: select
+              name: enablePathStyleAccess
+              label: Endpoint 访问风格
+              options:
+                - label: Virtual Hosted Style
+                  value: false
+                - label: Path Style
+                  value: true
+              value: false
              validation: required
            - $formkit: text
-              name: urlSuffix
+              name: endpoint
+              label: EndPoint
+              placeholder: 请填写不带bucket-name的Endpoint
+              validation: required
+              help: 协议头请在上方设置，此处无需以"http://"或"https://"开头，系统会自动拼接
+            - $formkit: password
+              name: accessKey
+              label: Access Key ID
+              placeholder: 存储桶用户标识(用户名)
+              validation: required
+            - $formkit: password
+              name: accessSecret
+              label: Access Key Secret
+              placeholder: 存储桶密钥(密码)
+              validation: required
+            - $formkit: text
+              name: region
+              label: Region
+              placeholder: 如不填写，则默认为"Auto"
+              help: 若Region为Auto无法使用，才需要填写对应Region
+            - $formkit: text
+              name: location
+              label: 上传目录
+              placeholder: 如不填写，则默认上传到根目录
+              help: 支持的占位符请查阅：https://github.com/halo-dev/plugin-s3#上传目录
+            - $formkit: select
+              name: randomFilenameMode
+              label: 上传时重命名文件方式
+              options:
+                - label: 保留原文件名
+                  value: none
+                - label: 自定义（请在下方输入自定义模板）
+                  value: custom
+                - label: 使用UUID
+                  value: uuid
+                - label: 使用毫秒时间戳
+                  value: timestampMs
+                - label: 使用原文件名 + 随机字母
+                  value: withString
+                - label: 使用日期 + 随机字母
+                  value: dateWithString
+                - label: 使用日期时间 + 随机字母
+                  value: datetimeWithString
+                - label: 使用随机字母
+                  value: string
+              validation: required
+            - $formkit: number
+              name: randomStringLength
+              key: randomStringLength
+              label: 随机字母长度
+              min: 4
+              max: 16
+              if: "$randomFilenameMode == 'dateWithString' || $randomFilenameMode == 'datetimeWithString' || $randomFilenameMode == 'withString' || $randomFilenameMode == 'string'"
+              help: 支持4~16位, 默认为8位
+            - $formkit: text
+              name: customTemplate
+              key: customTemplate
+              label: 自定义文件名模板
+              if: "$randomFilenameMode == 'custom'"
+              value: "${origin-filename}"
+              help: 支持的占位符请查阅：https://github.com/halo-dev/plugin-s3#自定义文件名模板
+            - $formkit: select
+              name: duplicateFilenameHandling
+              label: 重复文件名处理方式
+              options:
+                - label: 加随机字母数字后缀
+                  value: randomAlphanumeric
+                - label: 加随机字母后缀
+                  value: randomAlphabetic
+                - label: 报错不上传
+                  value: exception
+              validation: required
+            - $formkit: select
+              name: protocol
+              label: 绑定域名协议
+              options:
+                - label: HTTPS
+                  value: https
+                - label: HTTP
+                  value: http
+              validation: required
+            - $formkit: text
+              name: domain
+              label: 绑定域名（CDN域名）
+              placeholder: 如不设置，那么将使用 Bucket + EndPoint 作为域名
+              help: 协议头请在上方设置，此处无需以"http://"或"https://"开头，系统会自动拼接
+            - $formkit: repeater
+              name: urlSuffixes
              label: 网址后缀
-              placeholder: 例如：?imageMogr2/format/webp
-              validation: required
+              help: 用于对指定文件类型的网址添加后缀处理参数，优先级从上到下只取第一个匹配项
+              value: [ ]
+              min: 0
+              children:
+                - $formkit: text
+                  name: fileSuffix
+                  label: 文件后缀
+                  placeholder: 以半角逗号分隔，例如：jpg,jpeg,png,gif
+                  validation: required
+                - $formkit: text
+                  name: urlSuffix
+                  label: 网址后缀
+                  placeholder: 例如：?imageMogr2/format/webp
+                  validation: required
--- a/src/main/resources/extensions/s3os-role-template.yaml
+++ b/src/main/resources/extensions/s3os-role-template.yaml
@@ -39,3 +39,16 @@ rules:
  - apiGroups: [ "s3os.halo.run" ]
    resources: [ "attachments" ]
    verbs: [ "delete" ]
+---
+apiVersion: v1alpha1
+kind: "Role"
+metadata:
+  name: role-template-s3os-policy-config-validation
+  labels:
+    halo.run/role-template: "true"
+    rbac.authorization.halo.run/aggregate-to-role-template-manage-configmaps: "true"
+rules:
+  - apiGroups: ["s3os.halo.run"]
+    resources: ["policies/validation"]
+    resourceNames: ["s3"]
+    verbs: [ "create" ]
--- a/src/main/resources/plugin.yaml
+++ b/src/main/resources/plugin.yaml
@@ -4,7 +4,7 @@ metadata:
  name: PluginS3ObjectStorage
 spec:
  enabled: true
-  requires: ">=2.13.0"
+  requires: ">=2.14.0"
  author:
    name: Halo OSS Team
    website: https://github.com/halo-dev
--- a/src/main/resources/validation.jpg
+++ b/src/main/resources/validation.jpg