chore: remove deprecated plugin wrapper (#161)

### What this PR does?
移除对已过时的 PluginWrapper 的引用，Halo 2.18.0 版本后将不在支持从 BasePlugin 中获取 PluginWrapper ，也不再支持依赖注入 PluginWrapper，使用 PluginContext 代替

see also https://github.com/halo-dev/halo/pull/6243 for more details

```release-note
None
```

.github/workflows/cd.yaml

@@ -0,0 +1,17 @@
+name: CD
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  cd:
+    uses: halo-sigs/reusable-workflows/.github/workflows/plugin-cd.yaml@v1
+    secrets:
+      halo-username: ${{ secrets.HALO_USERNAME }}
+      halo-password: ${{ secrets.HALO_PASSWORD }}
+    permissions:
+      contents: write
+    with:
+      app-id: app-Qxhpp

.github/workflows/ci.yaml

@@ -0,0 +1,13 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  ci:
+    uses: halo-sigs/reusable-workflows/.github/workflows/plugin-ci.yaml@v1

.github/workflows/workflow.yaml

@@ -1,138 +0,0 @@
-name: Build Plugin JAR File
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - "**"
-      - "!**.md"
-  release:
-    types:
-      - created
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "**"
-      - "!**.md"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Set up JDK 17
-        uses: actions/setup-java@v2
-        with:
-          distribution: 'temurin'
-          cache: 'gradle'
-          java-version: 17
-      - name: Set up Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - uses: pnpm/action-setup@v2.0.1
-        name: Install pnpm
-        id: pnpm-install
-        with:
-          version: 8
-          run_install: false
-      - name: Get pnpm store directory
-        id: pnpm-cache
-        run: |
-          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
-      - uses: actions/cache@v3
-        name: Setup pnpm cache
-        with:
-          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
-          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/widget/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-store-
-      - name: Install Frontend Dependencies
-        run: |
-          ./gradlew pnpmInstall
-      - name: Build with Gradle
-        run: |
-          # Set the version with tag name when releasing
-          version=${{ github.event.release.tag_name }}
-          version=${version#v}
-          sed -i "s/version=.*-SNAPSHOT$/version=$version/1" gradle.properties
-          ./gradlew clean build -x test
-      - name: Archive plugin-s3 jar
-        uses: actions/upload-artifact@v2
-        with:
-          name: plugin-s3
-          path: |
-            build/libs/*.jar
-          retention-days: 1
-
-  github-release:
-    runs-on: ubuntu-latest
-    needs: build
-    if: github.event_name == 'release'
-    steps:
-      - name: Download plugin-s3 jar
-        uses: actions/download-artifact@v2
-        with:
-          name: plugin-s3
-          path: build/libs
-      - name: Get Name of Artifact
-        id: get_artifact
-        run: |
-          ARTIFACT_PATHNAME=$(ls build/libs/*.jar | head -n 1)
-          ARTIFACT_NAME=$(basename ${ARTIFACT_PATHNAME})
-          echo "Artifact pathname: ${ARTIFACT_PATHNAME}"
-          echo "Artifact name: ${ARTIFACT_NAME}"
-          echo "ARTIFACT_PATHNAME=${ARTIFACT_PATHNAME}" >> $GITHUB_ENV
-          echo "ARTIFACT_NAME=${ARTIFACT_NAME}" >> $GITHUB_ENV
-          echo "RELEASE_ID=${{ github.event.release.id }}" >> $GITHUB_ENV
-      - name: Upload a Release Asset
-        uses: actions/github-script@v2
-        if: github.event_name == 'release'
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            console.log('environment', process.versions);
-
-            const fs = require('fs').promises;
-
-            const { repo: { owner, repo }, sha } = context;
-            console.log({ owner, repo, sha });
-
-            const releaseId = process.env.RELEASE_ID
-            const artifactPathName = process.env.ARTIFACT_PATHNAME
-            const artifactName = process.env.ARTIFACT_NAME
-            console.log('Releasing', releaseId, artifactPathName, artifactName)
-
-            await github.repos.uploadReleaseAsset({
-              owner, repo,
-              release_id: releaseId,
-              name: artifactName,
-              data: await fs.readFile(artifactPathName)
-            });
-
-  app-store-release:
-    runs-on: ubuntu-latest
-    needs: build
-    if: github.event_name == 'release'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Download plugin-s3 jar
-        uses: actions/download-artifact@v2
-        with:
-          name: plugin-s3
-          path: build/libs
-      - name: Sync to Halo App Store
-        uses: halo-sigs/app-store-release-action@main
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          app-id: ${{secrets.APP_ID}}
-          release-id: ${{ github.event.release.id }}
-          assets-dir: "build/libs"
-          halo-username: ${{ secrets.HALO_USERNAME }}
-          halo-password: ${{ secrets.HALO_PASSWORD }}

build.gradle

@@ -16,7 +16,7 @@ repositories {
 }
 
 dependencies {
-    implementation platform('run.halo.tools.platform:plugin:2.12.0-SNAPSHOT')
+    implementation platform('run.halo.tools.platform:plugin:2.14.0-SNAPSHOT')
     compileOnly 'run.halo.app:api'
 
     implementation platform('software.amazon.awssdk:bom:2.19.8')
@@ -36,7 +36,7 @@ configurations.runtimeClasspath {
 
 
 halo {
-    version = '2.12.1'
+    version = '2.14.0'
 }
 
 haloPlugin {
@@ -59,6 +59,10 @@ task buildFrontend(type: PnpmTask) {
     args = ['build']
 }
 
+tasks.named('buildFrontend') {
+    dependsOn 'pnpmInstall'
+}
+
 build {
     // build frontend before build
     tasks.getByName('compileJava').dependsOn('buildFrontend')

console/src/controller/s-3-link-controller/getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts

@@ -10,6 +10,7 @@ export function getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName(params: GetApisS3O
         continuationObject: params.continuationObject,
         pageSize: params.pageSize,
         unlinked: params.unlinked,
+        filePrefix: params.filePrefix,
     };
     return request.get<DeepRequired<S3ListResult>>(`/apis/s3os.halo.run/v1alpha1/objects/${params.policyName}`, {
         params: paramsInput,
@@ -22,4 +23,5 @@ interface GetApisS3OsHaloRunV1Alpha1ObjectsByPolicyNameParams {
     continuationObject?: any;
     pageSize: any;
     unlinked?: any;
+    filePrefix?: any;
 }

console/src/utils/request.ts

@@ -18,7 +18,9 @@ request.interceptors.response.use(
       return Promise.reject(error);
     }
     const { status } = errorResponse;
-    if (status !== 200) {
+    if (status === 400) {
+      Toast.error(errorResponse.data.detail);
+    } else if (status !== 200) {
       Toast.error("status: " + status);
     }
     return Promise.reject(error);

console/src/views/S3Link.vue

@@ -14,6 +14,7 @@ import {
   VTag,
 } from "@halo-dev/components";
 import CarbonFolderDetailsReference from "~icons/carbon/folder-details-reference";
+import IconErrorWarning from "~icons/ri/error-warning-line";
 import {computed, onMounted, ref, watch} from "vue";
 import {
   getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName,
@@ -27,10 +28,14 @@ const policyName = ref<string>("");
 const page = ref(1);
 const size = ref(50);
 const policyOptions = ref<{ label: string; value: string; attrs: any }[]>([{
-  label: "请选择策略",
+  label: "请选择存储策略",
   value: "",
   attrs: {disabled: true}
 }]);
+// update when fetch first page
+const filePrefix = ref<string>("");
+// update when user input
+const filePrefixBind = ref<string>("");
 const s3Objects = ref<S3ListResult>({
   objects: [],
   hasMore: false,
@@ -58,15 +63,15 @@ const selectedLinkedStatusItem = ref<boolean | undefined>(linkedStatusItems[0].v
 
 const emptyTips = computed(() => {
   if (isFetchingPolicies.value) {
-    return "正在加载策略";
+    return "正在加载存储策略";
   } else {
     if (policyOptions.value.length <= 1) {
-      return "没有可用的策略，请前往【附件】添加S3策略";
+      return "没有可用的存储策略，请前往【附件】添加S3存储策略";
     } else {
       if (!policyName.value) {
-        return "请在左上方选择策略";
+        return "请在左上方选择存储策略";
       } else {
-        return "该策略的 桶/文件夹 下没有文件";
+        return "该存储策略的 桶/文件夹 下没有文件";
       }
     }
   }
@@ -92,7 +97,7 @@ const fetchPolicies = async () => {
     const policiesData = await getApisS3OsHaloRunV1Alpha1PoliciesS3();
     if (policiesData.status == 200) {
       policyOptions.value = [{
-        label: "请选择策略",
+        label: "请选择存储策略",
         value: "",
         attrs: {disabled: true}
       }];
@@ -139,6 +144,8 @@ const clearTokenAndObject = () => {
   s3Objects.value.nextContinuationObject = "";
 };
 
+// filePrefix will not be updated from user input
+// if you want to update filePrefix, please call `handleFirstPage`
 const fetchObjects = async () => {
   if (!policyName.value) {
     return;
@@ -152,6 +159,7 @@ const fetchObjects = async () => {
       continuationToken: s3Objects.value.currentToken,
       continuationObject: s3Objects.value.currentContinuationObject,
       unlinked: selectedLinkedStatusItem.value,
+      filePrefix: filePrefix.value
     });
     if (objectsData.status == 200) {
       s3Objects.value = objectsData.data;
@@ -222,6 +230,7 @@ const handleFirstPage = () => {
   isFetching.value = true;
   page.value = 1;
   clearTokenAndObject();
+  filePrefix.value = filePrefixBind.value;
   fetchObjects();
 };
 
@@ -255,18 +264,26 @@ const handleModalClose = () => {
             <div class="flex w-full flex-1 items-center sm:w-auto">
               <div
                 v-if="!selectedFiles.length"
-                class="flex items-center gap-2"
+                class="flex flex-wrap items-center gap-2"
               >
-                策略:
+                <span class="whitespace-nowrap">存储策略:</span>
                 <FormKit
                   id="policyChoose"
-                  outer-class="!p-0 w-48"
+                  outer-class="!p-0"
+                  style="min-width: 10rem;"
                   v-model="policyName"
                   name="policyName"
                   type="select"
                   :options="policyOptions"
-                  @change="fetchObjects()"
+                  @change="handleFirstPage"
                 ></FormKit>
+                <icon-error-warning v-if="!policyName" class="text-red-500"/>
+                <SearchInput
+                  v-model="filePrefixBind"
+                  v-if="policyName"
+                  placeholder="请输入文件名前缀搜索"
+                  @update:modelValue="handleFirstPage"
+                ></SearchInput>
               </div>
               <VSpace v-else>
                 <VButton type="primary" @click="handleLink">

gradle.properties

@@ -1 +1 @@
-version=1.8.0-SNAPSHOT
+version=1.9.0-SNAPSHOT

src/main/java/run/halo/s3os/FilePathUtils.java

@@ -6,6 +6,6 @@ import lombok.experimental.UtilityClass;
 public class FilePathUtils {
 
     public static String getFilePathByPlaceholder(String filePath) {
-        return PlaceholderReplacer.replacePlaceholders(filePath, "");
+        return PlaceholderReplacer.replacePlaceholders(filePath, null);
     }
 }

src/main/java/run/halo/s3os/LinkRequest.java

@@ -1,13 +1,12 @@
 package run.halo.s3os;
 
+import java.util.Set;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 
-import java.util.List;
-
 @Data
 @RequiredArgsConstructor
 public class LinkRequest {
     private String policyName;
-    private List<String> objectKeys;
+    private Set<String> objectKeys;
 }

src/main/java/run/halo/s3os/PlaceholderReplacer.java

@@ -69,7 +69,7 @@ public class PlaceholderReplacer {
 
     private static String currentMillisecond(Map<String, String> reusableParams) {
         LocalDateTime time = LocalDateTime.parse(reusableParams.get("time"));
-        return String.valueOf(time.getNano() / 1000000);
+        return String.format("%03d", time.getNano() / 1000000);
     }
 
     private static String currentSecond(Map<String, String> reusableParams) {

src/main/java/run/halo/s3os/PolicyConfigValidationController.java

@@ -0,0 +1,123 @@
+package run.halo.s3os;
+
+import static run.halo.s3os.S3OsAttachmentHandler.MULTIPART_MIN_PART_SIZE;
+import static run.halo.s3os.S3OsAttachmentHandler.checkResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.buffer.DataBuffer;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.core.io.buffer.DefaultDataBufferFactory;
+import org.springframework.util.StringUtils;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import run.halo.app.infra.utils.PathUtils;
+import run.halo.app.plugin.ApiVersion;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
+
+@ApiVersion("s3os.halo.run/v1alpha1")
+@RestController
+@RequiredArgsConstructor
+@Slf4j
+public class PolicyConfigValidationController {
+    private final S3OsAttachmentHandler handler;
+
+    @PostMapping("/policies/s3/validation")
+    public Mono<Void> validatePolicyConfig(@RequestBody S3OsProperties properties) {
+        var filename = "halo-s3-plugin-test-file-" + System.currentTimeMillis() + ".jpg";
+        var content = readImage();
+        return Mono.using(() -> handler.buildS3Client(properties),
+                client -> {
+                    var uploadState =
+                        new S3OsAttachmentHandler.UploadState(properties, filename, false);
+
+                    return handler.checkFileExistsAndRename(uploadState, client)
+                        // init multipart upload
+                        .flatMap(state -> Mono.fromCallable(() -> client.createMultipartUpload(
+                            CreateMultipartUploadRequest.builder()
+                                .bucket(properties.getBucket())
+                                .contentType(state.contentType)
+                                .key(state.objectKey)
+                                .build())))
+                        .doOnNext((response) -> {
+                            checkResult(response, "createMultipartUpload");
+                            uploadState.uploadId = response.uploadId();
+                        })
+                        .thenMany(handler.reshape(content, MULTIPART_MIN_PART_SIZE))
+                        // buffer to part
+                        .windowUntil((buffer) -> {
+                            uploadState.buffered += buffer.readableByteCount();
+                            if (uploadState.buffered >= MULTIPART_MIN_PART_SIZE) {
+                                uploadState.buffered = 0;
+                                return true;
+                            } else {
+                                return false;
+                            }
+                        })
+                        // upload part
+                        .concatMap((window) -> window.collectList().flatMap((bufferList) -> {
+                            var buffer = S3OsAttachmentHandler.concatBuffers(bufferList);
+                            return handler.uploadPart(uploadState, buffer, client);
+                        }))
+                        .reduce(uploadState, (state, completedPart) -> {
+                            state.completedParts.put(completedPart.partNumber(), completedPart);
+                            return state;
+                        })
+                        // complete multipart upload
+                        .flatMap((state) -> Mono.just(client.completeMultipartUpload(
+                            CompleteMultipartUploadRequest
+                                .builder()
+                                .bucket(properties.getBucket())
+                                .uploadId(state.uploadId)
+                                .multipartUpload(CompletedMultipartUpload.builder()
+                                    .parts(state.completedParts.values())
+                                    .build())
+                                .key(state.objectKey)
+                                .build())
+                        ))
+                        // get object metadata
+                        .flatMap((response) -> {
+                            checkResult(response, "completeUpload");
+                            return Mono.just(client.headObject(
+                                HeadObjectRequest.builder()
+                                    .bucket(properties.getBucket())
+                                    .key(uploadState.objectKey)
+                                    .build()
+                            ));
+                        })
+                        // check object metadata
+                        .doOnNext((response) -> {
+                            checkResult(response, "headObject");
+                        })
+                        // delete object
+                        .flatMap((response) -> Mono.just(client.deleteObject(
+                            software.amazon.awssdk.services.s3.model.DeleteObjectRequest.builder()
+                                .bucket(properties.getBucket())
+                                .key(uploadState.objectKey)
+                                .build()
+                        )))
+                        .doOnNext((response) -> checkResult(response, "deleteObject"))
+                        .then();
+                },
+                SdkAutoCloseable::close)
+            .onErrorMap(S3ExceptionHandler::map);
+    }
+
+    private Flux<DataBuffer> readImage() {
+        DefaultResourceLoader resourceLoader = new DefaultResourceLoader(this.getClass()
+            .getClassLoader());
+        String path = PathUtils.combinePath("validation.jpg");
+        String simplifyPath = StringUtils.cleanPath(path);
+        Resource resource = resourceLoader.getResource(simplifyPath);
+        return DataBufferUtils.read(resource, new DefaultDataBufferFactory(), 1024);
+    }
+}

src/main/java/run/halo/s3os/S3LinkController.java

@@ -1,30 +1,22 @@
 package run.halo.s3os;
 
 import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
-import run.halo.app.core.extension.attachment.Attachment;
 import run.halo.app.core.extension.attachment.Policy;
-import run.halo.app.extension.ReactiveExtensionClient;
 import run.halo.app.plugin.ApiVersion;
 
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
 @ApiVersion("s3os.halo.run/v1alpha1")
 @RestController
 @RequiredArgsConstructor
 public class S3LinkController {
     private final S3LinkService s3LinkService;
-    private final ReactiveExtensionClient client;
-
-    /**
-     * Map of linking file, used as a lock, key is policyName/objectKey, value is policyName/objectKey.
-     */
-    private final Map<String, Object> linkingFile = new ConcurrentHashMap<>();
 
     @GetMapping("/policies/s3")
     public Flux<Policy> listS3Policies() {
@@ -36,50 +28,19 @@ public class S3LinkController {
         @RequestParam(name = "continuationToken", required = false) String continuationToken,
         @RequestParam(name = "continuationObject", required = false) String continuationObject,
         @RequestParam(name = "pageSize") Integer pageSize,
-        @RequestParam(name = "unlinked", required = false, defaultValue = "false")
-        Boolean unlinked) {
+        @RequestParam(name = "unlinked", required = false, defaultValue = "false") Boolean unlinked,
+        @RequestParam(name = "filePrefix", required = false) String filePrefix) {
         if (unlinked) {
             return s3LinkService.listObjectsUnlinked(policyName, continuationToken,
-                continuationObject, pageSize);
+                continuationObject, pageSize, filePrefix);
         } else {
-            return s3LinkService.listObjects(policyName, continuationToken, pageSize);
+            return s3LinkService.listObjects(policyName, continuationToken, pageSize, filePrefix);
         }
     }
 
     @PostMapping("/attachments/link")
     public Mono<LinkResult> addAttachmentRecord(@RequestBody LinkRequest linkRequest) {
-        return Flux.fromIterable(linkRequest.getObjectKeys())
-            .filter(objectKey -> linkingFile.put(linkRequest.getPolicyName() + "/" + objectKey,
-                linkRequest.getPolicyName() + "/" + objectKey) == null)
-            .collectList()
-            .flatMap(operableObjectKeys -> client.list(Attachment.class,
-                    attachment -> Objects.equals(attachment.getSpec().getPolicyName(),
-                        linkRequest.getPolicyName())
-                        && StringUtils.isNotEmpty(attachment.getMetadata().getAnnotations()
-                        .get(S3OsAttachmentHandler.OBJECT_KEY))
-                        && linkRequest.getObjectKeys().contains(attachment.getMetadata()
-                        .getAnnotations().get(S3OsAttachmentHandler.OBJECT_KEY)),
-                    null)
-                .collectList()
-                .flatMap(existingAttachments -> Flux.fromIterable(linkRequest.getObjectKeys())
-                    .flatMap((objectKey) -> {
-                        if (operableObjectKeys.contains(objectKey) && existingAttachments.stream()
-                            .noneMatch(attachment -> Objects.equals(
-                                attachment.getMetadata().getAnnotations().get(
-                                    S3OsAttachmentHandler.OBJECT_KEY), objectKey))) {
-                            return s3LinkService
-                                .addAttachmentRecord(linkRequest.getPolicyName(), objectKey)
-                                .onErrorResume((throwable) -> Mono.just(
-                                    new LinkResult.LinkResultItem(objectKey, false,
-                                        throwable.getMessage())));
-                        } else {
-                            return Mono.just(new LinkResult.LinkResultItem(objectKey, false,
-                                "附件库中已存在该对象"));
-                        }
-                    })
-                    .doOnNext(linkResultItem -> linkingFile.remove(
-                        linkRequest.getPolicyName() + "/" + linkResultItem.getObjectKey()))
-                    .collectList()
-                    .map(LinkResult::new)));
+        return s3LinkService.addAttachmentRecords(linkRequest.getPolicyName(),
+            linkRequest.getObjectKeys());
     }
 }

src/main/java/run/halo/s3os/S3LinkService.java

@@ -1,6 +1,7 @@
 package run.halo.s3os;
 
 
+import java.util.Set;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import run.halo.app.core.extension.attachment.Policy;
@@ -9,10 +10,10 @@ public interface S3LinkService {
     Flux<Policy> listS3Policies();
 
     Mono<S3ListResult> listObjects(String policyName, String continuationToken,
-        Integer pageSize);
+        Integer pageSize, String filePrefix);
 
-    Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName, String objectKey);
+    Mono<LinkResult> addAttachmentRecords(String policyName, Set<String> objectKeys);
 
     Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
-        String continuationObject, Integer pageSize);
+        String continuationObject, Integer pageSize, String filePrefix);
 }

src/main/java/run/halo/s3os/S3LinkServiceImpl.java

@@ -3,7 +3,11 @@ package run.halo.s3os;
 import static run.halo.s3os.S3OsAttachmentHandler.OBJECT_KEY;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
@@ -11,6 +15,7 @@ import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.data.domain.Sort;
 import org.springframework.http.HttpStatus;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.context.ReactiveSecurityContextHolder;
@@ -23,7 +28,11 @@ import reactor.core.scheduler.Schedulers;
 import run.halo.app.core.extension.attachment.Attachment;
 import run.halo.app.core.extension.attachment.Policy;
 import run.halo.app.extension.ConfigMap;
+import run.halo.app.extension.ListOptions;
+import run.halo.app.extension.MetadataUtil;
 import run.halo.app.extension.ReactiveExtensionClient;
+import run.halo.app.extension.index.query.QueryFactory;
+import run.halo.app.extension.router.selector.FieldSelector;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
 import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
@@ -37,16 +46,21 @@ public class S3LinkServiceImpl implements S3LinkService {
     private final ReactiveExtensionClient client;
     private final S3OsAttachmentHandler handler;
 
+    /**
+     * Map of linking file, used as a lock, key is policyName/objectKey, value is policyName/objectKey.
+     */
+    private final Map<String, Object> linkingFile = new ConcurrentHashMap<>();
+
 
     @Override
     public Flux<Policy> listS3Policies() {
         return client.list(Policy.class, (policy) -> "s3os".equals(
-            policy.getSpec().getTemplateName()), null);
+            policy.getSpec().getTemplateName()), Comparator.naturalOrder());
     }
 
     @Override
     public Mono<S3ListResult> listObjects(String policyName, String continuationToken,
-        Integer pageSize) {
+        Integer pageSize, String filePrefix) {
         return client.fetch(Policy.class, policyName)
             .flatMap((policy) -> {
                 var configMapName = policy.getSpec().getConfigMapName();
@@ -56,11 +70,11 @@ public class S3LinkServiceImpl implements S3LinkService {
                 var properties = handler.getProperties(configMap);
                 var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation());
                 return Mono.using(() -> handler.buildS3Client(properties),
+                        // 执行 listObjects
                         (s3Client) -> Mono.fromCallable(
                             () -> s3Client.listObjectsV2(ListObjectsV2Request.builder()
                                 .bucket(properties.getBucket())
-                                .prefix(StringUtils.isNotEmpty(finalLocation)
-                                    ? finalLocation + "/" : null)
+                                .prefix(buildPrefix(finalLocation, filePrefix))
                                 .delimiter("/")
                                 .maxKeys(pageSize)
                                 .continuationToken(StringUtils.isNotEmpty(continuationToken)
@@ -69,13 +83,16 @@ public class S3LinkServiceImpl implements S3LinkService {
                         S3Client::close)
                     .flatMap(listObjectsV2Response -> {
                         List<S3Object> contents = listObjectsV2Response.contents();
+                        // 过滤掉目录并转换为ObjectVo
                         var objectVos = contents
                             .stream().map(S3ListResult.ObjectVo::fromS3Object)
                             .filter(objectVo -> !objectVo.getKey().endsWith("/"))
                             .collect(Collectors.toMap(S3ListResult.ObjectVo::getKey, o -> o));
-                        return client.list(Attachment.class,
-                                attachment -> policyName.equals(
-                                    attachment.getSpec().getPolicyName()), null)
+                        // 获取已经关联的附件并标记
+                        ListOptions listOptions = new ListOptions();
+                        listOptions.setFieldSelector(
+                            FieldSelector.of(QueryFactory.equal("spec.policyName", policyName)));
+                        return client.listAll(Attachment.class, listOptions, Sort.unsorted())
                             .doOnNext(attachment -> {
                                 S3ListResult.ObjectVo objectVo =
                                     objectVos.get(attachment.getMetadata().getAnnotations()
@@ -95,9 +112,62 @@ public class S3LinkServiceImpl implements S3LinkService {
             .onErrorMap(S3ExceptionHandler::map);
     }
 
+    @Override
+    public Mono<LinkResult> addAttachmentRecords(String policyName, Set<String> objectKeys) {
+        return getOperableObjectKeys(objectKeys, policyName)
+            .flatMap(operableObjectKeys -> getExistingAttachments(objectKeys, policyName)
+                .flatMap(existingAttachments -> getLinkResultItems(objectKeys, operableObjectKeys,
+                    existingAttachments, policyName)
+                    .collectList()
+                    .map(LinkResult::new)));
+    }
+
+    private Mono<Set<String>> getOperableObjectKeys(Set<String> objectKeys, String policyName) {
+        return Flux.fromIterable(objectKeys)
+            .filter(objectKey ->
+                linkingFile.put(policyName + "/" + objectKey, policyName + "/" + objectKey) == null)
+            .collect(Collectors.toSet());
+    }
+
+    private Mono<Set<String>> getExistingAttachments(Set<String> objectKeys,
+                                                          String policyName) {
+        ListOptions listOptions = new ListOptions();
+        listOptions.setFieldSelector(
+            FieldSelector.of(QueryFactory.equal("spec.policyName", policyName)));
+        return client.listAll(Attachment.class, listOptions, Sort.unsorted())
+            .filter(attachment -> StringUtils.isNotBlank(
+                MetadataUtil.nullSafeAnnotations(attachment).get(S3OsAttachmentHandler.OBJECT_KEY))
+                && objectKeys.contains(
+                MetadataUtil.nullSafeAnnotations(attachment).get(S3OsAttachmentHandler.OBJECT_KEY)))
+            .map(attachment -> MetadataUtil.nullSafeAnnotations(attachment)
+                .get(S3OsAttachmentHandler.OBJECT_KEY))
+            .collect(Collectors.toSet());
+    }
+
+    private Flux<LinkResult.LinkResultItem> getLinkResultItems(Set<String> objectKeys,
+                                                               Set<String> operableObjectKeys,
+                                                               Set<String> existingAttachments,
+                                                               String policyName) {
+        return Flux.fromIterable(objectKeys)
+            .flatMap((objectKey) -> {
+                if (operableObjectKeys.contains(objectKey) &&
+                    !existingAttachments.contains(objectKey)) {
+                    return addAttachmentRecord(policyName, objectKey)
+                        .onErrorResume((throwable) -> Mono.just(
+                            new LinkResult.LinkResultItem(objectKey, false,
+                                throwable.getMessage())));
+                } else {
+                    return Mono.just(
+                        new LinkResult.LinkResultItem(objectKey, false, "附件库中已存在该对象"));
+                }
+            })
+            .doFinally(signalType -> operableObjectKeys.forEach(
+                objectKey -> linkingFile.remove(policyName + "/" + objectKey)));
+    }
+
     @Override
     public Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
-        String continuationObject, Integer pageSize) {
+        String continuationObject, Integer pageSize, String filePrefix) {
         // TODO 优化成查一次数据库
         return Mono.defer(() -> {
             List<S3ListResult.ObjectVo> s3Objects = new ArrayList<>();
@@ -106,7 +176,8 @@ public class S3LinkServiceImpl implements S3LinkService {
 
             return Flux.defer(() -> Flux.just(
                     new TokenState(null, currToken.get() == null ? "" : currToken.get())))
-                .flatMap(tokenState -> listObjects(policyName, tokenState.nextToken, pageSize))
+                .flatMap(tokenState -> listObjects(policyName, tokenState.nextToken,
+                    pageSize, filePrefix))
                 .flatMap(s3ListResult -> {
                     var filteredObjects = s3ListResult.getObjects();
                     if (!continuationObjectMatched.get()) {
@@ -151,8 +222,6 @@ public class S3LinkServiceImpl implements S3LinkService {
     record TokenState(String currToken, String nextToken) {
     }
 
-
-    @Override
     public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
         String objectKey) {
         return authenticationConsumer(authentication -> client.fetch(Policy.class, policyName)
@@ -203,4 +272,18 @@ public class S3LinkServiceImpl implements S3LinkService {
             .flatMap(func);
     }
 
+    String buildPrefix(String finalLocation, String filePrefix) {
+        if (StringUtils.isBlank(finalLocation) && StringUtils.isBlank(filePrefix)) {
+            return null;
+        }
+        StringBuilder sb = new StringBuilder();
+        if (StringUtils.isNotBlank(finalLocation)) {
+            sb.append(finalLocation).append("/");
+        }
+        if (StringUtils.isNotBlank(filePrefix)) {
+            sb.append(filePrefix);
+        }
+        return sb.toString();
+    }
+
 }

src/main/java/run/halo/s3os/S3OsAttachmentHandler.java

@@ -390,7 +390,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
             SdkAutoCloseable::close);
     }
 
-    private Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
+    Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
                                                S3Client s3client) {
         return Mono.defer(() -> {
                 // deduplication of uploading files
@@ -437,7 +437,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
     }
 
 
-    private Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
+    Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
                                    S3Client s3client) {
         final int partNumber = ++uploadState.partCounter;
         return Mono.just(s3client.uploadPart(UploadPartRequest.builder()
@@ -457,7 +457,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
             });
     }
 
-    private static void checkResult(SdkResponse result, String operation) {
+    static void checkResult(SdkResponse result, String operation) {
         log.info("operation: {}, result: {}", operation, result);
         if (result.sdkHttpResponse() == null || !result.sdkHttpResponse().isSuccessful()) {
             log.error("Failed to upload object, response: {}", result.sdkHttpResponse());
@@ -465,7 +465,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
         }
     }
 
-    private static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
+    static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
         int partSize = 0;
         for (DataBuffer b : buffers) {
             partSize += b.readableByteCount();

src/main/java/run/halo/s3os/S3OsPlugin.java

@@ -1,8 +1,8 @@
 package run.halo.s3os;
 
-import org.pf4j.PluginWrapper;
 import org.springframework.stereotype.Component;
 import run.halo.app.plugin.BasePlugin;
+import run.halo.app.plugin.PluginContext;
 
 /**
  * @author johnniang
@@ -11,8 +11,8 @@ import run.halo.app.plugin.BasePlugin;
 @Component
 public class S3OsPlugin extends BasePlugin {
 
-    public S3OsPlugin(PluginWrapper wrapper) {
-        super(wrapper);
+    public S3OsPlugin(PluginContext pluginContext) {
+        super(pluginContext);
     }
 
     @Override

src/main/resources/extensions/policy-template-s3os.yaml

@@ -3,7 +3,7 @@ kind: PolicyTemplate
 metadata:
   name: s3os
 spec:
-  displayName: S3 Object Storage
+  displayName: S3 对象存储
   settingName: s3os-policy-template-setting
 ---
 apiVersion: v1alpha1
@@ -14,6 +14,10 @@ spec:
   forms:
     - group: default
       formSchema:
+        - $formkit: verificationForm
+          action: "/apis/s3os.halo.run/v1alpha1/policies/s3/validation"
+          label: 对象存储验证
+          children:
             - $formkit: text
               name: bucket
               label: Bucket 桶名称

src/main/resources/extensions/s3os-role-template.yaml

@@ -39,3 +39,17 @@ rules:
   - apiGroups: [ "s3os.halo.run" ]
     resources: [ "attachments" ]
     verbs: [ "delete" ]
+---
+apiVersion: v1alpha1
+kind: "Role"
+metadata:
+  name: role-template-s3os-policy-config-validation
+  labels:
+    halo.run/role-template: "true"
+    halo.run/hidden: "true"
+    rbac.authorization.halo.run/aggregate-to-role-template-manage-configmaps: "true"
+rules:
+  - apiGroups: ["s3os.halo.run"]
+    resources: ["policies/validation"]
+    resourceNames: ["s3"]
+    verbs: [ "create" ]

src/main/resources/extensions/settings.yaml

@@ -7,8 +7,5 @@ spec:
     - group: basic
       label: 使用提示
       formSchema:
-        - $formkit: text
-          help: 请前往 “附件 - 存储策略” 添加策略
-          label: 此处不用设置，请前往 “附件 - 存储策略” 添加策略
-          name: text
-          placeholder: 此处不用设置，请前往 “附件 - 存储策略” 添加策略
+        - $el: p
+          children: 请前往 “附件 - 存储策略” 添加策略

src/main/resources/plugin.yaml

@@ -4,14 +4,16 @@ metadata:
   name: PluginS3ObjectStorage
 spec:
   enabled: true
-  requires: ">=2.12.0"
+  requires: ">=2.14.0"
   author:
-    name: Halo OSS Team
+    name: Halo
     website: https://github.com/halo-dev
   logo: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAABUdJREFUeF7tmmtoXFUQx3+zMYGi/RJRULRFsL6oim2klVK0CrVqKxjoJ0Vt1bR7t6lURagiFlGpFPogydklVo1i/VB84QMfqFijpflQrI+IYj7FF+0HMbUEI82O3NNtzG6y956bvXd3IXvgksCd+c/M/8w598ycFWb5kFkePw0CGhkwyxloLIFZngCNTbCxBBpLoI4Y0A6aydNKC62kaAXGSDHCv4zQzIh0MRa3uzVdAppmMSmWoVyL0IZyWUiA3wL7gI/FcCQOMmpCQCHwDpSOCoIYQOmWLK9WgFHdr4Bu4wyOsR14uBKni3SFdxhnl+T4fCaYVcsAzbCEPNsRbpiJow46u8TwkINcCX9RNWYgr2nWIrwAzJ2BehSVAVpYJbv5y1Up8QwoBL/f1aGY5JaJ4aALVqIEqMc24EkXRxKQmSeGX8JwEyNAPR4AesMcSPD9ECdpk15GgmwkQoB2spxxPgWaEwzQBfptMdxRfQI8PgJWuniYuIyyQbLlMzH2DFCPDUAu8cDcDRwWQ1s58VgJ0E7OYZyvgAXu/lVBMiAL4iXA4z5gbxVCimpiQAxLp1OKl4AMfSj3RPWuKvJ5FkmOr0ttxUuAx9/AWVUJKKoR5QnJ8nRsBOh65tJiy9nFYJ+fwB586nUcEsN1FROgHncDG2EqWIWR/wpcUCFGsPooc6SPfyYLOS8BzbAI5RlgVQJObrXdH9iZAPb/kCkul25+jEyAejboDxJyLiMG42Orx2fAioTs+LC3iOHDSARohs0oexJxStgtPWw5jZ34IUpJS7b4kBa4BDTDTSifJBI8vC+G1ZOxdQtzGGMIOD8hm1vF2I7UxAgmwLNpn8SaHyTPjZLjWGmg6tl9YCIrYiVCuVOyvOZEQGG3fzlWB06BjZJipXTbI/OUoWmuQvgmAbs+5HIxfOlKwCFgSQKOrBNDXxCuerwH3Ba77ZPMl16GQwnQjSwkxXexO6A8JdnwDpF6tANvxG1fzNQu+LR7QEKtrFfEuNcJ6vEzcHGMJEzZdH3scgRkC6e9eOwr/Qi3iuGEK6B6PAo85yofKid40oMfV9EoR8CbENxKCjVYvNCWSg8DUXQKn8TRKDqBstOs/6AMiLelVaYUDQtO0zyPcH+YnMP7d8Vw+3Ry5TLgJeBeB2BXEX8mDwDHEUZQWzafEBNcPca2FwlrpMd+WaaM6QnI8CyKX6AkN/KsmHyfZ9voQjfKUbDPOMoCxF6TVzLKzn75JZBmOcIXlVgN0y39JGka/yOVDtOL/D5g9ssS4L9Qz5aNl0Y26KZwRAzXTBZVj8PAIjd1Z6ntYoIzuWwtoBkeK9T/ztacBUurwA1cQRODzvpugvvEcFeYaHkCHuFMRvGPwwvDQCK+P0qexZLjt9N6mmE9am+P4xnKn5LlbBew4GowzTqEF12AnGXytEuOtyaC30aKY7YRcr0zRrDg92K40hUrtCWmGfajrHUFDJHzf9tzc8na9zu1j8eCL/RKj72Zch6hBPhI6tEFbHJGnV5wSEzxjVGMrbZBlB2SJXL57kRAgQS/G7wDODcyEUKX9LC5ZOb9+wO/lV5J9+c4sIsmdkoX/v+RhzMBloRNXESeB8E+4UNtSbtHsvSXCqt/1oB2xNYc88PBiiT8uuIgTeyVLn6IqFskHomAiY0rwyUoa1BWI3YGzyu8+wPld1K8Tp5+yeL/ri90aCdXM26JuBCYN/GcOjYPIwzbv3CYJg5IF/4dQixjRgTEYrlOQBoE1MlE1MyNRgbUjPo6MdzIgDqZiJq50ciAmlFfJ4YbGVAnE1EzN2Z9BvwH8mZfUOuuxhIAAAAASUVORK5CYII=
   settingName: s3os-settings
   configMapName: s3os-configMap
-  homepage: https://github.com/halo-dev/plugin-s3
+  homepage: https://www.halo.run/store/apps/app-Qxhpp
+  repo: https://github.com/halo-dev/plugin-s3
+  issues: https://github.com/halo-dev/plugin-s3/issues
   displayName: "对象存储（Amazon S3 协议）"
   description: "提供兼容 Amazon S3 协议的对象存储策略，兼容阿里云、腾讯云、七牛云等"
   license: