mirror of
https://github.com/halo-dev/plugin-s3.git
synced 2025-10-15 06:39:08 +00:00
Compare commits
6 Commits
Author | SHA1 | Date | |
---|---|---|---|
![]() |
5e7e6620fd | ||
![]() |
47b6a37d0a | ||
![]() |
68b1a88b14 | ||
![]() |
f4ec56b7bc | ||
![]() |
c5d4e719a7 | ||
![]() |
034b3f3ded |
17
.github/workflows/cd.yaml
vendored
Normal file
17
.github/workflows/cd.yaml
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
name: CD
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
|
||||
jobs:
|
||||
cd:
|
||||
uses: halo-sigs/reusable-workflows/.github/workflows/plugin-cd.yaml@v1
|
||||
secrets:
|
||||
halo-username: ${{ secrets.HALO_USERNAME }}
|
||||
halo-password: ${{ secrets.HALO_PASSWORD }}
|
||||
permissions:
|
||||
contents: write
|
||||
with:
|
||||
app-id: app-Qxhpp
|
13
.github/workflows/ci.yaml
vendored
Normal file
13
.github/workflows/ci.yaml
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
uses: halo-sigs/reusable-workflows/.github/workflows/plugin-ci.yaml@v1
|
138
.github/workflows/workflow.yaml
vendored
138
.github/workflows/workflow.yaml
vendored
@@ -1,138 +0,0 @@
|
||||
name: Build Plugin JAR File
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "**"
|
||||
- "!**.md"
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "**"
|
||||
- "!**.md"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
cache: 'gradle'
|
||||
java-version: 17
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 18
|
||||
- uses: pnpm/action-setup@v2.0.1
|
||||
name: Install pnpm
|
||||
id: pnpm-install
|
||||
with:
|
||||
version: 8
|
||||
run_install: false
|
||||
- name: Get pnpm store directory
|
||||
id: pnpm-cache
|
||||
run: |
|
||||
echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
|
||||
- uses: actions/cache@v3
|
||||
name: Setup pnpm cache
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/widget/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
- name: Install Frontend Dependencies
|
||||
run: |
|
||||
./gradlew pnpmInstall
|
||||
- name: Build with Gradle
|
||||
run: |
|
||||
# Set the version with tag name when releasing
|
||||
version=${{ github.event.release.tag_name }}
|
||||
version=${version#v}
|
||||
sed -i "s/version=.*-SNAPSHOT$/version=$version/1" gradle.properties
|
||||
./gradlew clean build -x test
|
||||
- name: Archive plugin-s3 jar
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: plugin-s3
|
||||
path: |
|
||||
build/libs/*.jar
|
||||
retention-days: 1
|
||||
|
||||
github-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: github.event_name == 'release'
|
||||
steps:
|
||||
- name: Download plugin-s3 jar
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: plugin-s3
|
||||
path: build/libs
|
||||
- name: Get Name of Artifact
|
||||
id: get_artifact
|
||||
run: |
|
||||
ARTIFACT_PATHNAME=$(ls build/libs/*.jar | head -n 1)
|
||||
ARTIFACT_NAME=$(basename ${ARTIFACT_PATHNAME})
|
||||
echo "Artifact pathname: ${ARTIFACT_PATHNAME}"
|
||||
echo "Artifact name: ${ARTIFACT_NAME}"
|
||||
echo "ARTIFACT_PATHNAME=${ARTIFACT_PATHNAME}" >> $GITHUB_ENV
|
||||
echo "ARTIFACT_NAME=${ARTIFACT_NAME}" >> $GITHUB_ENV
|
||||
echo "RELEASE_ID=${{ github.event.release.id }}" >> $GITHUB_ENV
|
||||
- name: Upload a Release Asset
|
||||
uses: actions/github-script@v2
|
||||
if: github.event_name == 'release'
|
||||
with:
|
||||
github-token: ${{secrets.GITHUB_TOKEN}}
|
||||
script: |
|
||||
console.log('environment', process.versions);
|
||||
|
||||
const fs = require('fs').promises;
|
||||
|
||||
const { repo: { owner, repo }, sha } = context;
|
||||
console.log({ owner, repo, sha });
|
||||
|
||||
const releaseId = process.env.RELEASE_ID
|
||||
const artifactPathName = process.env.ARTIFACT_PATHNAME
|
||||
const artifactName = process.env.ARTIFACT_NAME
|
||||
console.log('Releasing', releaseId, artifactPathName, artifactName)
|
||||
|
||||
await github.repos.uploadReleaseAsset({
|
||||
owner, repo,
|
||||
release_id: releaseId,
|
||||
name: artifactName,
|
||||
data: await fs.readFile(artifactPathName)
|
||||
});
|
||||
|
||||
app-store-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: github.event_name == 'release'
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: true
|
||||
- name: Download plugin-s3 jar
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: plugin-s3
|
||||
path: build/libs
|
||||
- name: Sync to Halo App Store
|
||||
uses: halo-sigs/app-store-release-action@main
|
||||
with:
|
||||
github-token: ${{secrets.GITHUB_TOKEN}}
|
||||
app-id: ${{secrets.APP_ID}}
|
||||
release-id: ${{ github.event.release.id }}
|
||||
assets-dir: "build/libs"
|
||||
halo-username: ${{ secrets.HALO_USERNAME }}
|
||||
halo-password: ${{ secrets.HALO_PASSWORD }}
|
@@ -16,7 +16,7 @@ repositories {
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation platform('run.halo.tools.platform:plugin:2.12.0-SNAPSHOT')
|
||||
implementation platform('run.halo.tools.platform:plugin:2.14.0-SNAPSHOT')
|
||||
compileOnly 'run.halo.app:api'
|
||||
|
||||
implementation platform('software.amazon.awssdk:bom:2.19.8')
|
||||
@@ -36,7 +36,7 @@ configurations.runtimeClasspath {
|
||||
|
||||
|
||||
halo {
|
||||
version = '2.12.1'
|
||||
version = '2.14.0'
|
||||
}
|
||||
|
||||
haloPlugin {
|
||||
@@ -59,6 +59,10 @@ task buildFrontend(type: PnpmTask) {
|
||||
args = ['build']
|
||||
}
|
||||
|
||||
tasks.named('buildFrontend') {
|
||||
dependsOn 'pnpmInstall'
|
||||
}
|
||||
|
||||
build {
|
||||
// build frontend before build
|
||||
tasks.getByName('compileJava').dependsOn('buildFrontend')
|
||||
|
@@ -1 +1 @@
|
||||
version=1.8.0-SNAPSHOT
|
||||
version=1.9.0-SNAPSHOT
|
||||
|
@@ -6,6 +6,6 @@ import lombok.experimental.UtilityClass;
|
||||
public class FilePathUtils {
|
||||
|
||||
public static String getFilePathByPlaceholder(String filePath) {
|
||||
return PlaceholderReplacer.replacePlaceholders(filePath, "");
|
||||
return PlaceholderReplacer.replacePlaceholders(filePath, null);
|
||||
}
|
||||
}
|
||||
|
@@ -1,13 +1,12 @@
|
||||
package run.halo.s3os;
|
||||
|
||||
import java.util.Set;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class LinkRequest {
|
||||
private String policyName;
|
||||
private List<String> objectKeys;
|
||||
private Set<String> objectKeys;
|
||||
}
|
@@ -0,0 +1,123 @@
|
||||
package run.halo.s3os;
|
||||
|
||||
import static run.halo.s3os.S3OsAttachmentHandler.MULTIPART_MIN_PART_SIZE;
|
||||
import static run.halo.s3os.S3OsAttachmentHandler.checkResult;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.DefaultResourceLoader;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.buffer.DataBuffer;
|
||||
import org.springframework.core.io.buffer.DataBufferUtils;
|
||||
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import run.halo.app.infra.utils.PathUtils;
|
||||
import run.halo.app.plugin.ApiVersion;
|
||||
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
|
||||
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.utils.SdkAutoCloseable;
|
||||
|
||||
@ApiVersion("s3os.halo.run/v1alpha1")
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PolicyConfigValidationController {
|
||||
private final S3OsAttachmentHandler handler;
|
||||
|
||||
@PostMapping("/policies/s3/validation")
|
||||
public Mono<Void> validatePolicyConfig(@RequestBody S3OsProperties properties) {
|
||||
var filename = "halo-s3-plugin-test-file-" + System.currentTimeMillis() + ".jpg";
|
||||
var content = readImage();
|
||||
return Mono.using(() -> handler.buildS3Client(properties),
|
||||
client -> {
|
||||
var uploadState =
|
||||
new S3OsAttachmentHandler.UploadState(properties, filename, false);
|
||||
|
||||
return handler.checkFileExistsAndRename(uploadState, client)
|
||||
// init multipart upload
|
||||
.flatMap(state -> Mono.fromCallable(() -> client.createMultipartUpload(
|
||||
CreateMultipartUploadRequest.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.contentType(state.contentType)
|
||||
.key(state.objectKey)
|
||||
.build())))
|
||||
.doOnNext((response) -> {
|
||||
checkResult(response, "createMultipartUpload");
|
||||
uploadState.uploadId = response.uploadId();
|
||||
})
|
||||
.thenMany(handler.reshape(content, MULTIPART_MIN_PART_SIZE))
|
||||
// buffer to part
|
||||
.windowUntil((buffer) -> {
|
||||
uploadState.buffered += buffer.readableByteCount();
|
||||
if (uploadState.buffered >= MULTIPART_MIN_PART_SIZE) {
|
||||
uploadState.buffered = 0;
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
// upload part
|
||||
.concatMap((window) -> window.collectList().flatMap((bufferList) -> {
|
||||
var buffer = S3OsAttachmentHandler.concatBuffers(bufferList);
|
||||
return handler.uploadPart(uploadState, buffer, client);
|
||||
}))
|
||||
.reduce(uploadState, (state, completedPart) -> {
|
||||
state.completedParts.put(completedPart.partNumber(), completedPart);
|
||||
return state;
|
||||
})
|
||||
// complete multipart upload
|
||||
.flatMap((state) -> Mono.just(client.completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest
|
||||
.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.uploadId(state.uploadId)
|
||||
.multipartUpload(CompletedMultipartUpload.builder()
|
||||
.parts(state.completedParts.values())
|
||||
.build())
|
||||
.key(state.objectKey)
|
||||
.build())
|
||||
))
|
||||
// get object metadata
|
||||
.flatMap((response) -> {
|
||||
checkResult(response, "completeUpload");
|
||||
return Mono.just(client.headObject(
|
||||
HeadObjectRequest.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.key(uploadState.objectKey)
|
||||
.build()
|
||||
));
|
||||
})
|
||||
// check object metadata
|
||||
.doOnNext((response) -> {
|
||||
checkResult(response, "headObject");
|
||||
})
|
||||
// delete object
|
||||
.flatMap((response) -> Mono.just(client.deleteObject(
|
||||
software.amazon.awssdk.services.s3.model.DeleteObjectRequest.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.key(uploadState.objectKey)
|
||||
.build()
|
||||
)))
|
||||
.doOnNext((response) -> checkResult(response, "deleteObject"))
|
||||
.then();
|
||||
},
|
||||
SdkAutoCloseable::close)
|
||||
.onErrorMap(S3ExceptionHandler::map);
|
||||
}
|
||||
|
||||
private Flux<DataBuffer> readImage() {
|
||||
DefaultResourceLoader resourceLoader = new DefaultResourceLoader(this.getClass()
|
||||
.getClassLoader());
|
||||
String path = PathUtils.combinePath("validation.jpg");
|
||||
String simplifyPath = StringUtils.cleanPath(path);
|
||||
Resource resource = resourceLoader.getResource(simplifyPath);
|
||||
return DataBufferUtils.read(resource, new DefaultDataBufferFactory(), 1024);
|
||||
}
|
||||
}
|
@@ -1,30 +1,22 @@
|
||||
package run.halo.s3os;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import run.halo.app.core.extension.attachment.Attachment;
|
||||
import run.halo.app.core.extension.attachment.Policy;
|
||||
import run.halo.app.extension.ReactiveExtensionClient;
|
||||
import run.halo.app.plugin.ApiVersion;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@ApiVersion("s3os.halo.run/v1alpha1")
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class S3LinkController {
|
||||
private final S3LinkService s3LinkService;
|
||||
private final ReactiveExtensionClient client;
|
||||
|
||||
/**
|
||||
* Map of linking file, used as a lock, key is policyName/objectKey, value is policyName/objectKey.
|
||||
*/
|
||||
private final Map<String, Object> linkingFile = new ConcurrentHashMap<>();
|
||||
|
||||
@GetMapping("/policies/s3")
|
||||
public Flux<Policy> listS3Policies() {
|
||||
@@ -48,38 +40,7 @@ public class S3LinkController {
|
||||
|
||||
@PostMapping("/attachments/link")
|
||||
public Mono<LinkResult> addAttachmentRecord(@RequestBody LinkRequest linkRequest) {
|
||||
return Flux.fromIterable(linkRequest.getObjectKeys())
|
||||
.filter(objectKey -> linkingFile.put(linkRequest.getPolicyName() + "/" + objectKey,
|
||||
linkRequest.getPolicyName() + "/" + objectKey) == null)
|
||||
.collectList()
|
||||
.flatMap(operableObjectKeys -> client.list(Attachment.class,
|
||||
attachment -> Objects.equals(attachment.getSpec().getPolicyName(),
|
||||
linkRequest.getPolicyName())
|
||||
&& StringUtils.isNotEmpty(attachment.getMetadata().getAnnotations()
|
||||
.get(S3OsAttachmentHandler.OBJECT_KEY))
|
||||
&& linkRequest.getObjectKeys().contains(attachment.getMetadata()
|
||||
.getAnnotations().get(S3OsAttachmentHandler.OBJECT_KEY)),
|
||||
null)
|
||||
.collectList()
|
||||
.flatMap(existingAttachments -> Flux.fromIterable(linkRequest.getObjectKeys())
|
||||
.flatMap((objectKey) -> {
|
||||
if (operableObjectKeys.contains(objectKey) && existingAttachments.stream()
|
||||
.noneMatch(attachment -> Objects.equals(
|
||||
attachment.getMetadata().getAnnotations().get(
|
||||
S3OsAttachmentHandler.OBJECT_KEY), objectKey))) {
|
||||
return s3LinkService
|
||||
.addAttachmentRecord(linkRequest.getPolicyName(), objectKey)
|
||||
.onErrorResume((throwable) -> Mono.just(
|
||||
new LinkResult.LinkResultItem(objectKey, false,
|
||||
throwable.getMessage())));
|
||||
} else {
|
||||
return Mono.just(new LinkResult.LinkResultItem(objectKey, false,
|
||||
"附件库中已存在该对象"));
|
||||
}
|
||||
})
|
||||
.doOnNext(linkResultItem -> linkingFile.remove(
|
||||
linkRequest.getPolicyName() + "/" + linkResultItem.getObjectKey()))
|
||||
.collectList()
|
||||
.map(LinkResult::new)));
|
||||
return s3LinkService.addAttachmentRecords(linkRequest.getPolicyName(),
|
||||
linkRequest.getObjectKeys());
|
||||
}
|
||||
}
|
||||
|
@@ -1,6 +1,7 @@
|
||||
package run.halo.s3os;
|
||||
|
||||
|
||||
import java.util.Set;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import run.halo.app.core.extension.attachment.Policy;
|
||||
@@ -11,7 +12,7 @@ public interface S3LinkService {
|
||||
Mono<S3ListResult> listObjects(String policyName, String continuationToken,
|
||||
Integer pageSize);
|
||||
|
||||
Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName, String objectKey);
|
||||
Mono<LinkResult> addAttachmentRecords(String policyName, Set<String> objectKeys);
|
||||
|
||||
Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
|
||||
String continuationObject, Integer pageSize);
|
||||
|
@@ -4,6 +4,9 @@ import static run.halo.s3os.S3OsAttachmentHandler.OBJECT_KEY;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Function;
|
||||
@@ -23,7 +26,11 @@ import reactor.core.scheduler.Schedulers;
|
||||
import run.halo.app.core.extension.attachment.Attachment;
|
||||
import run.halo.app.core.extension.attachment.Policy;
|
||||
import run.halo.app.extension.ConfigMap;
|
||||
import run.halo.app.extension.ListOptions;
|
||||
import run.halo.app.extension.MetadataUtil;
|
||||
import run.halo.app.extension.ReactiveExtensionClient;
|
||||
import run.halo.app.extension.index.query.QueryFactory;
|
||||
import run.halo.app.extension.router.selector.FieldSelector;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
@@ -37,6 +44,11 @@ public class S3LinkServiceImpl implements S3LinkService {
|
||||
private final ReactiveExtensionClient client;
|
||||
private final S3OsAttachmentHandler handler;
|
||||
|
||||
/**
|
||||
* Map of linking file, used as a lock, key is policyName/objectKey, value is policyName/objectKey.
|
||||
*/
|
||||
private final Map<String, Object> linkingFile = new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
@Override
|
||||
public Flux<Policy> listS3Policies() {
|
||||
@@ -73,9 +85,10 @@ public class S3LinkServiceImpl implements S3LinkService {
|
||||
.stream().map(S3ListResult.ObjectVo::fromS3Object)
|
||||
.filter(objectVo -> !objectVo.getKey().endsWith("/"))
|
||||
.collect(Collectors.toMap(S3ListResult.ObjectVo::getKey, o -> o));
|
||||
return client.list(Attachment.class,
|
||||
attachment -> policyName.equals(
|
||||
attachment.getSpec().getPolicyName()), null)
|
||||
ListOptions listOptions = new ListOptions();
|
||||
listOptions.setFieldSelector(
|
||||
FieldSelector.of(QueryFactory.equal("spec.policyName", policyName)));
|
||||
return client.listAll(Attachment.class, listOptions, null)
|
||||
.doOnNext(attachment -> {
|
||||
S3ListResult.ObjectVo objectVo =
|
||||
objectVos.get(attachment.getMetadata().getAnnotations()
|
||||
@@ -95,6 +108,59 @@ public class S3LinkServiceImpl implements S3LinkService {
|
||||
.onErrorMap(S3ExceptionHandler::map);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LinkResult> addAttachmentRecords(String policyName, Set<String> objectKeys) {
|
||||
return getOperableObjectKeys(objectKeys, policyName)
|
||||
.flatMap(operableObjectKeys -> getExistingAttachments(objectKeys, policyName)
|
||||
.flatMap(existingAttachments -> getLinkResultItems(objectKeys, operableObjectKeys,
|
||||
existingAttachments, policyName)
|
||||
.collectList()
|
||||
.map(LinkResult::new)));
|
||||
}
|
||||
|
||||
private Mono<Set<String>> getOperableObjectKeys(Set<String> objectKeys, String policyName) {
|
||||
return Flux.fromIterable(objectKeys)
|
||||
.filter(objectKey ->
|
||||
linkingFile.put(policyName + "/" + objectKey, policyName + "/" + objectKey) == null)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private Mono<Set<String>> getExistingAttachments(Set<String> objectKeys,
|
||||
String policyName) {
|
||||
ListOptions listOptions = new ListOptions();
|
||||
listOptions.setFieldSelector(
|
||||
FieldSelector.of(QueryFactory.equal("spec.policyName", policyName)));
|
||||
return client.listAll(Attachment.class, listOptions, null)
|
||||
.filter(attachment -> StringUtils.isNotBlank(
|
||||
MetadataUtil.nullSafeAnnotations(attachment).get(S3OsAttachmentHandler.OBJECT_KEY))
|
||||
&& objectKeys.contains(
|
||||
MetadataUtil.nullSafeAnnotations(attachment).get(S3OsAttachmentHandler.OBJECT_KEY)))
|
||||
.map(attachment -> MetadataUtil.nullSafeAnnotations(attachment)
|
||||
.get(S3OsAttachmentHandler.OBJECT_KEY))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private Flux<LinkResult.LinkResultItem> getLinkResultItems(Set<String> objectKeys,
|
||||
Set<String> operableObjectKeys,
|
||||
Set<String> existingAttachments,
|
||||
String policyName) {
|
||||
return Flux.fromIterable(objectKeys)
|
||||
.flatMap((objectKey) -> {
|
||||
if (operableObjectKeys.contains(objectKey) &&
|
||||
!existingAttachments.contains(objectKey)) {
|
||||
return addAttachmentRecord(policyName, objectKey)
|
||||
.onErrorResume((throwable) -> Mono.just(
|
||||
new LinkResult.LinkResultItem(objectKey, false,
|
||||
throwable.getMessage())));
|
||||
} else {
|
||||
return Mono.just(
|
||||
new LinkResult.LinkResultItem(objectKey, false, "附件库中已存在该对象"));
|
||||
}
|
||||
})
|
||||
.doFinally(signalType -> operableObjectKeys.forEach(
|
||||
objectKey -> linkingFile.remove(policyName + "/" + objectKey)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
|
||||
String continuationObject, Integer pageSize) {
|
||||
@@ -151,8 +217,6 @@ public class S3LinkServiceImpl implements S3LinkService {
|
||||
record TokenState(String currToken, String nextToken) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
|
||||
String objectKey) {
|
||||
return authenticationConsumer(authentication -> client.fetch(Policy.class, policyName)
|
||||
|
@@ -390,8 +390,8 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
|
||||
SdkAutoCloseable::close);
|
||||
}
|
||||
|
||||
private Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
|
||||
S3Client s3client) {
|
||||
Mono<UploadState> checkFileExistsAndRename(UploadState uploadState,
|
||||
S3Client s3client) {
|
||||
return Mono.defer(() -> {
|
||||
// deduplication of uploading files
|
||||
if (uploadingFile.put(uploadState.getUploadingMapKey(),
|
||||
@@ -437,8 +437,8 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
|
||||
}
|
||||
|
||||
|
||||
private Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
|
||||
S3Client s3client) {
|
||||
Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer,
|
||||
S3Client s3client) {
|
||||
final int partNumber = ++uploadState.partCounter;
|
||||
return Mono.just(s3client.uploadPart(UploadPartRequest.builder()
|
||||
.bucket(uploadState.properties.getBucket())
|
||||
@@ -457,7 +457,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
|
||||
});
|
||||
}
|
||||
|
||||
private static void checkResult(SdkResponse result, String operation) {
|
||||
static void checkResult(SdkResponse result, String operation) {
|
||||
log.info("operation: {}, result: {}", operation, result);
|
||||
if (result.sdkHttpResponse() == null || !result.sdkHttpResponse().isSuccessful()) {
|
||||
log.error("Failed to upload object, response: {}", result.sdkHttpResponse());
|
||||
@@ -465,7 +465,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
|
||||
}
|
||||
}
|
||||
|
||||
private static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
|
||||
static ByteBuffer concatBuffers(List<DataBuffer> buffers) {
|
||||
int partSize = 0;
|
||||
for (DataBuffer b : buffers) {
|
||||
partSize += b.readableByteCount();
|
||||
|
@@ -3,7 +3,7 @@ kind: PolicyTemplate
|
||||
metadata:
|
||||
name: s3os
|
||||
spec:
|
||||
displayName: S3 Object Storage
|
||||
displayName: S3 对象存储
|
||||
settingName: s3os-policy-template-setting
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
@@ -14,130 +14,134 @@ spec:
|
||||
forms:
|
||||
- group: default
|
||||
formSchema:
|
||||
- $formkit: text
|
||||
name: bucket
|
||||
label: Bucket 桶名称
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: endpointProtocol
|
||||
label: Endpoint 访问协议
|
||||
options:
|
||||
- label: HTTPS
|
||||
value: https
|
||||
- label: HTTP
|
||||
value: http
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: enablePathStyleAccess
|
||||
label: Endpoint 访问风格
|
||||
options:
|
||||
- label: Virtual Hosted Style
|
||||
value: false
|
||||
- label: Path Style
|
||||
value: true
|
||||
value: false
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: endpoint
|
||||
label: EndPoint
|
||||
placeholder: 请填写不带bucket-name的Endpoint
|
||||
validation: required
|
||||
help: 协议头请在上方设置,此处无需以"http://"或"https://"开头,系统会自动拼接
|
||||
- $formkit: password
|
||||
name: accessKey
|
||||
label: Access Key ID
|
||||
placeholder: 存储桶用户标识(用户名)
|
||||
validation: required
|
||||
- $formkit: password
|
||||
name: accessSecret
|
||||
label: Access Key Secret
|
||||
placeholder: 存储桶密钥(密码)
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: region
|
||||
label: Region
|
||||
placeholder: 如不填写,则默认为"Auto"
|
||||
help: 若Region为Auto无法使用,才需要填写对应Region
|
||||
- $formkit: text
|
||||
name: location
|
||||
label: 上传目录
|
||||
placeholder: 如不填写,则默认上传到根目录
|
||||
help: 支持的占位符请查阅:https://github.com/halo-dev/plugin-s3#上传目录
|
||||
- $formkit: select
|
||||
name: randomFilenameMode
|
||||
label: 上传时重命名文件方式
|
||||
options:
|
||||
- label: 保留原文件名
|
||||
value: none
|
||||
- label: 自定义(请在下方输入自定义模板)
|
||||
value: custom
|
||||
- label: 使用UUID
|
||||
value: uuid
|
||||
- label: 使用毫秒时间戳
|
||||
value: timestampMs
|
||||
- label: 使用原文件名 + 随机字母
|
||||
value: withString
|
||||
- label: 使用日期 + 随机字母
|
||||
value: dateWithString
|
||||
- label: 使用日期时间 + 随机字母
|
||||
value: datetimeWithString
|
||||
- label: 使用随机字母
|
||||
value: string
|
||||
validation: required
|
||||
- $formkit: number
|
||||
name: randomStringLength
|
||||
key: randomStringLength
|
||||
label: 随机字母长度
|
||||
min: 4
|
||||
max: 16
|
||||
if: "$randomFilenameMode == 'dateWithString' || $randomFilenameMode == 'datetimeWithString' || $randomFilenameMode == 'withString' || $randomFilenameMode == 'string'"
|
||||
help: 支持4~16位, 默认为8位
|
||||
- $formkit: text
|
||||
name: customTemplate
|
||||
key: customTemplate
|
||||
label: 自定义文件名模板
|
||||
if: "$randomFilenameMode == 'custom'"
|
||||
value: "${origin-filename}"
|
||||
help: 支持的占位符请查阅:https://github.com/halo-dev/plugin-s3#自定义文件名模板
|
||||
- $formkit: select
|
||||
name: duplicateFilenameHandling
|
||||
label: 重复文件名处理方式
|
||||
options:
|
||||
- label: 加随机字母数字后缀
|
||||
value: randomAlphanumeric
|
||||
- label: 加随机字母后缀
|
||||
value: randomAlphabetic
|
||||
- label: 报错不上传
|
||||
value: exception
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: protocol
|
||||
label: 绑定域名协议
|
||||
options:
|
||||
- label: HTTPS
|
||||
value: https
|
||||
- label: HTTP
|
||||
value: http
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: domain
|
||||
label: 绑定域名(CDN域名)
|
||||
placeholder: 如不设置,那么将使用 Bucket + EndPoint 作为域名
|
||||
help: 协议头请在上方设置,此处无需以"http://"或"https://"开头,系统会自动拼接
|
||||
- $formkit: repeater
|
||||
name: urlSuffixes
|
||||
label: 网址后缀
|
||||
help: 用于对指定文件类型的网址添加后缀处理参数,优先级从上到下只取第一个匹配项
|
||||
value: [ ]
|
||||
min: 0
|
||||
- $formkit: verificationForm
|
||||
action: "/apis/s3os.halo.run/v1alpha1/policies/s3/validation"
|
||||
label: 对象存储验证
|
||||
children:
|
||||
- $formkit: text
|
||||
name: fileSuffix
|
||||
label: 文件后缀
|
||||
placeholder: 以半角逗号分隔,例如:jpg,jpeg,png,gif
|
||||
name: bucket
|
||||
label: Bucket 桶名称
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: endpointProtocol
|
||||
label: Endpoint 访问协议
|
||||
options:
|
||||
- label: HTTPS
|
||||
value: https
|
||||
- label: HTTP
|
||||
value: http
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: enablePathStyleAccess
|
||||
label: Endpoint 访问风格
|
||||
options:
|
||||
- label: Virtual Hosted Style
|
||||
value: false
|
||||
- label: Path Style
|
||||
value: true
|
||||
value: false
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: urlSuffix
|
||||
name: endpoint
|
||||
label: EndPoint
|
||||
placeholder: 请填写不带bucket-name的Endpoint
|
||||
validation: required
|
||||
help: 协议头请在上方设置,此处无需以"http://"或"https://"开头,系统会自动拼接
|
||||
- $formkit: password
|
||||
name: accessKey
|
||||
label: Access Key ID
|
||||
placeholder: 存储桶用户标识(用户名)
|
||||
validation: required
|
||||
- $formkit: password
|
||||
name: accessSecret
|
||||
label: Access Key Secret
|
||||
placeholder: 存储桶密钥(密码)
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: region
|
||||
label: Region
|
||||
placeholder: 如不填写,则默认为"Auto"
|
||||
help: 若Region为Auto无法使用,才需要填写对应Region
|
||||
- $formkit: text
|
||||
name: location
|
||||
label: 上传目录
|
||||
placeholder: 如不填写,则默认上传到根目录
|
||||
help: 支持的占位符请查阅:https://github.com/halo-dev/plugin-s3#上传目录
|
||||
- $formkit: select
|
||||
name: randomFilenameMode
|
||||
label: 上传时重命名文件方式
|
||||
options:
|
||||
- label: 保留原文件名
|
||||
value: none
|
||||
- label: 自定义(请在下方输入自定义模板)
|
||||
value: custom
|
||||
- label: 使用UUID
|
||||
value: uuid
|
||||
- label: 使用毫秒时间戳
|
||||
value: timestampMs
|
||||
- label: 使用原文件名 + 随机字母
|
||||
value: withString
|
||||
- label: 使用日期 + 随机字母
|
||||
value: dateWithString
|
||||
- label: 使用日期时间 + 随机字母
|
||||
value: datetimeWithString
|
||||
- label: 使用随机字母
|
||||
value: string
|
||||
validation: required
|
||||
- $formkit: number
|
||||
name: randomStringLength
|
||||
key: randomStringLength
|
||||
label: 随机字母长度
|
||||
min: 4
|
||||
max: 16
|
||||
if: "$randomFilenameMode == 'dateWithString' || $randomFilenameMode == 'datetimeWithString' || $randomFilenameMode == 'withString' || $randomFilenameMode == 'string'"
|
||||
help: 支持4~16位, 默认为8位
|
||||
- $formkit: text
|
||||
name: customTemplate
|
||||
key: customTemplate
|
||||
label: 自定义文件名模板
|
||||
if: "$randomFilenameMode == 'custom'"
|
||||
value: "${origin-filename}"
|
||||
help: 支持的占位符请查阅:https://github.com/halo-dev/plugin-s3#自定义文件名模板
|
||||
- $formkit: select
|
||||
name: duplicateFilenameHandling
|
||||
label: 重复文件名处理方式
|
||||
options:
|
||||
- label: 加随机字母数字后缀
|
||||
value: randomAlphanumeric
|
||||
- label: 加随机字母后缀
|
||||
value: randomAlphabetic
|
||||
- label: 报错不上传
|
||||
value: exception
|
||||
validation: required
|
||||
- $formkit: select
|
||||
name: protocol
|
||||
label: 绑定域名协议
|
||||
options:
|
||||
- label: HTTPS
|
||||
value: https
|
||||
- label: HTTP
|
||||
value: http
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: domain
|
||||
label: 绑定域名(CDN域名)
|
||||
placeholder: 如不设置,那么将使用 Bucket + EndPoint 作为域名
|
||||
help: 协议头请在上方设置,此处无需以"http://"或"https://"开头,系统会自动拼接
|
||||
- $formkit: repeater
|
||||
name: urlSuffixes
|
||||
label: 网址后缀
|
||||
placeholder: 例如:?imageMogr2/format/webp
|
||||
validation: required
|
||||
help: 用于对指定文件类型的网址添加后缀处理参数,优先级从上到下只取第一个匹配项
|
||||
value: [ ]
|
||||
min: 0
|
||||
children:
|
||||
- $formkit: text
|
||||
name: fileSuffix
|
||||
label: 文件后缀
|
||||
placeholder: 以半角逗号分隔,例如:jpg,jpeg,png,gif
|
||||
validation: required
|
||||
- $formkit: text
|
||||
name: urlSuffix
|
||||
label: 网址后缀
|
||||
placeholder: 例如:?imageMogr2/format/webp
|
||||
validation: required
|
@@ -39,3 +39,16 @@ rules:
|
||||
- apiGroups: [ "s3os.halo.run" ]
|
||||
resources: [ "attachments" ]
|
||||
verbs: [ "delete" ]
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: "Role"
|
||||
metadata:
|
||||
name: role-template-s3os-policy-config-validation
|
||||
labels:
|
||||
halo.run/role-template: "true"
|
||||
rbac.authorization.halo.run/aggregate-to-role-template-manage-configmaps: "true"
|
||||
rules:
|
||||
- apiGroups: ["s3os.halo.run"]
|
||||
resources: ["policies/validation"]
|
||||
resourceNames: ["s3"]
|
||||
verbs: [ "create" ]
|
@@ -4,7 +4,7 @@ metadata:
|
||||
name: PluginS3ObjectStorage
|
||||
spec:
|
||||
enabled: true
|
||||
requires: ">=2.12.0"
|
||||
requires: ">=2.14.0"
|
||||
author:
|
||||
name: Halo OSS Team
|
||||
website: https://github.com/halo-dev
|
||||
|
BIN
src/main/resources/validation.jpg
Normal file
BIN
src/main/resources/validation.jpg
Normal file
Binary file not shown.
After Width: | Height: | Size: 18 KiB |
Reference in New Issue
Block a user