Complete S3 protocol adaptation

This commit is contained in:
longjuan
2022-12-16 13:36:18 +08:00
parent 535643d14e
commit 2808db4876
10 changed files with 184 additions and 141 deletions

View File

@@ -3,7 +3,7 @@ plugins {
id 'java'
}
group 'run.halo.alioss'
group 'run.halo.s3os'
sourceCompatibility = JavaVersion.VERSION_17
repositories {
@@ -27,7 +27,8 @@ dependencies {
compileOnly files("lib/halo-2.0.0-SNAPSHOT-plain.jar")
implementation "com.aliyun.oss:aliyun-sdk-oss:3.15.0"
implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000')
implementation 'com.amazonaws:aws-java-sdk-s3'
implementation "javax.xml.bind:jaxb-api:2.3.1"
implementation "javax.activation:activation:1.1.1"
implementation "org.glassfish.jaxb:jaxb-runtime:2.3.3"

View File

@@ -7,5 +7,5 @@ pluginManagement {
gradlePluginPortal()
}
}
rootProject.name = 'plugin-alioss'
rootProject.name = 'halo-plugin-s3os'

View File

@@ -1,36 +0,0 @@
package run.halo.alioss;
import lombok.Data;
import org.springframework.util.StringUtils;
@Data
class AliOssProperties {
private String bucket;
private String endpoint;
private String accessKey;
private String accessSecret;
private String location;
private Protocol protocol = Protocol.https;
private String domain;
private String allowExtensions;
public String getObjectName(String filename) {
var objectName = filename;
if (StringUtils.hasText(getLocation())) {
objectName = getLocation() + "/" + objectName;
}
return objectName;
}
enum Protocol {
http, https
}
}

View File

@@ -1,24 +1,17 @@
package run.halo.alioss;
package run.halo.s3os;
import com.aliyun.oss.ClientException;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.internal.OSSHeaders;
import com.aliyun.oss.model.CannedAccessControlList;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PutObjectRequest;
import com.aliyun.oss.model.PutObjectResult;
import com.aliyun.oss.model.StorageClass;
import com.aliyun.oss.model.VoidResult;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.pf4j.Extension;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.web.util.UriUtils;
@@ -34,11 +27,19 @@ import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.Metadata;
import run.halo.app.infra.utils.JsonUtils;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
@Slf4j
@Extension
public class AliOssAttachmentHandler implements AttachmentHandler {
public class S3OsAttachmentHandler implements AttachmentHandler {
private static final String OBJECT_KEY = "alioss.plugin.halo.run/object-key";
private static final String OBJECT_KEY = "s3os.plugin.halo.run/object-key";
@Override
public Mono<Attachment> upload(UploadContext uploadContext) {
@@ -60,38 +61,36 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
}
var objectName = annotations.get(OBJECT_KEY);
var properties = getProperties(deleteContext.configMap());
var oss = buildOss(properties);
var client = buildOsClient(properties);
ossExecute(() -> {
log.info("{}/{} is being deleted from AliOSS", properties.getBucket(),
log.info("{}/{} is being deleted from S3ObjectStorage", properties.getBucket(),
objectName);
VoidResult result = oss.deleteObject(properties.getBucket(), objectName);
if (log.isDebugEnabled()) {
debug(result);
}
log.info("{}/{} was deleted successfully from AliOSS", properties.getBucket(),
client.deleteObject(properties.getBucket(), objectName);
log.info("{}/{} was deleted successfully from S3ObjectStorage", properties.getBucket(),
objectName);
return result;
}, oss::shutdown);
return null;
}, client::shutdown);
}).map(DeleteContext::attachment);
}
<T> T ossExecute(Supplier<T> runnable, Runnable finalizer) {
try {
return runnable.get();
} catch (OSSException oe) {
} catch (AmazonServiceException ase) {
log.error("""
Caught an OSSException, which means your request made it to OSS, but was
Caught an AmazonServiceException, which means your request made it to S3ObjectStorage, but was
rejected with an error response for some reason.
Error message: {}, error code: {}, request id: {}, host id: {}
""", oe.getErrorCode(), oe.getErrorCode(), oe.getRequestId(), oe.getHostId());
throw Exceptions.propagate(oe);
} catch (ClientException ce) {
Error message: {}
""", ase.getMessage());
throw Exceptions.propagate(ase);
} catch (SdkClientException sce) {
log.error("""
Caught an ClientException, which means the client encountered a serious internal
problem while trying to communicate with OSS, such as not being able to access
Caught an SdkClientException, which means the client encountered a serious internal
problem while trying to communicate with S3ObjectStorage, such as not being able to access
the network.
""");
throw Exceptions.propagate(ce);
Error message: {}
""", sce.getMessage());
throw Exceptions.propagate(sce);
} finally {
if (finalizer != null) {
finalizer.run();
@@ -99,16 +98,20 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
}
}
AliOssProperties getProperties(ConfigMap configMap) {
S3OsProperties getProperties(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, AliOssProperties.class);
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}
Attachment buildAttachment(UploadContext uploadContext, AliOssProperties properties,
Attachment buildAttachment(UploadContext uploadContext, S3OsProperties properties,
ObjectDetail objectDetail) {
var host = properties.getBucket() + "." + properties.getEndpoint();
var externalLink =
properties.getProtocol() + "://" + host + "/" + objectDetail.objectName();
String externalLink;
if (StringUtils.isBlank(properties.getDomain())) {
var host = properties.getBucket() + "." + properties.getEndpoint();
externalLink = properties.getProtocol() + "://" + host + "/" + objectDetail.objectName();
} else {
externalLink = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectDetail.objectName();
}
var metadata = new Metadata();
metadata.setName(UUID.randomUUID().toString());
@@ -128,14 +131,20 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
return attachment;
}
OSS buildOss(AliOssProperties properties) {
return new OSSClientBuilder().build(properties.getEndpoint(), properties.getAccessKey(),
properties.getAccessSecret());
AmazonS3 buildOsClient(S3OsProperties properties) {
return AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(properties.getAccessKey(), properties.getAccessSecret())))
.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(properties.getEndpoint(),properties.getRegion()))
.withPathStyleAccessEnabled(false)
.withChunkedEncodingDisabled(true)
.build();
}
Mono<ObjectDetail> upload(UploadContext uploadContext, AliOssProperties properties) {
Mono<ObjectDetail> upload(UploadContext uploadContext, S3OsProperties properties) {
return Mono.fromCallable(() -> {
var client = buildOss(properties);
var client = buildOsClient(properties);
// build object name
var objectName = properties.getObjectName(uploadContext.file().filename());
@@ -152,14 +161,11 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
}).subscribe(DataBufferUtils.releaseConsumer());
final var bucket = properties.getBucket();
log.info("Uploading {} into AliOSS {}/{}/{}", uploadContext.file().filename(),
log.info("Uploading {} into S3ObjectStorage {}/{}/{}", uploadContext.file().filename(),
properties.getEndpoint(), bucket, objectName);
var request = new PutObjectRequest(bucket, objectName, pis);
var metadata = new ObjectMetadata();
metadata.setHeader(OSSHeaders.OSS_STORAGE_CLASS, StorageClass.Standard.toString());
metadata.setObjectAcl(CannedAccessControlList.PublicRead);
request.setMetadata(metadata);
var request = new PutObjectRequest(bucket, objectName, pis,
new ObjectMetadata());
return ossExecute(() -> {
var result = client.putObject(request);
@@ -174,20 +180,11 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
void debug(PutObjectResult result) {
log.debug("""
PutObjectResult: request id: {}, version id: {}, server CRC: {},
client CRC: {}, etag: {}, response status: {}, response headers: {}, response body: {}
""", result.getRequestId(), result.getVersionId(), result.getServerCRC(),
result.getClientCRC(), result.getETag(), result.getResponse().getStatusCode(),
result.getResponse().getHeaders(), result.getResponse().getErrorResponseAsString());
}
void debug(VoidResult result) {
log.debug("""
VoidResult: request id: {}, server CRC: {},
client CRC: {}, response status: {}, response headers: {}, response body: {}
""", result.getRequestId(), result.getServerCRC(), result.getClientCRC(),
result.getResponse().getStatusCode(), result.getResponse().getHeaders(),
result.getResponse().getErrorResponseAsString());
PutObjectResult: VersionId: {}, ETag: {}, ContentMd5: {}, ExpirationTime: {}, ExpirationTimeRuleId: {},
response RawMetadata: {}, UserMetadata: {}
""", result.getVersionId(), result.getETag(), result.getContentMd5(), result.getExpirationTime(),
result.getExpirationTimeRuleId(), result.getMetadata().getRawMetadata(),
result.getMetadata().getUserMetadata());
}
boolean shouldHandle(Policy policy) {
@@ -196,7 +193,7 @@ public class AliOssAttachmentHandler implements AttachmentHandler {
return false;
}
String templateName = policy.getSpec().getTemplateName();
return "alioss".equals(templateName);
return "s3os".equals(templateName);
}
record ObjectDetail(String bucketName, String objectName, ObjectMetadata objectMetadata) {

View File

@@ -1,4 +1,4 @@
package run.halo.alioss;
package run.halo.s3os;
import org.pf4j.PluginWrapper;
import org.springframework.stereotype.Component;
@@ -9,9 +9,9 @@ import run.halo.app.plugin.BasePlugin;
* @since 2.0.0
*/
@Component
public class AliOSSPlugin extends BasePlugin {
public class S3OsPlugin extends BasePlugin {
public AliOSSPlugin(PluginWrapper wrapper) {
public S3OsPlugin(PluginWrapper wrapper) {
super(wrapper);
}

View File

@@ -0,0 +1,76 @@
package run.halo.s3os;
import lombok.Data;
import org.springframework.util.StringUtils;
@Data
class S3OsProperties {
private String bucket;
private String endpoint;
private String accessKey;
private String accessSecret;
/**
* 开头结尾已去除"/"
*/
private String location;
private Protocol protocol = Protocol.https;
/**
* 不包含协议头
*/
private String domain;
private String allowExtensions;
private String region = "Auto";
public String getObjectName(String filename) {
var objectName = filename;
if (StringUtils.hasText(getLocation())) {
objectName = getLocation() + "/" + objectName;
}
return objectName;
}
enum Protocol {
http, https
}
public void setDomain(String domain) {
if (domain.toLowerCase().startsWith("http://")){
domain = domain.substring(7);
} else if (domain.toLowerCase().startsWith("https://")) {
domain = domain.substring(8);
}
this.domain = domain;
}
public void setLocation(String location) {
final var fileSeparator = "/";
if (StringUtils.hasText(location)) {
if (location.startsWith(fileSeparator)) {
location = location.substring(1);
}
if (location.endsWith(fileSeparator)) {
location = location.substring(0, location.length() - 1);
}
} else {
location = "";
}
this.location = location;
}
public void setRegion(String region) {
if (!StringUtils.hasText(region)) {
this.region = "Auto";
}else {
this.region = region;
}
}
}

View File

@@ -1,22 +1,22 @@
apiVersion: storage.halo.run/v1alpha1
kind: PolicyTemplate
metadata:
name: alioss
name: s3os
spec:
displayName: Aliyun OSS
settingName: alioss-policy-template-setting
displayName: S3ObjectStorage
settingName: s3os-policy-template-setting
---
apiVersion: v1alpha1
kind: Setting
metadata:
name: alioss-policy-template-setting
name: s3os-policy-template-setting
spec:
forms:
- group: default
formSchema:
- $formkit: text
name: bucket
label: Bucket
label: Bucket 桶名称
validation: required
- $formkit: text
name: endpoint
@@ -30,6 +30,11 @@ spec:
name: accessSecret
label: Access Secret
validation: required
- $formkit: text
name: region
label: Region
placeholder: 如不填写,则默认为"Auto"
help: 若Region为Auto无法使用才需要填写对应Region
- $formkit: text
name: location
label: 上传目录
@@ -44,8 +49,9 @@ spec:
value: http
- $formkit: text
name: domain
label: 绑定域名
label: 绑定域名CDN域名
placeholder: 如不设置,那么将使用 Bucket + EndPoint 作为域名
help: 协议头请在上方设置,此处无需以"http://"或"https://"开头,系统会自动拼接
- $formkit: textarea
name: allow_extensions
label: 允许上传的文件类型

View File

@@ -1,14 +1,14 @@
apiVersion: v1alpha1
kind: Setting
metadata:
name: alioss-settings
name: s3os-settings
spec:
forms:
- group: basic
label: 基本设置
formSchema:
- $formkit: text
help: This will be used for your account.
label: Email
name: email
validation: required|email
help: 请前往 “附件 - 存储策略” 添加策略
label: 此处不用设置,请前往 “附件 - 存储策略” 添加策略
name: text
placeholder: 此处不用设置,请前往 “附件 - 存储策略” 添加策略

View File

@@ -1,19 +1,19 @@
apiVersion: plugin.halo.run/v1alpha1
kind: Plugin
metadata:
name: PluginAliOSS
name: PluginS3ObjectStorage
spec:
enabled: true
version: 1.0.0
requires: ">=2.0.0"
author:
name: Halo OSS Team
website: https://github.com/halo-dev
logo: data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+QWxpYmFiYSBDbG91ZDwvdGl0bGU+PHBhdGggZmlsbD0iI0ZGNkEwMCIgZD0iTTMuOTk2IDQuNTE3aDUuMjkxTDguMDEgNi4zMjQgNC4xNTMgNy41MDZhMS42NjggMS42NjggMCAwIDAtMS4xNjUgMS42MDF2NS43ODZhMS42NjggMS42NjggMCAwIDAgMS4xNjUgMS42bDMuODU3IDEuMTgzIDEuMjc3IDEuODA3SDMuOTk2QTMuOTk2IDMuOTk2IDAgMCAxIDAgMTUuNDg3VjguNTEzYTMuOTk2IDMuOTk2IDAgMCAxIDMuOTk2LTMuOTk2bTE2LjAwOCAwaC01LjI5MWwxLjI3NyAxLjgwNyAzLjg1NyAxLjE4MmMuNzE1LjIyNyAxLjE3Ljg4OSAxLjE2NSAxLjYwMXY1Ljc4NmExLjY2OCAxLjY2OCAwIDAgMS0xLjE2NSAxLjZsLTMuODU3IDEuMTgzLTEuMjc3IDEuODA3aDUuMjkxQTMuOTk2IDMuOTk2IDAgMCAwIDI0IDE1LjQ4N1Y4LjUxM2EzLjk5NiAzLjk5NiAwIDAgMC0zLjk5Ni0zLjk5Nm0tNC4wMDcgOC4zNDVIOC4wMDJ2LTEuODA0aDcuOTk1WiIvPjwvc3ZnPg==
settingName: alioss-settings
configMapName: alioss-configMap
homepage: https://github.com/halo-sigs/plugin-alioss
displayName: "阿里云 OSS"
description: "提供阿里云 OSS 的存储策略"
name: longjuan
website: https://github.com/longjuan
logo: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAABUdJREFUeF7tmmtoXFUQx3+zMYGi/RJRULRFsL6oim2klVK0CrVqKxjoJ0Vt1bR7t6lURagiFlGpFPogydklVo1i/VB84QMfqFijpflQrI+IYj7FF+0HMbUEI82O3NNtzG6y956bvXd3IXvgksCd+c/M/8w598ycFWb5kFkePw0CGhkwyxloLIFZngCNTbCxBBpLoI4Y0A6aydNKC62kaAXGSDHCv4zQzIh0MRa3uzVdAppmMSmWoVyL0IZyWUiA3wL7gI/FcCQOMmpCQCHwDpSOCoIYQOmWLK9WgFHdr4Bu4wyOsR14uBKni3SFdxhnl+T4fCaYVcsAzbCEPNsRbpiJow46u8TwkINcCX9RNWYgr2nWIrwAzJ2BehSVAVpYJbv5y1Up8QwoBL/f1aGY5JaJ4aALVqIEqMc24EkXRxKQmSeGX8JwEyNAPR4AesMcSPD9ECdpk15GgmwkQoB2spxxPgWaEwzQBfptMdxRfQI8PgJWuniYuIyyQbLlMzH2DFCPDUAu8cDcDRwWQ1s58VgJ0E7OYZyvgAXu/lVBMiAL4iXA4z5gbxVCimpiQAxLp1OKl4AMfSj3RPWuKvJ5FkmOr0ttxUuAx9/AWVUJKKoR5QnJ8nRsBOh65tJiy9nFYJ+fwB586nUcEsN1FROgHncDG2EqWIWR/wpcUCFGsPooc6SPfyYLOS8BzbAI5RlgVQJObrXdH9iZAPb/kCkul25+jEyAejboDxJyLiMG42Orx2fAioTs+LC3iOHDSARohs0oexJxStgtPWw5jZ34IUpJS7b4kBa4BDTDTSifJBI8vC+G1ZOxdQtzGGMIOD8hm1vF2I7UxAgmwLNpn8SaHyTPjZLjWGmg6tl9YCIrYiVCuVOyvOZEQGG3fzlWB06BjZJipXTbI/OUoWmuQvgmAbs+5HIxfOlKwCFgSQKOrBNDXxCuerwH3Ba77ZPMl16GQwnQjSwkxXexO6A8JdnwDpF6tANvxG1fzNQu+LR7QEKtrFfEuNcJ6vEzcHGMJEzZdH3scgRkC6e9eOwr/Qi3iuGEK6B6PAo85yofKid40oMfV9EoR8CbENxKCjVYvNCWSg8DUXQKn8TRKDqBstOs/6AMiLelVaYUDQtO0zyPcH+YnMP7d8Vw+3Ry5TLgJeBeB2BXEX8mDwDHEUZQWzafEBNcPca2FwlrpMd+WaaM6QnI8CyKX6AkN/KsmHyfZ9voQjfKUbDPOMoCxF6TVzLKzn75JZBmOcIXlVgN0y39JGka/yOVDtOL/D5g9ssS4L9Qz5aNl0Y26KZwRAzXTBZVj8PAIjd1Z6ntYoIzuWwtoBkeK9T/ztacBUurwA1cQRODzvpugvvEcFeYaHkCHuFMRvGPwwvDQCK+P0qexZLjt9N6mmE9am+P4xnKn5LlbBew4GowzTqEF12AnGXytEuOtyaC30aKY7YRcr0zRrDg92K40hUrtCWmGfajrHUFDJHzf9tzc8na9zu1j8eCL/RKj72Zch6hBPhI6tEFbHJGnV5wSEzxjVGMrbZBlB2SJXL57kRAgQS/G7wDODcyEUKX9LC5ZOb9+wO/lV5J9+c4sIsmdkoX/v+RhzMBloRNXESeB8E+4UNtSbtHsvSXCqt/1oB2xNYc88PBiiT8uuIgTeyVLn6IqFskHomAiY0rwyUoa1BWI3YGzyu8+wPld1K8Tp5+yeL/ri90aCdXM26JuBCYN/GcOjYPIwzbv3CYJg5IF/4dQixjRgTEYrlOQBoE1MlE1MyNRgbUjPo6MdzIgDqZiJq50ciAmlFfJ4YbGVAnE1EzN2Z9BvwH8mZfUOuuxhIAAAAASUVORK5CYII=
settingName: s3os-settings
configMapName: s3os-configMap
homepage: https://github.com/longjuan/halo-plugin-s3os
displayName: "对象存储Amazon S3协议"
description: "提供兼容 Amazon S3 协议的对象存储策略,兼容阿里云、腾讯云、七牛云等"
license:
- name: "MIT"

View File

@@ -1,4 +1,4 @@
package run.halo.alioss;
package run.halo.s3os;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -8,24 +8,23 @@ import static org.mockito.Mockito.when;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import run.halo.app.core.extension.attachment.Policy;
import run.halo.app.core.extension.attachment.Policy.PolicySpec;
class AliOssAttachmentHandlerTest {
class S3OsAttachmentHandlerTest {
AliOssAttachmentHandler handler;
S3OsAttachmentHandler handler;
@BeforeEach
void setUp() {
handler = new AliOssAttachmentHandler();
handler = new S3OsAttachmentHandler();
}
@Test
void acceptHandlingWhenPolicyTemplateIsExpected() {
var policy = mock(Policy.class);
var spec = mock(PolicySpec.class);
var spec = mock(Policy.PolicySpec.class);
when(policy.getSpec()).thenReturn(spec);
when(spec.getTemplateName()).thenReturn("alioss");
when(spec.getTemplateName()).thenReturn("s3os");
assertTrue(handler.shouldHandle(policy));
when(spec.getTemplateName()).thenReturn("invalid");