perf: auto rename attachment if it exists (#22)

Fixes https://github.com/halo-dev/halo/issues/3337 不更新依赖了，直接复制了FileNameUtils 在有image.png的情况下再同时粘贴两张截图，期望两张都能被上传且被自动重命名。 ![image](https://user-images.githubusercontent.com/28662535/220059741-da25a490-6f6a-4172-a393-aa3f84ab6b38.png) ![image](https://user-images.githubusercontent.com/28662535/220059786-24cda2bb-6faa-4377-8eb8-a70920916f3d.png) ```release-note 文件存在时自动重命名 ```
2025-10-16 15:30:00 +00:00 · 2023-02-25 10:38:14 +08:00
parent 459cc1cf94
commit c635ebede8
2 changed files with 197 additions and 114 deletions
--- a/src/main/java/run/halo/s3os/FileNameUtils.java
+++ b/src/main/java/run/halo/s3os/FileNameUtils.java
@@ -0,0 +1,44 @@
+package run.halo.s3os;
+
+import com.google.common.io.Files;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public final class FileNameUtils {
+
+    private FileNameUtils() {
+    }
+
+    public static String removeFileExtension(String filename, boolean removeAllExtensions) {
+        if (filename == null || filename.isEmpty()) {
+            return filename;
+        }
+        var extPattern = "(?<!^)[.]" + (removeAllExtensions ? ".*" : "[^.]*$");
+        return filename.replaceAll(extPattern, "");
+    }
+
+    /**
+     * Append random string after file name.
+     * <pre>
+     * Case 1: halo.run -> halo-xyz.run
+     * Case 2: .run -> xyz.run
+     * Case 3: halo -> halo-xyz
+     * </pre>
+     *
+     * @param filename is name of file.
+     * @param length is for generating random string with specific length.
+     * @return File name with random string.
+     */
+    public static String randomFileName(String filename, int length) {
+        var nameWithoutExt = Files.getNameWithoutExtension(filename);
+        var ext = Files.getFileExtension(filename);
+        var random = RandomStringUtils.randomAlphabetic(length).toLowerCase();
+        if (StringUtils.isBlank(nameWithoutExt)) {
+            return random + "." + ext;
+        }
+        if (StringUtils.isBlank(ext)) {
+            return nameWithoutExt + "-" + random;
+        }
+        return nameWithoutExt + "-" + random + "." + ext;
+    }
+}
--- a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
+++ b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
@@ -1,6 +1,5 @@
 package run.halo.s3os;

-import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.pf4j.Extension;
@@ -10,7 +9,9 @@ import org.springframework.http.MediaTypeFactory;
 import org.springframework.web.server.ServerErrorException;
 import org.springframework.web.server.ServerWebInputException;
 import org.springframework.web.util.UriUtils;
+import reactor.core.Exceptions;
 import reactor.core.publisher.Mono;
+import reactor.util.retry.Retry;
 import run.halo.app.core.extension.attachment.Attachment;
 import run.halo.app.core.extension.attachment.Attachment.AttachmentSpec;
 import run.halo.app.core.extension.attachment.Constant;
@@ -31,6 +32,7 @@ import software.amazon.awssdk.services.s3.model.*;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -50,8 +52,8 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
        return Mono.just(uploadContext).filter(context -> this.shouldHandle(context.policy()))
                .flatMap(context -> {
                    final var properties = getProperties(context.configMap());
-                    return upload(context, properties).map(
-                            objectDetail -> this.buildAttachment(context, properties, objectDetail));
+                    return upload(context, properties)
+                            .map(objectDetail -> this.buildAttachment(properties, objectDetail));
                });
    }

@@ -87,32 +89,33 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
        return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
    }

-    Attachment buildAttachment(UploadContext uploadContext, S3OsProperties properties,
-                               ObjectDetail objectDetail) {
+    Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) {
        String externalLink;
        if (StringUtils.isBlank(properties.getDomain())) {
            var host = properties.getBucket() + "." + properties.getEndpoint();
-            externalLink = properties.getProtocol() + "://" + host + "/" + objectDetail.objectKey();
+            externalLink = properties.getProtocol() + "://" + host + "/" + objectDetail.uploadState.objectKey;
        } else {
-            externalLink = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectDetail.objectKey();
+            externalLink = properties.getProtocol() + "://" + properties.getDomain() + "/"
+                    + objectDetail.uploadState.objectKey;
        }

        var metadata = new Metadata();
        metadata.setName(UUID.randomUUID().toString());
        metadata.setAnnotations(
-                Map.of(OBJECT_KEY, objectDetail.objectKey(), Constant.EXTERNAL_LINK_ANNO_KEY,
+                Map.of(OBJECT_KEY, objectDetail.uploadState.objectKey, Constant.EXTERNAL_LINK_ANNO_KEY,
                        UriUtils.encodePath(externalLink, StandardCharsets.UTF_8)));

        var objectMetadata = objectDetail.objectMetadata();
        var spec = new AttachmentSpec();
        spec.setSize(objectMetadata.contentLength());
-        spec.setDisplayName(uploadContext.file().filename());
+        spec.setDisplayName(objectDetail.uploadState.fileName);
        spec.setMediaType(objectMetadata.contentType());

        var attachment = new Attachment();
        attachment.setMetadata(metadata);
        attachment.setSpec(spec);
-        log.info("Upload object {} to bucket {} successfully", objectDetail.objectKey(), properties.getBucket());
+        log.info("Upload object {} to bucket {} successfully", objectDetail.uploadState.objectKey,
+                properties.getBucket());
        return attachment;
    }

@@ -130,108 +133,130 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
    }

    Mono<ObjectDetail> upload(UploadContext uploadContext, S3OsProperties properties) {
-        var originFilename = uploadContext.file().filename();
-        var objectKey = properties.getObjectName(originFilename);
-        var contentType = MediaTypeFactory.getMediaType(originFilename)
-                .orElse(MediaType.APPLICATION_OCTET_STREAM).toString();
-        var uploadingMapKey = properties.getBucket() + "/" + objectKey;
-        // deduplication of uploading files
-        if (uploadingFile.put(uploadingMapKey, uploadingMapKey) != null) {
-            return Mono.error(new ServerWebInputException("文件 " + originFilename + " 已存在，建议更名后重试。"));
-        }
-
-        var s3client = buildS3AsyncClient(properties);
-
-        var uploadState = new UploadState(properties.getBucket(), objectKey);
-
-        return Mono
-                // check whether file exists
-                .fromFuture(s3client.headObject(HeadObjectRequest.builder()
-                        .bucket(properties.getBucket())
-                        .key(objectKey)
-                        .build()))
-                .onErrorResume(NoSuchKeyException.class, e -> {
-                    var builder = HeadObjectResponse.builder();
-                    builder.sdkHttpResponse(SdkHttpResponse.builder().statusCode(404).build());
-                    return Mono.just(builder.build());
-                })
-                .flatMap(response -> {
-                    if (response != null && response.sdkHttpResponse() != null && response.sdkHttpResponse().isSuccessful()) {
-                        return Mono.error(new ServerWebInputException("文件 " + originFilename + " 已存在，建议更名后重试。"));
-                    }else {
-                        return Mono.just(uploadState);
-                    }
-                })
-                // init multipart upload
-                .flatMap(state -> Mono.fromFuture(s3client.createMultipartUpload(
-                        CreateMultipartUploadRequest.builder()
-                                .bucket(properties.getBucket())
-                                .contentType(contentType)
-                                .key(objectKey)
-                                .build())))
-                .flatMapMany((response) -> {
-                    checkResult(response, "createMultipartUpload");
-                    uploadState.setUploadId(response.uploadId());
-                    return uploadContext.file().content();
-                })
-                // buffer to part
-                .windowUntil((buffer) -> {
-                    uploadState.buffered += buffer.readableByteCount();
-                    if (uploadState.buffered >= MULTIPART_MIN_PART_SIZE) {
-                        uploadState.buffered = 0;
-                        return true;
-                    } else {
-                        return false;
-                    }
-                })
-                // upload part
-                .concatMap((window) -> window.collectList().flatMap((bufferList) -> {
-                    var buffer = S3OsAttachmentHandler.concatBuffers(bufferList);
-                    return uploadPart(uploadState, buffer, s3client);
-                }))
-                .reduce(uploadState, (state, completedPart) -> {
-                    state.completedParts.put(completedPart.partNumber(), completedPart);
-                    return state;
-                })
-                // complete multipart upload
-                .flatMap((state) -> Mono
-                        .fromFuture(s3client.completeMultipartUpload(CompleteMultipartUploadRequest.builder()
-                                .bucket(state.bucket)
-                                .uploadId(state.uploadId)
-                                .multipartUpload(CompletedMultipartUpload.builder()
-                                        .parts(state.completedParts.values())
-                                        .build())
-                                .key(state.objectKey)
-                                .build())
-                        ))
-                // get object metadata
-                .flatMap((response) -> {
-                    checkResult(response, "completeUpload");
-                    return Mono.fromFuture(s3client.headObject(
-                            HeadObjectRequest.builder()
-                                    .bucket(properties.getBucket())
-                                    .key(objectKey)
-                                    .build()
-                    ));
-                })
-                // build object detail
-                .map((response) -> {
-                    checkResult(response, "getMetadata");
-                    return new ObjectDetail(properties.getBucket(), objectKey, response);
-                })
-                // close client
-                .doFinally((signalType) -> {
-                    uploadingFile.remove(uploadingMapKey);
-                    s3client.close();
+        return Mono.zip(Mono.just(new UploadState(properties, uploadContext.file().filename())),
+                        Mono.just(buildS3AsyncClient(properties)))
+                .flatMap(tuple -> {
+                    var uploadState = tuple.getT1();
+                    var s3client = tuple.getT2();
+                    return checkFileExistsAndRename(uploadState, s3client)
+                            // init multipart upload
+                            .flatMap(state -> Mono.fromFuture(s3client.createMultipartUpload(
+                                    CreateMultipartUploadRequest.builder()
+                                            .bucket(properties.getBucket())
+                                            .contentType(state.contentType)
+                                            .key(state.objectKey)
+                                            .build())))
+                            .flatMapMany((response) -> {
+                                checkResult(response, "createMultipartUpload");
+                                uploadState.uploadId = response.uploadId();
+                                return uploadContext.file().content();
+                            })
+                            // buffer to part
+                            .windowUntil((buffer) -> {
+                                uploadState.buffered += buffer.readableByteCount();
+                                if (uploadState.buffered >= MULTIPART_MIN_PART_SIZE) {
+                                    uploadState.buffered = 0;
+                                    return true;
+                                } else {
+                                    return false;
+                                }
+                            })
+                            // upload part
+                            .concatMap((window) -> window.collectList().flatMap((bufferList) -> {
+                                var buffer = S3OsAttachmentHandler.concatBuffers(bufferList);
+                                return uploadPart(uploadState, buffer, s3client);
+                            }))
+                            .reduce(uploadState, (state, completedPart) -> {
+                                state.completedParts.put(completedPart.partNumber(), completedPart);
+                                return state;
+                            })
+                            // complete multipart upload
+                            .flatMap((state) -> Mono
+                                    .fromFuture(s3client.completeMultipartUpload(
+                                            CompleteMultipartUploadRequest
+                                                    .builder()
+                                                    .bucket(properties.getBucket())
+                                                    .uploadId(state.uploadId)
+                                                    .multipartUpload(CompletedMultipartUpload.builder()
+                                                            .parts(state.completedParts.values())
+                                                            .build())
+                                                    .key(state.objectKey)
+                                                    .build())
+                                    ))
+                            // get object metadata
+                            .flatMap((response) -> {
+                                checkResult(response, "completeUpload");
+                                return Mono.fromFuture(s3client.headObject(
+                                        HeadObjectRequest.builder()
+                                                .bucket(properties.getBucket())
+                                                .key(uploadState.objectKey)
+                                                .build()
+                                ));
+                            })
+                            // build object detail
+                            .map((response) -> {
+                                checkResult(response, "getMetadata");
+                                return new ObjectDetail(uploadState, response);
+                            })
+                            // close client
+                            .doFinally((signalType) -> {
+                                if (uploadState.needRemoveMapKey) {
+                                    uploadingFile.remove(uploadState.getUploadingMapKey());
+                                }
+                                s3client.close();
+                            });
                });
    }

+    private Mono<UploadState> checkFileExistsAndRename(UploadState uploadState, S3AsyncClient s3client) {
+        return Mono.defer(() -> {
+                    // deduplication of uploading files
+                    if (uploadingFile.put(uploadState.getUploadingMapKey(), uploadState.getUploadingMapKey()) != null) {
+                        return Mono.error(new FileAlreadyExistsException("文件 " + uploadState.objectKey
+                                + " 已存在，建议更名后重试。[local]"));
+                    }
+                    uploadState.needRemoveMapKey = true;
+                    // check whether file exists
+                    return Mono
+                            .fromFuture(s3client.headObject(HeadObjectRequest.builder()
+                                    .bucket(uploadState.properties.getBucket())
+                                    .key(uploadState.objectKey)
+                                    .build()))
+                            .onErrorResume(NoSuchKeyException.class, e -> {
+                                var builder = HeadObjectResponse.builder();
+                                builder.sdkHttpResponse(SdkHttpResponse.builder().statusCode(404).build());
+                                return Mono.just(builder.build());
+                            })
+                            .flatMap(response -> {
+                                if (response != null && response.sdkHttpResponse() != null
+                                        && response.sdkHttpResponse().isSuccessful()) {
+                                    return Mono.error(new FileAlreadyExistsException("文件 " + uploadState.objectKey
+                                            + " 已存在，建议更名后重试。[remote]"));
+                                } else {
+                                    return Mono.just(uploadState);
+                                }
+                            });
+                })
+                .retryWhen(Retry.max(3)
+                        .filter(FileAlreadyExistsException.class::isInstance)
+                        .doAfterRetry((retrySignal) -> {
+                            if (uploadState.needRemoveMapKey) {
+                                uploadingFile.remove(uploadState.getUploadingMapKey());
+                                uploadState.needRemoveMapKey = false;
+                            }
+                            uploadState.randomFileName();
+                        })
+                )
+                .onErrorMap(Exceptions::isRetryExhausted,
+                        throwable -> new ServerWebInputException(throwable.getCause().getMessage()));
+    }
+

    private Mono<CompletedPart> uploadPart(UploadState uploadState, ByteBuffer buffer, S3AsyncClient s3client) {
        final int partNumber = ++uploadState.partCounter;
        return Mono
                .fromFuture(s3client.uploadPart(UploadPartRequest.builder()
-                                .bucket(uploadState.bucket)
+                                .bucket(uploadState.properties.getBucket())
                                .key(uploadState.objectKey)
                                .partNumber(partNumber)
                                .uploadId(uploadState.uploadId)
@@ -262,9 +287,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
        }

        ByteBuffer partData = ByteBuffer.allocate(partSize);
-        buffers.forEach((buffer) -> {
-            partData.put(buffer.toByteBuffer());
-        });
+        buffers.forEach((buffer) -> partData.put(buffer.toByteBuffer()));

        // Reset read pointer to first byte
        partData.rewind();
@@ -282,21 +305,37 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
        return "s3os".equals(templateName);
    }

-    record ObjectDetail(String bucketName, String objectKey, HeadObjectResponse objectMetadata) {
+    record ObjectDetail(UploadState uploadState, HeadObjectResponse objectMetadata) {
    }

-    @Data
    static class UploadState {
-        String bucket;
-        String objectKey;
+        final S3OsProperties properties;
+        final String originalFileName;
        String uploadId;
        int partCounter;
        Map<Integer, CompletedPart> completedParts = new HashMap<>();
        int buffered = 0;
+        String contentType;
+        String fileName;
+        String objectKey;
+        boolean needRemoveMapKey = false;

-        UploadState(String bucket, String objectKey) {
-            this.bucket = bucket;
-            this.objectKey = objectKey;
+        public UploadState(S3OsProperties properties, String fileName) {
+            this.properties = properties;
+            this.originalFileName = fileName;
+            this.fileName = fileName;
+            this.objectKey = properties.getObjectName(fileName);
+            this.contentType = MediaTypeFactory.getMediaType(fileName)
+                    .orElse(MediaType.APPLICATION_OCTET_STREAM).toString();
+        }
+
+        public String getUploadingMapKey() {
+            return properties.getBucket() + "/" + objectKey;
+        }
+
+        public void randomFileName() {
+            this.fileName = FileNameUtils.randomFileName(originalFileName, 4);
+            this.objectKey = properties.getObjectName(fileName);
        }
    }