feat: Associate files originally in s3 (#59)

```release-note
关联从其他渠道上传至 s3 的文件
```
【插件】->【对象存储(Amazon S3 协议)】->【关联s3文件】
![image](https://github.com/halo-sigs/plugin-s3/assets/28662535/83096e23-d362-4924-b2f0-6984683d87cf)
![image](https://github.com/halo-sigs/plugin-s3/assets/28662535/6dd74842-f9a5-4e3c-9a93-36b4b188ecf6)

目前待优化的功能:
- [ ] 列出未关联的文件时会查询多次数据库,期望优化成只查询一次,但是代码改动较大。
This commit is contained in:
longjuan
2023-09-12 00:04:09 +08:00
committed by GitHub
parent 1b8bef991e
commit 91a61fd1d2
43 changed files with 4872 additions and 21 deletions

View File

@@ -1,30 +1,22 @@
plugins {
id "io.github.guqing.plugin-development" version "0.0.7-SNAPSHOT"
id "io.freefair.lombok" version "8.0.0-rc2"
id 'java'
id "com.github.node-gradle.node" version "3.3.0"
id "io.freefair.lombok" version "8.0.1"
id "run.halo.plugin.devtools" version "0.0.6"
}
group 'run.halo.s3os'
sourceCompatibility = JavaVersion.VERSION_17
repositories {
mavenCentral()
maven { url 'https://s01.oss.sonatype.org/content/repositories/releases' }
maven { url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' }
mavenCentral()
}
jar {
enabled = true
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
from {
configurations.runtimeClasspath.collect {
it.isDirectory() ? it : zipTree(it)
}
}
maven { url 'https://repo.spring.io/milestone' }
}
dependencies {
implementation platform('run.halo.tools.platform:plugin:2.5.0-SNAPSHOT')
implementation platform('run.halo.tools.platform:plugin:2.9.0-SNAPSHOT')
compileOnly 'run.halo.app:api'
implementation platform('software.amazon.awssdk:bom:2.19.8')
@@ -38,6 +30,38 @@ dependencies {
testImplementation 'io.projectreactor:reactor-test'
}
halo {
version = '2.9.0'
}
haloPlugin {
watchDomains {
consoleSource {
files files('console/src/')
}
}
}
test {
useJUnitPlatform()
}
node {
nodeProjectDir = file("${project.projectDir}/console")
}
task buildFrontend(type: NpxTask) {
command = 'pnpm'
args = ['build']
}
task pnpmInstall(type: NpxTask) {
command = "pnpm"
args = ["install"]
}
build {
// build frontend before build
tasks.getByName('compileJava').dependsOn('buildFrontend')
tasks.getByName("buildFrontend").dependsOn("pnpmInstall")
}

12
console/.editorconfig Normal file
View File

@@ -0,0 +1,12 @@
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = false
insert_final_newline = true

15
console/.eslintrc.cjs Normal file
View File

@@ -0,0 +1,15 @@
/* eslint-env node */
require("@rushstack/eslint-patch/modern-module-resolution");
module.exports = {
root: true,
extends: [
"plugin:vue/vue3-essential",
"eslint:recommended",
"@vue/eslint-config-typescript/recommended",
"@vue/eslint-config-prettier",
],
env: {
"vue/setup-compiler-macros": true,
},
};

28
console/.gitignore vendored Normal file
View File

@@ -0,0 +1,28 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
.DS_Store
dist
dist-ssr
coverage
*.local
/cypress/videos/
/cypress/screenshots/
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

6
console/env.d.ts vendored Normal file
View File

@@ -0,0 +1,6 @@
/// <reference types="vite/client" />
declare module "*.vue" {
import Vue from "vue";
export default Vue;
}

45
console/package.json Normal file
View File

@@ -0,0 +1,45 @@
{
"name": "@halo-dev/plugin-starter",
"version": "1.0.0",
"private": true,
"scripts": {
"dev": "vite build --watch",
"build": "vite build",
"preview": "vite preview --port 4173",
"test:unit": "vitest --environment jsdom",
"type-check": "vue-tsc --noEmit -p tsconfig.vitest.json --composite false",
"lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore"
},
"dependencies": {
"@halo-dev/components": "^1.5.0",
"@halo-dev/console-shared": "^2.8.0",
"axios": "^1.4.0",
"canvas-confetti": "^1.6.0",
"path-browserify": "^1.0.1",
"vue": "^3.2.41"
},
"devDependencies": {
"@iconify/json": "^2.2.18",
"@rushstack/eslint-patch": "^1.2.0",
"@types/canvas-confetti": "^1.6.0",
"@types/jsdom": "^20.0.0",
"@types/node": "^16.18.0",
"@vitejs/plugin-vue": "^3.1.2",
"@vitejs/plugin-vue-jsx": "^2.0.1",
"@vue/eslint-config-prettier": "^7.0.0",
"@vue/eslint-config-typescript": "^11.0.2",
"@vue/test-utils": "^2.2.0",
"@vue/tsconfig": "^0.1.3",
"eslint": "^8.26.0",
"eslint-plugin-vue": "^9.6.0",
"jsdom": "^19.0.0",
"npm-run-all": "^4.1.5",
"prettier": "^2.7.1",
"sass": "^1.58.0",
"typescript": "~4.7.4",
"unplugin-icons": "^0.15.2",
"vite": "^3.1.8",
"vitest": "^0.24.3",
"vue-tsc": "^1.0.9"
}
}

3576
console/pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 5.5 KiB

View File

@@ -0,0 +1 @@
export * from "./s-3-link-controller";

View File

@@ -0,0 +1,25 @@
import request from "@/utils/request";
import { S3ListResult, DeepRequired } from "../../interface";
/**
* /apis/s3os.halo.run/v1alpha1/objects/{policyName}
*/
export function getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName(params: GetApisS3OsHaloRunV1Alpha1ObjectsByPolicyNameParams) {
const paramsInput = {
continuationToken: params.continuationToken,
continuationObject: params.continuationObject,
pageSize: params.pageSize,
unlinked: params.unlinked,
};
return request.get<DeepRequired<S3ListResult>>(`/apis/s3os.halo.run/v1alpha1/objects/${params.policyName}`, {
params: paramsInput,
});
}
interface GetApisS3OsHaloRunV1Alpha1ObjectsByPolicyNameParams {
policyName: any;
continuationToken?: any;
continuationObject?: any;
pageSize: any;
unlinked?: any;
}

View File

@@ -0,0 +1,9 @@
import request from "@/utils/request";
import { DeepRequired } from "../../interface";
/**
* /apis/s3os.halo.run/v1alpha1/policies/s3
*/
export function getApisS3OsHaloRunV1Alpha1PoliciesS3() {
return request.get<DeepRequired<any>>(`/apis/s3os.halo.run/v1alpha1/policies/s3`);
}

View File

@@ -0,0 +1,3 @@
export * from "./postApisS3OsHaloRunV1Alpha1AttachmentsLink";
export * from "./getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName";
export * from "./getApisS3OsHaloRunV1Alpha1PoliciesS3";

View File

@@ -0,0 +1,9 @@
import request from "@/utils/request";
import { LinkResult, DeepRequired, LinkRequest } from "../../interface";
/**
* /apis/s3os.halo.run/v1alpha1/attachments/link
*/
export function postApisS3OsHaloRunV1Alpha1AttachmentsLink(input: LinkRequest) {
return request.post<DeepRequired<LinkResult>>(`/apis/s3os.halo.run/v1alpha1/attachments/link`, input);
}

21
console/src/index.ts Normal file
View File

@@ -0,0 +1,21 @@
import {definePlugin} from "@halo-dev/console-shared";
import type {PluginTab} from "@halo-dev/console-shared";
import HomeView from "./views/HomeView.vue";
import {markRaw} from "vue";
export default definePlugin({
components: {},
routes: [],
extensionPoints: {
"plugin:self:tabs:create": () : PluginTab[] => {
return [
{
id: "s3-link",
label: "关联S3文件",
component: markRaw(HomeView),
permissions: []
},
];
},
},
});

View File

@@ -0,0 +1,4 @@
export interface LinkRequest {
objectKeys?: string[];
policyName?: string;
}

View File

@@ -0,0 +1,5 @@
import { LinkResultItem } from "../../interface";
export interface LinkResult {
items?: LinkResultItem[];
}

View File

@@ -0,0 +1,5 @@
export interface LinkResultItem {
message?: string;
objectKey?: string;
success?: boolean;
}

View File

@@ -0,0 +1,20 @@
export interface Metadata {
annotations?: MetadataAnnotations;
creationTimestamp?: string;
deletionTimestamp?: string;
finalizers?: string[];
/** The name field will be generated automatically according to the given generateName field */
generateName?: string;
labels?: MetadataLabels;
/** Metadata name */
name: string;
version?: number;
}
export interface MetadataAnnotations {
[key: string]: any;
}
export interface MetadataLabels {
[key: string]: any;
}

View File

@@ -0,0 +1,6 @@
export interface ObjectVo {
displayName?: string;
isLinked?: boolean;
key?: string;
lastModified?: string;
}

View File

@@ -0,0 +1,8 @@
import { Metadata, PolicySpec } from "../../interface";
export interface Policy {
apiVersion: string;
kind: string;
metadata: Metadata;
spec: PolicySpec;
}

View File

@@ -0,0 +1,8 @@
export interface PolicySpec {
/** Reference name of ConfigMap extension */
configMapName?: string;
/** Display name of policy */
displayName: string;
/** Reference name of PolicyTemplate */
templateName: string;
}

View File

@@ -0,0 +1,10 @@
import { ObjectVo } from "../../interface";
export interface S3ListResult {
currentContinuationObject?: string;
currentToken?: string;
hasMore?: boolean;
nextContinuationObject?: string;
nextToken?: string;
objects?: ObjectVo[];
}

View File

@@ -0,0 +1,11 @@
export * from "./apiTypes/LinkRequest";
export * from "./apiTypes/LinkResult";
export * from "./apiTypes/LinkResultItem";
export * from "./apiTypes/Metadata";
export * from "./apiTypes/ObjectVo";
export * from "./apiTypes/Policy";
export * from "./apiTypes/PolicySpec";
export * from "./apiTypes/S3ListResult";
export type Primitive = undefined | null | boolean | string | number | symbol;
export type DeepRequired<T> = T extends Primitive ? T : keyof T extends never ? T : { [K in keyof T]-?: DeepRequired<T[K]> };

View File

@@ -0,0 +1,7 @@
{
"docsUrl": "http://localhost:8090/v3/api-docs/extension-api",
"includeTags": ["s-3-link-controller"],
"excludeTags": [],
"axiosInstanceUrl": "@/utils/request",
"prefix": ""
}

View File

@@ -0,0 +1,31 @@
import axios from "axios";
import {Toast} from "@halo-dev/components";
const baseURL = import.meta.env.VITE_API_URL;
const request = axios.create({
baseURL,
withCredentials: true,
});
// 非200状态码就弹窗
request.interceptors.response.use(
(response) => {
return response;
},
async (error) => {
const errorResponse = error.response;
if (!errorResponse) {
return Promise.reject(error);
}
const { status } = errorResponse;
if (status !== 200) {
Toast.error("status: " + status);
}
return Promise.reject(error);
}
);
request.defaults.headers.common["X-Requested-With"] = "XMLHttpRequest";
// TODO 使用halo console 中的axios https://github.com/halo-dev/halo/issues/3979
export default request;

View File

@@ -0,0 +1,457 @@
<script setup lang="ts">
import {
IconRefreshLine,
Toast,
VButton,
VCard,
VEmpty,
VEntity,
VEntityField,
VLoading,
VModal,
VPageHeader,
VSpace,
VTag,
} from "@halo-dev/components";
import CarbonFolderDetailsReference from "~icons/carbon/folder-details-reference";
import {computed, onMounted, ref, watch} from "vue";
import {
getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName,
getApisS3OsHaloRunV1Alpha1PoliciesS3,
postApisS3OsHaloRunV1Alpha1AttachmentsLink,
} from "@/controller";
import type {ObjectVo, S3ListResult, Policy, LinkResultItem} from "@/interface";
const selectedFiles = ref<string[]>([]);
const policyName = ref<string>("");
const page = ref(1);
const size = ref(50);
const policyOptions = ref<{ label: string; value: string; attrs: any }[]>([{
label: "请选择策略",
value: "",
attrs: {disabled: true}
}]);
const s3Objects = ref<S3ListResult>({
objects: [],
hasMore: false,
currentToken: "",
nextToken: "",
currentContinuationObject: "",
nextContinuationObject: "",
});
// view state
const isFetching = ref(false);
const isShowModal = ref(false);
const isLinking = ref(false);
const isFetchingPolicies = ref(true);
const linkTips = ref("");
const linkFailedTable = ref<LinkResultItem[]>([]);
const linkedStatusItems: { label: string; value?: boolean }[] = [
{label: "全部"},
{label: "未关联", value: true},
];
// action state
const checkedAll = ref(false);
const selectedLinkedStatusItem = ref<boolean | undefined>(linkedStatusItems[0].value);
const emptyTips = computed(() => {
if (isFetchingPolicies.value) {
return "正在加载策略";
} else {
if (policyOptions.value.length <= 1) {
return "没有可用的策略请前往【附件】添加S3策略";
} else {
if (!policyName.value) {
return "请在左上方选择策略";
} else {
return "该策略的 桶/文件夹 下没有文件";
}
}
}
});
const handleCheckAllChange = (e: Event) => {
const {checked} = e.target as HTMLInputElement;
if (checked) {
selectedFiles.value =
s3Objects.value.objects?.filter(file => !file.isLinked).map((file) => {
return file.key || "";
}) || [];
} else {
selectedFiles.value.length = 0;
checkedAll.value = false;
}
};
const fetchPolicies = async () => {
try {
const policiesData = await getApisS3OsHaloRunV1Alpha1PoliciesS3();
if (policiesData.status == 200) {
policyOptions.value = [{
label: "请选择策略",
value: "",
attrs: {disabled: true}
}];
policiesData.data.forEach((policy: Policy) => {
policyOptions.value.push({
label: policy.spec.displayName,
value: policy.metadata.name,
attrs: {}
});
});
}
} catch (error) {
console.error(error);
}
isFetchingPolicies.value = false;
};
onMounted(() => {
fetchPolicies();
});
watch(selectedFiles, (newValue) => {
checkedAll.value = s3Objects.value.objects?.filter(file => !file.isLinked)
.filter(file => !newValue.includes(file.key || "")).length == 0
&& s3Objects.value.objects?.length != 0;
});
watch(selectedLinkedStatusItem, () => {
handleFirstPage();
});
const changeNextTokenAndObject = () => {
s3Objects.value.currentToken = s3Objects.value.nextToken;
s3Objects.value.currentContinuationObject = s3Objects.value.nextContinuationObject;
s3Objects.value.nextToken = "";
s3Objects.value.nextContinuationObject = "";
};
const clearTokenAndObject = () => {
s3Objects.value.currentToken = "";
s3Objects.value.currentContinuationObject = "";
s3Objects.value.nextToken = "";
s3Objects.value.nextContinuationObject = "";
};
const fetchObjects = async () => {
if (!policyName.value) {
return;
}
isFetching.value = true;
s3Objects.value.objects = [];
try {
const objectsData = await getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName({
policyName: policyName.value,
pageSize: size.value,
continuationToken: s3Objects.value.currentToken,
continuationObject: s3Objects.value.currentContinuationObject,
unlinked: selectedLinkedStatusItem.value,
});
if (objectsData.status == 200) {
s3Objects.value = objectsData.data;
if (s3Objects.value.objects?.length == 0 && s3Objects.value.hasMore && s3Objects.value.nextToken) {
changeNextTokenAndObject();
await fetchObjects();
} else if (s3Objects.value.objects?.length == 0 && !s3Objects.value.hasMore && page.value > 1) {
page.value = 1;
clearTokenAndObject();
await fetchObjects();
Toast.warning("最后一页为空,已返回第一页");
}
}
} catch (error) {
console.error(error);
}
selectedFiles.value.length = 0;
checkedAll.value = false;
isFetching.value = false;
};
const checkSelection = (file: ObjectVo) => {
return selectedFiles.value.includes(file.key || "");
};
const handleLink = async () => {
isLinking.value = true;
isShowModal.value = true;
linkTips.value = `正在关联${selectedFiles.value.length}个文件`;
linkFailedTable.value = [];
const linkResult = await postApisS3OsHaloRunV1Alpha1AttachmentsLink({
policyName: policyName.value,
objectKeys: selectedFiles.value
});
const successCount = linkResult.data.items.filter(item => item.success).length;
const failedCount = linkResult.data.items.filter(item => !item.success).length;
linkTips.value = `关联成功${successCount}个文件,关联失败${failedCount}个文件`;
if (failedCount > 0) {
linkFailedTable.value = linkResult.data.items.filter(item => !item.success);
}
isLinking.value = false;
};
const selectOneAndLink = (file: ObjectVo) => {
selectedFiles.value = [file.key || ""];
handleLink();
};
const handleNextPage = () => {
if (!policyName.value) {
return;
}
if (s3Objects.value.hasMore) {
isFetching.value = true;
page.value += 1;
changeNextTokenAndObject();
fetchObjects();
}
};
const handleFirstPage = () => {
if (!policyName.value) {
return;
}
isFetching.value = true;
page.value = 1;
clearTokenAndObject();
fetchObjects();
};
const handleModalClose = () => {
isShowModal.value = false;
fetchObjects();
};
</script>
<template>
<VPageHeader title="关联S3文件(Beta)">
<template #icon>
<CarbonFolderDetailsReference class="mr-2 self-center"/>
</template>
</VPageHeader>
<div class="m-0 md:m-4">
<VCard :body-class="['!p-0']">
<template #header>
<div class="block w-full bg-gray-50 px-4 py-3">
<div
class="relative flex flex-col items-start sm:flex-row sm:items-center"
>
<div
v-permission="['system:users:aaa']"
class="mr-4 hidden items-center sm:flex"
>
<input
v-model="checkedAll"
class="h-4 w-4 rounded border-gray-300 text-indigo-600"
type="checkbox"
@change="handleCheckAllChange"
/>
</div>
<div class="flex w-full flex-1 items-center sm:w-auto">
<div
v-if="!selectedFiles.length"
class="flex items-center gap-2"
>
策略:
<FormKit
id="policyChoose"
outer-class="!p-0 w-48"
v-model="policyName"
name="policyName"
type="select"
:options="policyOptions"
@change="fetchObjects()"
></FormKit>
</div>
<VSpace v-else>
<VButton type="primary" @click="handleLink">
关联
</VButton>
</VSpace>
</div>
<div class="mt-4 flex sm:mt-0">
<VSpace spacing="lg">
<FilterCleanButton
v-if="selectedLinkedStatusItem != linkedStatusItems[0].value"
@click="selectedLinkedStatusItem = linkedStatusItems[0].value"
/>
<FilterDropdown
v-model="selectedLinkedStatusItem"
:label="$t('core.common.filters.labels.status')"
:items="linkedStatusItems"
/>
<div class="flex flex-row gap-2">
<div
class="group cursor-pointer rounded p-1 hover:bg-gray-200"
@click="fetchObjects()"
>
<IconRefreshLine
v-tooltip="$t('core.common.buttons.refresh')"
:class="{
'animate-spin text-gray-900': isFetching,
}"
class="h-4 w-4 text-gray-600 group-hover:text-gray-900"
/>
</div>
</div>
</VSpace>
</div>
</div>
</div>
</template>
<VLoading v-if="isFetching"/>
<Transition v-else-if="!s3Objects.objects?.length" appear name="fade">
<VEmpty
message="空空如也"
:title="emptyTips"
>
</VEmpty>
</Transition>
<Transition v-else appear name="fade">
<ul
class="box-border h-full w-full divide-y divide-gray-100"
role="list"
>
<li v-for="(file, index) in s3Objects.objects" :key="index">
<VEntity :is-selected="checkSelection(file)">
<template
#checkbox
>
<input
v-model="selectedFiles"
:value="file.key || ''"
class="h-4 w-4 rounded border-gray-300 text-indigo-600"
name="post-checkbox"
:disabled="file.isLinked"
type="checkbox"
/>
</template>
<template #start>
<VEntityField>
<template #description>
<AttachmentFileTypeIcon
:display-ext="false"
:file-name="file.displayName || ''"
:width="8"
:height="8"
/>
</template>
</VEntityField>
<VEntityField
:title="file.displayName || ''"
:description="file.key || ''"
/>
</template>
<template #end>
<VEntityField>
<template #description>
<VTag :theme="file.isLinked ? 'default':'primary'">
{{
file.isLinked ? '已关联' : '未关联'
}}
</VTag>
</template>
</VEntityField>
<VEntityField>
<template #description>
<VButton
:disabled="file.isLinked || false"
@click="selectOneAndLink(file)"
>
关联
</VButton>
</template>
</VEntityField>
</template>
</VEntity>
</li>
</ul>
</Transition>
<template #footer>
<div class="bg-white sm:flex sm:items-center justify-between">
<div class="inline-flex items-center gap-5">
<span class="text-xs text-gray-500 hidden md:flex"> {{ s3Objects.objects?.length }} 项数据</span>
<span class="text-xs text-gray-500 hidden md:flex">已自动过滤文件夹对象页面实际显示数量少为正常现象</span>
</div>
<div class="inline-flex items-center gap-5">
<div class="inline-flex items-center gap-2">
<VButton size="small" @click="handleFirstPage" :disabled="!policyName">返回第一页</VButton>
<span class="text-sm text-gray-500"> {{ page }} </span>
<VButton size="small" @click="handleNextPage" :disabled="!s3Objects.hasMore || isFetching || !policyName">
下一页
</VButton>
</div>
<div class="inline-flex items-center gap-2">
<select
v-model="size"
class="h-8 border outline-none rounded-base px-2 text-gray-800 text-sm border-gray-300"
@change="handleFirstPage"
>
<option
v-for="(sizeOption, index) in [20, 50, 100, 200]"
:key="index"
:value="sizeOption"
>
{{ sizeOption }}
</option>
</select>
<span class="text-sm text-gray-500">/</span>
</div>
</div>
</div>
</template>
</VCard>
</div>
<VModal
:visible="isShowModal"
:fullscreen="false"
:title="'关联结果'"
:width="500"
:mount-to-body="true"
@close="handleModalClose"
>
<template #footer>
<VSpace>
<VButton
:loading="isLinking"
type="primary"
@click="handleModalClose"
>
确定
</VButton>
</VSpace>
</template>
<div class="flex flex-col">
{{ linkTips }}
<table v-if="linkFailedTable.length != 0">
<tr>
<th class="border border-black font-normal">失败对象</th>
<th class="border border-black font-normal">失败原因</th>
</tr>
<tr v-for="failedInfo in linkFailedTable" :key="failedInfo.objectKey">
<th class="border border-black font-normal">{{failedInfo.objectKey}}</th>
<th class="border border-black font-normal">{{failedInfo.message}}</th>
</tr>
</table>
</div>
</VModal>
</template>
<style lang="scss" scoped>
</style>

13
console/tsconfig.app.json Normal file
View File

@@ -0,0 +1,13 @@
{
"extends": "@vue/tsconfig/tsconfig.web.json",
"include": ["./env.d.ts", "./src/**/*", "./src/**/*.vue"],
"exclude": ["./src/**/__tests__/*"],
"compilerOptions": {
"composite": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
},
"types": ["unplugin-icons/types/vue"]
}
}

View File

@@ -0,0 +1,8 @@
{
"extends": "@vue/tsconfig/tsconfig.node.json",
"include": ["vite.config.*", "vitest.config.*", "cypress.config.*"],
"compilerOptions": {
"composite": true,
"types": ["node"]
}
}

14
console/tsconfig.json Normal file
View File

@@ -0,0 +1,14 @@
{
"files": [],
"references": [
{
"path": "./tsconfig.config.json"
},
{
"path": "./tsconfig.app.json"
},
{
"path": "./tsconfig.vitest.json"
}
]
}

View File

@@ -0,0 +1,9 @@
{
"extends": "./tsconfig.app.json",
"exclude": [],
"compilerOptions": {
"composite": true,
"lib": [],
"types": ["node", "jsdom"]
}
}

47
console/vite.config.ts Normal file
View File

@@ -0,0 +1,47 @@
import { fileURLToPath, URL } from "url";
import { defineConfig } from "vite";
import Vue from "@vitejs/plugin-vue";
import VueJsx from "@vitejs/plugin-vue-jsx";
import Icons from "unplugin-icons/vite";
const pluginEntryName = "PluginS3ObjectStorage";
// https://vitejs.dev/config/
export default defineConfig({
plugins: [Vue(), VueJsx(), Icons({ compiler: "vue3" })],
resolve: {
alias: {
"@": fileURLToPath(new URL("./src", import.meta.url)),
},
},
build: {
outDir: fileURLToPath(
new URL("../src/main/resources/console", import.meta.url)
),
emptyOutDir: true,
lib: {
entry: "src/index.ts",
name: pluginEntryName,
formats: ["iife"],
fileName: () => "main.js",
},
rollupOptions: {
external: [
"vue",
"@halo-dev/console-shared",
"@halo-dev/components",
"vue-router",
],
output: {
globals: {
vue: "Vue",
"vue-router": "VueRouter",
"@halo-dev/components": "HaloComponents",
"@halo-dev/console-shared": "HaloConsoleShared",
},
extend: true,
},
},
},
});

View File

@@ -1 +1 @@
version=1.4.1-SNAPSHOT
version=1.5.0-SNAPSHOT

View File

@@ -91,4 +91,18 @@ public final class FileNameUtils {
return random + "." + extension;
}
}
/**
* Extracts the file name from an Amazon S3 object key.
*
* @param objectKey The Amazon S3 object key from which to extract the file name.
* @return The extracted file name.
*/
public static String extractFileNameFromS3Key(String objectKey) {
int lastSlashIndex = objectKey.lastIndexOf("/");
if (lastSlashIndex >= 0 && lastSlashIndex < objectKey.length() - 1) {
return objectKey.substring(lastSlashIndex + 1);
}
return objectKey;
}
}

View File

@@ -0,0 +1,13 @@
package run.halo.s3os;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import java.util.List;
@Data
@RequiredArgsConstructor
public class LinkRequest {
private String policyName;
private List<String> objectKeys;
}

View File

@@ -0,0 +1,24 @@
package run.halo.s3os;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
@Data
@AllArgsConstructor
public class LinkResult {
private List<LinkResultItem> items;
@Data
@AllArgsConstructor
public static class LinkResultItem {
private String objectKey;
private Boolean success;
private String message;
}
}

View File

@@ -0,0 +1,85 @@
package run.halo.s3os;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.web.bind.annotation.*;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.core.extension.attachment.Attachment;
import run.halo.app.core.extension.attachment.Policy;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.plugin.ApiVersion;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@ApiVersion("s3os.halo.run/v1alpha1")
@RestController
@RequiredArgsConstructor
public class S3LinkController {
private final S3LinkService s3LinkService;
private final ReactiveExtensionClient client;
/**
* Map of linking file, used as a lock, key is policyName/objectKey, value is policyName/objectKey.
*/
private final Map<String, Object> linkingFile = new ConcurrentHashMap<>();
@GetMapping("/policies/s3")
public Flux<Policy> listS3Policies() {
return s3LinkService.listS3Policies();
}
@GetMapping("/objects/{policyName}")
public Mono<S3ListResult> listObjects(@PathVariable String policyName,
@RequestParam(name = "continuationToken", required = false) String continuationToken,
@RequestParam(name = "continuationObject", required = false) String continuationObject,
@RequestParam(name = "pageSize") Integer pageSize,
@RequestParam(name = "unlinked", required = false, defaultValue = "false")
Boolean unlinked) {
if (unlinked) {
return s3LinkService.listObjectsUnlinked(policyName, continuationToken,
continuationObject, pageSize);
} else {
return s3LinkService.listObjects(policyName, continuationToken, pageSize);
}
}
@PostMapping("/attachments/link")
public Mono<LinkResult> addAttachmentRecord(@RequestBody LinkRequest linkRequest) {
return Flux.fromIterable(linkRequest.getObjectKeys())
.filter(objectKey -> linkingFile.put(linkRequest.getPolicyName() + "/" + objectKey,
linkRequest.getPolicyName() + "/" + objectKey) == null)
.collectList()
.flatMap(operableObjectKeys -> client.list(Attachment.class,
attachment -> Objects.equals(attachment.getSpec().getPolicyName(),
linkRequest.getPolicyName())
&& StringUtils.isNotEmpty(attachment.getMetadata().getAnnotations()
.get(S3OsAttachmentHandler.OBJECT_KEY))
&& linkRequest.getObjectKeys().contains(attachment.getMetadata()
.getAnnotations().get(S3OsAttachmentHandler.OBJECT_KEY)),
null)
.collectList()
.flatMap(existingAttachments -> Flux.fromIterable(linkRequest.getObjectKeys())
.flatMap((objectKey) -> {
if (operableObjectKeys.contains(objectKey) && existingAttachments.stream()
.noneMatch(attachment -> Objects.equals(
attachment.getMetadata().getAnnotations().get(
S3OsAttachmentHandler.OBJECT_KEY), objectKey))) {
return s3LinkService
.addAttachmentRecord(linkRequest.getPolicyName(), objectKey)
.onErrorResume((throwable) -> Mono.just(
new LinkResult.LinkResultItem(objectKey, false,
throwable.getMessage())));
} else {
return Mono.just(new LinkResult.LinkResultItem(objectKey, false,
"附件库中已存在该对象"));
}
})
.doOnNext(linkResultItem -> linkingFile.remove(
linkRequest.getPolicyName() + "/" + linkResultItem.getObjectKey()))
.collectList()
.map(LinkResult::new)));
}
}

View File

@@ -0,0 +1,18 @@
package run.halo.s3os;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.core.extension.attachment.Policy;
public interface S3LinkService {
Flux<Policy> listS3Policies();
Mono<S3ListResult> listObjects(String policyName, String continuationToken,
Integer pageSize);
Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName, String objectKey);
Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
String continuationObject, Integer pageSize);
}

View File

@@ -0,0 +1,215 @@
package run.halo.s3os;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpStatus;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.ReactiveSecurityContextHolder;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.util.UriUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import run.halo.app.core.extension.attachment.Attachment;
import run.halo.app.core.extension.attachment.Constant;
import run.halo.app.core.extension.attachment.Policy;
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.infra.utils.JsonUtils;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.S3Object;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
import static run.halo.s3os.S3OsAttachmentHandler.OBJECT_KEY;
@Service
@RequiredArgsConstructor
@Slf4j
public class S3LinkServiceImpl implements S3LinkService {
private final ReactiveExtensionClient client;
private final S3OsAttachmentHandler handler;
@Override
public Flux<Policy> listS3Policies() {
return client.list(Policy.class, (policy) -> "s3os".equals(
policy.getSpec().getTemplateName()), null);
}
@Override
public Mono<S3ListResult> listObjects(String policyName, String continuationToken,
Integer pageSize) {
return client.fetch(Policy.class, policyName)
.flatMap((policy) -> {
var configMapName = policy.getSpec().getConfigMapName();
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap((configMap) -> {
var properties = handler.getProperties(configMap);
return Mono.using(() -> handler.buildS3Client(properties),
(s3Client) -> Mono.fromCallable(
() -> s3Client.listObjectsV2(ListObjectsV2Request.builder()
.bucket(properties.getBucket())
.prefix(StringUtils.isNotEmpty(properties.getLocation())
? properties.getLocation() + "/" : null)
.delimiter("/")
.maxKeys(pageSize)
.continuationToken(StringUtils.isNotEmpty(continuationToken)
? continuationToken : null)
.build())).subscribeOn(Schedulers.boundedElastic()),
S3Client::close)
.flatMap(listObjectsV2Response -> {
List<S3Object> contents = listObjectsV2Response.contents();
var objectVos = contents
.stream().map(S3ListResult.ObjectVo::fromS3Object)
.filter(objectVo -> !objectVo.getKey().endsWith("/"))
.collect(Collectors.toMap(S3ListResult.ObjectVo::getKey, o -> o));
return client.list(Attachment.class,
attachment -> policyName.equals(
attachment.getSpec().getPolicyName()), null)
.doOnNext(attachment -> {
S3ListResult.ObjectVo objectVo =
objectVos.get(attachment.getMetadata().getAnnotations()
.getOrDefault(OBJECT_KEY, ""));
if (objectVo != null) {
objectVo.setIsLinked(true);
}
})
.then()
.thenReturn(new S3ListResult(new ArrayList<>(objectVos.values()),
listObjectsV2Response.continuationToken(),
null, null,
listObjectsV2Response.nextContinuationToken(),
listObjectsV2Response.isTruncated()));
});
});
}
@Override
public Mono<S3ListResult> listObjectsUnlinked(String policyName, String continuationToken,
String continuationObject, Integer pageSize) {
// TODO 优化成查一次数据库
return Mono.defer(() -> {
List<S3ListResult.ObjectVo> s3Objects = new ArrayList<>();
AtomicBoolean continuationObjectMatched = new AtomicBoolean(false);
AtomicReference<String> currToken = new AtomicReference<>(continuationToken);
return Flux.defer(() -> Flux.just(
new TokenState(null, currToken.get() == null ? "" : currToken.get())))
.flatMap(tokenState -> listObjects(policyName, tokenState.nextToken, pageSize))
.flatMap(s3ListResult -> {
var filteredObjects = s3ListResult.getObjects();
if (!continuationObjectMatched.get()) {
// 判断s3ListResult.getObjects()里是否有continuationObject
var continuationObjectVo = s3ListResult.getObjects().stream()
.filter(objectVo -> objectVo.getKey().equals(continuationObject))
.findFirst();
if (continuationObjectVo.isPresent()) {
s3Objects.clear();
// 删除continuationObject及之前的所有对象
filteredObjects = s3ListResult.getObjects().stream()
.dropWhile(objectVo -> !objectVo.getKey()
.equals(continuationObject))
.skip(1)
.toList();
continuationObjectMatched.set(true);
}
}
filteredObjects = filteredObjects.stream()
.filter(objectVo -> !objectVo.getIsLinked())
.toList();
s3Objects.addAll(filteredObjects);
currToken.set(s3ListResult.getNextToken());
return Mono.just(new TokenState(s3ListResult.getCurrentToken(),
s3ListResult.getNextToken()));
})
.repeat()
.takeUntil(
tokenState -> tokenState.nextToken() == null || s3Objects.size() >= pageSize)
.last()
.map(tokenState -> {
var limitedObjects = s3Objects.stream().limit(pageSize).toList();
return new S3ListResult(limitedObjects, continuationToken, continuationObject,
!limitedObjects.isEmpty() ? limitedObjects.get(limitedObjects.size() - 1)
.getKey() : null, tokenState.currToken,
limitedObjects.size() == pageSize);
});
});
}
record TokenState(String currToken, String nextToken) {
}
@Override
public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
String objectKey) {
return authenticationConsumer(authentication -> client.fetch(Policy.class, policyName)
// TODO 检查是否已经存在
.flatMap((policy) -> {
var configMapName = policy.getSpec().getConfigMapName();
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap(configMap -> {
var properties = handler.getProperties(configMap);
return Mono.using(() -> handler.buildS3Client(properties),
(s3Client) -> Mono.fromCallable(
() -> s3Client.headObject(
HeadObjectRequest.builder()
.bucket(properties.getBucket())
.key(objectKey)
.build()))
.subscribeOn(Schedulers.boundedElastic()),
S3Client::close)
.map(headObjectResponse -> {
var objectDetail = new S3OsAttachmentHandler.ObjectDetail(
new S3OsAttachmentHandler.UploadState(properties,
FileNameUtils.extractFileNameFromS3Key(objectKey)),
headObjectResponse);
return handler.buildAttachment(properties, objectDetail);
})
.doOnNext(attachment -> {
var spec = attachment.getSpec();
if (spec == null) {
spec = new Attachment.AttachmentSpec();
attachment.setSpec(spec);
}
spec.setOwnerName(authentication.getName());
spec.setPolicyName(policyName);
})
.flatMap(client::create)
.thenReturn(new LinkResult.LinkResultItem(objectKey, true, null));
}))
.onErrorResume(throwable ->
Mono.just(new LinkResult.LinkResultItem(objectKey, false, throwable.getMessage())));
}
private <T> Mono<T> authenticationConsumer(Function<Authentication, Mono<T>> func) {
return ReactiveSecurityContextHolder.getContext()
.switchIfEmpty(Mono.error(() -> new ResponseStatusException(HttpStatus.UNAUTHORIZED,
"Authentication required.")))
.map(SecurityContext::getAuthentication)
.flatMap(func);
}
}

View File

@@ -0,0 +1,38 @@
package run.halo.s3os;
import lombok.AllArgsConstructor;
import lombok.Data;
import software.amazon.awssdk.services.s3.model.S3Object;
import java.time.Instant;
import java.util.List;
@Data
@AllArgsConstructor
public class S3ListResult {
private List<ObjectVo> objects;
private String currentToken;
private String currentContinuationObject;
private String nextContinuationObject;
private String nextToken;
private Boolean hasMore;
@Data
@AllArgsConstructor
public static class ObjectVo {
private String key;
private Instant lastModified;
private Boolean isLinked;
private String displayName;
public static ObjectVo fromS3Object(S3Object s3Object) {
final var key = s3Object.key();
final var displayName = key.substring(key.lastIndexOf("/") + 1);
return new ObjectVo(key, s3Object.lastModified(), false, displayName);
}
}
}

View File

@@ -64,8 +64,12 @@ import software.amazon.awssdk.utils.SdkAutoCloseable;
@Extension
public class S3OsAttachmentHandler implements AttachmentHandler {
private static final String OBJECT_KEY = "s3os.plugin.halo.run/object-key";
private static final int MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
public static final String OBJECT_KEY = "s3os.plugin.halo.run/object-key";
public static final int MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
/**
* Map to store uploading file, used as a lock, key is bucket/objectKey, value is bucket/objectKey.
*/
private final Map<String, Object> uploadingFile = new ConcurrentHashMap<>();
@Override
@@ -193,7 +197,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
return attachment;
}
private String getObjectURL(S3OsProperties properties, String objectKey) {
String getObjectURL(S3OsProperties properties, String objectKey) {
String objectURL;
if (StringUtils.isBlank(properties.getDomain())) {
String host;

File diff suppressed because one or more lines are too long

View File

@@ -5,7 +5,7 @@ metadata:
spec:
forms:
- group: basic
label: 基本设置
label: 使用提示
formSchema:
- $formkit: text
help: 请前往 “附件 - 存储策略” 添加策略

View File

@@ -4,8 +4,7 @@ metadata:
name: PluginS3ObjectStorage
spec:
enabled: true
version: 1.4.1
requires: ">=2.5.0"
requires: ">=2.9.0"
author:
name: longjuan
website: https://github.com/longjuan