S3 sdk (#6215)

* refactor: fastgpt object storage & global proxy (#6155) * feat: migrate to fastgpt storage sdk * chore: rename env variable * chore: move to sdk dir * docs: object storage * CHORE * chore: storage mocks * chore: update docker-compose * fix: global proxy agent * fix: update COS proxy * refactor: use fetch instead of http.request * fix: axios request base url * fix: axios proxy request behavior * fix: bumps axios * fix: patch axios for proxy * fix: replace axios with proxied axios * fix: upload txt file encoding * clean code * fix: use "minio" for minio adapter (#6205) * fix: use minio client to delete files when using minio vendor (#6206) * doc * feat: filter citations and add response button control (#6170) * feat: filter citations and add response button control * i18n * fix * fix test * perf: chat api code * fix: workflow edge overlap and auto-align in folded loop nodes (#6204) * fix: workflow edge overlap and auto-align in folded loop nodes * sort * fix * fix edge * fix icon * perf: s3 file name * perf: admin get app api * perf: catch user error * fix: refactor useOrg hook to use debounced search key (#6180) * chore: comment minio adapter (#6207) * chore: filename with suffix random id * perf: s3 storage code * fix: encode filename when copy object --------- Co-authored-by: archer <545436317@qq.com> * fix: node card link * json * perf: chat index; * index * chat item soft delete (#6216) * chat item soft delete * temp * fix * remove code * perf: delete chat item --------- Co-authored-by: archer <545436317@qq.com> * feat: select wheather filter sensitive info when export apps (#6222) * fix some bugs (#6210) * fix v4.14.5 bugs * type * fix * fix * custom feedback * fix * code * fix * remove invalid function --------- Co-authored-by: archer <545436317@qq.com> * perf: test * fix file default local upload (#6223) * docs: improve object storage introduction (#6224) * doc --------- Co-authored-by: roy <whoeverimf5@gmail.com> Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
2026-05-05 01:02:59 +08:00 · 2026-01-09 18:25:02 +08:00
parent 56ec86e2fe
commit c93c3937e1
281 changed files with 11562 additions and 2527 deletions
@@ -1,27 +1,68 @@
 import { vi } from 'vitest';
+import { createVitestStorageMock } from '../../../sdk/storage/src/testing/vitestMock';
+
+const mockStorageByBucket = new Map<string, ReturnType<typeof createVitestStorageMock>>();
+const getMockStorage = (bucketName: string) => {
+  const existing = mockStorageByBucket.get(bucketName);
+  if (existing) return existing;
+  const storage = createVitestStorageMock({
+    vi,
+    bucketName,
+    baseUrl: 'http://localhost:9000'
+  });
+  mockStorageByBucket.set(bucketName, storage);
+  return storage;
+};

 // Create mock S3 bucket object for global use
-const createMockS3Bucket = () => ({
-  name: 'mock-bucket',
-  client: {},
-  externalClient: {},
-  exist: vi.fn().mockResolvedValue(true),
-  delete: vi.fn().mockResolvedValue(undefined),
-  putObject: vi.fn().mockResolvedValue(undefined),
-  getFileStream: vi.fn().mockResolvedValue(null),
-  statObject: vi.fn().mockResolvedValue({ size: 0, etag: 'mock-etag' }),
-  move: vi.fn().mockResolvedValue(undefined),
-  copy: vi.fn().mockResolvedValue(undefined),
-  addDeleteJob: vi.fn().mockResolvedValue(undefined),
-  createPostPresignedUrl: vi.fn().mockResolvedValue({
-    url: 'http://localhost:9000/mock-bucket',
-    fields: { key: 'mock-key' },
-    maxSize: 100 * 1024 * 1024
-  }),
-  createExternalUrl: vi.fn().mockResolvedValue('http://localhost:9000/mock-bucket/mock-key'),
-  createGetPresignedUrl: vi.fn().mockResolvedValue('http://localhost:9000/mock-bucket/mock-key'),
-  createPublicUrl: vi.fn((key: string) => `http://localhost:9000/mock-bucket/${key}`)
-});
+const createMockS3Bucket = (bucketName = 'mock-bucket') => {
+  const client = getMockStorage(bucketName);
+  const externalClient = getMockStorage(bucketName);
+
+  return {
+    name: bucketName,
+    client,
+    externalClient,
+    exist: vi.fn().mockResolvedValue(true),
+    delete: vi.fn().mockResolvedValue(undefined),
+    putObject: vi.fn(async (key: string, body: any) => {
+      await client.uploadObject({ key, body });
+    }),
+    getFileStream: vi.fn(async (key: string) => {
+      const res = await client.downloadObject({ key });
+      return res.body;
+    }),
+    statObject: vi.fn(async (key: string) => {
+      const meta = await client.getObjectMetadata({ key });
+      return {
+        size: meta.contentLength ?? 0,
+        etag: meta.etag ?? 'mock-etag'
+      };
+    }),
+    move: vi.fn(async ({ from, to }: { from: string; to: string }) => {
+      await client.copyObjectInSelfBucket({ sourceKey: from, targetKey: to });
+      await client.deleteObject({ key: from });
+    }),
+    copy: vi.fn(async ({ from, to }: { from: string; to: string }) => {
+      await client.copyObjectInSelfBucket({ sourceKey: from, targetKey: to });
+    }),
+    addDeleteJob: vi.fn().mockResolvedValue(undefined),
+    createPostPresignedUrl: vi.fn().mockResolvedValue({
+      url: 'http://localhost:9000/mock-bucket',
+      fields: { key: 'mock-key' },
+      maxSize: 100 * 1024 * 1024
+    }),
+    createExternalUrl: vi.fn(async (key: string) => {
+      const { url } = await externalClient.generatePresignedGetUrl({ key });
+      return url;
+    }),
+    createGetPresignedUrl: vi.fn(async (key: string) => {
+      const { url } = await client.generatePresignedGetUrl({ key });
+      return url;
+    }),
+    createPublicUrl: vi.fn((key: string) => externalClient.generatePublicGetUrl({ key }).url)
+  };
+};

 // Initialize global s3BucketMap early to prevent any real S3 connections
 const mockBucket = createMockS3Bucket();
@@ -68,27 +109,36 @@ const createMockBucketClass = (defaultName: string) => {
  return class MockS3Bucket {
    public name: string;
    public options: any;
-    public client = {};
-    public externalClient = {};
+    public client = getMockStorage(defaultName);
+    public externalClient = getMockStorage(defaultName);

    constructor(bucket?: string, options?: any) {
      this.name = bucket || defaultName;
      this.options = options || {};
+      this.client = getMockStorage(this.name);
+      this.externalClient = getMockStorage(this.name);
    }

    async exist() {
      return true;
    }
    async delete() {}
-    async putObject() {}
+    async putObject(key: string, body: any) {
+      await this.client.uploadObject({ key, body });
+    }
    async getFileStream() {
      return null;
    }
    async statObject() {
      return { size: 0, etag: 'mock-etag' };
    }
-    async move() {}
-    async copy() {}
+    async move({ from, to }: { from: string; to: string }) {
+      await this.client.copyObjectInSelfBucket({ sourceKey: from, targetKey: to });
+      await this.client.deleteObject({ key: from });
+    }
+    async copy({ from, to }: { from: string; to: string }) {
+      await this.client.copyObjectInSelfBucket({ sourceKey: from, targetKey: to });
+    }
    async addDeleteJob() {}
    async createPostPresignedUrl(params: any, options?: any) {
      return {
@@ -98,13 +148,21 @@ const createMockBucketClass = (defaultName: string) => {
      };
    }
    async createExternalUrl(params: any) {
-      return `http://localhost:9000/mock-bucket/${params.key}`;
+      const { url } = await this.externalClient.generatePresignedGetUrl({
+        key: params.key,
+        expiredSeconds: params.expires
+      });
+      return url;
    }
    async createGetPresignedUrl(params: any) {
-      return `http://localhost:9000/mock-bucket/${params.key}`;
+      const { url } = await this.client.generatePresignedGetUrl({
+        key: params.key,
+        expiredSeconds: params.expires
+      });
+      return url;
    }
    createPublicUrl(objectKey: string) {
-      return `http://localhost:9000/mock-bucket/${objectKey}`;
+      return this.externalClient.generatePublicGetUrl({ key: objectKey }).url;
    }
  };
 };