chore: add extra sub path for custom S3 (#6339)

* fix: storage sdk * chore: add extra sub path for custom S3
2026-02-27 01:02:22 +08:00 · 2026-01-30 11:56:53 +08:00
parent cdb896ebf9
commit 6213f0001d
12 changed files with 133 additions and 36 deletions
--- a/packages/service/common/s3/buckets/private.ts
+++ b/packages/service/common/s3/buckets/private.ts
@@ -20,9 +20,10 @@ export class S3PrivateBucket extends S3BaseBucket {
          region,
          vendor,
          credentials,
-          forcePathStyle: true,
          endpoint: options.endpoint!,
-          maxRetries: options.maxRetries!
+          maxRetries: options.maxRetries!,
+          forcePathStyle: options.forcePathStyle,
+          publicAccessExtraSubPath: options.publicAccessExtraSubPath
        } as Omit<IAwsS3CompatibleStorageOptions, 'bucket'>;
        return {
          config,
@@ -38,7 +39,8 @@ export class S3PrivateBucket extends S3BaseBucket {
          credentials,
          endpoint: options.endpoint!,
          maxRetries: options.maxRetries!,
-          forcePathStyle: options.forcePathStyle
+          forcePathStyle: options.forcePathStyle,
+          publicAccessExtraSubPath: options.publicAccessExtraSubPath
        } as Omit<IAwsS3CompatibleStorageOptions, 'bucket'>;
        return {
          config,
--- a/packages/service/common/s3/buckets/public.ts
+++ b/packages/service/common/s3/buckets/public.ts
@@ -21,9 +21,10 @@ export class S3PublicBucket extends S3BaseBucket {
          region,
          vendor,
          credentials,
-          forcePathStyle: true,
          endpoint: options.endpoint!,
-          maxRetries: options.maxRetries!
+          maxRetries: options.maxRetries!,
+          forcePathStyle: options.forcePathStyle,
+          publicAccessExtraSubPath: options.publicAccessExtraSubPath
        } as Omit<IAwsS3CompatibleStorageOptions, 'bucket'>;
        return {
          config,
@@ -39,7 +40,8 @@ export class S3PublicBucket extends S3BaseBucket {
          credentials,
          endpoint: options.endpoint!,
          maxRetries: options.maxRetries!,
-          forcePathStyle: options.forcePathStyle
+          forcePathStyle: options.forcePathStyle,
+          publicAccessExtraSubPath: options.publicAccessExtraSubPath
        } as Omit<IAwsS3CompatibleStorageOptions, 'bucket'>;
        return {
          config,
--- a/packages/service/common/s3/constants.ts
+++ b/packages/service/common/s3/constants.ts
@@ -47,7 +47,7 @@ export function createDefaultStorageOptions() {
    case 'minio': {
      return {
        vendor: 'minio',
-        forcePathStyle: true,
+        forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : false,
        externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
        endpoint: process.env.STORAGE_S3_ENDPOINT || 'http://localhost:9000',
        region: process.env.STORAGE_REGION || 'us-east-1',
@@ -59,7 +59,8 @@ export function createDefaultStorageOptions() {
        },
        maxRetries: process.env.STORAGE_S3_MAX_RETRIES
          ? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
-          : 3
+          : 3,
+        publicAccessExtraSubPath: process.env.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH || undefined
      } satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
        publicBucket: string;
        privateBucket: string;
@@ -82,7 +83,8 @@ export function createDefaultStorageOptions() {
        },
        maxRetries: process.env.STORAGE_S3_MAX_RETRIES
          ? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
-          : 3
+          : 3,
+        publicAccessExtraSubPath: process.env.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH || undefined
      } satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
        publicBucket: string;
        privateBucket: string;
--- a/packages/service/package.json
+++ b/packages/service/package.json
@@ -3,7 +3,7 @@
  "version": "1.0.0",
  "type": "module",
  "dependencies": {
-    "@fastgpt-sdk/storage": "0.6.6",
+    "@fastgpt-sdk/storage": "^0.6.15",
    "@fastgpt/global": "workspace:*",
    "@maxmind/geoip2-node": "^6.3.4",
    "@modelcontextprotocol/sdk": "^1.25.2",
--- a/packages/web/common/file/utils.ts
+++ b/packages/web/common/file/utils.ts
@@ -141,6 +141,7 @@ export const putFileToS3 = async ({
  maxSize?: number;
  t: any;
 }) => {
+  console.log(headers);
  try {
    const res = await axios.put(url, file, {
      headers: {
@@ -153,6 +154,6 @@ export const putFileToS3 = async ({
      onSuccess?.();
    }
  } catch (error) {
-    Promise.reject(parseS3UploadError({ t, error, maxSize }));
+    return Promise.reject(parseS3UploadError({ t, error, maxSize }));
  }
 };
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -133,8 +133,8 @@ importers:
  packages/service:
    dependencies:
      '@fastgpt-sdk/storage':
-        specifier: 0.6.6
-        version: 0.6.6(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)
+        specifier: ^0.6.15
+        version: 0.6.15(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)
      '@fastgpt/global':
        specifier: workspace:*
        version: link:../global
@@ -537,8 +537,8 @@ importers:
        specifier: 11.11.0
        version: 11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1)
      '@fastgpt-sdk/storage':
-        specifier: 0.6.6
-        version: 0.6.6(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)
+        specifier: ^0.6.15
+        version: 0.6.15(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)
      '@fastgpt/global':
        specifier: workspace:*
        version: link:../../packages/global
@@ -2608,8 +2608,8 @@ packages:
  '@fastgpt-sdk/plugin@0.2.17':
    resolution: {integrity: sha512-TU93FD9JIeAV+isoLVVbW+yX14J27Kgd5Sn8LPvYWkrorUEtWeVfd8rOzh/KXPd43hCgD3bSDZ1W3hC06Spnog==}

-  '@fastgpt-sdk/storage@0.6.6':
-    resolution: {integrity: sha512-nG+DWwyJS6upfEbJ+FgTc8CGhzpW1JiPEWGO1luhm6lgYRitm0HBBeX3m0j17lYTa2S0ZTDRGJgnXYzOWpYOLg==}
+  '@fastgpt-sdk/storage@0.6.15':
+    resolution: {integrity: sha512-oPbm6EtXQ3ysad/OebF2ovwbIax6PeCvYqA3cGAVEHEJMBU3633ktl1ZaIIkmyjWJLsABZpMf6m7lPBMyISGrA==}
    engines: {node: '>=20'}

  '@fastify/accept-negotiator@1.1.0':
@@ -14724,7 +14724,7 @@ snapshots:
    transitivePeerDependencies:
      - '@types/node'

-  '@fastgpt-sdk/storage@0.6.6(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)':
+  '@fastgpt-sdk/storage@0.6.15(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)':
    dependencies:
      '@aws-sdk/client-s3': 3.948.0
      '@aws-sdk/lib-storage': 3.948.0(@aws-sdk/client-s3@3.948.0)
@@ -14759,7 +14759,7 @@ snapshots:
      - tsx
      - yaml

-  '@fastgpt-sdk/storage@0.6.6(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)':
+  '@fastgpt-sdk/storage@0.6.15(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)':
    dependencies:
      '@aws-sdk/client-s3': 3.948.0
      '@aws-sdk/lib-storage': 3.948.0(@aws-sdk/client-s3@3.948.0)
@@ -20136,8 +20136,8 @@ snapshots:
      '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2)
      eslint: 8.57.1
      eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.57.1)
-      eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1)
+      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1)
+      eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1)
      eslint-plugin-jsx-a11y: 6.10.2(eslint@8.57.1)
      eslint-plugin-react: 7.37.4(eslint@8.57.1)
      eslint-plugin-react-hooks: 5.0.0-canary-7118f5dd7-20230705(eslint@8.57.1)
@@ -20156,6 +20156,21 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

+  eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1):
+    dependencies:
+      '@nolyfill/is-core-module': 1.0.39
+      debug: 4.4.0
+      eslint: 8.57.1
+      get-tsconfig: 4.10.0
+      is-bun-module: 1.3.0
+      oxc-resolver: 5.0.0
+      stable-hash: 0.0.5
+      tinyglobby: 0.2.12
+    optionalDependencies:
+      eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1)
+    transitivePeerDependencies:
+      - supports-color
+
  eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0):
    dependencies:
      '@nolyfill/is-core-module': 1.0.39
@@ -20197,6 +20212,17 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

+  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1):
+    dependencies:
+      debug: 3.2.7
+    optionalDependencies:
+      '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2)
+      eslint: 8.57.1
+      eslint-import-resolver-node: 0.3.9
+      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1)
+    transitivePeerDependencies:
+      - supports-color
+
  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1):
    dependencies:
      debug: 3.2.7
@@ -20237,6 +20263,35 @@ snapshots:
      - eslint-import-resolver-webpack
      - supports-color

+  eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1):
+    dependencies:
+      '@rtsao/scc': 1.1.0
+      array-includes: 3.1.8
+      array.prototype.findlastindex: 1.2.6
+      array.prototype.flat: 1.3.3
+      array.prototype.flatmap: 1.3.3
+      debug: 3.2.7
+      doctrine: 2.1.0
+      eslint: 8.57.1
+      eslint-import-resolver-node: 0.3.9
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1)
+      hasown: 2.0.2
+      is-core-module: 2.16.1
+      is-glob: 4.0.3
+      minimatch: 3.1.2
+      object.fromentries: 2.0.8
+      object.groupby: 1.0.3
+      object.values: 1.2.1
+      semver: 6.3.1
+      string.prototype.trimend: 1.0.9
+      tsconfig-paths: 3.15.0
+    optionalDependencies:
+      '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2)
+    transitivePeerDependencies:
+      - eslint-import-resolver-typescript
+      - eslint-import-resolver-webpack
+      - supports-color
+
  eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1):
    dependencies:
      '@rtsao/scc': 1.1.0
--- a/projects/app/.env.template
+++ b/projects/app/.env.template
@@ -46,6 +46,7 @@ STORAGE_EXTERNAL_ENDPOINT=
 STORAGE_S3_ENDPOINT=http://localhost:9000
 STORAGE_S3_FORCE_PATH_STYLE=true
 STORAGE_S3_MAX_RETRIES=3
+STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH=

 # Redis URL
 REDIS_URL=redis://default:mypassword@localhost:6379
--- a/projects/app/package.json
+++ b/projects/app/package.json
@@ -20,7 +20,7 @@
    "@dagrejs/dagre": "^1.1.4",
    "@emotion/react": "11.11.1",
    "@emotion/styled": "11.11.0",
-    "@fastgpt-sdk/storage": "0.6.6",
+    "@fastgpt-sdk/storage": "^0.6.15",
    "@fastgpt/global": "workspace:*",
    "@fastgpt/service": "workspace:*",
    "@fastgpt/web": "workspace:*",
--- a/sdk/storage/package.json
+++ b/sdk/storage/package.json
@@ -1,7 +1,7 @@
 {
  "name": "@fastgpt-sdk/storage",
  "private": false,
-  "version": "0.6.6",
+  "version": "0.6.15",
  "type": "module",
  "main": "./dist/index.js",
  "types": "./dist/index.d.ts",
--- a/sdk/storage/src/adapters/aws-s3.adapter.ts
+++ b/sdk/storage/src/adapters/aws-s3.adapter.ts
@@ -42,7 +42,7 @@ import type {
 import { Upload } from '@aws-sdk/lib-storage';
 import { EmptyObjectError } from '../errors';
 import type { Readable } from 'node:stream';
-import { camelCase, chunk, isNotNil, kebabCase } from 'es-toolkit';
+import { camelCase, chunk, isNotNil, kebabCase, trim } from 'es-toolkit';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants';

@@ -299,29 +299,42 @@ export class AwsS3StorageAdapter implements IStorage {
      }
    }

-    if (contentType) {
-      meta['Content-Type'] = contentType;
-    }
+    const convertToS3Headers = (meta: Record<string, string>) => {
+      return Object.keys(meta)
+        .filter((key) => key !== 'Content-Type')
+        .map((key) => `x-amz-meta-${key}`);
+    };

    const url = await getSignedUrl(
      this.client,
      new PutObjectCommand({
        Bucket: this.options.bucket,
        Key: key,
-        Metadata: meta
+        Metadata: meta,
+        ContentType: contentType
      }),
      {
-        expiresIn
+        expiresIn,
+        unhoistableHeaders: new Set(convertToS3Headers(meta))
      }
    );

+    const headers: Record<string, string> = {};
+    for (const [key, value] of Object.entries(meta)) {
+      if (key.toLowerCase() === 'content-type') {
+        continue;
+      }
+      headers[`x-amz-meta-${key}`] = value;
+    }
+    if (contentType) {
+      headers['Content-Type'] = contentType;
+    }
+
    return {
      key,
      url: url,
      bucket: this.options.bucket,
-      metadata: {
-        'Content-Type': meta['Content-Type']
-      }
+      metadata: headers
    };
  }

@@ -353,10 +366,21 @@ export class AwsS3StorageAdapter implements IStorage {

    let url: string;
    if (this.options.forcePathStyle) {
-      url = `${this.options.endpoint}/${this.options.bucket}/${key}`;
+      if (this.options.publicAccessExtraSubPath) {
+        url = `${this.options.endpoint}/${trim(this.options.publicAccessExtraSubPath, '/')}/${this.options.bucket}/${key}`;
+      } else {
+        url = `${this.options.endpoint}/${this.options.bucket}/${key}`;
+      }
    } else {
      const endpoint = new URL(this.options.endpoint);
-      url = `${endpoint.protocol}//${this.options.bucket}.${endpoint.host}/${key}`;
+      const protocol = endpoint.protocol;
+      const host = endpoint.host;
+
+      if (this.options.publicAccessExtraSubPath) {
+        url = `${protocol}//${this.options.bucket}.${host}/${trim(this.options.publicAccessExtraSubPath, '/')}/${key}`;
+      } else {
+        url = `${protocol}//${this.options.bucket}.${host}/${key}`;
+      }
    }

    return {
--- a/sdk/storage/src/adapters/minio.adapter.ts
+++ b/sdk/storage/src/adapters/minio.adapter.ts
@@ -44,7 +44,9 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
      throw new Error('Invalid storage vendor: expected "minio"');
    }

-    options.forcePathStyle = true;
+    // NOTE:
+    // Maybe some self-hosted MinIO services don't support path style access,
+    // options.forcePathStyle = true;
    super(options);

    // 解析 endpoint URL
@@ -59,7 +61,7 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
      accessKey: options.credentials.accessKeyId,
      secretKey: options.credentials.secretAccessKey,
      region: options.region,
-      pathStyle: true // MinIO 强制使用 path style
+      pathStyle: options.forcePathStyle
    });
  }

--- a/sdk/storage/src/interface.ts
+++ b/sdk/storage/src/interface.ts
@@ -68,6 +68,14 @@ export interface ICommonStorageOptions {
    /** SecretAccessKey / SecretKey / SK */
    secretAccessKey: string;
  };
+
+  /**
+   * 公共访问时额外添加的子路径，可选。
+   *
+   * 说明：
+   * - 用于在公共访问时添加额外的前缀，例如 `/sub-path`。
+   */
+  publicAccessExtraSubPath?: string;
 }

 /**