diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts index e210e485b7..b3e5450427 100644 --- a/packages/service/common/s3/buckets/private.ts +++ b/packages/service/common/s3/buckets/private.ts @@ -20,9 +20,10 @@ export class S3PrivateBucket extends S3BaseBucket { region, vendor, credentials, - forcePathStyle: true, endpoint: options.endpoint!, - maxRetries: options.maxRetries! + maxRetries: options.maxRetries!, + forcePathStyle: options.forcePathStyle, + publicAccessExtraSubPath: options.publicAccessExtraSubPath } as Omit; return { config, @@ -38,7 +39,8 @@ export class S3PrivateBucket extends S3BaseBucket { credentials, endpoint: options.endpoint!, maxRetries: options.maxRetries!, - forcePathStyle: options.forcePathStyle + forcePathStyle: options.forcePathStyle, + publicAccessExtraSubPath: options.publicAccessExtraSubPath } as Omit; return { config, diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts index f0eff9ea03..cf73c500b6 100644 --- a/packages/service/common/s3/buckets/public.ts +++ b/packages/service/common/s3/buckets/public.ts @@ -21,9 +21,10 @@ export class S3PublicBucket extends S3BaseBucket { region, vendor, credentials, - forcePathStyle: true, endpoint: options.endpoint!, - maxRetries: options.maxRetries! + maxRetries: options.maxRetries!, + forcePathStyle: options.forcePathStyle, + publicAccessExtraSubPath: options.publicAccessExtraSubPath } as Omit; return { config, @@ -39,7 +40,8 @@ export class S3PublicBucket extends S3BaseBucket { credentials, endpoint: options.endpoint!, maxRetries: options.maxRetries!, - forcePathStyle: options.forcePathStyle + forcePathStyle: options.forcePathStyle, + publicAccessExtraSubPath: options.publicAccessExtraSubPath } as Omit; return { config, diff --git a/packages/service/common/s3/constants.ts b/packages/service/common/s3/constants.ts index 3773ecf56f..470d5d17eb 100644 --- a/packages/service/common/s3/constants.ts +++ b/packages/service/common/s3/constants.ts @@ -47,7 +47,7 @@ export function createDefaultStorageOptions() { case 'minio': { return { vendor: 'minio', - forcePathStyle: true, + forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : false, externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined, endpoint: process.env.STORAGE_S3_ENDPOINT || 'http://localhost:9000', region: process.env.STORAGE_REGION || 'us-east-1', @@ -59,7 +59,8 @@ export function createDefaultStorageOptions() { }, maxRetries: process.env.STORAGE_S3_MAX_RETRIES ? parseInt(process.env.STORAGE_S3_MAX_RETRIES) - : 3 + : 3, + publicAccessExtraSubPath: process.env.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH || undefined } satisfies Omit & { publicBucket: string; privateBucket: string; @@ -82,7 +83,8 @@ export function createDefaultStorageOptions() { }, maxRetries: process.env.STORAGE_S3_MAX_RETRIES ? parseInt(process.env.STORAGE_S3_MAX_RETRIES) - : 3 + : 3, + publicAccessExtraSubPath: process.env.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH || undefined } satisfies Omit & { publicBucket: string; privateBucket: string; diff --git a/packages/service/package.json b/packages/service/package.json index 8ad6c20beb..832ebcc35d 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "type": "module", "dependencies": { - "@fastgpt-sdk/storage": "0.6.6", + "@fastgpt-sdk/storage": "^0.6.15", "@fastgpt/global": "workspace:*", "@maxmind/geoip2-node": "^6.3.4", "@modelcontextprotocol/sdk": "^1.25.2", diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 2f474feea0..897bc6a0df 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -141,6 +141,7 @@ export const putFileToS3 = async ({ maxSize?: number; t: any; }) => { + console.log(headers); try { const res = await axios.put(url, file, { headers: { @@ -153,6 +154,6 @@ export const putFileToS3 = async ({ onSuccess?.(); } } catch (error) { - Promise.reject(parseS3UploadError({ t, error, maxSize })); + return Promise.reject(parseS3UploadError({ t, error, maxSize })); } }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d35c52039c..376f17d029 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -133,8 +133,8 @@ importers: packages/service: dependencies: '@fastgpt-sdk/storage': - specifier: 0.6.6 - version: 0.6.6(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1) + specifier: ^0.6.15 + version: 0.6.15(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1) '@fastgpt/global': specifier: workspace:* version: link:../global @@ -537,8 +537,8 @@ importers: specifier: 11.11.0 version: 11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1) '@fastgpt-sdk/storage': - specifier: 0.6.6 - version: 0.6.6(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1) + specifier: ^0.6.15 + version: 0.6.15(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1) '@fastgpt/global': specifier: workspace:* version: link:../../packages/global @@ -2608,8 +2608,8 @@ packages: '@fastgpt-sdk/plugin@0.2.17': resolution: {integrity: sha512-TU93FD9JIeAV+isoLVVbW+yX14J27Kgd5Sn8LPvYWkrorUEtWeVfd8rOzh/KXPd43hCgD3bSDZ1W3hC06Spnog==} - '@fastgpt-sdk/storage@0.6.6': - resolution: {integrity: sha512-nG+DWwyJS6upfEbJ+FgTc8CGhzpW1JiPEWGO1luhm6lgYRitm0HBBeX3m0j17lYTa2S0ZTDRGJgnXYzOWpYOLg==} + '@fastgpt-sdk/storage@0.6.15': + resolution: {integrity: sha512-oPbm6EtXQ3ysad/OebF2ovwbIax6PeCvYqA3cGAVEHEJMBU3633ktl1ZaIIkmyjWJLsABZpMf6m7lPBMyISGrA==} engines: {node: '>=20'} '@fastify/accept-negotiator@1.1.0': @@ -14724,7 +14724,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@fastgpt-sdk/storage@0.6.6(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)': + '@fastgpt-sdk/storage@0.6.15(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)': dependencies: '@aws-sdk/client-s3': 3.948.0 '@aws-sdk/lib-storage': 3.948.0(@aws-sdk/client-s3@3.948.0) @@ -14759,7 +14759,7 @@ snapshots: - tsx - yaml - '@fastgpt-sdk/storage@0.6.6(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)': + '@fastgpt-sdk/storage@0.6.15(@opentelemetry/api@1.9.0)(@types/node@24.0.13)(jiti@2.6.0)(lightningcss@1.30.1)(proxy-agent@6.5.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.1)': dependencies: '@aws-sdk/client-s3': 3.948.0 '@aws-sdk/lib-storage': 3.948.0(@aws-sdk/client-s3@3.948.0) @@ -20136,8 +20136,8 @@ snapshots: '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.57.1) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) eslint-plugin-jsx-a11y: 6.10.2(eslint@8.57.1) eslint-plugin-react: 7.37.4(eslint@8.57.1) eslint-plugin-react-hooks: 5.0.0-canary-7118f5dd7-20230705(eslint@8.57.1) @@ -20156,6 +20156,21 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1): + dependencies: + '@nolyfill/is-core-module': 1.0.39 + debug: 4.4.0 + eslint: 8.57.1 + get-tsconfig: 4.10.0 + is-bun-module: 1.3.0 + oxc-resolver: 5.0.0 + stable-hash: 0.0.5 + tinyglobby: 0.2.12 + optionalDependencies: + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + transitivePeerDependencies: + - supports-color + eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0): dependencies: '@nolyfill/is-core-module': 1.0.39 @@ -20197,6 +20212,17 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + dependencies: + debug: 3.2.7 + optionalDependencies: + '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2) + eslint: 8.57.1 + eslint-import-resolver-node: 0.3.9 + eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1) + transitivePeerDependencies: + - supports-color + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1): dependencies: debug: 3.2.7 @@ -20237,6 +20263,35 @@ snapshots: - eslint-import-resolver-webpack - supports-color + eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + dependencies: + '@rtsao/scc': 1.1.0 + array-includes: 3.1.8 + array.prototype.findlastindex: 1.2.6 + array.prototype.flat: 1.3.3 + array.prototype.flatmap: 1.3.3 + debug: 3.2.7 + doctrine: 2.1.0 + eslint: 8.57.1 + eslint-import-resolver-node: 0.3.9 + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + hasown: 2.0.2 + is-core-module: 2.16.1 + is-glob: 4.0.3 + minimatch: 3.1.2 + object.fromentries: 2.0.8 + object.groupby: 1.0.3 + object.values: 1.2.1 + semver: 6.3.1 + string.prototype.trimend: 1.0.9 + tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.8.2) + transitivePeerDependencies: + - eslint-import-resolver-typescript + - eslint-import-resolver-webpack + - supports-color + eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1): dependencies: '@rtsao/scc': 1.1.0 diff --git a/projects/app/.env.template b/projects/app/.env.template index 9f9ec5c8ae..9c1bbe2cc2 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -46,6 +46,7 @@ STORAGE_EXTERNAL_ENDPOINT= STORAGE_S3_ENDPOINT=http://localhost:9000 STORAGE_S3_FORCE_PATH_STYLE=true STORAGE_S3_MAX_RETRIES=3 +STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH= # Redis URL REDIS_URL=redis://default:mypassword@localhost:6379 diff --git a/projects/app/package.json b/projects/app/package.json index 60c0d16178..cf01068569 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -20,7 +20,7 @@ "@dagrejs/dagre": "^1.1.4", "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", - "@fastgpt-sdk/storage": "0.6.6", + "@fastgpt-sdk/storage": "^0.6.15", "@fastgpt/global": "workspace:*", "@fastgpt/service": "workspace:*", "@fastgpt/web": "workspace:*", diff --git a/sdk/storage/package.json b/sdk/storage/package.json index b66ae8d837..3da850aff7 100644 --- a/sdk/storage/package.json +++ b/sdk/storage/package.json @@ -1,7 +1,7 @@ { "name": "@fastgpt-sdk/storage", "private": false, - "version": "0.6.6", + "version": "0.6.15", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/sdk/storage/src/adapters/aws-s3.adapter.ts b/sdk/storage/src/adapters/aws-s3.adapter.ts index ef2297295b..f590344c99 100644 --- a/sdk/storage/src/adapters/aws-s3.adapter.ts +++ b/sdk/storage/src/adapters/aws-s3.adapter.ts @@ -42,7 +42,7 @@ import type { import { Upload } from '@aws-sdk/lib-storage'; import { EmptyObjectError } from '../errors'; import type { Readable } from 'node:stream'; -import { camelCase, chunk, isNotNil, kebabCase } from 'es-toolkit'; +import { camelCase, chunk, isNotNil, kebabCase, trim } from 'es-toolkit'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants'; @@ -299,29 +299,42 @@ export class AwsS3StorageAdapter implements IStorage { } } - if (contentType) { - meta['Content-Type'] = contentType; - } + const convertToS3Headers = (meta: Record) => { + return Object.keys(meta) + .filter((key) => key !== 'Content-Type') + .map((key) => `x-amz-meta-${key}`); + }; const url = await getSignedUrl( this.client, new PutObjectCommand({ Bucket: this.options.bucket, Key: key, - Metadata: meta + Metadata: meta, + ContentType: contentType }), { - expiresIn + expiresIn, + unhoistableHeaders: new Set(convertToS3Headers(meta)) } ); + const headers: Record = {}; + for (const [key, value] of Object.entries(meta)) { + if (key.toLowerCase() === 'content-type') { + continue; + } + headers[`x-amz-meta-${key}`] = value; + } + if (contentType) { + headers['Content-Type'] = contentType; + } + return { key, url: url, bucket: this.options.bucket, - metadata: { - 'Content-Type': meta['Content-Type'] - } + metadata: headers }; } @@ -353,10 +366,21 @@ export class AwsS3StorageAdapter implements IStorage { let url: string; if (this.options.forcePathStyle) { - url = `${this.options.endpoint}/${this.options.bucket}/${key}`; + if (this.options.publicAccessExtraSubPath) { + url = `${this.options.endpoint}/${trim(this.options.publicAccessExtraSubPath, '/')}/${this.options.bucket}/${key}`; + } else { + url = `${this.options.endpoint}/${this.options.bucket}/${key}`; + } } else { const endpoint = new URL(this.options.endpoint); - url = `${endpoint.protocol}//${this.options.bucket}.${endpoint.host}/${key}`; + const protocol = endpoint.protocol; + const host = endpoint.host; + + if (this.options.publicAccessExtraSubPath) { + url = `${protocol}//${this.options.bucket}.${host}/${trim(this.options.publicAccessExtraSubPath, '/')}/${key}`; + } else { + url = `${protocol}//${this.options.bucket}.${host}/${key}`; + } } return { diff --git a/sdk/storage/src/adapters/minio.adapter.ts b/sdk/storage/src/adapters/minio.adapter.ts index 4ba7f1003f..07e8fa08a8 100644 --- a/sdk/storage/src/adapters/minio.adapter.ts +++ b/sdk/storage/src/adapters/minio.adapter.ts @@ -44,7 +44,9 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage throw new Error('Invalid storage vendor: expected "minio"'); } - options.forcePathStyle = true; + // NOTE: + // Maybe some self-hosted MinIO services don't support path style access, + // options.forcePathStyle = true; super(options); // 解析 endpoint URL @@ -59,7 +61,7 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage accessKey: options.credentials.accessKeyId, secretKey: options.credentials.secretAccessKey, region: options.region, - pathStyle: true // MinIO 强制使用 path style + pathStyle: options.forcePathStyle }); } diff --git a/sdk/storage/src/interface.ts b/sdk/storage/src/interface.ts index f51a750702..d1c8407fdf 100644 --- a/sdk/storage/src/interface.ts +++ b/sdk/storage/src/interface.ts @@ -68,6 +68,14 @@ export interface ICommonStorageOptions { /** SecretAccessKey / SecretKey / SK */ secretAccessKey: string; }; + + /** + * 公共访问时额外添加的子路径,可选。 + * + * 说明: + * - 用于在公共访问时添加额外的前缀,例如 `/sub-path`。 + */ + publicAccessExtraSubPath?: string; } /**