diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 1e205a3b01..b14e601e46 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -106,9 +106,7 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据 ## 代码规范 -- 采用 DDD 架构 + 模块划分的代码风格，按业务进行划分，然后再按 controller + service + entity 划分。 -- 使用 type 进行类型声明。 -- function props 数量不能超过 2 个，多参数采用对象传递。 +[FastGPT 代码规范](./skills/system-pr_review/style/syntax.md) ## 运行要求 diff --git a/.claude/design/api/index.md b/.claude/design/api/index.md index 9157b7c1b8..389cbda564 100644 --- a/.claude/design/api/index.md +++ b/.claude/design/api/index.md @@ -57,14 +57,14 @@ export const CreateDatasetBodySchema = z.object({ description: '知识库名称' }) }); -export type CreateDatasetBody = z.infer; +export type CreateDatasetBodyType = z.infer; // 出参 Schema export const CreateDatasetResponseSchema = ObjectIdSchema.meta({ example: '68ad85a7463006c963799a05', description: '新创建的知识库 ID' }); -export type CreateDatasetResponse = z.infer; +export type CreateDatasetResponseType = z.infer; ``` ### 步骤 2: 实现 API 路由 @@ -77,13 +77,13 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { CreateDatasetBodySchema, CreateDatasetResponseSchema, - type CreateDatasetResponse + type CreateDatasetResponseType } from '@fastgpt/global/openapi/core/dataset/api'; // ❌ 不要在路由文件中重导出类型别名 -// export type DatasetCreateBody = CreateDatasetBody; +// export type DatasetCreateBodyType = CreateDatasetBodyType; -async function handler(req: ApiRequestProps): Promise { +async function handler(req: ApiRequestProps): Promise { // 1. 入参验证 const { parentId, name } = CreateDatasetBodySchema.parse(req.body); @@ -148,10 +148,10 @@ export const DatasetPath: OpenAPIPath = { ```typescript // ✅ 正确: 从 openapi 导入 -import type { CreateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; +import type { CreateDatasetBodyType } from '@fastgpt/global/openapi/core/dataset/api'; // ❌ 错误: 从路由文件导入 -import type { DatasetCreateBody } from '@/pages/api/core/dataset/create'; +import type { DatasetCreateBodyType } from '@/pages/api/core/dataset/create'; // ❌ 错误: 从旧的 global 文件导入 import type { CreateDatasetParams } from '@/global/core/dataset/api'; @@ -164,12 +164,12 @@ import type { CreateDatasetParams } from '@/global/core/dataset/api'; ```typescript import createHandler from '@/pages/api/core/dataset/create'; import type { - CreateDatasetBody, - CreateDatasetResponse + CreateDatasetBodyType, + CreateDatasetResponseType } from '@fastgpt/global/openapi/core/dataset/api'; import { Call } from '@test/utils/request'; -const res = await Call(createHandler, { +const res = await Call(createHandler, { auth: users.members[0], body: { name: 'test', intro: 'intro', avatar: 'avatar', type: DatasetTypeEnum.dataset } }); @@ -304,8 +304,8 @@ grep -r "CreateDatasetParams" projects/app/src/ ```typescript // ❌ 删除这些 export type DatasetCreateQuery = {}; -export type DatasetCreateBody = CreateDatasetBody; -export type DatasetCreateResponse = CreateDatasetResponse; +export type DatasetCreateBodyType = CreateDatasetBodyType; +export type DatasetCreateResponse = CreateDatasetResponseType; ``` ## 审查检查清单 diff --git a/.claude/skills/system-pr_review/SKILL.md b/.claude/skills/system-pr_review/SKILL.md index a33902b496..646ce7e51b 100644 --- a/.claude/skills/system-pr_review/SKILL.md +++ b/.claude/skills/system-pr_review/SKILL.md @@ -11,21 +11,41 @@ description: 当用户传入一个 review 的 pr 链接时候，触发该 skill ## 步骤 0：拉取代码 -```bash -# 检出 PR 分支到本地 -gh pr checkout +使用以下命令**无需切换分支**，直接使用 PR 编号即可： +```bash # 获取 PR 基本信息 -gh pr view --json number,title,body,author,state,headRefName,baseRefName,additions,deletions,files +gh pr view --json number,title,body,author,state,headRefName,baseRefName,additions,deletions,files # 获取完整 diff -gh pr diff +gh pr diff # 查看 commit 历史 -gh pr view --json commits --jq '.commits[].messageHeadline' +gh pr view --json commits --jq '.commits[].messageHeadline' # 检查 CI 状态 -gh pr checks +gh pr checks +``` + +如需在本地运行 **tsc / 单元测试**，使用 `git worktree` 创建独立目录，**不影响当前分支**： + +```bash +# 1. 拉取 PR 代码到临时分支 +git fetch upstream pull//head:pr/ + +# 2. 在独立目录检出（与当前工作区完全隔离） +git worktree add ~/pr-worktrees/pr- pr/ + +# 3. 进入该目录安装依赖、运行测试 +cd ~/pr-worktrees/pr- +pnpm install +pnpm tsc --noEmit # 类型检查 +pnpm test # 单元测试 + +# 4. 审查完毕后清理 +cd - +git worktree remove ~/pr-worktrees/pr- +git branch -D pr/ ``` --- @@ -91,6 +111,7 @@ gh pr checks - [] [包结构规范](./style/package.md) - [] [日志规范](./style/logger.md) - [] [Service 解耦规范](./style/service-decoupling.md) +- [] [语法风格规范](./style/syntax.md) --- diff --git a/.claude/skills/system-pr_review/style/syntax.md b/.claude/skills/system-pr_review/style/syntax.md new file mode 100644 index 0000000000..6b24b145ca --- /dev/null +++ b/.claude/skills/system-pr_review/style/syntax.md @@ -0,0 +1,277 @@ +# 代码规范 + +## 基础代码组织模式 + +采用 DDD 架构，按业务域 → 子功能 → 固定文件三层划分。 + +### 目录结构 + +``` +packages/ +├── global/core/ # 类型、常量（前后端共享） +│ ├── app/ +│ │ ├── type.ts # 顶层聚合类型 +│ │ ├── constants.ts +│ │ ├── workflow/ +│ │ │ ├── type.ts +│ │ │ └── constants.ts +│ │ ├── version/ +│ │ │ └── type.ts +│ │ └── evaluation/ +│ │ └── type.ts +│ ├── chat/ +│ ├── dataset/ +│ └── plugin/ +│ +└── service/core/ # 后端业务逻辑（不可在前端引用） + ├── app/ + │ ├── schema.ts # App 主表 Mongoose Schema + │ ├── entity.ts # findById / create / updateById 等基础操作封装 + │ ├── service.ts # 聚合业务逻辑（跨子功能协调），不允许互相引用，只允许单向依赖，跨 service 的协调需由上层通过 props 传入另一个 service 或者衍生方法 + │ ├── utils.ts # 纯函数工具，无副作用，可独立单测 + │ ├── version/ + │ │ ├── schema.ts + │ │ ├── entity.ts + │ │ ├── service.ts + │ │ └── utils.ts + │ ├── evaluation/ + │ │ ├── schema.ts # 合并多个 schema 到单文件 + │ │ ├── entity.ts + │ │ ├── service.ts + │ │ └── utils.ts + │ ├── logs/ + │ └── tool/ + │ ├── service.ts + │ └── utils.ts + ├── chat/ + ├── dataset/ + └── plugin/ +``` + +### 叶子目录固定文件说明 + +| 文件 | 职责 | +|------|------| +| `schema.ts` | Mongoose Schema 定义，导出 Model 和 SchemaType | +| `entity.ts` | 数据访问封装：`findById`、`create`、`updateById` 等基础操作 | +| `service.ts` | 业务逻辑：调用 entity，跨模块协调，处理业务规则 | +| `utils.ts` | 纯函数工具，无副作用，可独立单测 | + +```typescript +// entity.ts 示例 —— 只做数据访问，不含业务判断 +export const findAppById = (id: string) => + MongoApp.findById(id).lean(); +export const createApp = (data: AppCreateParams, session?: ClientSession) => + MongoApp.create([data], { session }); + +// service.ts 示例 —— 调用 entity，处理业务规则 +export const createAppAndInitVersion = async (data: AppCreateParams, session?: ClientSession) => { + const app = await createApp(data, session); + await createVersion({ appId: app._id, ... }, session); + return app; +}; + +// service 需协同，通过 props 传入另一个 service 或者衍生方法。 +const service1 = xxxx +const service2 = (props: {id:string; service1: typeof service1 }) => { + const data = findAppById(id) + return props.service1(data); +}; +``` + +### 层级约束 + +- `global/core/` 只放类型和常量，**禁止**引入 mongoose、服务端 SDK +- `service/core/` 只在服务端使用，**禁止**被 `packages/web/` 或前端页面直接引用 +- 子功能目录不超过 **3 层**嵌套 +- 一个目录内无需拆子功能时，直接放 `schema.ts` + `entity.ts` + `service.ts` + `utils.ts` +- 多个 schema 文件（如 `evalSchema.ts` + `evalItemSchema.ts`）**合并**到单个 `schema.ts` + +## 使用 `type` 进行类型声明，不使用 `interface` + +```typescript +// ❌ 不好的实践 +interface User { + id: string; + name: string; +} + +// ✅ 好的实践 +type User = { + id: string; + name: string; +} +``` + +--- + +## 使用 IIFE 写法来取代 if/else 进行变量条件赋值。 + +```typescript +// ❌ 不好的实践 +if (condition) { + value = true; +} else { + value = false; +} + +// ✅ 好的实践 +const value = (() => { + if (condition) { + return true; + } + return false; +})(); +``` + +--- + +## 类型推导：Zod schema 同时承担校验和类型 + +用 `z.infer` 从 schema 推导类型，不重复手写相同结构的 type。 + +```typescript +// ❌ 不好的实践 +type MessageParam = { role: 'user' | 'assistant'; content: string }; +const MessageParamSchema = z.object({ role: z.enum(['user', 'assistant']), content: z.string() }); + +// ✅ 好的实践 +export const MessageParamSchema = z.discriminatedUnion('role', [...]); +export type MessageParam = z.infer; +``` + +--- + +## 可选链调用回调 + +用 `?.()` 调用可选回调，取代 `if (fn) fn()` 的冗余写法。 + +```typescript +// ❌ 不好的实践 +if (onProgress) { + onProgress({ phase: 'creatingContainer' }); +} + +// ✅ 好的实践 +onProgress?.({ phase: 'creatingContainer' }); +``` + +--- + +## 空值合并取默认值 + +用 `??` 取代 `||` 处理默认值，避免 `0`、`false`、`''` 被错误覆盖。 + +```typescript +// ❌ 不好的实践 +const version = lastVersion?.version || 0; // version 为 0 时被误覆盖 +const text = item?.value || ''; + +// ✅ 好的实践 +const version = (lastVersion?.version ?? -1) + 1; +const text = item?.value ?? ''; +``` + +--- + +## 解构重命名 + +同名变量来自多个来源时，解构时重命名，避免命名冲突。 + +```typescript +// ❌ 不好的实践 +const r1 = await getSkillGuidance(...); +const r2 = await createLLMResponse(...); +const inputTokens = r1.usage.inputTokens + r2.usage.inputTokens; + +// ✅ 好的实践 +const { usage: guidanceUsage } = await getSkillGuidance(...); +const { usage: generateUsage } = await createLLMResponse(...); +const inputTokens = guidanceUsage.inputTokens + generateUsage.inputTokens; +``` + +--- + +## 类型守卫 + +用 `is` 关键字收窄 `unknown` / `any` 类型，替代强制断言。 + +```typescript +// ❌ 不好的实践 +function process(value: unknown) { + const n = value as number; // 不安全 +} + +// ✅ 好的实践 +const isValidNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); + +if (isValidNumber(value)) { + // 此处 value 安全收窄为 number +} +``` + +--- + +## 非关键清理用 `.catch()` 链 + +次要的清理操作（不影响主流程）用 `.catch()` 吞掉错误，不污染主 try/catch。 + +```typescript +// ❌ 不好的实践 +try { + await client.delete(); +} catch { + // 清理失败，主流程中断 +} + +// ✅ 好的实践 +await client.delete().catch(() => {}); +``` + +--- + +## 函数参数不超过 2 个，多参数用对象传递 + +独立参数不超过 2 个，超过时改为对象参数，便于扩展且无需关心顺序。 + +```typescript +// ❌ 不好的实践 +function createVersion(skillId: string, teamId: string, tmbId: string, version: number) {} + +// ✅ 好的实践 +function createVersion(data: { skillId: string; teamId: string; tmbId: string; version: number }) {} +``` + +--- + +## 数据写操作函数支持可选 session 参数 + +涉及数据库写操作的函数统一支持可选的 `session` 参数，便于上层组合事务。事务统一通过 `mongoSessionRun` 发起，内部自动处理 startTransaction / commit / abort / retry。 + +```typescript +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; +import { type ClientSession } from '@fastgpt/service/common/mongo'; + +// entity.ts —— 基础操作透传 session +export const createVersion = (data: CreateVersionData, session?: ClientSession) => + MongoAppVersion.create([data], { session }); + +// service.ts —— 需要事务时用 mongoSessionRun 包裹，外部已有 session 时直接传入 +export const createAppAndInitVersion = async ( + data: AppCreateParams, + session?: ClientSession +) => { + const create = async (session: ClientSession) => { + const app = await createApp(data, session); + await createVersion({ appId: app._id, version: 0 }, session); + return app; + }; + + if (session) { + return create(session); + } else { + return mongoSessionRun(create); + } +}; +``` diff --git a/document/content/docs/openapi/dataset.mdx b/document/content/docs/openapi/dataset.mdx index 40a109dd55..9dfe4c52a5 100644 --- a/document/content/docs/openapi/dataset.mdx +++ b/document/content/docs/openapi/dataset.mdx @@ -257,7 +257,7 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete? | 参数 | 说明 | 必填 | | ---------------- | ----------------------------------------------------------------------------------------------------------- | ---- | | datasetId | 知识库ID | ✅ | -| parentId： | 父级ID，不填则默认为根目录 | | +| parentId | 父级ID，不填则默认为根目录 | | | trainingType | 数据处理方式。chunk: 按文本长度进行分割;qa: 问答对提取 | ✅ | | indexPrefixTitle | 是否自动生成标题索引 | | | customPdfParse | 是否开启PDF增强解析, 默认 false: 关闭;true: 开启; | | @@ -378,10 +378,7 @@ data 为集合的 ID。 "data": { "collectionId": "65abcfab9d1448617cba5f0d", "results": { - "insertLen": 5, // 分割成多少段 - "overToken": [], - "repeat": [], - "error": [] + "insertLen": 5 } } } @@ -440,10 +437,7 @@ data 为集合的 ID。 "data": { "collectionId": "65abd0ad9d1448617cba6031", "results": { - "insertLen": 1, - "overToken": [], - "repeat": [], - "error": [] + "insertLen": 1 } } } @@ -492,10 +486,7 @@ data 为集合的 ID。 "data": { "collectionId": "65abc044e4704bac793fbd81", "results": { - "insertLen": 1, - "overToken": [], - "repeat": [], - "error": [] + "insertLen": 1 } } } @@ -561,10 +552,7 @@ data 为集合的 ID。 "data": { "collectionId": "65abc044e4704bac793fbd81", "results": { - "insertLen": 1, - "overToken": [], - "repeat": [], - "error": [] + "insertLen": 1 } } } @@ -624,10 +612,7 @@ data 为集合的 ID。 "data": { "collectionId": "6646fcedfabd823cdc6de746", "results": { - "insertLen": 1, - "overToken": [], - "repeat": [], - "error": [] + "insertLen": 1 } } } @@ -992,10 +977,7 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/data/push "code": 200, "statusText": "", "data": { - "insertLen": 1, // 最终插入成功的数量 - "overToken": [], // 超出 token 的 - "repeat": [], // 重复的数量 - "error": [] // 其他错误 + "insertLen": 1 // 最终插入成功的数量 } } ``` diff --git a/document/content/docs/self-host/upgrading/4-14/41411.mdx b/document/content/docs/self-host/upgrading/4-14/41411.mdx index 8a0dbdb835..ea74df358f 100644 --- a/document/content/docs/self-host/upgrading/4-14/41411.mdx +++ b/document/content/docs/self-host/upgrading/4-14/41411.mdx @@ -9,9 +9,10 @@ description: 'FastGPT V4.14.11 更新说明' ## ⚙️ 优化 +1. 对大量接口增加了 zod 参数校验，减少攻击和错误参数类型风险。 ## 🐛 修复 1. 对话 Agent 模式，模型存在刷新后被重置问题。 2. 部分接口未正确进行权限校验。 -3. 修复部分接口 nosql 注入分析。 \ No newline at end of file +3. API 推送接口，计费异常。 \ No newline at end of file diff --git a/document/data/doc-last-modified.json b/document/data/doc-last-modified.json index 0a95908929..6bc5525baa 100644 --- a/document/data/doc-last-modified.json +++ b/document/data/doc-last-modified.json @@ -143,8 +143,8 @@ "document/content/docs/openapi/index.mdx": "2026-02-12T18:45:30+08:00", "document/content/docs/openapi/intro.en.mdx": "2026-02-26T22:14:30+08:00", "document/content/docs/openapi/intro.mdx": "2026-02-12T18:45:30+08:00", - "document/content/docs/openapi/share.en.mdx": "2026-02-26T22:14:30+08:00", - "document/content/docs/openapi/share.mdx": "2026-02-12T18:45:30+08:00", + "document/content/docs/openapi/share.en.mdx": "2026-04-10T22:55:44+08:00", + "document/content/docs/openapi/share.mdx": "2026-04-10T22:55:44+08:00", "document/content/docs/self-host/config/json.en.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/config/json.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/config/model/intro.en.mdx": "2026-03-30T10:05:42+08:00", @@ -222,7 +222,7 @@ "document/content/docs/self-host/upgrading/4-14/4141.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/41410.en.mdx": "2026-03-31T23:15:29+08:00", "document/content/docs/self-host/upgrading/4-14/41410.mdx": "2026-04-08T16:15:25+08:00", - "document/content/docs/self-host/upgrading/4-14/41411.mdx": "2026-04-10T13:58:10+08:00", + "document/content/docs/self-host/upgrading/4-14/41411.mdx": "2026-04-12T10:39:41+08:00", "document/content/docs/self-host/upgrading/4-14/4142.en.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4142.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4143.en.mdx": "2026-03-03T17:39:47+08:00", diff --git a/packages/global/common/file/s3/type.ts b/packages/global/common/file/s3/type.ts new file mode 100644 index 0000000000..a4cfe4d3b5 --- /dev/null +++ b/packages/global/common/file/s3/type.ts @@ -0,0 +1,11 @@ +import z from 'zod'; + +export const CreatePostPresignedUrlResponseSchema = z.object({ + url: z.string().nonempty(), + key: z.string().nonempty(), + headers: z.record(z.string(), z.string()), + maxSize: z.number().positive().optional() // bytes +}); +export type CreatePostPresignedUrlResponseType = z.infer< + typeof CreatePostPresignedUrlResponseSchema +>; diff --git a/packages/global/common/file/type.ts b/packages/global/common/file/type.ts index 85291c2b16..d4cde8e047 100644 --- a/packages/global/common/file/type.ts +++ b/packages/global/common/file/type.ts @@ -1,9 +1,12 @@ -import type { BucketNameEnum } from './constants'; +import { BucketNameEnum } from './constants'; +import { ObjectIdSchema } from '../type/mongo'; +import z from 'zod'; -export type FileTokenQuery = { - bucketName: `${BucketNameEnum}`; - teamId: string; - uid: string; // tmbId/ share uid/ teamChat uid - fileId: string; - customExpireMinutes?: number; -}; +const FileTokenQuerySchema = z.object({ + bucketName: z.enum(BucketNameEnum), + teamId: ObjectIdSchema, + uid: z.string().nonempty(), + fileId: z.string().nonempty(), + customExpireMinutes: z.number().optional() +}); +export type FileTokenQuery = z.infer; diff --git a/packages/global/common/parentFolder/type.ts b/packages/global/common/parentFolder/type.ts index 3b29cc4624..a1891cf883 100644 --- a/packages/global/common/parentFolder/type.ts +++ b/packages/global/common/parentFolder/type.ts @@ -3,25 +3,33 @@ import z from 'zod'; export const ParentIdSchema = z.string().nullish(); export type ParentIdType = string | null | undefined; -export type GetPathProps = { - sourceId?: ParentIdType; - type: 'current' | 'parent'; -}; +export const GetPathPropsSchema = z.object({ + sourceId: ParentIdSchema.optional(), + type: z.enum(['current', 'parent']).optional() +}); +export type GetPathProps = z.infer; -export type ParentTreePathItemType = { - parentId: ParentIdType; - parentName: string; -}; +export const ParentTreePathItemSchema = z.object({ + parentId: ParentIdSchema, + parentName: z.string() +}); +export type ParentTreePathItemType = z.infer; -export type GetResourceFolderListProps = { - parentId: ParentIdType; -}; -export type GetResourceFolderListItemResponse = { - name: string; - id: string; -}; +export const GetResourceFolderListPropsSchema = z.object({ + parentId: ParentIdSchema +}); +export type GetResourceFolderListProps = z.infer; -export type GetResourceListItemResponse = GetResourceFolderListItemResponse & { - avatar: string; - isFolder: boolean; -}; +export const GetResourceFolderListItemResponseSchema = z.object({ + name: z.string(), + id: z.string() +}); +export type GetResourceFolderListItemResponse = z.infer< + typeof GetResourceFolderListItemResponseSchema +>; + +export const GetResourceListItemResponseSchema = GetResourceFolderListItemResponseSchema.extend({ + avatar: z.string(), + isFolder: z.boolean() +}); +export type GetResourceListItemResponse = z.infer; diff --git a/packages/global/core/ai/constants.ts b/packages/global/core/ai/constants.ts index af474fd05b..6f620cfe8b 100644 --- a/packages/global/core/ai/constants.ts +++ b/packages/global/core/ai/constants.ts @@ -1,5 +1,5 @@ import { i18nT } from '../../../web/i18n/utils'; -import type { CompletionUsage } from './type'; +import type { CompletionUsage } from './llm/type'; import type { LLMModelItemType, EmbeddingModelItemType, STTModelType } from './model.schema'; export const getLLMDefaultUsage = (): CompletionUsage => { diff --git a/packages/global/core/ai/llm/type.ts b/packages/global/core/ai/llm/type.ts new file mode 100644 index 0000000000..f4ab83609b --- /dev/null +++ b/packages/global/core/ai/llm/type.ts @@ -0,0 +1,249 @@ +import type openai from 'openai'; +import type { Stream } from 'openai/streaming'; +import z from 'zod'; + +/* 通用类型 */ +export const ChatCompletionContentPartTextSchema = z.object({ + type: z.literal('text'), + text: z.string(), + key: z.string().optional() +}); +// tool function +export const ChatCompletionMessageToolCallFunctionSchema = z.object({ + arguments: z.string().meta({ description: '工具参数' }), + name: z.string().meta({ description: '工具名称' }) +}); + +// Function call message +export const ChatCompletionMessageFunctionCallSchema = + ChatCompletionMessageToolCallFunctionSchema.extend({ + id: z.string().optional().meta({ description: '工具调用 ID' }), + toolName: z.string().optional().meta({ description: '工具名称' }), + toolAvatar: z.string().optional().meta({ description: '工具头像' }) + }); +export type ChatCompletionMessageFunctionCall = z.infer< + typeof ChatCompletionMessageFunctionCallSchema +>; + +/** + * System message: 对齐 openai SDK 的 ChatCompletionSystemMessageParam + * content 仅允许字符串或纯文本 part 数组 + */ +export const ChatCompletionSystemMessageParamSchema = z.object({ + role: z.literal('system'), + content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]), + name: z.string().optional() +}); +export type ChatCompletionSystemMessageParam = z.infer< + typeof ChatCompletionSystemMessageParamSchema +>; + +/* ---------- User Input message: ChatCompletionContentPart schemas ---------- + * openai SDK 不导出 runtime zod schema，这里手写对齐 SDK 的联合类型， + * 并加上 FastGPT 的扩展字段：所有分支可选 `key`，以及自定义 `file_url` 分支。 + * 外部再扩展新分支： + * z.discriminatedUnion('type', [ + * ...ChatCompletionContentPartSchema.options, + * MyCustomPartSchema + * ]) + */ +export const ChatCompletionContentPartImageSchema = z.object({ + type: z.literal('image_url'), + image_url: z.object({ + url: z.string(), + detail: z.enum(['auto', 'low', 'high']).optional() + }), + key: z.string().optional() +}); +export const ChatCompletionContentPartInputAudioSchema = z.object({ + type: z.literal('input_audio'), + input_audio: z.object({ + data: z.string(), + format: z.enum(['wav', 'mp3']) + }), + key: z.string().optional() +}); +// SDK 的 `file` 分支（base64 / file_id 输入） +export const ChatCompletionContentPartFileSchema = z.object({ + type: z.literal('file'), + file: z.object({ + file_data: z.string().optional(), + file_id: z.string().optional(), + filename: z.string().optional() + }), + key: z.string().optional() +}); +// FastGPT 自定义扩展：外链文件 +export const ChatCompletionContentPartFileTypeSchema = z.object({ + type: z.literal('file_url'), + name: z.string(), + url: z.string(), + key: z.string().optional() +}); +export const ChatCompletionContentPartSchema = z.discriminatedUnion('type', [ + ChatCompletionContentPartTextSchema, + ChatCompletionContentPartImageSchema, + ChatCompletionContentPartInputAudioSchema, + ChatCompletionContentPartFileSchema, + ChatCompletionContentPartFileTypeSchema +]); +export type ChatCompletionContentPart = z.infer; +export type ChatCompletionContentPartText = z.infer; + +export const ChatCompletionUserMessageParamSchema = z.object({ + role: z.literal('user'), + content: z.union([z.string(), z.array(ChatCompletionContentPartSchema)]), + name: z.string().optional() +}); +export type ChatCompletionUserMessageParam = z.infer; + +/* ========= User end ===== */ + +/** + * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name` + * SDK content 仅允许纯文本 part + */ +export const ChatCompletionToolMessageParamSchema = z.object({ + role: z.literal('tool'), + content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]), + tool_call_id: z.string(), + name: z.string().optional() +}); +export type ChatCompletionToolMessageParam = z.infer; + +/** + * Function message: 对齐 openai SDK 的 ChatCompletionFunctionMessageParam + * SDK 已标记 deprecated，保留用于旧接口兼容 + */ +export const ChatCompletionFunctionMessageParamSchema = z.object({ + role: z.literal('function'), + content: z.string().nullable(), + name: z.string() +}); +export type ChatCompletionFunctionMessageParam = z.infer< + typeof ChatCompletionFunctionMessageParamSchema +>; + +/** + * Assistant message: 对齐 openai SDK 的 ChatCompletionAssistantMessageParam + * - content: 文本或 text/refusal part 数组，可空 + * - tool_calls / function_call（已废弃）/ audio / refusal 全部可选 + * - 新增 FastGPT 扩展 `interactive` 字段 + */ +// SDK 的 refusal content part（仅出现在 assistant 消息里） +export const ChatCompletionContentPartRefusalSchema = z.object({ + type: z.literal('refusal'), + refusal: z.string() +}); +export type ChatCompletionContentPartRefusal = z.infer< + typeof ChatCompletionContentPartRefusalSchema +>; +// SDK 的 ChatCompletionMessageToolCall（目前 type 只有 'function' 一种） +export const ChatCompletionMessageToolCallSchema = z.object({ + id: z.string(), + type: z.literal('function'), + function: ChatCompletionMessageToolCallFunctionSchema +}); +export type ChatCompletionMessageToolCall = z.infer; + +export const ChatCompletionAssistantMessageParamSchema = z.object({ + role: z.literal('assistant'), + content: z + .union([ + z.string(), + z.array( + z.discriminatedUnion('type', [ + ChatCompletionContentPartTextSchema, + ChatCompletionContentPartRefusalSchema + ]) + ) + ]) + .nullish() + .meta({ + description: 'Assistant message content', + example: 'Hello, how are you?' + }), + tool_calls: z.array(ChatCompletionMessageToolCallSchema).optional().meta({ + description: '工具调用' + }), + // FastGPT 自定义扩展。为避免与 workflow/interactive 形成循环依赖，此处用 z.any() 占位， + // 真实类型见 packages/global/core/workflow/template/system/interactive/type.ts:WorkflowInteractiveResponseType + interactive: z.any().optional().meta({ + description: '交互式响应（FastGPT 自定义扩展）' + }), + // 下面的几个，目前系统没用到 + audio: z.object({ id: z.string() }).nullish(), + function_call: ChatCompletionMessageToolCallFunctionSchema.nullish().meta({ + description: '函数调用', + deprecated: true + }), + name: z.string().optional(), + refusal: z.string().nullish() +}); +export type ChatCompletionAssistantMessageParam = z.infer< + typeof ChatCompletionAssistantMessageParamSchema +>; + +/* =====Assistant end ===== */ + +/** + * Developer message: 对齐 openai SDK 的 ChatCompletionDeveloperMessageParam + * o1+ 模型用 developer 消息代替 system + */ +export const ChatCompletionDeveloperMessageParamSchema = z.object({ + role: z.literal('developer'), + content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]), + name: z.string().optional() +}); +export type ChatCompletionDeveloperMessageParam = z.infer< + typeof ChatCompletionDeveloperMessageParamSchema +>; + +/** + * ChatCompletionMessageParam: 6 个 role 的 discriminated union + * 每个分支附加 FastGPT 全局扩展字段：reasoning_content / dataId / hideInUI + */ +const messageParamExtraFields = { + reasoning_content: z.string().optional(), + dataId: z.string().optional(), + hideInUI: z.boolean().optional() +}; +export const ChatCompletionMessageParamSchema = z.discriminatedUnion('role', [ + ChatCompletionSystemMessageParamSchema.extend(messageParamExtraFields), + ChatCompletionDeveloperMessageParamSchema.extend(messageParamExtraFields), + ChatCompletionUserMessageParamSchema.extend(messageParamExtraFields), + ChatCompletionAssistantMessageParamSchema.extend(messageParamExtraFields), + ChatCompletionToolMessageParamSchema.extend(messageParamExtraFields), + ChatCompletionFunctionMessageParamSchema.extend(messageParamExtraFields) +]); +export type ChatCompletionMessageParam = z.infer; + +/* ========= Message end ===== */ + +/* ===== 一些自定义扩展类型 ===== */ +// Stream response +export type StreamResponseType = Stream< + openai.Chat.Completions.ChatCompletionChunk & { error?: any } +>; +export type UnStreamResponseType = openai.Chat.Completions.ChatCompletion & { + error?: any; +}; + +export const CompletionFinishReasonSchema = z + .union([ + z.enum(['error', 'close', 'stop', 'length', 'tool_calls', 'content_filter', 'function_call']), + z.literal(null), + z.undefined() + ]) + .meta({ description: '模型完成原因' }); +export type CompletionFinishReason = z.infer; + +// export type { Stream }; +export * from 'openai'; +export * from 'openai/resources'; + +export type PromptTemplateItem = { + title: string; + desc: string; + value: Record; +}; diff --git a/packages/global/core/ai/prompt/AIChat.ts b/packages/global/core/ai/prompt/AIChat.ts index 19db35e251..d701ce9e6c 100644 --- a/packages/global/core/ai/prompt/AIChat.ts +++ b/packages/global/core/ai/prompt/AIChat.ts @@ -1,5 +1,5 @@ /* v8 ignore file */ -import { type PromptTemplateItem } from '../type'; +import { type PromptTemplateItem } from '../llm/type'; import { i18nT } from '../../../../web/i18n/utils'; import { getPromptByVersion } from './utils'; diff --git a/packages/global/core/ai/sandbox/constants.ts b/packages/global/core/ai/sandbox/constants.ts index e595eb907e..9e76409d1e 100644 --- a/packages/global/core/ai/sandbox/constants.ts +++ b/packages/global/core/ai/sandbox/constants.ts @@ -1,6 +1,6 @@ import type { I18nStringType } from '../../../common/i18n/type'; import { hashStr } from '../../../common/string/tools'; -import type { ChatCompletionTool } from '../type'; +import type { ChatCompletionTool } from '../llm/type'; import { z } from 'zod'; // ---- 沙盒状态 ---- diff --git a/packages/global/core/ai/type.ts b/packages/global/core/ai/type.ts deleted file mode 100644 index 32b9adc34e..0000000000 --- a/packages/global/core/ai/type.ts +++ /dev/null @@ -1,98 +0,0 @@ -import openai from 'openai'; -import type { - ChatCompletion as SdkChatCompletion, - ChatCompletionMessageToolCall, - ChatCompletionMessageParam as OpenaiChatCompletionMessageParam, - ChatCompletionContentPart as SdkChatCompletionContentPart, - ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam, - ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam, - ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam -} from 'openai/resources'; -import type { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type'; -import type { Stream } from 'openai/streaming'; -import z from 'zod'; - -// Extension of ChatCompletionMessageParam, Add file url type -export type ChatCompletionContentPartFile = { - type: 'file_url'; - name: string; - url: string; - key?: string; -}; -// Rewrite ChatCompletionContentPart, Add file type -export type ChatCompletionContentPart = - | (SdkChatCompletionContentPart & { key?: string }) - | ChatCompletionContentPartFile; -type CustomChatCompletionUserMessageParam = Omit & { - role: 'user'; - content: string | Array; -}; -export type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & { - role: 'tool'; - name?: string; -}; -type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & { - role: 'assistant'; - interactive?: WorkflowInteractiveResponseType; -}; - -export type ChatCompletionMessageParam = ( - | Exclude< - SdkChatCompletionMessageParam, - | SdkChatCompletionUserMessageParam - | SdkChatCompletionToolMessageParam - | SdkChatCompletionAssistantMessageParam - > - | CustomChatCompletionUserMessageParam - | CustomChatCompletionToolMessageParam - | CustomChatCompletionAssistantMessageParam -) & { - reasoning_content?: string; - dataId?: string; - hideInUI?: boolean; -}; -export type SdkChatCompletionMessageParam = OpenaiChatCompletionMessageParam; - -/* ToolChoice and functionCall extension */ -export type ChatCompletionAssistantToolParam = { - role: 'assistant'; - tool_calls: ChatCompletionMessageToolCall[]; -}; - -export type ChatCompletionMessageFunctionCall = - SdkChatCompletionAssistantMessageParam.FunctionCall & { - id?: string; - toolName?: string; - toolAvatar?: string; - }; - -// Stream response -export type StreamChatType = Stream; -export type UnStreamChatType = openai.Chat.Completions.ChatCompletion; - -// UnStream response -export type ChatCompletion = SdkChatCompletion & { - error?: any; -}; - -export const CompletionFinishReasonSchema = z - .union([ - z.enum(['error', 'close', 'stop', 'length', 'tool_calls', 'content_filter', 'function_call']), - z.literal(null), - z.undefined() - ]) - .meta({ description: '模型完成原因' }); -export type CompletionFinishReason = z.infer; - -export type { Stream }; - -export default openai; -export * from 'openai'; -export * from 'openai/resources'; - -// Other -export type PromptTemplateItem = { - title: string; - desc: string; - value: Record; -}; diff --git a/packages/global/core/chat/adapt.ts b/packages/global/core/chat/adapt.ts index 67e3bd513d..53df999de7 100644 --- a/packages/global/core/chat/adapt.ts +++ b/packages/global/core/chat/adapt.ts @@ -17,7 +17,7 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, ChatCompletionToolMessageParam -} from '../../core/ai/type'; +} from '../ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constants'; import { getPlanCallResponseText } from './utils'; diff --git a/packages/global/core/chat/helperBot/adaptor.ts b/packages/global/core/chat/helperBot/adaptor.ts index c4b68742a5..33edc98f37 100644 --- a/packages/global/core/chat/helperBot/adaptor.ts +++ b/packages/global/core/chat/helperBot/adaptor.ts @@ -1,10 +1,5 @@ import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants'; -import type { - ChatCompletionContentPart, - ChatCompletionMessageParam, - ChatCompletionMessageToolCall, - ChatCompletionToolMessageParam -} from '../../ai/type'; +import type { ChatCompletionContentPart, ChatCompletionMessageParam } from '../../ai/llm/type'; import { ChatFileTypeEnum, ChatRoleEnum } from '../constants'; import type { HelperBotChatItemType } from './type'; import { simpleUserContentPart } from '../adapt'; diff --git a/packages/global/core/dataset/api.ts b/packages/global/core/dataset/api.ts deleted file mode 100644 index 2126ef8102..0000000000 --- a/packages/global/core/dataset/api.ts +++ /dev/null @@ -1,131 +0,0 @@ -import type { ChunkSettingsType, DatasetDataIndexItemType } from './type'; -import type { DatasetCollectionTypeEnum, DatasetCollectionDataProcessModeEnum } from './constants'; -import type { ParentIdType } from '../../common/parentFolder/type'; -import type { APIFileItemType } from './apiDataset/type'; - -/* ================= collection ===================== */ -// Input + store params -type DatasetCollectionStoreDataType = ChunkSettingsType & { - parentId?: ParentIdType; - metadata?: Record; - - customPdfParse?: boolean; -}; - -// create collection params -export type CreateDatasetCollectionParams = DatasetCollectionStoreDataType & { - datasetId: string; - name: string; - type: DatasetCollectionTypeEnum; - - fileId?: string; - rawLink?: string; - externalFileId?: string; - externalFileUrl?: string; - apiFileId?: string; - apiFileParentId?: string; //when file is imported by folder, the parentId is the folderId - - rawTextLength?: number; - hashRawText?: string; - - tags?: string[]; - - createTime?: Date; - updateTime?: Date; -}; - -export type ApiCreateDatasetCollectionParams = DatasetCollectionStoreDataType & { - datasetId: string; - tags?: string[]; -}; -export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - name: string; - text: string; -}; -export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - link: string; -}; -export type ApiDatasetCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - name: string; - apiFileId: string; -}; -export type ApiDatasetCreateDatasetCollectionV2Params = ApiCreateDatasetCollectionParams & { - apiFiles: APIFileItemType[]; -}; -export type FileIdCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - fileId: string; -}; -export type reTrainingDatasetFileCollectionParams = DatasetCollectionStoreDataType & { - datasetId: string; - collectionId: string; -}; -export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - fileMetadata?: Record; - collectionMetadata?: Record; -}; -export type CsvTableCreateDatasetCollectionParams = { - datasetId: string; - parentId?: string; - fileId: string; -}; -export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - externalFileId?: string; - externalFileUrl: string; - filename?: string; -}; -export type ImageCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { - collectionName: string; -}; - -/* ================= tag ===================== */ -export type CreateDatasetCollectionTagParams = { - datasetId: string; - tag: string; -}; -export type AddTagsToCollectionsParams = { - originCollectionIds: string[]; - collectionIds: string[]; - datasetId: string; - tag: string; -}; -export type UpdateDatasetCollectionTagParams = { - datasetId: string; - tagId: string; - tag: string; -}; - -/* ================= data ===================== */ -export type PgSearchRawType = { - id: string; - collection_id: string; - score: number; -}; -export type PushDatasetDataChunkProps = { - q?: string; - a?: string; - imageId?: string; - chunkIndex?: number; - indexes?: Omit[]; -}; - -export type PostDatasetSyncParams = { - datasetId: string; -}; - -export type PushDatasetDataProps = { - collectionId: string; - data: PushDatasetDataChunkProps[]; - trainingType?: DatasetCollectionDataProcessModeEnum; - indexSize?: number; - autoIndexes?: boolean; - imageIndex?: boolean; - prompt?: string; - - billId?: string; - - // Abandon - trainingMode?: DatasetCollectionDataProcessModeEnum; -}; -export type PushDatasetDataResponse = { - insertLen: number; -}; diff --git a/packages/global/core/dataset/controller.ts b/packages/global/core/dataset/controller.ts deleted file mode 100644 index ec724e8c63..0000000000 --- a/packages/global/core/dataset/controller.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { DatasetDataIndexItemType, DatasetDataSchemaType } from './type'; - -export type CreateDatasetDataProps = { - teamId: string; - tmbId: string; - datasetId: string; - collectionId: string; - chunkIndex?: number; - q: string; - a?: string; - imageId?: string; - indexes?: Omit[]; - indexPrefix?: string; -}; - -export type UpdateDatasetDataProps = { - dataId: string; - - q: string; - a?: string; - indexes?: (Omit & { - dataId?: string; // pg data id - })[]; - imageId?: string; - indexPrefix?: string; -}; - -export type PatchIndexesProps = - | { - type: 'create'; - index: Omit & { - dataId?: string; - }; - } - | { - type: 'update'; - index: DatasetDataIndexItemType; - } - | { - type: 'delete'; - index: DatasetDataIndexItemType; - } - | { - type: 'unChange'; - index: DatasetDataIndexItemType; - }; diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index c20710970e..b0d64874b5 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -65,7 +65,7 @@ export const DatasetSchema = z .object({ _id: ObjectIdSchema.meta({ description: '数据集 ID' }), parentId: ParentIdSchema.meta({ description: '父级 ID' }), - userId: ObjectIdSchema.meta({ description: '用户 ID' }), + userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), updateTime: z.date().meta({ description: '更新时间' }), @@ -143,7 +143,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({ metadata: z.record(z.string(), z.any()).optional().meta({ description: '其他元数据' }), customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }), - trainingType: z.enum(DatasetCollectionDataProcessModeEnum).meta({ description: '训练类型' }) + trainingType: z + .enum(DatasetCollectionDataProcessModeEnum) + .optional() + .meta({ description: '训练类型' }) }); export type DatasetCollectionSchemaType = z.infer; @@ -157,10 +160,20 @@ export type DatasetCollectionTagsSchemaType = z.infer; export const DatasetDataFieldSchema = z.object({ @@ -177,7 +190,7 @@ export type DatasetDataHistoryType = z.infer; export const DatasetDataSchema = DatasetDataFieldSchema.extend({ _id: ObjectIdSchema.meta({ description: '数据 ID' }), - userId: ObjectIdSchema.meta({ description: '用户 ID' }), + userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), @@ -206,7 +219,6 @@ export type DatasetDataTextSchemaType = z.infer; /* ===== Training ===== */ export const DatasetTrainingSchema = z.object({ _id: ObjectIdSchema.meta({ description: '训练 ID' }), - userId: ObjectIdSchema.meta({ description: '用户 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), @@ -227,7 +239,9 @@ export const DatasetTrainingSchema = z.object({ .array(DatasetDataIndexItemSchema.omit({ dataId: true })) .meta({ description: '向量索引' }), retryCount: z.number().meta({ description: '重试次数' }), - errorMsg: z.string().optional().meta({ description: '错误信息' }) + errorMsg: z.string().optional().meta({ description: '错误信息' }), + + userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true }) }); export type DatasetTrainingSchemaType = z.infer; @@ -322,6 +336,50 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({ }); export type DatasetDataItemType = z.infer; +// Update dataset data +export const UpdateDatasetDataPropsSchema = z.object({ + dataId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据 ID' + }), + q: z.string().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + a: z.string().optional().meta({ + example: 'FastGPT 是一个 AI Agent 构建平台', + description: '回答/补充文本' + }), + indexes: z.array(DatasetDataIndexOptionalSchema).optional().meta({ + description: '向量索引列表' + }), + imageId: z.string().optional().meta({ + description: '图片 ID' + }), + indexPrefix: z.string().optional().meta({ + description: '索引前缀标题' + }) +}); +export type UpdateDatasetDataPropsType = z.infer; + +// Create dataset data +export const CreateDatasetDataPropsSchema = z.object({ + teamId: ObjectIdSchema.meta({ description: '团队 ID' }), + tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), + datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), + collectionId: ObjectIdSchema.meta({ description: '集合 ID' }), + chunkIndex: z.int().min(0).optional().meta({ description: '块索引' }), + q: z.string().meta({ description: '问题/主文本' }), + a: z.string().optional().meta({ description: '回答/补充文本' }), + imageId: z.string().optional().meta({ description: '图片 ID' }), + indexes: z + .array(DatasetDataIndexItemSchema.omit({ dataId: true })) + .optional() + .meta({ description: '向量索引列表' }), + indexPrefix: z.string().optional().meta({ description: '索引前缀标题' }) +}); +export type CreateDatasetDataPropsType = z.infer; + /* --------------- file ---------------------- */ export const DatasetFileSchema = z.object({ _id: ObjectIdSchema.meta({ description: '文件 ID' }), diff --git a/packages/global/core/dataset/v2/api.ts b/packages/global/core/dataset/v2/api.ts deleted file mode 100644 index fdcd354052..0000000000 --- a/packages/global/core/dataset/v2/api.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ObjectIdSchema } from '../../../common/type/mongo'; -import z from 'zod'; - -export const PresignDatasetFileGetUrlSchema = z.union([ - z.object({ - key: z - .string() - .nonempty() - .refine((key) => key.startsWith('dataset/'), { - message: 'Invalid key format: must start with "dataset/"' - }) - .transform((k) => decodeURIComponent(k)), - preview: z.boolean().optional() - }), - z.object({ - collectionId: ObjectIdSchema - // datasetId: ObjectIdSchema - }) -]); -export type PresignDatasetFileGetUrlParams = z.infer; - -export const PresignDatasetFilePostUrlSchema = z.object({ - filename: z.string().min(1), - datasetId: ObjectIdSchema -}); -export type PresignDatasetFilePostUrlParams = z.infer; - -export const ShortPreviewLinkSchema = z.object({ - k: z - .string() - .nonempty() - .transform((k) => `chat:temp_file:${decodeURIComponent(k)}`) -}); -export type ShortPreviewLinkParams = z.infer; diff --git a/packages/global/core/workflow/node/agent/skillTools.ts b/packages/global/core/workflow/node/agent/skillTools.ts index c365f21008..96077748bd 100644 --- a/packages/global/core/workflow/node/agent/skillTools.ts +++ b/packages/global/core/workflow/node/agent/skillTools.ts @@ -1,5 +1,5 @@ import z from 'zod'; -import type { ChatCompletionTool } from '../../../ai/type'; +import type { ChatCompletionTool } from '../../../ai/llm/type'; export enum SandboxToolIds { readFile = 'sandbox_read_file', diff --git a/packages/global/core/workflow/runtime/type.ts b/packages/global/core/workflow/runtime/type.ts index 98413fced1..08a9eaca6a 100644 --- a/packages/global/core/workflow/runtime/type.ts +++ b/packages/global/core/workflow/runtime/type.ts @@ -18,7 +18,7 @@ import { ReadFileNodeResponseSchema } from '../template/system/readFiles/type'; import type { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type'; import type { AiChatQuoteRoleType } from '../template/system/aiChat/type'; import type { OpenaiAccountType } from '../../../support/user/team/type'; -import { CompletionFinishReasonSchema } from '../../ai/type'; +import { CompletionFinishReasonSchema } from '../../ai/llm/type'; import type { InteractiveNodeResponseType, WorkflowInteractiveResponseType diff --git a/packages/global/core/workflow/template/system/interactive/type.ts b/packages/global/core/workflow/template/system/interactive/type.ts index ae7b8e8e6c..d0b43cf9bb 100644 --- a/packages/global/core/workflow/template/system/interactive/type.ts +++ b/packages/global/core/workflow/template/system/interactive/type.ts @@ -1,10 +1,10 @@ import { NodeOutputItemSchema } from '../../../runtime/type'; import { FlowNodeInputTypeEnum } from '../../../../../core/workflow/node/constant'; import { WorkflowIOValueTypeEnum } from '../../../../../core/workflow/constants'; -import type { ChatCompletionMessageParam } from '../../../../ai/type'; import { AppFileSelectConfigTypeSchema } from '../../../../app/type/config.schema'; import { RuntimeEdgeItemTypeSchema } from '../../../type/edge'; import z from 'zod'; +import { ChatCompletionMessageParamSchema } from '../../../../ai/llm/type'; export const InteractiveBasicTypeSchema = z.object({ entryNodeIds: z.array(z.string()), @@ -42,21 +42,12 @@ export const ToolCallChildrenInteractiveSchema = z.object({ params: z.object({ childrenResponse: z.any(), toolParams: z.object({ - memoryRequestMessages: z.array(z.any()), // 这轮工具中，产生的新的 messages + memoryRequestMessages: z.array(ChatCompletionMessageParamSchema), // 这轮工具中，产生的新的 messages toolCallId: z.string() // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response }) }) }); -export type ToolCallChildrenInteractive = InteractiveNodeType & { - type: 'toolChildrenInteractive'; - params: { - childrenResponse: WorkflowInteractiveResponseType; - toolParams: { - memoryRequestMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages - toolCallId: string; // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response - }; - }; -}; +export type ToolCallChildrenInteractive = z.infer; // Loop bode export const LoopInteractiveSchema = z.object({ diff --git a/packages/global/openapi/api.ts b/packages/global/openapi/api.ts index e08ea67350..de6b886915 100644 --- a/packages/global/openapi/api.ts +++ b/packages/global/openapi/api.ts @@ -16,7 +16,12 @@ export type PaginationProps = T & { pageNum: number | string; }>; -export const PaginationResponseSchema = (itemSchema: T) => +export const PaginationResponseSchema = ( + itemSchema: T +): z.ZodObject<{ + total: z.ZodDefault>; + list: z.ZodDefault>>; +}> => z.object({ total: z.number().optional().default(0), list: z.array(itemSchema).optional().default([]) diff --git a/packages/global/openapi/core/ai/sandbox/api.ts b/packages/global/openapi/core/ai/sandbox/api.ts index cb29592833..c2c5e3f6df 100644 --- a/packages/global/openapi/core/ai/sandbox/api.ts +++ b/packages/global/openapi/core/ai/sandbox/api.ts @@ -52,7 +52,7 @@ export type SandboxReadBody = z.infer; export const SandboxReadResponseSchema = z .string() - .openapi({ format: 'binary', description: '文件内容流' }); + .meta({ format: 'binary', description: '文件内容流' }); /** * 下载文件或目录 - 请求体（响应为文件流或 ZIP） @@ -64,7 +64,7 @@ export type SandboxDownloadBody = z.input; export const SandboxDownloadResponseSchema = z .string() - .openapi({ format: 'binary', description: '文件流或 ZIP 包' }); + .meta({ format: 'binary', description: '文件流或 ZIP 包' }); /** * 检查沙盒是否存在 diff --git a/packages/global/openapi/core/chat/completion/api.ts b/packages/global/openapi/core/chat/completion/api.ts new file mode 100644 index 0000000000..64476680a3 --- /dev/null +++ b/packages/global/openapi/core/chat/completion/api.ts @@ -0,0 +1,144 @@ +import z from 'zod'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { ChatCompletionMessageParamSchema } from '../../../../core/ai/llm/type'; +import { getNanoid } from '../../../../common/string/tools'; +import { AppChatConfigTypeSchema, AppSchemaTypeSchema } from '../../../../core/app/type'; +import { AuthUserTypeEnum } from '../../../../support/permission/constant'; +import { OutLinkChatAuthSchema } from '../../../../support/permission/chat'; +import { StoreNodeItemTypeSchema } from '../../../../core/workflow/type/node'; +import { StoreEdgeItemTypeSchema } from '../../../../core/workflow/type/edge'; + +const WebCompletionsSchema = z.object({ + chatId: z + .string() + .max(1024) + .optional() + .meta({ description: '聊天ID, 传入的话会自动获取历史记录，不传入则认为是新对话' }), + appId: ObjectIdSchema.optional(), + customUid: z.string().max(1024).optional().meta({ description: '自定义用户ID(分享链接)' }), + metadata: z.record(z.string(), z.any()).optional().meta({ description: '元数据' }) +}); + +// completions 接口实际上并没有用完所有字段，所以这里就取局部即可 +const ChatCompletionCreateParamsSchema = z.object({ + messages: z.array(ChatCompletionMessageParamSchema).optional().default([]).meta({ + description: '消息列表' + }), + stream: z.boolean().optional().default(false).meta({ + description: '是否流式返回' + }) +}); + +export const CompletionsPropsSchema = OutLinkChatAuthSchema.extend(WebCompletionsSchema.shape) + .extend(ChatCompletionCreateParamsSchema.shape) + .extend({ + variables: z.record(z.string(), z.any()).optional().default({}).meta({ + description: '全局变量或插件输入' + }), + responseChatItemId: z + .string() + .optional() + .default(() => getNanoid()) + .meta({ + description: '自定义响应的 assistant 的消息 ID，如果不传入，则自动生成一个' + }), + detail: z.boolean().optional().default(false).meta({ + description: '是否返回详细信息，包括 reasoning_content, tool_calls, usage 等' + }), + retainDatasetCite: z.boolean().optional().default(false).meta({ + description: '是否保留数据集引用' + }), + showSkillReferences: z.boolean().optional().default(false).meta({ + description: '是否显示技能引用' + }) + }); +export type CompletionsProps = z.infer; + +/* =============== Response =============== */ + +const ChatCompletionResponseMessageSchema = z.object({ + role: z.literal('assistant').meta({ description: '消息角色' }), + content: z.any().meta({ + description: + '消息内容。普通对话为字符串；detail=true 或工作流命中交互节点时，可能为带 type 字段的对象数组（type 取值: text / interactive / tool / file / reasoning）' + }), + reasoning_content: z.string().optional().meta({ description: '思考过程内容（仅推理模型有）' }) +}); + +const ChatCompletionChoiceSchema = z.object({ + message: ChatCompletionResponseMessageSchema.meta({ description: '助手消息' }), + finish_reason: z.string().meta({ description: '完成原因，例如 stop' }), + index: z.number().meta({ description: '选项索引' }) +}); + +const ChatCompletionUsageSchema = z.object({ + prompt_tokens: z + .literal(1) + .meta({ description: '固定为 1，需要从 detail 中计算每一个节点的token数' }), + completion_tokens: z + .literal(1) + .meta({ description: '固定为 1，需要从 detail 中计算每一个节点的token数' }), + total_tokens: z + .literal(1) + .meta({ description: '固定为 1，需要从 detail 中计算每一个节点的token数' }) +}); + +export const CompletionsResponseSchema = z.object({ + id: z.string().meta({ description: '对话 ID（chatId）' }), + model: z.literal('').meta({ description: '模型名称，v1 接口固定为空字符串' }), + usage: ChatCompletionUsageSchema.meta({ + description: 'Token 用量。v1 接口为占位值，需要时请从 responseData 计算' + }), + choices: z.array(ChatCompletionChoiceSchema).meta({ description: '回复选项列表' }), + responseData: z.array(z.any()).optional().meta({ + description: + '各节点详细响应数据（仅 detail=true 时返回）。每项是一个节点的执行结果，常见字段如 moduleName / moduleType / runningTime / quoteList 等' + }), + newVariables: z + .record(z.string(), z.any()) + .optional() + .meta({ description: '工作流执行后更新的变量（仅 detail=true 时返回）' }) +}); +export type CompletionsResponseType = z.infer; + +export const AuthResponseSchema = z.object({ + teamId: ObjectIdSchema.meta({ description: '团队ID' }), + tmbId: ObjectIdSchema.meta({ description: '团队成员ID' }), + app: AppSchemaTypeSchema.meta({ description: '应用' }), + showCite: z.boolean().default(false).optional().meta({ + description: '是否显示引用' + }), + showRunningStatus: z.boolean().default(false).optional().meta({ + description: '是否显示运行状态' + }), + showSkillReferences: z.boolean().default(false).optional().meta({ + description: '是否显示技能引用' + }), + authType: z.enum(AuthUserTypeEnum).meta({ description: '认证类型' }), + apikey: z.string().optional().meta({ description: 'API密钥' }), + responseAllData: z.boolean().meta({ + description: '是否返回所有数据' + }), + outLinkUserId: z.string().optional().meta({ description: '外部链接用户ID' }), + sourceName: z.string().optional().meta({ description: '来源名称' }) +}); +export type AuthResponseType = z.infer; + +/* ====== Chat test ====== */ +export const ChatTestPropsSchema = z.object({ + messages: z.array(ChatCompletionMessageParamSchema).meta({ description: '消息列表' }), + responseChatItemId: z + .string() + .optional() + .meta({ description: '自定义响应的 assistant 的消息 ID，如果不传入，则自动生成一个' }), + nodes: z.array(StoreNodeItemTypeSchema).meta({ description: '节点列表' }), + edges: z.array(StoreEdgeItemTypeSchema).meta({ description: '边列表' }), + chatConfig: AppChatConfigTypeSchema.meta({ description: '聊天配置' }), + variables: z.record(z.string(), z.any()).optional().default({}).meta({ + description: '全局变量或插件输入' + }), + appId: ObjectIdSchema.meta({ description: '应用ID' }), + appName: z.string().meta({ description: '应用名称' }), + chatId: z.string().meta({ description: '聊天ID' }) +}); +export type ChatTestPropsType = z.infer; diff --git a/packages/global/openapi/core/chat/completion/index.ts b/packages/global/openapi/core/chat/completion/index.ts new file mode 100644 index 0000000000..7b8fcd527c --- /dev/null +++ b/packages/global/openapi/core/chat/completion/index.ts @@ -0,0 +1,455 @@ +import type { OpenAPIPath } from '../../../type'; +import { TagsMap } from '../../../tag'; +import { ChatTestPropsSchema, CompletionsPropsSchema, CompletionsResponseSchema } from './api'; + +/* =============== Request examples =============== */ + +const basicRequestExample = { + chatId: 'my_chatId', + stream: false, + detail: false, + responseChatItemId: 'my_responseChatItemId', + variables: { + uid: 'asdfadsfasfd2323', + name: '张三' + }, + messages: [ + { + role: 'user', + content: '导演是谁' + } + ] +}; + +const imageFileRequestExample = { + chatId: 'abcd', + stream: false, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: '导演是谁' }, + { type: 'image_url', image_url: { url: '图片链接' } }, + { + type: 'file_url', + name: '文件名', + url: '文档链接，支持 txt md html word pdf ppt csv excel' + } + ] + } + ] +}; + +const userSelectInteractiveRequestExample = { + stream: true, + detail: true, + chatId: '22222231', + messages: [{ role: 'user', content: 'Confirm' }] +}; + +const userInputInteractiveRequestExample = { + stream: true, + detail: true, + chatId: '22231', + messages: [ + { + role: 'user', + content: '{"测试 1":"这是输入框的内容","测试 2":666}' + } + ] +}; + +/* =============== Response examples =============== */ + +// detail=false, stream=false +const detailFalseStreamFalseExample = { + id: 'adsfasf', + model: '', + usage: { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 1 + }, + choices: [ + { + message: { + role: 'assistant', + content: '电影《铃芽之旅》的导演是新海诚。' + }, + finish_reason: 'stop', + index: 0 + } + ] +}; + +// detail=true, stream=false +const detailTrueStreamFalseExample = { + responseData: [ + { + moduleName: 'Dataset Search', + price: 1.2000000000000002, + model: 'Embedding-2', + tokens: 6, + similarity: 0.61, + limit: 3 + }, + { + moduleName: 'AI Chat', + price: 454.5, + model: 'FastAI-4k', + tokens: 303, + question: '导演是谁', + answer: '电影《铃芽之旅》的导演是新海诚。', + maxToken: 2050, + quoteList: [ + { + dataset_id: '646627f4f7b896cfd8910e38', + id: '8099', + q: '本作的主人公是谁？', + a: '本作的主人公是名叫铃芽的少女。', + source: '手动修改' + }, + { + dataset_id: '646627f4f7b896cfd8910e38', + id: '19339', + q: '电影《铃芽之旅》的导演是谁？', + a: '电影《铃芽之旅》的导演是新海诚。', + source: '手动修改' + } + ], + completeMessages: [ + { + obj: 'System', + value: + '下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》的导演是谁？\n电影《铃芽之旅》的导演是新海诚。]\n' + }, + { obj: 'Human', value: '导演是谁' }, + { obj: 'AI', value: '电影《铃芽之旅》的导演是新海诚。' } + ] + } + ], + newVariables: { + uid: 'asdfadsfasfd2323', + name: '张三' + }, + id: '', + model: '', + usage: { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 1 + }, + choices: [ + { + message: { + role: 'assistant', + content: '电影《铃芽之旅》的导演是新海诚。' + }, + finish_reason: 'stop', + index: 0 + } + ] +}; + +// 交互节点-用户选择 (非流式响应,从 choices 中获取 type=interactive) +const interactiveUserSelectResponseExample = { + id: 'chatId', + model: '', + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 }, + choices: [ + { + message: { + role: 'assistant', + content: [ + { + type: 'interactive', + interactive: { + type: 'userSelect', + params: { + description: '测试', + userSelectOptions: [ + { value: 'Confirm', key: 'option1' }, + { value: 'Cancel', key: 'option2' } + ] + } + } + } + ] + }, + finish_reason: 'stop', + index: 0 + } + ] +}; + +// 交互节点-表单输入 (非流式响应) +const interactiveUserInputResponseExample = { + id: 'chatId', + model: '', + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 }, + choices: [ + { + message: { + role: 'assistant', + content: [ + { + type: 'interactive', + interactive: { + type: 'userInput', + params: { + description: '测试', + inputForm: [ + { + type: 'input', + key: '测试 1', + label: '测试 1', + description: '', + value: '', + defaultValue: '', + valueType: 'string', + required: false, + list: [{ label: '', value: '' }] + }, + { + type: 'numberInput', + key: '测试 2', + label: '测试 2', + description: '', + value: '', + defaultValue: '', + valueType: 'number', + required: false, + list: [{ label: '', value: '' }] + } + ] + } + } + } + ] + }, + finish_reason: 'stop', + index: 0 + } + ] +}; + +// detail=false, stream=true +// 注：示例为简化版，省略了 object/created 等占位字段；真实响应每行还会带这些字段 +const detailFalseStreamTrueExample = `data: {"choices":[{"delta":{"content":""}}]} +data: {"choices":[{"delta":{"content":"电影"}}]} +data: {"choices":[{"delta":{"content":"《铃芽之旅》"}}]} +data: {"choices":[{"delta":{"content":"的导演"}}]} +data: {"choices":[{"delta":{"content":"是新海诚。"}}]} +data: {"choices":[{"delta":{},"finish_reason":"stop"}]} +data: [DONE]`; + +// detail=true, stream=true +const detailTrueStreamTrueExample = `event: flowNodeStatus +data: {"status":"running","name":"知识库搜索"} + +event: flowNodeStatus +data: {"status":"running","name":"AI 对话"} + +event: answer +data: {"choices":[{"delta":{"content":"电影"}}]} + +event: answer +data: {"choices":[{"delta":{"content":"《铃芽之旅》"}}]} + +event: answer +data: {"choices":[{"delta":{"content":"的导演是新海诚。"}}]} + +event: answer +data: {"choices":[{"delta":{},"finish_reason":"stop"}]} + +event: answer +data: [DONE] + +event: flowResponses +data: [{"moduleName":"知识库搜索","runningTime":1.78}, ...]`; + +// 交互节点 stream 响应 +const interactiveStreamExample = `event: interactive +data: {"interactive":{"type":"userSelect","params":{...}}} + +event: answer +data: [DONE]`; + +/* =============== OpenAPI Path =============== */ + +export const ChatCompletionPath: OpenAPIPath = { + '/v1/chat/completions': { + post: { + tags: [TagsMap.chatController], + summary: '请求对话 Agent 和工作流', + description: `v1 对话接口兼容 GPT 的接口。如果你的项目使用的是标准的 GPT 官方接口，可以直接通过修改 BaseUrl 和 Authorization 来访问 FastGPT 应用。 + +**注意事项** + +- 该接口的 API Key 需使用「应用特定的 key」，否则会报错。 +- 传入的 \`model\`、\`temperature\` 等参数字段均无效，这些字段由编排决定，不会根据 API 参数改变。 +- 不会返回实际消耗 \`Token\` 值。如果需要，可以设置 \`detail=true\`，并手动计算 \`responseData\` 里的 \`tokens\` 值。 + +**chatId 行为** + +- 不传入（或为空）：不使用 FastGPT 提供的上下文功能，完全通过传入的 \`messages\` 构建上下文。 +- 非空字符串：使用 chatId 进行对话，自动从 FastGPT 数据库取历史记录，并使用 \`messages\` 数组最后一个内容作为用户问题，其余 message 会被忽略。请自行确保 chatId 唯一，长度小于 250。 + +**stream / detail 组合** + +- \`detail=false, stream=false\`：返回精简 JSON（仅 \`choices/usage/id\` 等）。 +- \`detail=false, stream=true\`：返回兼容 GPT 的 SSE 流。 +- \`detail=true, stream=false\`：在 JSON 中额外包含 \`responseData\`（各节点详细信息）和 \`newVariables\`。 +- \`detail=true, stream=true\`：返回多 event 的 SSE 流（\`answer\` / \`flowNodeStatus\` / \`flowResponses\` 等）。 + +**event 取值**（\`stream=true\` 场景下，\`detail=true\` 才会有非 answer 的 event） + +- \`answer\`：返回给客户端的文本（最终会算作回答）。 +- \`fastAnswer\`：指定回复返回给客户端的文本（最终会算作回答）。 +- \`toolCall\` / \`toolParams\` / \`toolResponse\`：工具相关。 +- \`flowNodeStatus\`：运行到的节点状态。 +- \`flowResponses\`：节点完整响应。 +- \`updateVariables\`：更新变量。 +- \`interactive\`：交互节点配置。 +- \`error\`：报错。 + +**交互节点** + +如果工作流中包含交互节点，需要设置 \`detail=true\`： + +- \`stream=true\`：可从 \`event=interactive\` 数据中获取交互节点的配置。 +- \`stream=false\`：可从 \`choices\` 中获取 \`type=interactive\` 的元素。 + +接收到交互节点信息后，可以根据数据进行 UI 渲染并引导用户输入/选择，然后再次调用本接口继续工作流： + +- 用户选择：直接将选择结果作为 user message 的 content 传入。 +- 表单输入：将输入内容以对象形式序列化为字符串，作为 user message 的 content 传入；务必确保 \`chatId\` 一致。 + +--- + +## SSE 响应示例（\`stream=true\`） + +OpenAPI 渲染器对 \`text/event-stream\` 示例支持有限，因此 SSE 示例在此以 markdown 形式给出。 + +> 下方示例为简化版本（省略了 \`id\`/\`object\`/\`created\` 等占位字段，\`...\` 表示省略的内容）。\`interactive.params\` 的完整结构请参考下方 Responses → application/json 中的 \`interactiveUserSelect\` / \`interactiveUserInput\` example。 + +### \`detail=false, stream=true\` + +兼容 GPT 的 SSE 流，仅包含 \`data\`（无 \`event\`）： + +\`\`\`text +${detailFalseStreamTrueExample} +\`\`\` + +### \`detail=true, stream=true\` + +包含 \`flowNodeStatus\` / \`answer\` / \`flowResponses\` 等多种 event： + +\`\`\`text +${detailTrueStreamTrueExample} +\`\`\` + +### 交互节点 stream 响应 + +\`detail=true, stream=true\` 时，工作流命中交互节点： + +\`\`\`text +${interactiveStreamExample} +\`\`\` +`, + requestBody: { + content: { + 'application/json': { + schema: CompletionsPropsSchema, + examples: { + basic: { + summary: '基础请求示例', + value: basicRequestExample + }, + imageFile: { + summary: '图片/文件请求示例', + description: + '仅 messages 有部分区别，其他参数一致。目前不支持上传文件，需自行上传到对象存储后传入文件链接', + value: imageFileRequestExample + }, + interactiveUserSelect: { + summary: '交互节点-用户选择 继续运行', + description: '直接传递选择结果作为 user message 的 content', + value: userSelectInteractiveRequestExample + }, + interactiveUserInput: { + summary: '交互节点-表单输入 继续运行', + description: + '将表单输入内容以对象形式序列化成字符串，作为 user message 的 content。对象 key 对应表单 key', + value: userInputInteractiveRequestExample + } + } + } + } + }, + responses: { + 200: { + content: { + 'application/json': { + schema: CompletionsResponseSchema, + examples: { + detailFalseStreamFalse: { + summary: 'detail=false, stream=false 响应', + description: '精简 JSON 响应，仅包含基础字段', + value: detailFalseStreamFalseExample + }, + detailTrueStreamFalse: { + summary: 'detail=true, stream=false 响应', + description: + '在精简响应基础上额外包含 responseData（各节点详细信息）和 newVariables', + value: detailTrueStreamFalseExample + }, + interactiveUserSelect: { + summary: '交互节点-用户选择 响应', + description: + '工作流命中用户选择交互节点。从 choices[].message.content 中获取 type=interactive 的元素', + value: interactiveUserSelectResponseExample + }, + interactiveUserInput: { + summary: '交互节点-表单输入 响应', + description: + '工作流命中表单输入交互节点。从 choices[].message.content 中获取 type=interactive 的元素', + value: interactiveUserInputResponseExample + } + } + }, + 'text/event-stream': { + examples: {} + } + } + } + } + } + }, + '/core/chat/chatTest': { + post: { + tags: [TagsMap.chatController], + summary: '测试对话（调试）', + description: `调试运行 Agent / 工作流。接收完整的节点、边和聊天配置，按测试模式执行一次工作流，通过 SSE 流式返回运行结果与节点状态。仅用于 FastGPT 编排页面的调试预览，不建议作为对外接口使用。 + +响应为 SSE 流，event 结构与 \`/v1/chat/completions\` \`detail=true, stream=true\` 一致，包括 \`answer\` / \`flowNodeStatus\` / \`flowResponses\` / \`interactive\` 等。`, + requestBody: { + content: { + 'application/json': { + schema: ChatTestPropsSchema + } + } + }, + responses: { + 200: { + description: 'SSE 流式响应', + content: { + 'text/event-stream': { + examples: {} + } + } + } + } + } + } +}; diff --git a/packages/global/openapi/core/chat/file/index.ts b/packages/global/openapi/core/chat/file/index.ts index 239035e43d..5314389140 100644 --- a/packages/global/openapi/core/chat/file/index.ts +++ b/packages/global/openapi/core/chat/file/index.ts @@ -1,7 +1,7 @@ import type { OpenAPIPath } from '../../../type'; import { TagsMap } from '../../../tag'; import { PresignChatFilePostUrlSchema, PresignChatFileGetUrlSchema } from './api'; -import { CreatePostPresignedUrlResultSchema } from '../../../../../service/common/s3/type'; +import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type'; import { z } from 'zod'; export const ChatFilePath: OpenAPIPath = { @@ -22,7 +22,7 @@ export const ChatFilePath: OpenAPIPath = { description: '成功上传对话文件预签名 URL', content: { 'application/json': { - schema: CreatePostPresignedUrlResultSchema + schema: CreatePostPresignedUrlResponseSchema } } } diff --git a/packages/global/openapi/core/chat/index.ts b/packages/global/openapi/core/chat/index.ts index 63a2c7ab6d..250339dbab 100644 --- a/packages/global/openapi/core/chat/index.ts +++ b/packages/global/openapi/core/chat/index.ts @@ -11,6 +11,7 @@ import { ChatInputGuidePath } from './inputGuide/index'; import { OutLinkChatPath } from './outLink/index'; import { ChatRecordPath } from './record/index'; import { ChatFilePath } from './file'; +import { ChatCompletionPath } from './completion'; export const ChatPath: OpenAPIPath = { ...ChatFeedbackPath, @@ -23,6 +24,7 @@ export const ChatPath: OpenAPIPath = { ...ChatInputGuidePath, ...OutLinkChatPath, ...ChatRecordPath, + ...ChatCompletionPath, '/core/chat/recentlyUsed': { get: { diff --git a/packages/global/openapi/core/dataset/api.ts b/packages/global/openapi/core/dataset/api.ts index a86e5339b5..fc46cdec17 100644 --- a/packages/global/openapi/core/dataset/api.ts +++ b/packages/global/openapi/core/dataset/api.ts @@ -411,3 +411,41 @@ export const ExportDatasetQuerySchema = z.object({ }) }); export type ExportDatasetQuery = z.infer; + +/* ============================================================================ + * API: 获取知识库引用权限 + * Route: GET /api/core/dataset/getPermission + * ============================================================================ */ +export const GetDatasetPermissionQuerySchema = z.object({ + id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }) +}); +export type GetDatasetPermissionQuery = z.infer; + +export const GetDatasetPermissionResponseSchema = z.object({ + datasetName: z.string().meta({ + example: '产品文档知识库', + description: '知识库名称' + }), + permission: z.object({ + hasWritePer: z.boolean().meta({ + example: true, + description: '是否有写权限' + }), + hasReadPer: z.boolean().meta({ + example: true, + description: '是否有读权限' + }) + }) +}); +export type GetDatasetPermissionResponse = z.infer; + +/* ============================================================================ + * 数据集同步入参 + * ============================================================================ */ +export const PostDatasetSyncBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }) +}); +export type PostDatasetSyncParams = z.infer; diff --git a/packages/global/openapi/core/dataset/apiDataset/api.ts b/packages/global/openapi/core/dataset/apiDataset/api.ts new file mode 100644 index 0000000000..09e137805c --- /dev/null +++ b/packages/global/openapi/core/dataset/apiDataset/api.ts @@ -0,0 +1,102 @@ +import { z } from 'zod'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { ParentIdSchema } from '../../../../common/parentFolder/type'; +import { + APIFileItemSchema, + ApiDatasetServerSchema +} from '../../../../core/dataset/apiDataset/type'; + +/* ============================================================================ + * API: 获取第三方知识库目录（仅文件夹） + * Route: POST /api/core/dataset/apiDataset/getCatalog + * ============================================================================ */ +export const GetApiDatasetCatalogBodySchema = z.object({ + searchKey: z.string().optional().meta({ + example: '产品文档', + description: '搜索关键词' + }), + parentId: ParentIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '父级节点 ID，不传或 null 表示根目录' + }), + apiDatasetServer: ApiDatasetServerSchema.optional().meta({ + description: '第三方知识库服务器配置（API/飞书/语雀）' + }) +}); +export type GetApiDatasetCatalogBody = z.infer; + +export const GetApiDatasetCatalogResponseSchema = z.array(APIFileItemSchema).meta({ + description: '目录列表（仅包含 hasChild = true 的节点）' +}); +export type GetApiDatasetCatalogResponse = z.infer; + +/* ============================================================================ + * API: 获取第三方知识库节点完整路径 + * Route: POST /api/core/dataset/apiDataset/getPathNames + * ============================================================================ */ +export const GetApiDatasetPathNamesBodySchema = z.object({ + datasetId: ObjectIdSchema.optional().meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID，传入时从知识库配置中读取 apiDatasetServer' + }), + parentId: ParentIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '当前节点 ID，不传或 null 时返回空字符串' + }), + apiDatasetServer: ApiDatasetServerSchema.optional().meta({ + description: '第三方知识库服务器配置，datasetId 不传时必须提供' + }) +}); +export type GetApiDatasetPathNamesBody = z.infer; + +export const GetApiDatasetPathNamesResponseSchema = z.string().meta({ + example: '/根目录/产品文档/介绍', + description: '节点的完整路径字符串' +}); +export type GetApiDatasetPathNamesResponse = z.infer; + +/* ============================================================================ + * API: 获取第三方知识库文件列表 + * Route: POST /api/core/dataset/apiDataset/list + * ============================================================================ */ +export const GetApiDatasetFileListBodySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + searchKey: z.string().optional().meta({ + example: '产品文档', + description: '搜索关键词' + }), + parentId: ParentIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '父级节点 ID，不传或 null 表示根目录' + }) +}); +export type GetApiDatasetFileListBody = z.infer; + +export const GetApiDatasetFileListResponseSchema = z.array(APIFileItemSchema).meta({ + description: '文件/文件夹列表' +}); +export type GetApiDatasetFileListResponse = z.infer; + +/* ============================================================================ + * API: 获取第三方知识库已存在的 apiFileId 列表 + * Route: GET /api/core/dataset/apiDataset/listExistId + * ============================================================================ */ +export const GetApiDatasetFileListExistIdQuerySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }) +}); +export type GetApiDatasetFileListExistIdQuery = z.infer< + typeof GetApiDatasetFileListExistIdQuerySchema +>; + +export const GetApiDatasetFileListExistIdResponseSchema = z.array(z.string()).meta({ + description: '已存在集合对应的 apiFileId 列表' +}); +export type GetApiDatasetFileListExistIdResponse = z.infer< + typeof GetApiDatasetFileListExistIdResponseSchema +>; diff --git a/packages/global/openapi/core/dataset/apiDataset/index.ts b/packages/global/openapi/core/dataset/apiDataset/index.ts new file mode 100644 index 0000000000..6846a5807a --- /dev/null +++ b/packages/global/openapi/core/dataset/apiDataset/index.ts @@ -0,0 +1,84 @@ +import type { OpenAPIPath } from '../../../type'; +import { TagsMap } from '../../../tag'; +import { + GetApiDatasetCatalogBodySchema, + GetApiDatasetFileListBodySchema, + GetApiDatasetFileListExistIdQuerySchema, + GetApiDatasetPathNamesBodySchema +} from './api'; + +export const ApiDatasetPath: OpenAPIPath = { + '/core/dataset/apiDataset/getCatalog': { + post: { + summary: '获取第三方知识库目录', + description: + '列出第三方知识库（API/飞书/语雀）的目录节点，仅返回包含子节点的文件夹，用于构建目录选择器', + tags: [TagsMap.datasetApiDataset], + requestBody: { + content: { + 'application/json': { + schema: GetApiDatasetCatalogBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回目录节点列表' + } + } + } + }, + '/core/dataset/apiDataset/getPathNames': { + post: { + summary: '获取第三方知识库节点路径', + description: '根据节点 ID 沿父级链向上查找，拼接出完整路径字符串，用于面包屑或路径展示', + tags: [TagsMap.datasetApiDataset], + requestBody: { + content: { + 'application/json': { + schema: GetApiDatasetPathNamesBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回节点的完整路径' + } + } + } + }, + '/core/dataset/apiDataset/list': { + post: { + summary: '获取第三方知识库文件列表', + description: '列出指定知识库下的第三方文件/文件夹，支持关键词与父级筛选', + tags: [TagsMap.datasetApiDataset], + requestBody: { + content: { + 'application/json': { + schema: GetApiDatasetFileListBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回文件/文件夹列表' + } + } + } + }, + '/core/dataset/apiDataset/listExistId': { + get: { + summary: '获取已导入的第三方文件 ID 列表', + description: '返回指定知识库下已创建集合所对应的 apiFileId，用于导入时的去重判断', + tags: [TagsMap.datasetApiDataset], + requestParams: { + query: GetApiDatasetFileListExistIdQuerySchema + }, + responses: { + 200: { + description: '成功返回已存在的 apiFileId 列表' + } + } + } + } +}; diff --git a/packages/global/openapi/core/dataset/collection/api.ts b/packages/global/openapi/core/dataset/collection/api.ts index 44e0b2f003..a5559601ff 100644 --- a/packages/global/openapi/core/dataset/collection/api.ts +++ b/packages/global/openapi/core/dataset/collection/api.ts @@ -1,6 +1,23 @@ -import { ParentIdSchema } from '../../../../common/parentFolder/type'; +import { + GetPathPropsSchema, + ParentIdSchema, + ParentTreePathItemSchema +} from '../../../../common/parentFolder/type'; import { ObjectIdSchema } from '../../../../common/type/mongo'; import { OutLinkChatAuthSchema } from '../../../../support/permission/chat'; +import { + DatasetCollectionSyncResultEnum, + DatasetCollectionTypeEnum, + DatasetCollectionDataProcessModeEnum, + TrainingModeEnum +} from '../../../../core/dataset/constants'; +import { + ChunkSettingsSchema, + DatasetCollectionItemSchema, + DatasetCollectionSchema +} from '../../../../core/dataset/type'; +import { PermissionSchema } from '../../../../support/permission/controller'; +import { PaginationResponseSchema, PaginationSchema } from '../../../api'; import z from 'zod'; // ============= Scroll Collections ============= @@ -62,3 +79,130 @@ const ChatExportSchema = OutLinkChatAuthSchema.extend({ export const ExportCollectionBodySchema = z.union([BasicExportSchema, ChatExportSchema]); export type ExportCollectionBodyType = z.infer; + +// ============= Delete Collection ============= +export const DeleteCollectionQuerySchema = z.object({ + id: z.string().optional().meta({ description: '单个集合 ID（与 body.collectionIds 二选一）' }) +}); +export type DeleteCollectionQueryType = z.infer; + +export const DeleteCollectionBodySchema = z.object({ + collectionIds: z.array(z.string()).optional().meta({ description: '集合 ID 列表' }) +}); +export type DeleteCollectionBodyType = z.infer; + +// ============= Get Collection Detail ============= +export const GetCollectionDetailQuerySchema = z.object({ + id: z.string().meta({ description: '集合 ID' }) +}); +export type GetCollectionDetailQueryType = z.infer; + +export const GetCollectionDetailResponseSchema = DatasetCollectionItemSchema.meta({ + description: '集合详情' +}); +export type GetCollectionDetailResponseType = z.infer; + +// ============= List Collections V2 ============= +export const ListCollectionV2BodySchema = PaginationSchema.extend({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: z.string().nullable().optional().default(null).meta({ description: '父级目录 ID' }), + searchText: z.string().max(100).optional().default('').meta({ description: '搜索文本' }), + selectFolder: z.boolean().optional().default(false).meta({ description: '只返回文件夹' }), + filterTags: z.array(z.string()).optional().default([]).meta({ description: '过滤标签' }), + simple: z.boolean().optional().default(false).meta({ description: '简单模式（不统计数量）' }) +}); +export type ListCollectionV2BodyType = z.infer; + +// ============= List Collections V2 Response ============= +export const DatasetCollectionsListItemSchema = z.object({ + _id: ObjectIdSchema.meta({ description: '集合 ID' }), + parentId: DatasetCollectionSchema.shape.parentId, + tmbId: DatasetCollectionSchema.shape.tmbId, + name: DatasetCollectionSchema.shape.name, + type: DatasetCollectionSchema.shape.type, + createTime: DatasetCollectionSchema.shape.createTime, + updateTime: DatasetCollectionSchema.shape.updateTime, + forbid: DatasetCollectionSchema.shape.forbid, + trainingType: DatasetCollectionSchema.shape.trainingType, + tags: z.array(z.string()).optional().meta({ description: '标签' }), + + externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), + + fileId: z.string().optional().meta({ description: '文件 ID' }), + rawLink: z.string().optional().meta({ description: '原始链接' }), + permission: PermissionSchema, + dataAmount: z.number().meta({ description: '数据数量' }), + trainingAmount: z.number().meta({ description: '训练数量' }), + hasError: z.boolean().optional().meta({ description: '是否错误' }) +}); +export type DatasetCollectionsListItemType = z.infer; +export const ListCollectionV2ResponseSchema = PaginationResponseSchema( + DatasetCollectionsListItemSchema +); +export type ListCollectionV2ResponseType = z.infer; + +// ============= Get Collection Paths ============= +export const GetCollectionPathsQuerySchema = GetPathPropsSchema; +export type GetCollectionPathsQueryType = z.infer; + +export const GetCollectionPathsResponseSchema = z.array(ParentTreePathItemSchema); +export type GetCollectionPathsResponseType = z.infer; + +// ============= Read Collection Source ============= +export const ReadCollectionSourceBodySchema = OutLinkChatAuthSchema.extend({ + collectionId: ObjectIdSchema.meta({ description: '集合 ID' }), + appId: ObjectIdSchema.optional().meta({ description: '应用 ID（对话中使用）' }), + chatId: ObjectIdSchema.optional().meta({ description: '对话 ID（对话中使用）' }), + chatItemDataId: z.string().optional().meta({ description: '对话消息 ID（对话中使用）' }) +}); +export type ReadCollectionSourceBodyType = z.infer; + +export const ReadCollectionSourceResponseSchema = z.object({ + type: z.literal('url').meta({ description: '资源类型' }), + value: z.string().meta({ description: '资源 URL' }) +}); +export type ReadCollectionSourceResponseType = z.infer; + +// ============= Training Detail ============= +export const GetCollectionTrainingDetailQuerySchema = z.object({ + collectionId: z.string().meta({ description: '集合 ID' }) +}); +export type GetCollectionTrainingDetailQueryType = z.infer< + typeof GetCollectionTrainingDetailQuerySchema +>; + +const TrainingCountsSchema = z + .record(z.enum(TrainingModeEnum), z.number()) + .meta({ description: '各训练模式数量' }); + +export const GetCollectionTrainingDetailResponseSchema = z.object({ + trainingType: z + .enum(DatasetCollectionDataProcessModeEnum) + .optional() + .meta({ description: '训练类型' }), + advancedTraining: z + .object({ + customPdfParse: z.boolean().meta({ description: '自定义 PDF 解析' }), + imageIndex: z.boolean().meta({ description: '图片索引' }), + autoIndexes: z.boolean().meta({ description: '自动索引' }) + }) + .meta({ description: '高级训练配置' }), + queuedCounts: TrainingCountsSchema.meta({ description: '排队中数量' }), + trainingCounts: TrainingCountsSchema.meta({ description: '训练中数量' }), + errorCounts: TrainingCountsSchema.meta({ description: '错误数量' }), + trainedCount: z.number().meta({ description: '已训练数据量' }) +}); +export type GetCollectionTrainingDetailResponseType = z.infer< + typeof GetCollectionTrainingDetailResponseSchema +>; + +// ============= Sync Collection ============= +export const SyncCollectionBodySchema = z.object({ + collectionId: ObjectIdSchema.meta({ description: '集合 ID' }) +}); +export type SyncCollectionBodyType = z.infer; + +export const SyncCollectionResponseSchema = z.enum(DatasetCollectionSyncResultEnum).meta({ + description: '同步结果' +}); +export type SyncCollectionResponseType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/createApi.ts b/packages/global/openapi/core/dataset/collection/createApi.ts new file mode 100644 index 0000000000..4f0e9df2be --- /dev/null +++ b/packages/global/openapi/core/dataset/collection/createApi.ts @@ -0,0 +1,234 @@ +import z from 'zod'; +import { ChunkSettingsSchema } from '../../../../core/dataset/type'; +import { DatasetCollectionTypeEnum } from '../../../../core/dataset/constants'; +import { ParentIdSchema } from '../../../../common/parentFolder/type'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { APIFileItemSchema } from '../../../../core/dataset/apiDataset/type'; + +/* ============================================================================ + * 公共基础 Schema + * ============================================================================ */ + +// 集合存储数据基础 Schema（扩展自 ChunkSettings） +const DatasetCollectionStoreDataSchema = ChunkSettingsSchema.extend({ + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + metadata: z.record(z.string(), z.any()).optional().meta({ description: '元数据' }), + customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }) +}); + +// API 创建集合通用基础 Schema +export const ApiCreateCollectionBaseSchema = DatasetCollectionStoreDataSchema.extend({ + datasetId: z.string().meta({ description: '数据集 ID' }), + tags: z.array(z.string()).optional().meta({ description: '标签列表' }) +}); +export type ApiCreateDatasetCollectionParams = z.infer; + +// 集合创建带数据返回的 Response Schema（collectionId + insertResults） +export const CreateCollectionWithResultResponseSchema = z.object({ + collectionId: ObjectIdSchema.meta({ description: '新创建的集合 ID' }), + results: z + .object({ + insertLen: z.number().meta({ + example: 10, + description: '成功插入的数据条数' + }) + }) + .meta({ description: '数据插入结果' }) +}); +export type CreateCollectionWithResultResponseType = z.infer< + typeof CreateCollectionWithResultResponseSchema +>; + +/* ============================================================================ + * API: 创建集合（通用） + * Route: POST /core/dataset/collection/create + * ============================================================================ */ +export const CreateCollectionBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + name: z.string().meta({ description: '集合名称' }), + type: z + .enum([DatasetCollectionTypeEnum.folder, DatasetCollectionTypeEnum.virtual]) + .meta({ description: '集合类型（folder: 文件夹，virtual: 手动集合）' }), + tags: z.array(z.string()).optional().meta({ description: '标签列表' }) +}); +export type CreateCollectionBodyType = z.infer; + +export const CreateCollectionResponseSchema = ObjectIdSchema.meta({ + description: '新创建的集合 ID' +}); +export type CreateCollectionResponseType = z.infer; + +/* ============================================================================ + * API: 重新训练集合 + * Route: POST /core/dataset/collection/create/reTrainingCollection + * ============================================================================ */ +export const ReTrainingCollectionBodySchema = DatasetCollectionStoreDataSchema.extend({ + datasetId: z.string().meta({ description: '数据集 ID' }), + collectionId: z.string().meta({ description: '需要重新训练的集合 ID' }) +}); +export type ReTrainingCollectionBodyType = z.infer; + +export const ReTrainingCollectionResponseSchema = z.object({ + collectionId: ObjectIdSchema.meta({ description: '新集合 ID' }) +}); +export type ReTrainingCollectionResponseType = z.infer; + +/* ============================================================================ + * API: 通过文件 ID 创建集合 + * Route: POST /core/dataset/collection/create/fileId + * ============================================================================ */ +export const CreateCollectionByFileIdBodySchema = ApiCreateCollectionBaseSchema.extend({ + fileId: z.string().meta({ description: 'S3 文件对象键（必须是 dataset 路径下的文件）' }), + customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }) +}); +export type CreateCollectionByFileIdBodyType = z.infer; + +/* ============================================================================ + * API: 上传本地文件创建集合 + * Route: POST /core/dataset/collection/create/localFile + * Content-Type: multipart/form-data + * ============================================================================ */ +export const CreateCollectionByLocalFileBodySchema = ApiCreateCollectionBaseSchema; +export type CreateCollectionByLocalFileBodyType = z.infer< + typeof CreateCollectionByLocalFileBodySchema +>; + +// OpenAPI 文档专用：描述 multipart/form-data 的实际结构 +// file 字段为二进制文件；data 字段为 JSON 序列化的对象（encoding: application/json） +export const CreateCollectionByLocalFileFormSchema = z.object({ + file: z.any().meta({ format: 'binary', description: '上传的文件（二进制）' }), + data: CreateCollectionByLocalFileBodySchema.meta({ + description: '集合参数（JSON 序列化后传入）' + }) +}); + +/* ============================================================================ + * API: 通过链接创建集合 + * Route: POST /core/dataset/collection/create/link + * ============================================================================ */ +export const CreateLinkCollectionBodySchema = ApiCreateCollectionBaseSchema.extend({ + link: z.string().url().meta({ description: '链接 URL' }) +}); +export type CreateLinkCollectionBodyType = z.infer; + +/* ============================================================================ + * API: 通过文本创建集合 + * Route: POST /core/dataset/collection/create/text + * ============================================================================ */ +export const CreateTextCollectionBodySchema = ApiCreateCollectionBaseSchema.extend({ + name: z.string().meta({ description: '集合名称' }), + text: z.string().meta({ description: '文本内容' }) +}); +export type CreateTextCollectionBodyType = z.infer; + +/* ============================================================================ + * API: 通过 API 数据集创建集合（V1） + * Route: POST /core/dataset/collection/create/apiCollection + * ============================================================================ */ +export const CreateApiCollectionBodySchema = ApiCreateCollectionBaseSchema.extend({ + name: z.string().meta({ description: '集合名称' }), + apiFileId: z.string().meta({ description: 'API 文件 ID' }) +}); +export type CreateApiCollectionBodyType = z.infer; + +/* ============================================================================ + * API: 通过 API 数据集创建集合（V2，支持批量/文件夹） + * Route: POST /core/dataset/collection/create/apiCollectionV2 + * ============================================================================ */ +export const CreateApiCollectionV2BodySchema = ApiCreateCollectionBaseSchema.extend({ + apiFiles: z.array(APIFileItemSchema).meta({ description: 'API 文件列表（支持文件夹递归导入）' }) +}); +export type CreateApiCollectionV2BodyType = z.infer; + +/* ============================================================================ + * API: 上传图片集创建集合 + * Route: POST /core/dataset/collection/create/images + * Content-Type: multipart/form-data + * ============================================================================ */ +export const CreateImageCollectionBodySchema = ApiCreateCollectionBaseSchema.extend({ + collectionName: z.string().meta({ description: '集合名称' }) +}); +export type ImageCreateDatasetCollectionParams = z.infer; + +// OpenAPI 文档专用：实际 data 字段的内容结构 +export const CreateImageCollectionDataSchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + collectionName: z.string().meta({ description: '集合名称' }) +}); +export type CreateImageCollectionDataType = z.infer; +// handler 内 parse 用 +export const CreateImageCollectionFormSchema = CreateImageCollectionDataSchema; +export type CreateImageCollectionFormType = z.infer; + +// OpenAPI 文档专用：描述 multipart/form-data 的实际结构（多文件） +export const CreateImageCollectionMultipartSchema = z.object({ + file: z + .array(z.any().meta({ format: 'binary' })) + .meta({ description: '上传的图片文件列表（二进制，多选）' }), + data: CreateImageCollectionDataSchema.meta({ + description: '集合参数（JSON 序列化后传入）' + }) +}); + +/* ============================================================================ + * API: 导入备份 CSV 文件创建集合 + * Route: POST /core/dataset/collection/create/backup + * Content-Type: multipart/form-data + * ============================================================================ */ +// handler 内 parse 用 +export const CreateBackupCollectionFormSchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) +}); +export type CreateBackupCollectionFormType = z.infer; + +// OpenAPI 文档专用 +export const CreateBackupCollectionMultipartSchema = z.object({ + file: z.any().meta({ format: 'binary', description: '备份 CSV 文件（格式：q,a,indexes）' }), + data: CreateBackupCollectionFormSchema.meta({ description: '集合参数（JSON 序列化后传入）' }) +}); + +/* ============================================================================ + * API: 导入模板 CSV 文件创建集合 + * Route: POST /core/dataset/collection/create/template + * Content-Type: multipart/form-data + * ============================================================================ */ +// handler 内 parse 用 +export const CreateTemplateCollectionFormSchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) +}); +export type CreateTemplateCollectionFormType = z.infer; + +// OpenAPI 文档专用 +export const CreateTemplateCollectionMultipartSchema = z.object({ + file: z.any().meta({ format: 'binary', description: '模板 CSV 文件（格式：q,a,indexes）' }), + data: CreateTemplateCollectionFormSchema.meta({ description: '集合参数（JSON 序列化后传入）' }) +}); + +/* ============================================================================ + * API: 通过外部文件 URL 创建集合（已废弃） + * Route: POST /proApi/core/dataset/collection/create/externalFileUrl + * ============================================================================ */ +export const CreateExternalFileCollectionBodySchema = ApiCreateCollectionBaseSchema.extend({ + externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), + externalFileUrl: z.string().meta({ description: '外部文件 URL' }), + filename: z.string().optional().meta({ description: '文件名' }) +}); +export type ExternalFileCreateDatasetCollectionParams = z.infer< + typeof CreateExternalFileCollectionBodySchema +>; + +/* ============================================================================ + * CSV 表格集合入参 + * ============================================================================ */ +export const CsvTableCreateCollectionBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + fileId: z.string().meta({ description: '文件 ID' }) +}); +export type CsvTableCreateDatasetCollectionParams = z.infer< + typeof CsvTableCreateCollectionBodySchema +>; diff --git a/packages/global/openapi/core/dataset/collection/createPath.ts b/packages/global/openapi/core/dataset/collection/createPath.ts new file mode 100644 index 0000000000..7599180505 --- /dev/null +++ b/packages/global/openapi/core/dataset/collection/createPath.ts @@ -0,0 +1,287 @@ +import { TagsMap } from '../../../tag'; +import type { OpenAPIPath } from '../../../type'; +import { + CreateApiCollectionBodySchema, + CreateApiCollectionV2BodySchema, + CreateBackupCollectionMultipartSchema, + CreateCollectionBodySchema, + CreateCollectionByFileIdBodySchema, + CreateCollectionByLocalFileFormSchema, + CreateImageCollectionMultipartSchema, + CreateLinkCollectionBodySchema, + CreateTemplateCollectionMultipartSchema, + CreateTextCollectionBodySchema, + ReTrainingCollectionBodySchema +} from './createApi'; + +export const DatasetCollectionCreatePath: OpenAPIPath = { + /* ============================================================ + * 通用创建（直接写入集合记录，不触发训练） + * ============================================================ */ + '/core/dataset/collection/create': { + post: { + summary: '创建空集合/目录', + description: '创建空数据集合或者目录', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回新创建的集合 ID' + } + } + } + }, + + /* ============================================================ + * 重新训练已有集合 + * ============================================================ */ + '/core/dataset/collection/create/reTrainingCollection': { + post: { + summary: '重新训练集合', + description: '删除原集合并以新参数重新创建并训练，适用于调整分块策略后的重处理场景', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: ReTrainingCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回新集合 ID' + } + } + } + }, + + /* ============================================================ + * 通过已上传的文件 ID 创建集合 + * ============================================================ */ + '/core/dataset/collection/create/fileId': { + post: { + summary: '通过文件 ID 创建集合', + description: '使用已上传至 S3 的文件对象键创建集合并触发训练', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateCollectionByFileIdBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回集合 ID 及数据插入结果' + } + } + } + }, + + /* ============================================================ + * 上传本地文件创建集合（multipart/form-data） + * ============================================================ */ + '/core/dataset/collection/create/localFile': { + post: { + summary: '上传本地文件创建集合', + description: + '通过 multipart/form-data 上传文件，自动存储至 S3 后创建集合并触发训练。`file` 字段为二进制文件，`data` 字段为 JSON 序列化的集合参数对象', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'multipart/form-data': { + schema: CreateCollectionByLocalFileFormSchema, + encoding: { + data: { contentType: 'application/json' } + } + } + } + }, + responses: { + 200: { + description: '成功返回集合 ID 及数据插入结果' + } + } + } + }, + + /* ============================================================ + * 通过链接创建集合 + * ============================================================ */ + '/core/dataset/collection/create/link': { + post: { + summary: '通过链接创建集合', + description: '抓取指定 URL 内容创建集合并触发训练', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateLinkCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回集合 ID 及数据插入结果' + } + } + } + }, + + /* ============================================================ + * 通过文本创建集合 + * ============================================================ */ + '/core/dataset/collection/create/text': { + post: { + summary: '通过文本创建集合', + description: '将文本内容存储为文件后创建集合并触发训练', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateTextCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回集合 ID 及数据插入结果' + } + } + } + }, + + /* ============================================================ + * 通过 API 数据集创建集合（V1，单文件） + * ============================================================ */ + '/core/dataset/collection/create/apiCollection': { + post: { + summary: '通过 API 数据集创建集合（V1）', + description: '根据 apiFileId 从第三方 API 数据源拉取单个文件并创建集合', + deprecated: true, + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateApiCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功创建集合' + } + } + } + }, + + /* ============================================================ + * 通过 API 数据集创建集合（V2，批量/文件夹递归） + * ============================================================ */ + '/core/dataset/collection/create/apiCollectionV2': { + post: { + summary: '通过 API 数据集批量创建集合（V2）', + description: '支持传入文件列表或选择根目录，递归拉取 API 数据源文件并批量创建集合', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'application/json': { + schema: CreateApiCollectionV2BodySchema + } + } + }, + responses: { + 200: { + description: '成功批量创建集合' + } + } + } + }, + + /* ============================================================ + * 上传图片集创建集合（multipart/form-data） + * ============================================================ */ + '/core/dataset/collection/create/images': { + post: { + summary: '上传图片集创建集合', + description: + '通过 multipart/form-data 批量上传图片，使用 VLM 模型解析后创建集合。`file` 为多个图片文件（多选），`data` 为 JSON 序列化的集合参数对象', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'multipart/form-data': { + schema: CreateImageCollectionMultipartSchema, + encoding: { + data: { contentType: 'application/json' } + } + } + } + }, + responses: { + 200: { + description: '成功返回集合 ID 及数据插入结果' + } + } + } + }, + + /* ============================================================ + * 导入备份 CSV 文件创建集合（multipart/form-data） + * ============================================================ */ + '/core/dataset/collection/create/backup': { + post: { + summary: '导入备份 CSV 创建集合', + description: + '上传格式为 q,a,indexes 的 CSV 备份文件，恢复数据到知识库集合。`file` 为 CSV 文件，`data` 为 JSON 序列化的集合参数对象', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'multipart/form-data': { + schema: CreateBackupCollectionMultipartSchema, + encoding: { + data: { contentType: 'application/json' } + } + } + } + }, + responses: { + 200: { + description: '成功导入备份数据' + } + } + } + }, + + /* ============================================================ + * 导入模板 CSV 文件创建集合（multipart/form-data） + * ============================================================ */ + '/core/dataset/collection/create/template': { + post: { + summary: '导入模板 CSV 创建集合', + description: + '上传格式为 q,a,indexes 的 CSV 模板文件，批量导入数据到知识库集合。`file` 为 CSV 文件，`data` 为 JSON 序列化的集合参数对象', + tags: [TagsMap.datasetCollectionCrteate], + requestBody: { + content: { + 'multipart/form-data': { + schema: CreateTemplateCollectionMultipartSchema, + encoding: { + data: { contentType: 'application/json' } + } + } + } + }, + responses: { + 200: { + description: '成功导入模板数据' + } + } + } + } +}; diff --git a/packages/global/openapi/core/dataset/collection/index.ts b/packages/global/openapi/core/dataset/collection/index.ts index d30d7693e7..0e7729a578 100644 --- a/packages/global/openapi/core/dataset/collection/index.ts +++ b/packages/global/openapi/core/dataset/collection/index.ts @@ -1,12 +1,79 @@ import type { OpenAPIPath } from '../../../type'; import { TagsMap } from '../../../tag'; import { + DeleteCollectionBodySchema, + DeleteCollectionQuerySchema, ExportCollectionBodySchema, + GetCollectionDetailQuerySchema, + GetCollectionPathsQuerySchema, + GetCollectionTrainingDetailQuerySchema, + GetCollectionTrainingDetailResponseSchema, + ListCollectionV2BodySchema, + ReadCollectionSourceBodySchema, ScrollCollectionsBodySchema, + SyncCollectionBodySchema, UpdateDatasetCollectionBodySchema } from './api'; +import { DatasetCollectionCreatePath } from './createPath'; export const DatasetCollectionPath: OpenAPIPath = { + ...DatasetCollectionCreatePath, + '/core/dataset/collection/delete': { + delete: { + summary: '删除集合', + description: '删除一个或多个集合及其子集合，支持通过 query.id 或 body.collectionIds 指定', + tags: [TagsMap.datasetCollection], + requestParams: { + query: DeleteCollectionQuerySchema + }, + requestBody: { + content: { + 'application/json': { + schema: DeleteCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功删除集合' + } + } + } + }, + '/core/dataset/collection/detail': { + get: { + summary: '获取集合详情', + description: '获取集合详细信息，包括索引数量、错误数量、文件信息等', + tags: [TagsMap.datasetCollection], + requestParams: { + query: GetCollectionDetailQuerySchema + }, + responses: { + 200: { + description: '成功返回集合详情' + } + } + } + }, + '/core/dataset/collection/listV2': { + post: { + summary: '获取集合列表（分页）', + description: '获取数据集集合列表，支持分页、搜索、标签过滤', + tags: [TagsMap.datasetCollection], + requestBody: { + content: { + 'application/json': { + schema: ListCollectionV2BodySchema + } + } + }, + responses: { + 200: { + description: '成功返回集合列表和总数' + } + } + } + }, '/core/dataset/collection/scrollList': { post: { summary: '获取数据集集合列表（滚动分页）', @@ -45,6 +112,59 @@ export const DatasetCollectionPath: OpenAPIPath = { } } }, + '/core/dataset/collection/paths': { + get: { + summary: '获取集合面包屑路径', + description: '从指定集合向上递归获取父级路径链，用于面包屑导航', + tags: [TagsMap.datasetCollection], + requestParams: { + query: GetCollectionPathsQuerySchema + }, + responses: { + 200: { + description: '成功返回路径列表' + } + } + } + }, + '/core/dataset/collection/read': { + post: { + summary: '获取集合资源 URL', + description: '获取集合原始文件的访问 URL，支持直接鉴权和对话中鉴权两种模式', + tags: [TagsMap.datasetCollection], + requestBody: { + content: { + 'application/json': { + schema: ReadCollectionSourceBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回资源 URL' + } + } + } + }, + '/core/dataset/collection/sync': { + post: { + summary: '同步集合', + description: '重新拉取集合原始内容并更新数据，支持链接类型和 API 数据集类型', + tags: [TagsMap.datasetCollection], + requestBody: { + content: { + 'application/json': { + schema: SyncCollectionBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回同步结果（success / sameRaw / failed）' + } + } + } + }, '/core/dataset/collection/export': { post: { summary: '下载集合的所有数据块', @@ -63,5 +183,25 @@ export const DatasetCollectionPath: OpenAPIPath = { } } } + }, + '/core/dataset/collection/trainingDetail': { + get: { + summary: '获取集合训练详情', + description: '获取集合的训练状态，包括排队中、训练中、错误数量及已完成的数据量', + tags: [TagsMap.datasetCollection], + requestParams: { + query: GetCollectionTrainingDetailQuerySchema + }, + responses: { + 200: { + description: '成功返回训练详情', + content: { + 'application/json': { + schema: GetCollectionTrainingDetailResponseSchema + } + } + } + } + } } }; diff --git a/packages/global/openapi/core/dataset/collection/tagApi.ts b/packages/global/openapi/core/dataset/collection/tagApi.ts new file mode 100644 index 0000000000..d960826f61 --- /dev/null +++ b/packages/global/openapi/core/dataset/collection/tagApi.ts @@ -0,0 +1,36 @@ +import z from 'zod'; + +/* ============================================================================ + * API: 创建集合标签 + * Route: POST /proApi/core/dataset/tag/create + * ============================================================================ */ +export const CreateDatasetCollectionTagBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + tag: z.string().meta({ description: '标签名称' }) +}); +export type CreateDatasetCollectionTagParams = z.infer; + +/* ============================================================================ + * API: 批量为集合添加标签 + * Route: POST /proApi/core/dataset/tag/addToCollections + * ============================================================================ */ +export const AddTagsToCollectionsBodySchema = z.object({ + originCollectionIds: z + .array(z.string()) + .meta({ description: '来源集合 ID 列表（用于复制标签）' }), + collectionIds: z.array(z.string()).meta({ description: '目标集合 ID 列表' }), + datasetId: z.string().meta({ description: '数据集 ID' }), + tag: z.string().meta({ description: '标签名称' }) +}); +export type AddTagsToCollectionsParams = z.infer; + +/* ============================================================================ + * API: 更新集合标签 + * Route: POST /proApi/core/dataset/tag/update + * ============================================================================ */ +export const UpdateDatasetCollectionTagBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + tagId: z.string().meta({ description: '标签 ID' }), + tag: z.string().meta({ description: '新标签名称' }) +}); +export type UpdateDatasetCollectionTagParams = z.infer; diff --git a/packages/global/openapi/core/dataset/data/api.ts b/packages/global/openapi/core/dataset/data/api.ts index f0473c2c6e..93990d7421 100644 --- a/packages/global/openapi/core/dataset/data/api.ts +++ b/packages/global/openapi/core/dataset/data/api.ts @@ -1 +1,254 @@ import { z } from 'zod'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { + DatasetCollectionSchema, + DatasetDataIndexItemSchema, + DatasetDataItemSchema, + UpdateDatasetDataPropsSchema +} from '../../../../core/dataset/type'; +import { DatasetCollectionDataProcessModeEnum } from '../../../../core/dataset/constants'; +import { OutLinkChatAuthSchema } from '../../../../support/permission/chat'; +import { PaginationSchema, PaginationResponseSchema } from '../../../api'; + +const PushDataChunkSchema = z.object({ + q: z.string().optional().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + a: z.string().optional().meta({ + description: '回答/补充文本' + }), + imageId: z.string().optional().meta({ + description: '图片 ID' + }), + chunkIndex: z.number().optional().meta({ + example: 0, + description: '块索引' + }), + indexes: z + .array(DatasetDataIndexItemSchema.omit({ dataId: true })) + .optional() + .meta({ description: '额外向量索引' }) +}); +export type PushDataChunkType = z.infer; + +/* ============================================================================ + * API: 获取数据集数据详情 + * Route: GET /api/core/dataset/data/detail + * ============================================================================ */ +export const GetDatasetDataDetailQuerySchema = z.object({ + id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据 ID' + }) +}); +export type GetDatasetDataDetailQuery = z.infer; + +export const GetDatasetDataDetailResponseSchema = DatasetDataItemSchema; +export type GetDatasetDataDetailResponse = z.infer; + +/* ============================================================================ + * API: 更新数据集数据 + * Route: PUT /api/core/dataset/data/update + * ============================================================================ */ +export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema; +export type UpdateDatasetDataBody = z.infer; + +/* ============================================================================ + * API: 删除数据集数据 + * Route: DELETE /api/core/dataset/data/delete + * ============================================================================ */ +export const DeleteDatasetDataQuerySchema = z.object({ + id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据 ID' + }) +}); +export type DeleteDatasetDataQuery = z.infer; + +/* ============================================================================ + * API: 获取引用数据 + * Route: POST /api/core/dataset/data/getQuoteData + * ============================================================================ */ +export const GetQuoteDataBodySchema = z.union([ + z.object({ + id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据 ID' + }) + }), + OutLinkChatAuthSchema.extend({ + id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据 ID' + }), + // 对话模式下的额外字段（可选） + appId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a10', + description: '应用 ID（对话模式必填）' + }), + chatId: z.string().meta({ + example: '68ad85a7463006c963799a11', + description: '对话 ID（对话模式必填）' + }), + chatItemDataId: z.string().meta({ + example: '68ad85a7463006c963799a12', + description: '对话条目数据 ID（对话模式必填）' + }) + }) +]); +export type GetQuoteDataBody = z.infer; + +export const GetQuoteDataResponseSchema = z.object({ + q: z.string().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + a: z.string().optional().meta({ + example: 'FastGPT 是一个 AI Agent 构建平台', + description: '回答/补充文本' + }), + collection: DatasetCollectionSchema.meta({ + description: '所属集合信息' + }) +}); +export type GetQuoteDataResponse = z.infer; + +/* ============================================================================ + * API: 插入单条数据 + * Route: POST /api/core/dataset/data/insertData + * ============================================================================ */ +export const InsertDataBodySchema = PushDataChunkSchema.omit({ q: true }).extend({ + q: z.string().nonempty().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }) +}); +export type InsertDataBody = z.infer; + +export const InsertDataResponseSchema = ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a07', + description: '新插入的数据 ID' +}); +export type InsertDataResponse = z.infer; + +/* ============================================================================ + * API: 插入图片 + * Route: POST /api/core/dataset/data/insertImages (multipart/form-data) + * ============================================================================ */ +export const InsertImagesBodySchema = z.object({ + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + file: z + .array(z.any()) + .optional() + .meta({ + description: '图片文件列表，multipart/form-data 上传，每个 item 为 binary 文件', + items: { type: 'string', format: 'binary' } + }) +}); +export type InsertImagesBody = z.infer; + +export const InsertImagesResponseSchema = z.object({}); +export type InsertImagesResponse = z.infer; + +/* ============================================================================ + * API: 推送数据到训练队列 + * Route: POST /api/core/dataset/data/pushData + * ============================================================================ */ +export const PushDataBodySchema = z.object({ + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + data: z.array(PushDataChunkSchema).max(200).meta({ + description: '数据列表，最多 200 条' + }), + trainingType: z.enum(DatasetCollectionDataProcessModeEnum).optional().meta({ + description: '训练类型' + }), + indexSize: z.number().optional().meta({ + description: '索引大小限制' + }), + autoIndexes: z.boolean().optional().meta({ + description: '是否自动生成索引' + }), + imageIndex: z.boolean().optional().meta({ + description: '是否生成图片索引' + }), + prompt: z.string().optional().meta({ + description: '自定义提示词' + }), + billId: z.string().optional().meta({ + description: '账单 ID' + }), + + trainingMode: z.enum(DatasetCollectionDataProcessModeEnum).optional().meta({ + description: '训练类型', + deprecated: true + }) +}); +export type PushDataBody = z.infer; + +export const PushDataResponseSchema = z.object({ + insertLen: z.number().meta({ + example: 10, + description: '成功插入的数据条数' + }) +}); +export type PushDataResponseType = z.infer; + +/* ============================================================================ + * API: 获取数据列表 V2（推荐） + * Route: POST /api/core/dataset/data/v2/list + * ============================================================================ */ +export const GetDataListItemSchema = z.object({ + _id: ObjectIdSchema.meta({ description: '数据 ID' }), + datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), + collectionId: ObjectIdSchema.meta({ description: '集合 ID' }), + q: z.string().optional().meta({ description: '问题/主文本' }), + a: z.string().optional().meta({ description: '回答/补充文本' }), + imageId: z.string().optional().meta({ description: '图片 ID' }), + imageSize: z.number().optional().meta({ description: '图片大小（字节）' }), + imagePreviewUrl: z.string().optional().meta({ description: '图片预览 URL' }), + chunkIndex: z.number().optional().meta({ description: '块索引' }), + updated: z.boolean().optional().meta({ description: '是否已更新' }) +}); + +export const GetDatasetDataListBodySchema = PaginationSchema.extend({ + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + searchText: z.string().optional().meta({ + example: 'FastGPT', + description: '搜索关键词，按 q/a 字段模糊匹配' + }) +}); +export type GetDatasetDataListBody = z.infer; + +export const GetDatasetDataListResponseSchema = PaginationResponseSchema(GetDataListItemSchema); +export type GetDatasetDataListResponse = z.infer; + +/* ============================================================================ + * API: 获取数据列表（已废弃，使用 v2/list） + * Route: POST /api/core/dataset/data/list + * @deprecated + * ============================================================================ */ +export const GetDatasetDataListLegacyBodySchema = PaginationSchema.extend({ + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + searchText: z.string().optional().meta({ + example: 'FastGPT', + description: '搜索关键词' + }) +}); +export type GetDatasetDataListLegacyBody = z.infer; diff --git a/packages/global/openapi/core/dataset/data/index.ts b/packages/global/openapi/core/dataset/data/index.ts index 0d8e9403cc..ad70bd0f1f 100644 --- a/packages/global/openapi/core/dataset/data/index.ts +++ b/packages/global/openapi/core/dataset/data/index.ts @@ -1,4 +1,168 @@ import type { OpenAPIPath } from '../../../type'; import { TagsMap } from '../../../tag'; +import { + GetDatasetDataDetailQuerySchema, + UpdateDatasetDataBodySchema, + DeleteDatasetDataQuerySchema, + GetQuoteDataBodySchema, + InsertDataBodySchema, + InsertImagesBodySchema, + PushDataBodySchema, + GetDatasetDataListBodySchema, + GetDatasetDataListResponseSchema +} from './api'; -export const DatasetDataPath: OpenAPIPath = {}; +export const DatasetDataPath: OpenAPIPath = { + '/core/dataset/data/v2/list': { + post: { + summary: '获取数据列表', + description: '分页查询集合内的数据列表，支持关键词搜索，包含图片预览 URL', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'application/json': { + schema: GetDatasetDataListBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回分页数据列表', + content: { + 'application/json': { + schema: GetDatasetDataListResponseSchema + } + } + } + } + } + }, + '/core/dataset/data/detail': { + get: { + summary: '获取数据详情', + description: '获取单条数据集数据的详细信息，包括向量索引', + tags: [TagsMap.datasetData], + requestParams: { + query: GetDatasetDataDetailQuerySchema + }, + responses: { + 200: { + description: '成功返回数据详情' + } + } + } + }, + + '/core/dataset/data/update': { + put: { + summary: '更新数据', + description: '更新数据集数据的 q、a 和向量索引，触发重新向量化', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'application/json': { + schema: UpdateDatasetDataBodySchema + } + } + }, + responses: { + 200: { + description: '更新成功' + } + } + } + }, + + '/core/dataset/data/delete': { + delete: { + summary: '删除数据', + description: '删除指定数据集数据，需要写权限', + tags: [TagsMap.datasetData], + requestParams: { + query: DeleteDatasetDataQuerySchema + }, + responses: { + 200: { + description: '删除成功' + } + } + } + }, + + '/core/dataset/data/getQuoteData': { + post: { + summary: '获取引用数据', + description: '获取数据详情用于展示引用，支持直接访问或通过对话鉴权', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'application/json': { + schema: GetQuoteDataBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回引用数据和所属集合信息' + } + } + } + }, + '/core/dataset/data/insertData': { + post: { + summary: '插入单条数据', + description: '立即插入一条数据到数据集并生成向量索引', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'application/json': { + schema: InsertDataBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回新数据的 ID' + } + } + } + }, + '/core/dataset/data/insertImages': { + post: { + summary: '插入图片', + description: '上传图片文件并推送到训练队列（multipart/form-data）', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'multipart/form-data': { + schema: InsertImagesBodySchema + } + } + }, + responses: { + 200: { + description: '上传成功，图片已加入训练队列' + } + } + } + }, + '/core/dataset/data/pushData': { + post: { + summary: '推送数据到训练队列', + description: '批量推送数据到训练队列，最多 200 条', + tags: [TagsMap.datasetData], + requestBody: { + content: { + 'application/json': { + schema: PushDataBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回插入条数' + } + } + } + } +}; diff --git a/packages/global/openapi/core/dataset/file/api.ts b/packages/global/openapi/core/dataset/file/api.ts new file mode 100644 index 0000000000..a08f431888 --- /dev/null +++ b/packages/global/openapi/core/dataset/file/api.ts @@ -0,0 +1,78 @@ +import { z } from 'zod'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { DatasetSourceReadTypeEnum } from '../../../../core/dataset/constants'; +import { ChunkSettingsSchema } from '../../../../core/dataset/type'; +import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type'; + +/* ============================================================================ + * API: 预览文件分块 + * Route: POST /api/core/dataset/file/getPreviewChunks + * ============================================================================ */ +export const GetPreviewChunksBodySchema = ChunkSettingsSchema.extend({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + type: z.enum(DatasetSourceReadTypeEnum).meta({ + example: DatasetSourceReadTypeEnum.fileLocal, + description: '数据源读取类型' + }), + sourceId: z.string().nonempty().meta({ + example: '68ad85a7463006c963799a05', + description: '数据源 ID（文件 ID / 链接 / 外部文件 / API 文件等）' + }), + customPdfParse: z.boolean().optional().meta({ + description: '是否启用自定义 PDF 解析' + }), + overlapRatio: z.number().meta({ + example: 0.2, + description: '分块重叠比例' + }), + selector: z.string().optional().meta({ + example: 'body', + description: '网页抓取的 CSS 选择器' + }), + externalFileId: z.string().optional().meta({ + description: '外部文件标识' + }) +}); +export type GetPreviewChunksBody = z.infer; + +const PreviewChunkItemSchema = z.object({ + q: z.string().meta({ description: '主要文本' }), + a: z.string().meta({ description: '辅助文本' }) +}); + +export const GetPreviewChunksResponseSchema = z.object({ + chunks: z.array(PreviewChunkItemSchema).meta({ + description: '预览分块列表（最多 10 条）' + }), + total: z.number().meta({ + example: 42, + description: '分块总数' + }) +}); +export type GetPreviewChunksResponse = z.infer; + +/* ============================================================================ + * API: 获取知识库文件上传预签名 URL + * Route: POST /api/core/dataset/file/presignDatasetFilePostUrl + * ============================================================================ */ +export const PresignDatasetFilePostUrlBodySchema = z.object({ + filename: z.string().min(1).meta({ + example: '产品文档.pdf', + description: '待上传的文件名，不能为空' + }), + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '目标知识库 ID' + }) +}); +export type PresignDatasetFilePostUrlBody = z.infer; + +export const PresignDatasetFilePostUrlResponseSchema = CreatePostPresignedUrlResponseSchema.meta({ + description: 'S3 预签名上传 URL 及相关头信息' +}); +export type PresignDatasetFilePostUrlResponse = z.infer< + typeof PresignDatasetFilePostUrlResponseSchema +>; diff --git a/packages/global/openapi/core/dataset/file/index.ts b/packages/global/openapi/core/dataset/file/index.ts new file mode 100644 index 0000000000..6e5c69f2c8 --- /dev/null +++ b/packages/global/openapi/core/dataset/file/index.ts @@ -0,0 +1,59 @@ +import type { OpenAPIPath } from '../../../type'; +import { TagsMap } from '../../../tag'; +import { + GetPreviewChunksBodySchema, + GetPreviewChunksResponseSchema, + PresignDatasetFilePostUrlBodySchema, + PresignDatasetFilePostUrlResponseSchema +} from './api'; + +export const DatasetFilePath: OpenAPIPath = { + '/core/dataset/file/getPreviewChunks': { + post: { + summary: '预览文件分块', + description: '读取数据源并按给定分块参数预览生成的前 10 个分块，用于导入前校验', + tags: [TagsMap.datasetFile], + requestBody: { + content: { + 'application/json': { + schema: GetPreviewChunksBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回预览分块列表及总数', + content: { + 'application/json': { + schema: GetPreviewChunksResponseSchema + } + } + } + } + } + }, + '/core/dataset/file/presignDatasetFilePostUrl': { + post: { + summary: '获取知识库文件上传预签名 URL', + description: '为指定知识库生成 S3 上传预签名 URL，同时校验写权限并对上传频率进行限制', + tags: [TagsMap.datasetFile], + requestBody: { + content: { + 'application/json': { + schema: PresignDatasetFilePostUrlBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回预签名上传 URL、key、请求头和最大文件大小', + content: { + 'application/json': { + schema: PresignDatasetFilePostUrlResponseSchema + } + } + } + } + } + } +}; diff --git a/packages/global/openapi/core/dataset/index.ts b/packages/global/openapi/core/dataset/index.ts index 7b8f41da5e..399b58a53f 100644 --- a/packages/global/openapi/core/dataset/index.ts +++ b/packages/global/openapi/core/dataset/index.ts @@ -2,6 +2,9 @@ import type { OpenAPIPath } from '../../type'; import { TagsMap } from '../../tag'; import { DatasetDataPath } from './data'; import { DatasetCollectionPath } from './collection'; +import { ApiDatasetPath } from './apiDataset'; +import { DatasetFilePath } from './file'; +import { DatasetTrainingPath } from './training'; import { CreateDatasetBodySchema, CreateDatasetWithFilesBodySchema, @@ -13,7 +16,8 @@ import { ResumeDatasetInheritPermissionBodySchema, CreateDatasetFolderBodySchema, SearchDatasetTestBodySchema, - ExportDatasetQuerySchema + ExportDatasetQuerySchema, + GetDatasetPermissionQuerySchema } from './api'; export const DatasetPath: OpenAPIPath = { @@ -123,7 +127,6 @@ export const DatasetPath: OpenAPIPath = { } } }, - '/core/dataset/delete': { delete: { summary: '删除知识库', @@ -177,7 +180,6 @@ export const DatasetPath: OpenAPIPath = { } } }, - '/core/dataset/searchTest': { post: { summary: '搜索测试', @@ -197,7 +199,6 @@ export const DatasetPath: OpenAPIPath = { } } }, - '/core/dataset/exportAll': { get: { summary: '导出知识库全部数据', @@ -213,7 +214,25 @@ export const DatasetPath: OpenAPIPath = { } } }, + '/core/dataset/getPermission': { + get: { + summary: '获取知识库引用权限', + description: '获取当前用户对指定知识库的读写权限，鉴权失败时返回无权限结果而非报错', + tags: [TagsMap.datasetCommon], + requestParams: { + query: GetDatasetPermissionQuerySchema + }, + responses: { + 200: { + description: '成功返回知识库名称和读写权限，无访问权限时返回空名称和 false' + } + } + } + }, ...DatasetCollectionPath, - ...DatasetDataPath + ...DatasetDataPath, + ...ApiDatasetPath, + ...DatasetFilePath, + ...DatasetTrainingPath }; diff --git a/packages/global/openapi/core/dataset/training/api.ts b/packages/global/openapi/core/dataset/training/api.ts new file mode 100644 index 0000000000..5d6176397b --- /dev/null +++ b/packages/global/openapi/core/dataset/training/api.ts @@ -0,0 +1,176 @@ +import { z } from 'zod'; +import { ObjectIdSchema } from '../../../../common/type/mongo'; +import { TrainingModeEnum } from '../../../../core/dataset/constants'; +import { DatasetTrainingSchema } from '../../../../core/dataset/type'; +import { PaginationSchema, PaginationResponseSchema } from '../../../api'; + +/* ============================================================================ + * API: 更新训练数据（或重试所有错误数据） + * Route: PUT /api/core/dataset/training/updateTrainingData + * ============================================================================ */ +export const UpdateTrainingDataBodySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + dataId: ObjectIdSchema.optional().meta({ + example: '68ad85a7463006c963799a07', + description: '训练数据 ID，不传则重试集合内所有错误数据' + }), + q: z.string().optional().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + a: z.string().optional().meta({ + example: 'FastGPT 是一个 AI Agent 构建平台', + description: '回答/补充文本' + }), + chunkIndex: z.int().min(0).optional().meta({ + example: 0, + description: '块索引' + }) +}); +export type UpdateTrainingDataBody = z.infer; + +export const UpdateTrainingDataResponseSchema = z.object({}); +export type UpdateTrainingDataResponse = z.infer; + +/* ============================================================================ + * API: 重建数据集向量索引 + * Route: POST /api/core/dataset/training/rebuildEmbedding + * ============================================================================ */ +export const RebuildEmbeddingBodySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + vectorModel: z.string().meta({ + example: 'text-embedding-3-small', + description: '新的向量模型名称，不能与当前模型相同' + }) +}); +export type RebuildEmbeddingBody = z.infer; + +export const RebuildEmbeddingResponseSchema = z.object({}); +export type RebuildEmbeddingResponse = z.infer; + +/* ============================================================================ + * API: 删除训练数据 + * Route: POST /api/core/dataset/training/deleteTrainingData + * ============================================================================ */ +export const DeleteTrainingDataBodySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + dataId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a07', + description: '训练数据 ID' + }) +}); +export type DeleteTrainingDataBody = z.infer; + +export const DeleteTrainingDataResponseSchema = z.object({}); +export type DeleteTrainingDataResponse = z.infer; + +/* ============================================================================ + * API: 获取训练数据详情 + * Route: POST /api/core/dataset/training/getTrainingDataDetail + * ============================================================================ */ +export const GetTrainingDataDetailBodySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }), + dataId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a07', + description: '训练数据 ID' + }) +}); +export type GetTrainingDataDetailBody = z.infer; + +export const GetTrainingDataDetailResponseSchema = z + .object({ + _id: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a07', + description: '训练数据 ID' + }), + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }), + mode: z.enum(TrainingModeEnum).meta({ + example: TrainingModeEnum.chunk, + description: '训练模式' + }), + q: z.string().optional().meta({ + example: '什么是 FastGPT？', + description: '问题/主文本' + }), + a: z.string().optional().meta({ + example: 'FastGPT 是一个 AI Agent 构建平台', + description: '回答/补充文本' + }), + imagePreviewUrl: z.string().optional().meta({ + example: 'https://example.com/image.png', + description: '图片预览 URL（S3 签名链接，有效期30分钟）' + }) + }) + .nullish() + .meta({ description: '训练数据详情，数据不存在时为 null' }); +export type GetTrainingDataDetailResponse = z.infer; + +/* ============================================================================ + * API: 获取训练错误列表（分页） + * Route: POST /api/core/dataset/training/getTrainingError + * ============================================================================ */ +export const GetTrainingErrorBodySchema = PaginationSchema.extend({ + collectionId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a06', + description: '集合 ID' + }) +}); +export type GetTrainingErrorBody = z.infer; + +export const GetTrainingErrorResponseSchema = PaginationResponseSchema( + DatasetTrainingSchema.omit({ billId: true }).extend({ + billId: z.string().optional() + }) +); +export type GetTrainingErrorResponse = z.infer; + +/* ============================================================================ + * API: 获取数据集训练队列状态 + * Route: GET /api/core/dataset/training/getDatasetTrainingQueue + * ============================================================================ */ +export const GetDatasetTrainingQueueQuerySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '知识库 ID' + }) +}); +export type GetDatasetTrainingQueueQuery = z.infer; + +export const GetDatasetTrainingQueueResponseSchema = z.object({ + rebuildingCount: z.number().meta({ + example: 5, + description: '正在重建向量的数据条数' + }), + trainingCount: z.number().meta({ + example: 12, + description: '训练队列中的数据条数' + }) +}); +export type GetDatasetTrainingQueueResponse = z.infer; diff --git a/packages/global/openapi/core/dataset/training/index.ts b/packages/global/openapi/core/dataset/training/index.ts new file mode 100644 index 0000000000..34110a387d --- /dev/null +++ b/packages/global/openapi/core/dataset/training/index.ts @@ -0,0 +1,128 @@ +import type { OpenAPIPath } from '../../../type'; +import { TagsMap } from '../../../tag'; +import { + UpdateTrainingDataBodySchema, + RebuildEmbeddingBodySchema, + DeleteTrainingDataBodySchema, + GetTrainingDataDetailBodySchema, + GetTrainingErrorBodySchema, + GetDatasetTrainingQueueQuerySchema +} from './api'; + +export const DatasetTrainingPath: OpenAPIPath = { + '/core/dataset/training/updateTrainingData': { + put: { + summary: '更新训练数据', + description: '更新单条训练数据，或批量重试集合内所有错误数据（不传 dataId）', + tags: [TagsMap.datasetTraining], + requestBody: { + content: { + 'application/json': { + schema: UpdateTrainingDataBodySchema + } + } + }, + responses: { + 200: { + description: '更新成功' + } + } + } + }, + + '/core/dataset/training/rebuildEmbedding': { + post: { + summary: '重建数据集向量索引', + description: '切换向量模型并重建知识库所有数据的向量索引，需要所有者权限', + tags: [TagsMap.datasetTraining], + requestBody: { + content: { + 'application/json': { + schema: RebuildEmbeddingBodySchema + } + } + }, + responses: { + 200: { + description: '重建任务已启动' + } + } + } + }, + + '/core/dataset/training/deleteTrainingData': { + post: { + summary: '删除训练数据', + description: '删除指定的训练数据条目，需要管理权限', + tags: [TagsMap.datasetTraining], + requestBody: { + content: { + 'application/json': { + schema: DeleteTrainingDataBodySchema + } + } + }, + responses: { + 200: { + description: '删除成功' + } + } + } + }, + + '/core/dataset/training/getTrainingDataDetail': { + post: { + summary: '获取训练数据详情', + description: '获取单条训练数据的详细信息，包括图片预览 URL', + tags: [TagsMap.datasetTraining], + requestBody: { + content: { + 'application/json': { + schema: GetTrainingDataDetailBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回训练数据详情，数据不存在时返回空' + } + } + } + }, + + '/core/dataset/training/getTrainingError': { + post: { + summary: '获取训练错误列表', + description: '分页查询集合内训练失败的数据列表', + tags: [TagsMap.datasetTraining], + requestBody: { + content: { + 'application/json': { + schema: GetTrainingErrorBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回错误数据分页列表' + } + } + } + }, + + '/core/dataset/training/getDatasetTrainingQueue': { + get: { + summary: '获取训练队列状态', + description: '获取知识库当前的重建数量和训练队列数量', + tags: [TagsMap.datasetTraining], + requestParams: { + query: GetDatasetTrainingQueueQuerySchema + }, + responses: { + 200: { + description: '成功返回重建数量和训练队列数量' + } + } + } + } +}; diff --git a/packages/global/openapi/index.ts b/packages/global/openapi/index.ts index 1f0bc6b8e9..d4847d7ef9 100644 --- a/packages/global/openapi/index.ts +++ b/packages/global/openapi/index.ts @@ -54,7 +54,15 @@ export const openAPIDocument = createDocument({ }, { name: '知识库', - tags: [TagsMap.datasetCommon, TagsMap.datasetCollection] + tags: [ + TagsMap.datasetCommon, + TagsMap.datasetCollection, + TagsMap.datasetCollectionCrteate, + TagsMap.datasetData, + TagsMap.datasetFile, + TagsMap.datasetTraining, + TagsMap.datasetApiDataset + ] }, { name: '插件系统', diff --git a/packages/global/openapi/support/user/index.ts b/packages/global/openapi/support/user/index.ts index e1e13a2be3..8d13f89286 100644 --- a/packages/global/openapi/support/user/index.ts +++ b/packages/global/openapi/support/user/index.ts @@ -1,8 +1,10 @@ import { UserInformPath } from './inform'; import type { OpenAPIPath } from '../../type'; import { UserAccountPath } from './account'; +import { TeamPath } from './team'; export const UserPath: OpenAPIPath = { ...UserInformPath, - ...UserAccountPath + ...UserAccountPath, + ...TeamPath }; diff --git a/packages/global/openapi/support/user/team/api.ts b/packages/global/openapi/support/user/team/api.ts index ad0e91f12a..c062721c6f 100644 --- a/packages/global/openapi/support/user/team/api.ts +++ b/packages/global/openapi/support/user/team/api.ts @@ -1,4 +1,5 @@ import z from 'zod'; +import { LafAccountSchema, OpenaiAccountSchema } from '../../../../support/user/team/type'; export const TeamChangeOwnerBodySchema = z.object({ userId: z.string().describe("the New Owner's UserId.") @@ -8,3 +9,48 @@ export const TeamChangeOwnerResponseSchema = z.object(); export type TeamChangeOwnerBodyType = z.infer; export type TeamChangeOwnerResponseType = z.infer; + +/* ============================================================================ + * API: 更新团队信息 + * Route: POST /api/support/user/team/update + * ============================================================================ */ +export const UpdateTeamBodySchema = z.object({ + name: z.string().max(100).optional().meta({ + example: '我的团队', + description: '团队名称' + }), + avatar: z.string().optional().meta({ + description: '团队头像 URL' + }), + teamDomain: z.string().optional().meta({ + description: '团队域名' + }), + lafAccount: LafAccountSchema.optional().meta({ + description: 'Laf 账号配置' + }), + openaiAccount: OpenaiAccountSchema.optional().meta({ + description: 'OpenAI 账号配置' + }), + externalWorkflowVariable: z + .object({ + key: z + .string() + .regex(/^[a-zA-Z_]\w*$/, 'key 仅允许字母、数字、下划线，且不能以数字开头') + .meta({ + example: 'myVar', + description: '变量名，仅允许字母、数字、下划线，且不以数字开头' + }), + value: z.string().meta({ + example: 'myValue', + description: '变量值，为空字符串时删除该变量' + }) + }) + .optional() + .meta({ + description: '外部工作流变量（单次更新一个变量）' + }) +}); +export type UpdateTeamBodyType = z.infer; + +export const UpdateTeamResponseSchema = z.object({}); +export type UpdateTeamResponseType = z.infer; diff --git a/packages/global/openapi/support/user/team/index.ts b/packages/global/openapi/support/user/team/index.ts new file mode 100644 index 0000000000..03a586fe9e --- /dev/null +++ b/packages/global/openapi/support/user/team/index.ts @@ -0,0 +1,25 @@ +import type { OpenAPIPath } from '../../../type'; +import { TagsMap } from '../../../tag'; +import { UpdateTeamBodySchema } from './api'; + +export const TeamPath: OpenAPIPath = { + '/api/support/user/team/update': { + post: { + summary: '更新团队信息', + description: '更新团队名称、头像、域名、第三方账号（Laf/OpenAI）及外部工作流变量', + tags: [TagsMap.teamManage], + requestBody: { + content: { + 'application/json': { + schema: UpdateTeamBodySchema + } + } + }, + responses: { + 200: { + description: '更新成功' + } + } + } + } +}; diff --git a/packages/global/openapi/tag.ts b/packages/global/openapi/tag.ts index 4ff201a57d..97ce6e2e71 100644 --- a/packages/global/openapi/tag.ts +++ b/packages/global/openapi/tag.ts @@ -31,9 +31,11 @@ export const TagsMap = { // Dataset datasetCommon: '知识库管理', datasetCollection: '集合管理', - datasetCollectionController: '集合操作', + datasetCollectionCrteate: '知识库集合创建', datasetData: '数据管理', datasetTraining: '训练管理', + datasetApiDataset: 'API 数据集管理', + datasetFile: '文件管理', // Plugin pluginToolTag: '工具标签', @@ -43,6 +45,8 @@ export const TagsMap = { publishChannel: '发布渠道', /* Support */ + // Team + teamManage: '团队管理', // Wallet walletBill: '订单', walletDiscountCoupon: '优惠券', diff --git a/packages/service/common/logger/categories.ts b/packages/service/common/logger/categories.ts index 41111b6b30..528a7bd64c 100644 --- a/packages/service/common/logger/categories.ts +++ b/packages/service/common/logger/categories.ts @@ -64,7 +64,15 @@ export const LogCategories = { FILE: ['dataset', 'file'], FOLDER: ['dataset', 'folder'], QUEUES: ['dataset', 'queues'], - TRAINING: ['dataset', 'training'] + TRAINING: ['dataset', 'training'], + + FILE_PARSE: ['dataset', 'training', 'file-parse'], + EMBEDDING: ['dataset', 'training', 'embedding'], + QA: ['dataset', 'training', 'qa'], + IMAGE_PARSE: ['dataset', 'training', 'image-parse'], + IMAGE_INDEX: ['dataset', 'training', 'image-index'], + INDEX_EXTEND: ['dataset', 'training', 'index-extend'], + LLM_PARGRAPH: ['dataset', 'training', 'llm-pargraph'] }), AI: Object.assign(['ai'], { AGENT: ['ai', 'agent'], diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index 8e1a767df8..74fd77ac62 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -9,10 +9,10 @@ import { import { type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, - type CreatePostPresignedUrlResult, type createPreviewUrlParams, CreateGetPresignedUrlParamsSchema } from '../type'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; import { getSystemMaxFileSize, Mimes } from '../constants'; import path from 'node:path'; import { MongoS3TTL } from '../schema'; @@ -131,7 +131,7 @@ export class S3BaseBucket { async createPresignedPutUrl( params: CreatePostPresignedUrlParams, options: CreatePostPresignedUrlOptions = {} - ): Promise { + ): Promise { try { const { expiredHours, maxFileSize = getSystemMaxFileSize() } = options; const formatMaxFileSize = maxFileSize * 1024 * 1024; diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts index 2a1f500226..8aa73ee697 100644 --- a/packages/service/common/s3/type.ts +++ b/packages/service/common/s3/type.ts @@ -40,14 +40,6 @@ export const CreatePostPresignedUrlOptionsSchema = z.object({ }); export type CreatePostPresignedUrlOptions = z.infer; -export const CreatePostPresignedUrlResultSchema = z.object({ - url: z.string().nonempty(), - key: z.string().nonempty(), - headers: z.record(z.string(), z.string()), - maxSize: z.number().positive().optional() // bytes -}); -export type CreatePostPresignedUrlResult = z.infer; - export const CreateGetPresignedUrlParamsSchema = z.object({ key: z.string().nonempty(), expiredHours: z.number().positive().optional() diff --git a/packages/service/common/string/tiktoken/index.ts b/packages/service/common/string/tiktoken/index.ts index f3e7ebb403..8d0fefccf8 100644 --- a/packages/service/common/string/tiktoken/index.ts +++ b/packages/service/common/string/tiktoken/index.ts @@ -3,7 +3,7 @@ import { type ChatCompletionCreateParams, type ChatCompletionMessageParam, type ChatCompletionTool -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt'; import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type'; import { WorkerNameEnum, getWorkerController } from '../../../worker/utils'; diff --git a/packages/service/common/vectorDB/pg/index.ts b/packages/service/common/vectorDB/pg/index.ts index 629e08b082..db2db22718 100644 --- a/packages/service/common/vectorDB/pg/index.ts +++ b/packages/service/common/vectorDB/pg/index.ts @@ -1,7 +1,6 @@ /* pg vector crud */ import { DatasetVectorTableName, VectorVQ } from '../constants'; import { PgClient, connectPg } from './controller'; -import { type PgSearchRawType } from '@fastgpt/global/core/dataset/api'; import type { VectorControllerType } from '../type'; import dayjs from 'dayjs'; import { getLogger, LogCategories } from '../../logger'; @@ -168,7 +167,11 @@ export class PgVectorCtrl implements VectorControllerType { ) SELECT id, collection_id, score FROM relaxed_results ORDER BY score; COMMIT;` ); - const rows = results?.[results.length - 2]?.rows as PgSearchRawType[]; + const rows = results?.[results.length - 2]?.rows as { + id: string; + collection_id: string; + score: number; + }[]; if (!Array.isArray(rows)) { return { diff --git a/packages/service/core/agentSkills/skillMdBuilder.ts b/packages/service/core/agentSkills/skillMdBuilder.ts index d64f142046..6edcd75a55 100644 --- a/packages/service/core/agentSkills/skillMdBuilder.ts +++ b/packages/service/core/agentSkills/skillMdBuilder.ts @@ -6,7 +6,7 @@ */ import { createLLMResponse } from '../ai/llm/request'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { sliceJsonStr } from '@fastgpt/global/common/string/tools'; import json5 from 'json5'; diff --git a/packages/service/core/ai/functions/createQuestionGuide.ts b/packages/service/core/ai/functions/createQuestionGuide.ts index 06aaa3d9c6..df71991583 100644 --- a/packages/service/core/ai/functions/createQuestionGuide.ts +++ b/packages/service/core/ai/functions/createQuestionGuide.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { QuestionGuidePrompt, QuestionGuideFooterPrompt diff --git a/packages/service/core/ai/llm/agentCall/index.ts b/packages/service/core/ai/llm/agentCall/index.ts index 98e8c56ff3..e0c3de5a4a 100644 --- a/packages/service/core/ai/llm/agentCall/index.ts +++ b/packages/service/core/ai/llm/agentCall/index.ts @@ -3,7 +3,7 @@ import type { ChatCompletionTool, ChatCompletionMessageToolCall, CompletionFinishReason -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import type { ToolCallChildrenInteractive, diff --git a/packages/service/core/ai/llm/agentCall/utils.ts b/packages/service/core/ai/llm/agentCall/utils.ts index aeebba57b9..10d6fac64e 100644 --- a/packages/service/core/ai/llm/agentCall/utils.ts +++ b/packages/service/core/ai/llm/agentCall/utils.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; export const filterEmptyAssistantMessages = (messages: ChatCompletionMessageParam[]) => { return messages.filter((item) => { diff --git a/packages/service/core/ai/llm/compress/index.ts b/packages/service/core/ai/llm/compress/index.ts index db0428c6d4..ef9b1505d4 100644 --- a/packages/service/core/ai/llm/compress/index.ts +++ b/packages/service/core/ai/llm/compress/index.ts @@ -4,7 +4,7 @@ import { calculateCompressionThresholds } from './constants'; import type { CreateLLMResponseProps } from '../request'; import { createLLMResponse } from '../request'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { getCompressRequestMessagesPrompt } from './prompt'; import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'; diff --git a/packages/service/core/ai/llm/compress/prompt.ts b/packages/service/core/ai/llm/compress/prompt.ts index d2ad0a6fdc..f2f53eeba2 100644 --- a/packages/service/core/ai/llm/compress/prompt.ts +++ b/packages/service/core/ai/llm/compress/prompt.ts @@ -1,5 +1,5 @@ import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { calculateCompressionThresholds } from './constants'; export const getCompressRequestMessagesPrompt = async ({ diff --git a/packages/service/core/ai/llm/promptCall/index.ts b/packages/service/core/ai/llm/promptCall/index.ts index 0d4d31cdb8..2e7905c1f7 100644 --- a/packages/service/core/ai/llm/promptCall/index.ts +++ b/packages/service/core/ai/llm/promptCall/index.ts @@ -5,7 +5,7 @@ import type { ChatCompletionMessageToolCall, ChatCompletionSystemMessageParam, ChatCompletionTool -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { getPromptToolCallPrompt } from './prompt'; import { cloneDeep } from 'lodash'; diff --git a/packages/service/core/ai/llm/promptCall/prompt.ts b/packages/service/core/ai/llm/promptCall/prompt.ts index f87fc2a4b6..5c1a22a240 100644 --- a/packages/service/core/ai/llm/promptCall/prompt.ts +++ b/packages/service/core/ai/llm/promptCall/prompt.ts @@ -1,5 +1,5 @@ import { replaceVariable } from '@fastgpt/global/common/string/tools'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; export const getPromptToolCallPrompt = (tools: ChatCompletionTool['function'][]) => { const prompt = ` diff --git a/packages/service/core/ai/llm/request.ts b/packages/service/core/ai/llm/request.ts index 04dd1aa4ed..5a9a4dae6d 100644 --- a/packages/service/core/ai/llm/request.ts +++ b/packages/service/core/ai/llm/request.ts @@ -1,5 +1,5 @@ import type { - ChatCompletion, + ChatCompletionCreateParams, ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsStreaming, ChatCompletionMessageParam, @@ -7,9 +7,9 @@ import type { CompletionFinishReason, CompletionUsage, OpenAI, - StreamChatType, - UnStreamChatType -} from '@fastgpt/global/core/ai/type'; + StreamResponseType, + UnStreamResponseType +} from '@fastgpt/global/core/ai/llm/type'; import { computedMaxToken, computedTemperature, @@ -45,7 +45,9 @@ export type ResponseEvents = { onToolParam?: (e: { tool: ChatCompletionMessageToolCall; params: string }) => void; }; -export type CreateLLMResponseProps = { +export type CreateLLMResponseProps< + T extends ChatCompletionCreateParams = ChatCompletionCreateParams +> = { throwError?: boolean; userKey?: OpenaiAccountType; body: LLMRequestBodyType; @@ -77,7 +79,7 @@ type LLMResponse = { 底层封装 LLM 调用 帮助上层屏蔽 stream 和非 stream，以及 toolChoice 和 promptTool 模式。 工具调用无论哪种模式，都存 toolChoice 的格式，promptTool 通过修改 toolChoice 的结构，形成特定的 messages 进行调用。 */ -export const createLLMResponse = async ( +export const createLLMResponse = async ( args: CreateLLMResponseProps ): Promise => { // 生成唯一的请求追踪 ID @@ -264,8 +266,8 @@ export const createLLMResponse = async ( const outputTokens = usage?.completion_tokens || (await countGptMessagesTokens([assistantMessage])); - // 异步保存 LLM 请求追踪记录 - saveLLMRequestRecord({ + // 异步保存 LLM 请求追踪记录（fire-and-forget） + void saveLLMRequestRecord({ requestId, body: requestBody, response: { @@ -333,8 +335,8 @@ export const createLLMResponse = async ( completeMessages: [...requestMessages, assistantMessage] }; } catch (error) { - // 异步保存 LLM 请求追踪记录 - saveLLMRequestRecord({ + // 异步保存 LLM 请求追踪记录（fire-and-forget） + void saveLLMRequestRecord({ requestId, body: requestBody, response: { @@ -363,7 +365,8 @@ export const createLLMResponse = async ( } }; -type CompleteParams = Pick, 'body'> & ResponseEvents; +type CompleteParams = Pick, 'body'> & + ResponseEvents; type CompleteResponse = Pick< LLMResponse, @@ -382,7 +385,7 @@ export const createStreamResponse = async ({ onToolCall, onToolParam }: CompleteParams & { - response: StreamChatType; + response: StreamResponseType; isAborted?: CreateLLMResponseProps['isAborted']; }): Promise => { const { retainDatasetCite = true, tools, toolCallMode = 'toolChoice', model } = body; @@ -598,7 +601,7 @@ export const createCompleteResponse = async ({ onStreaming, onReasoning, onToolCall -}: CompleteParams & { response: ChatCompletion }): Promise => { +}: CompleteParams & { response: UnStreamResponseType }): Promise => { const { tools, toolCallMode = 'toolChoice', retainDatasetCite = true } = body; const modelData = getLLMModel(body.model); @@ -674,14 +677,11 @@ export const createCompleteResponse = async ({ }; }; -type CompletionsBodyType = - | ChatCompletionCreateParamsNonStreaming - | ChatCompletionCreateParamsStreaming; type InferCompletionsBody = T extends { stream: true } ? ChatCompletionCreateParamsStreaming : T extends { stream: false } ? ChatCompletionCreateParamsNonStreaming - : ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsStreaming; + : ChatCompletionCreateParams; type LLMRequestBodyType = Omit & { model: string | LLMModelItemType; @@ -698,7 +698,7 @@ type LLMRequestBodyType = Omit({ +const llmCompletionsBodyFormat = async ({ retainDatasetCite, useVision, requestOrigin, @@ -822,11 +822,11 @@ const createChatCompletion = async ({ options?: OpenAI.RequestOptions; }): Promise< | { - response: StreamChatType; + response: StreamResponseType; isStreamResponse: true; } | { - response: UnStreamChatType; + response: UnStreamResponseType; isStreamResponse: false; } > => { diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index 0bdaec72ef..fd76238076 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -4,9 +4,8 @@ import type { ChatCompletionContentPart, ChatCompletionContentPartRefusal, ChatCompletionContentPartText, - ChatCompletionMessageParam, - SdkChatCompletionMessageParam -} from '@fastgpt/global/core/ai/type'; + ChatCompletionMessageParam +} from '@fastgpt/global/core/ai/llm/type'; import { axios } from '../../../common/api/axios'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; @@ -376,9 +375,9 @@ export const loadRequestMessages = async ({ const loadMessages = ( await Promise.all( - mergeMessages.map(async (item, i) => { - delete item.dataId; - delete item.hideInUI; + mergeMessages.map(async (raw, i) => { + // 解构剥离系统内部字段，避免 mutate 调用方传入的 messages + const { dataId: _dataId, hideInUI: _hideInUI, ...item } = raw; if (item.role === ChatCompletionRequestMessageRoleEnum.System) { const content = parseSystemMessage(item.content); @@ -443,5 +442,5 @@ export const loadRequestMessages = async ({ ) ).filter(Boolean) as ChatCompletionMessageParam[]; - return loadMessages as SdkChatCompletionMessageParam[]; + return loadMessages; }; diff --git a/packages/service/core/ai/utils.ts b/packages/service/core/ai/utils.ts index 2024585921..d30fa230be 100644 --- a/packages/service/core/ai/utils.ts +++ b/packages/service/core/ai/utils.ts @@ -1,5 +1,5 @@ import { type LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; -import type { CompletionFinishReason, CompletionUsage } from '@fastgpt/global/core/ai/type'; +import type { CompletionFinishReason, CompletionUsage } from '@fastgpt/global/core/ai/llm/type'; import { getLLMDefaultUsage } from '@fastgpt/global/core/ai/constants'; import { removeDatasetCiteText } from '@fastgpt/global/core/ai/llm/utils'; import json5 from 'json5'; diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index d20be21973..18e2b21805 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -2,7 +2,6 @@ import { DatasetCollectionDataProcessModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; import { MongoDatasetCollection } from './schema'; import type { DatasetCollectionSchemaType, @@ -33,6 +32,10 @@ import { import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants'; import { getS3DatasetSource } from '../../../common/s3/sources/dataset'; import { removeS3TTL, isS3ObjectKey } from '../../../common/s3/utils'; +import type { + CreateCollectionWithResultResponseType, + ApiCreateDatasetCollectionParams +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; export const createCollectionAndInsertData = async ({ dataset, @@ -52,7 +55,7 @@ export const createCollectionAndInsertData = async ({ billId?: string; session?: ClientSession; -}) => { +}): Promise => { // Adapter 4.9.0 if (createCollectionParams.trainingType === DatasetCollectionDataProcessModeEnum.auto) { createCollectionParams.trainingType = DatasetCollectionDataProcessModeEnum.chunk; @@ -166,7 +169,7 @@ export const createCollectionAndInsertData = async ({ insertLen: predictDataLimitLength(trainingMode, chunks) }); - const fn = async (session: ClientSession) => { + const fn = async (session: ClientSession): Promise => { // 3. Create collection const { _id: collectionId } = await createOneCollection({ ...formatCreateCollectionParams, @@ -236,7 +239,9 @@ export const createCollectionAndInsertData = async ({ return { collectionId: String(collectionId), - insertResults + results: { + insertLen: insertResults.insertLen + } }; }; @@ -246,9 +251,21 @@ export const createCollectionAndInsertData = async ({ return mongoSessionRun(fn); }; -export type CreateOneCollectionParams = CreateDatasetCollectionParams & { +export type CreateOneCollectionParams = ApiCreateDatasetCollectionParams & { teamId: string; tmbId: string; + name: string; + type: DatasetCollectionTypeEnum; + fileId?: string; + rawLink?: string; + externalFileId?: string; + externalFileUrl?: string; + apiFileId?: string; + apiFileParentId?: string; + rawTextLength?: number; + hashRawText?: string; + createTime?: Date; + updateTime?: Date; session?: ClientSession; }; export async function createOneCollection({ session, ...props }: CreateOneCollectionParams) { diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 81c53ef3d9..91ebbb434a 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -218,7 +218,7 @@ export const getTrainingModeByCollection = ({ autoIndexes, imageIndex }: { - trainingType: DatasetCollectionDataProcessModeEnum; + trainingType?: DatasetCollectionDataProcessModeEnum; autoIndexes?: boolean; imageIndex?: boolean; }) => { diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts index c25b08f1fe..1c44f2d1ba 100644 --- a/packages/service/core/dataset/training/controller.ts +++ b/packages/service/core/dataset/training/controller.ts @@ -1,8 +1,8 @@ import { MongoDatasetTraining } from './schema'; import type { - PushDatasetDataChunkProps, - PushDatasetDataResponse -} from '@fastgpt/global/core/dataset/api'; + PushDataChunkType, + PushDataResponseType +} from '@fastgpt/global/openapi/core/dataset/data/api'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { type ClientSession } from '../../../common/mongo'; import { getLLMModel, getEmbeddingModel, getVlmModel } from '../../ai/model'; @@ -27,7 +27,7 @@ export const lockTrainingDataByTeamId = async (teamId: string): Promise => } catch (error) {} }; -export async function pushDataListToTrainingQueue({ +export const pushDataListToTrainingQueue = async ({ teamId, tmbId, datasetId, @@ -46,7 +46,7 @@ export async function pushDataListToTrainingQueue({ datasetId: string; collectionId: string; - data: PushDatasetDataChunkProps[]; + data: PushDataChunkType[]; mode?: TrainingModeEnum; agentModel: string; @@ -55,9 +55,9 @@ export async function pushDataListToTrainingQueue({ indexSize?: number; - billId?: string; + billId: string; session?: ClientSession; -}): Promise { +}): Promise => { const vectorModelData = getEmbeddingModel(vectorModel); if (!vectorModelData) { return Promise.reject(i18nT('common:error_embedding_not_config')); @@ -209,7 +209,7 @@ export async function pushDataListToTrainingQueue({ logger.info('Single transaction completed', { durationMs: Date.now() - start }); return { insertLen: insertedCount }; } -} +}; export const pushDatasetToParseQueue = async ({ teamId, diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index cd8c564bc2..61e1ae596a 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -35,7 +35,10 @@ const TrainingDataSchema = new Schema({ ref: DatasetColCollectionName, required: true }, - billId: String, + billId: { + type: String, + required: true + }, mode: { type: String, enum: Object.values(TrainingModeEnum), diff --git a/packages/service/core/workflow/dispatch/ai/agent/capability/type.ts b/packages/service/core/workflow/dispatch/ai/agent/capability/type.ts index a6fdcbe167..89a1f37d63 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/capability/type.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/capability/type.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import type { AIChatItemValueItemType } from '@fastgpt/global/core/chat/type'; export type CapabilityToolCallResult = { diff --git a/packages/service/core/workflow/dispatch/ai/agent/index.ts b/packages/service/core/workflow/dispatch/ai/agent/index.ts index 8c1b3cc82b..7a9ea40fa2 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/index.ts @@ -28,7 +28,7 @@ import type { DispatchPlanAgentResponse } from './sub/plan'; import { dispatchPlanAgent } from './sub/plan'; import { formatFileInput } from './sub/file/utils'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { masterCall } from './master/call'; import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type'; import { diff --git a/packages/service/core/workflow/dispatch/ai/agent/master/call.ts b/packages/service/core/workflow/dispatch/ai/agent/master/call.ts index d1293d55fa..18a5f06a22 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/master/call.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/master/call.ts @@ -1,4 +1,7 @@ -import type { ChatCompletionMessageParam, ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { + ChatCompletionMessageParam, + ChatCompletionTool +} from '@fastgpt/global/core/ai/llm/type'; import { runAgentCall } from '../../../../../ai/llm/agentCall'; import { chats2GPTMessages, runtimePrompt2ChatsValue } from '@fastgpt/global/core/chat/adapt'; import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/utils.ts index 26803653ba..56ecaee094 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/utils.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import z from 'zod'; import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/file/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/file/utils.ts index 0619b9c15f..7e16c646d1 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/file/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/file/utils.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { parseUrlToFileType } from '../../../../../utils/context'; import { getLogger, LogCategories } from '../../../../../../../common/logger'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/model/constants.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/model/constants.ts index 83bc030310..be6030e5c1 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/model/constants.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/model/constants.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; export const ModelAgentTool: ChatCompletionTool = { diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/model/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/model/index.ts index 6409f68e4b..4660342d87 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/model/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/model/index.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { createLLMResponse, type ResponseEvents } from '../../../../../../ai/llm/request'; import { getLLMModel } from '../../../../../../ai/model'; import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/ask/constants.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/ask/constants.ts index 7d8b26977d..30619b1059 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/ask/constants.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/ask/constants.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import z from 'zod'; import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/constants.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/constants.ts index 5b3ffbfd88..f25bc6e8e0 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/constants.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/constants.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds, systemSubInfo } from '@fastgpt/global/core/workflow/node/agent/constants'; import type { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type'; import { getNanoid } from '@fastgpt/global/common/string/tools'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/index.ts index 6a28f69b35..2131add59b 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/index.ts @@ -2,7 +2,7 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, ChatCompletionTool -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { createLLMResponse } from '../../../../../../ai/llm/request'; import { getInitialPlanPrompt, diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/prompt.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/prompt.ts index d540eebb80..7800df4c55 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/plan/prompt.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/plan/prompt.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'; import { AIAskTool } from './ask/constants'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/utils.ts index 636e0acfa5..7379d78e1d 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/utils.ts @@ -11,7 +11,7 @@ import { MongoApp } from '../../../../../../app/schema'; import { getMCPChildren } from '../../../../../../app/mcp'; import { getMCPToolRuntimeNode } from '@fastgpt/global/core/app/tool/mcpTool/utils'; import { getHTTPToolRuntimeNode } from '@fastgpt/global/core/app/tool/httpTool/utils'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io'; import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema'; import { diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/type.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/type.ts index b4d026da91..7baf337974 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/type.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/type.ts @@ -1,5 +1,5 @@ import type { StoreSecretValueType } from '@fastgpt/global/common/secret/type'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import type { SystemToolSecretInputTypeEnum } from '@fastgpt/global/core/app/tool/systemTool/constants'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; diff --git a/packages/service/core/workflow/dispatch/ai/agent/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/utils.ts index 33a99022af..856bd7fcc3 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/utils.ts @@ -2,7 +2,7 @@ import type { localeType } from '@fastgpt/global/common/i18n/type'; import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type'; import type { SubAppRuntimeType } from './type'; import { getAgentRuntimeTools } from './sub/tool/utils'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { readFileTool } from './sub/file/utils'; import { PlanAgentTool } from './sub/plan/constants'; import { datasetSearchTool } from './sub/dataset/utils'; diff --git a/packages/service/core/workflow/dispatch/ai/extract.ts b/packages/service/core/workflow/dispatch/ai/extract.ts index 66ceb8c943..13744a2968 100644 --- a/packages/service/core/workflow/dispatch/ai/extract.ts +++ b/packages/service/core/workflow/dispatch/ai/extract.ts @@ -23,7 +23,7 @@ const logger = getLogger(LogCategories.MODULE.WORKFLOW.AI); import { type ChatCompletionMessageParam, type ChatCompletionTool -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type'; import { diff --git a/packages/service/core/workflow/dispatch/ai/tool/toolCall.ts b/packages/service/core/workflow/dispatch/ai/tool/toolCall.ts index e3742a3d2e..4df498f675 100644 --- a/packages/service/core/workflow/dispatch/ai/tool/toolCall.ts +++ b/packages/service/core/workflow/dispatch/ai/tool/toolCall.ts @@ -2,7 +2,7 @@ import type { ChatCompletionMessageParam, ChatCompletionTool, CompletionFinishReason -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils'; import { runWorkflow } from '../../index'; diff --git a/packages/service/core/workflow/dispatch/ai/tool/type.ts b/packages/service/core/workflow/dispatch/ai/tool/type.ts index c02b0be1ad..71061c9bb8 100644 --- a/packages/service/core/workflow/dispatch/ai/tool/type.ts +++ b/packages/service/core/workflow/dispatch/ai/tool/type.ts @@ -1,7 +1,7 @@ import type { ChatCompletionMessageParam, CompletionFinishReason -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; diff --git a/packages/service/core/workflow/dispatch/ai/utils.ts b/packages/service/core/workflow/dispatch/ai/utils.ts index b72691408f..ae05e17002 100644 --- a/packages/service/core/workflow/dispatch/ai/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/utils.ts @@ -14,7 +14,7 @@ import type { McpToolDataType } from '@fastgpt/global/core/app/tool/mcpTool/type import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema'; import type { ToolNodeItemType } from './tool/type'; import json5 from 'json5'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; // Assistant process diff --git a/packages/service/worker/countGptMessagesTokens/index.ts b/packages/service/worker/countGptMessagesTokens/index.ts index 480cc3a809..9a999c06a4 100644 --- a/packages/service/worker/countGptMessagesTokens/index.ts +++ b/packages/service/worker/countGptMessagesTokens/index.ts @@ -6,7 +6,7 @@ import { type ChatCompletionContentPart, type ChatCompletionCreateParams, type ChatCompletionTool -} from '@fastgpt/global/core/ai/type'; +} from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { parentPort } from 'worker_threads'; import { getLogger, LogCategories } from '../../common/logger'; diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx index 1e56c71de1..449d19f0ef 100644 --- a/packages/web/common/file/hooks/useUploadAvatar.tsx +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -3,11 +3,11 @@ import { compressBase64Img } from '../img'; import { useToast } from '../../../hooks/useToast'; import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; -import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/type'; import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; export const useUploadAvatar = ( - api: (params: { filename: string }) => Promise, + api: (params: { filename: string }) => Promise, { onSuccess, maxW = 300, diff --git a/projects/app/src/components/Markdown/A.tsx b/projects/app/src/components/Markdown/A.tsx index 6839a1113a..eb79e7478c 100644 --- a/projects/app/src/components/Markdown/A.tsx +++ b/projects/app/src/components/Markdown/A.tsx @@ -16,7 +16,7 @@ import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useTranslation } from 'next-i18next'; import React, { useMemo } from 'react'; -import { getQuoteData } from '@/web/core/dataset/api'; +import { getQuoteData } from '@/web/core/dataset/api/data'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils'; import Markdown from '.'; diff --git a/projects/app/src/components/PromptTemplate/index.tsx b/projects/app/src/components/PromptTemplate/index.tsx index 4a525c5634..b0bee56d19 100644 --- a/projects/app/src/components/PromptTemplate/index.tsx +++ b/projects/app/src/components/PromptTemplate/index.tsx @@ -1,7 +1,7 @@ import React, { useState } from 'react'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { Box, Button, Flex, Grid, useTheme } from '@chakra-ui/react'; -import { type PromptTemplateItem } from '@fastgpt/global/core/ai/type'; +import { type PromptTemplateItem } from '@fastgpt/global/core/ai/llm/type'; import { ModalBody, ModalFooter } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; const PromptTemplate = ({ diff --git a/projects/app/src/components/core/chat/ChatContainer/type.ts b/projects/app/src/components/core/chat/ChatContainer/type.ts index 36567da8b4..d8dae16485 100644 --- a/projects/app/src/components/core/chat/ChatContainer/type.ts +++ b/projects/app/src/components/core/chat/ChatContainer/type.ts @@ -1,5 +1,5 @@ import type { StreamResponseType } from '@/web/common/api/fetch'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import type { ChatHistoryItemResType, StepTitleItemType, diff --git a/projects/app/src/components/core/chat/HelperBot/api.ts b/projects/app/src/components/core/chat/HelperBot/api.ts index fa5e61f5ae..5c16e678d1 100644 --- a/projects/app/src/components/core/chat/HelperBot/api.ts +++ b/projects/app/src/components/core/chat/HelperBot/api.ts @@ -6,7 +6,7 @@ import type { GetHelperBotFilePresignParamsType, GetHelperBotFilePreviewParamsType } from '@fastgpt/global/openapi/core/chat/helperBot/api'; -import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; export const getHelperBotChatRecords = (data: GetHelperBotChatRecordsParamsType) => GET('/core/chat/helperBot/getRecords', data); @@ -15,7 +15,7 @@ export const deleteHelperBotChatRecord = (data: DeleteHelperBotChatParamsType) = DELETE('/core/chat/helperBot/deleteRecord', data); export const getHelperBotFilePresign = (data: GetHelperBotFilePresignParamsType) => - POST('/core/chat/helperBot/getFilePresign', data); + POST('/core/chat/helperBot/getFilePresign', data); export const getHelperBotFilePreview = (data: GetHelperBotFilePreviewParamsType) => POST('/core/chat/helperBot/getFilePreview', data); diff --git a/projects/app/src/components/core/dataset/QuoteItem.tsx b/projects/app/src/components/core/dataset/QuoteItem.tsx index 8263e03629..6b5b3db6ce 100644 --- a/projects/app/src/components/core/dataset/QuoteItem.tsx +++ b/projects/app/src/components/core/dataset/QuoteItem.tsx @@ -8,12 +8,8 @@ import { useTranslation } from 'next-i18next'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import dynamic from 'next/dynamic'; import MyBox from '@fastgpt/web/components/common/MyBox'; -import { - DatasetCollectionTypeEnum, - SearchScoreTypeEnum, - SearchScoreTypeMap -} from '@fastgpt/global/core/dataset/constants'; -import type { readCollectionSourceBody } from '@/pages/api/core/dataset/collection/read'; +import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constants'; +import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import Markdown from '@/components/Markdown'; const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal')); @@ -100,7 +96,7 @@ const QuoteItem = ({ canDownloadSource?: boolean; canEditData?: boolean; canEditDataset?: boolean; -} & Omit) => { +} & Omit) => { const { t } = useTranslation(); const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); diff --git a/projects/app/src/components/core/dataset/RawSourceBox.tsx b/projects/app/src/components/core/dataset/RawSourceBox.tsx index 2455d7093c..7d71e9ab4a 100644 --- a/projects/app/src/components/core/dataset/RawSourceBox.tsx +++ b/projects/app/src/components/core/dataset/RawSourceBox.tsx @@ -5,11 +5,11 @@ import { useTranslation } from 'next-i18next'; import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; import MyIcon from '@fastgpt/web/components/common/Icon'; -import type { readCollectionSourceBody } from '@/pages/api/core/dataset/collection/read'; +import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import type { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; type Props = BoxProps & - readCollectionSourceBody & { + ReadCollectionSourceBodyType & { collectionType?: DatasetCollectionTypeEnum; sourceName?: string; sourceId?: string; diff --git a/projects/app/src/components/core/dataset/SelectModal.tsx b/projects/app/src/components/core/dataset/SelectModal.tsx index b855d2e3f5..177da51595 100644 --- a/projects/app/src/components/core/dataset/SelectModal.tsx +++ b/projects/app/src/components/core/dataset/SelectModal.tsx @@ -6,12 +6,10 @@ import { Box } from '@chakra-ui/react'; import FolderPath from '@/components/common/folder/Path'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; - -type PathItemType = { - parentId: ParentIdType; - parentName: string; -}; +import type { + ParentIdType, + ParentTreePathItemType +} from '@fastgpt/global/common/parentFolder/type'; const DatasetSelectContainer = ({ isOpen, @@ -24,7 +22,7 @@ const DatasetSelectContainer = ({ }: { isOpen: boolean; setParentId: Dispatch; - paths: PathItemType[]; + paths: ParentTreePathItemType[]; onClose: () => void; tips?: string | null; isLoading?: boolean; diff --git a/projects/app/src/global/core/dataset/api.ts b/projects/app/src/global/core/dataset/api.ts index 0ea7c91b9c..1a868c13ab 100644 --- a/projects/app/src/global/core/dataset/api.ts +++ b/projects/app/src/global/core/dataset/api.ts @@ -1,24 +1,7 @@ -import type { - PushDatasetDataChunkProps, - PushDatasetDataResponse -} from '@fastgpt/global/core/dataset/api'; -import type { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type'; - -/* ================= dataset ===================== */ - -export type RebuildEmbeddingProps = { - datasetId: string; - vectorModel: string; -}; +import type { PushDataResponseType } from '@fastgpt/global/openapi/core/dataset/data/api'; /* ================= collection ===================== */ export type CreateCollectionResponse = Promise<{ collectionId: string; - results: PushDatasetDataResponse; + results: PushDataResponseType; }>; - -/* ================= data ===================== */ -export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & { - collectionId: string; -}; diff --git a/projects/app/src/global/core/dataset/type.ts b/projects/app/src/global/core/dataset/type.ts deleted file mode 100644 index 13e328a0f9..0000000000 --- a/projects/app/src/global/core/dataset/type.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo'; -import { DatasetCollectionSchema } from '@fastgpt/global/core/dataset/type'; -import { PermissionSchema } from '@fastgpt/global/support/permission/controller'; -import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller'; -import z from 'zod'; - -/* ================= collection ===================== */ -export const DatasetCollectionsListItemSchema = z.object({ - _id: ObjectIdSchema.meta({ description: '集合 ID' }), - parentId: DatasetCollectionSchema.shape.parentId, - tmbId: DatasetCollectionSchema.shape.tmbId, - name: DatasetCollectionSchema.shape.name, - type: DatasetCollectionSchema.shape.type, - createTime: DatasetCollectionSchema.shape.createTime, - updateTime: DatasetCollectionSchema.shape.updateTime, - forbid: DatasetCollectionSchema.shape.forbid, - trainingType: DatasetCollectionSchema.shape.trainingType, - tags: z.array(z.string()).optional().meta({ description: '标签' }), - - externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), - - fileId: z.string().optional().meta({ description: '文件 ID' }), - rawLink: z.string().optional().meta({ description: '原始链接' }), - permission: PermissionSchema, - dataAmount: z.number().meta({ description: '数据数量' }), - trainingAmount: z.number().meta({ description: '训练数量' }), - hasError: z.boolean().optional().meta({ description: '是否错误' }) -}); -export type DatasetCollectionsListItemType = z.infer; - -/* ================= data ===================== */ -export const DatasetDataListItemSchema = z.object({ - _id: ObjectIdSchema.meta({ description: '数据 ID' }), - datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), - collectionId: ObjectIdSchema.meta({ description: '集合 ID' }), - q: z.string().optional().meta({ description: '问题' }), - a: z.string().optional().meta({ description: '答案' }), - imageId: z.string().optional().meta({ description: '图片 ID' }), - imageSize: z.number().optional().meta({ description: '图片大小' }), - imagePreviewUrl: z.string().optional().meta({ description: '图片预览 URL' }), - chunkIndex: z.number().optional().meta({ description: '块索引' }), - updated: z.boolean().optional().meta({ description: '是否更新' }) -}); -export type DatasetDataListItemType = z.infer; diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeCode/Copilot.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeCode/Copilot.tsx index fb1ac7dfa1..cb63d26603 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeCode/Copilot.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeCode/Copilot.tsx @@ -21,7 +21,7 @@ import { FlowNodeOutputTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; import { nanoid } from 'nanoid'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { JS_TEMPLATE, SandboxCodeTypeEnum diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SettingQuotePrompt.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SettingQuotePrompt.tsx index 624a1dd4c6..e559c1387d 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SettingQuotePrompt.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SettingQuotePrompt.tsx @@ -3,7 +3,7 @@ import type { RenderInputProps } from '../type'; import { Box, type BoxProps, Button, Flex, ModalFooter, useDisclosure } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { useForm } from 'react-hook-form'; -import { type PromptTemplateItem } from '@fastgpt/global/core/ai/type'; +import { type PromptTemplateItem } from '@fastgpt/global/core/ai/llm/type'; import { useTranslation } from 'next-i18next'; import { ModalBody } from '@chakra-ui/react'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx index 475e8ab9fa..40c95d489c 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx @@ -8,7 +8,7 @@ import DownloadButton from './DownloadButton'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { downloadFetch } from '@/web/common/system/utils'; import { useMemo, useState } from 'react'; -import { getDatasetDataPermission } from '@/web/core/dataset/api'; +import { getDatasetPermission } from '@/web/core/dataset/api'; import ScoreTag from './ScoreTag'; import { formatScore } from '@/components/core/dataset/QuoteItem'; import NavButton from './NavButton'; @@ -42,7 +42,7 @@ const CollectionReader = ({ const [quoteIndex, setQuoteIndex] = useState(0); // Get dataset permission - const { data: datasetData } = useRequest(async () => await getDatasetDataPermission(datasetId), { + const { data: datasetData } = useRequest(async () => await getDatasetPermission(datasetId), { manual: !userInfo || !datasetId, refreshDeps: [datasetId, userInfo] }); diff --git a/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx b/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx index da448f4dcc..b9b2be6601 100644 --- a/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx +++ b/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx @@ -3,14 +3,14 @@ import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { Flex, Input, Button, ModalBody, ModalFooter, Box } from '@chakra-ui/react'; import type { UseFormReturn } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api'; +import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api/apiDataset'; import type { GetResourceFolderListItemResponse, GetResourceFolderListProps, ParentIdType } from '@fastgpt/global/common/parentFolder/type'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import type { GetApiDatasetCataLogProps } from '@/pages/api/core/dataset/apiDataset/getCatalog'; +import type { GetApiDatasetCatalogBody } from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { useBoolean, useMemoizedFn, useMount } from 'ahooks'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; @@ -140,7 +140,7 @@ const ApiDatasetForm = ({ { - const params: GetApiDatasetCataLogProps = { + const params: GetApiDatasetCatalogBody = { parentId: e.parentId, apiDatasetServer }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx index ecb4bcb71b..45a001bcbf 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx @@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next'; import { Box, Button, HStack, ModalBody, ModalFooter, VStack } from '@chakra-ui/react'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIconButton from '@fastgpt/web/components/common/Icon/button'; -import { postBackupDatasetCollection } from '@/web/core/dataset/api'; +import { postBackupDatasetCollection } from '@/web/core/dataset/api/collection'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useContextSelector } from 'use-context-selector'; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx index c514015fec..1f1a450c7b 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx @@ -14,10 +14,11 @@ import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useDisclosure } from '@chakra-ui/react'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { checkTeamWebSyncLimit } from '@/web/support/user/team/api'; -import { getDatasetCollections, postDatasetSync } from '@/web/core/dataset/api'; +import { getDatasetCollections } from '@/web/core/dataset/api/collection'; +import { postDatasetSync } from '@/web/core/dataset/api'; import dynamic from 'next/dynamic'; import { usePagination } from '@fastgpt/web/hooks/usePagination'; -import { type DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { useRouter } from 'next/router'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { type WebsiteConfigFormType } from './WebsiteConfig'; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index bc8a29d420..d106bfa205 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -4,7 +4,7 @@ import { getDatasetCollectionPathById, postDatasetCollection, putDatasetCollectionById -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/collection'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyInput from '@/components/MyInput'; @@ -93,7 +93,13 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { } = useDisclosure(); const { runAsync: onCreateCollection } = useRequest( - async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => { + async ({ + name, + type + }: { + name: string; + type: DatasetCollectionTypeEnum.folder | DatasetCollectionTypeEnum.virtual; + }) => { const id = await postDatasetCollection({ parentId, datasetId: datasetDetail._id, diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index 966bd0e9c3..bf3a71ea33 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -14,14 +14,14 @@ import { getTagUsage, postAddTagsToCollections, updateDatasetCollectionTag -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/collection'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyInput from '@/components/MyInput'; import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm'; -import { type DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; const TagManageModal = ({ onClose }: { onClose: () => void }) => { const { t } = useTranslation(); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index 805dd42b57..91605834e6 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -2,7 +2,7 @@ import { Box, Checkbox, Flex, Input } from '@chakra-ui/react'; import MyPopover from '@fastgpt/web/components/common/MyPopover'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyBox from '@fastgpt/web/components/common/MyBox'; -import { putDatasetCollectionById } from '@/web/core/dataset/api'; +import { putDatasetCollectionById } from '@/web/core/dataset/api/collection'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useTranslation } from 'next-i18next'; @@ -13,7 +13,7 @@ import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; import { isEqual } from 'lodash'; -import { type DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; const TagsPopOver = ({ currentCollection, diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TemplateImportModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TemplateImportModal.tsx index 11854a00ea..c98a191ea8 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TemplateImportModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TemplateImportModal.tsx @@ -5,7 +5,7 @@ import { Box, Button, HStack, ModalBody, ModalFooter, VStack, Flex, Link } from import FileSelector, { type SelectFileItemType } from '@/components/Select/FileSelectorBox'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIconButton from '@fastgpt/web/components/common/Icon/button'; -import { postTemplateDatasetCollection } from '@/web/core/dataset/api'; +import { postTemplateDatasetCollection } from '@/web/core/dataset/api/collection'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useContextSelector } from 'use-context-selector'; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx index c0810ec2e1..a631c60eeb 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx @@ -17,22 +17,22 @@ import MyTag from '@fastgpt/web/components/common/Tag/index'; import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs'; import { useMemo, useState } from 'react'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { getDatasetCollectionTrainingDetail } from '@/web/core/dataset/api/collection'; import { deleteTrainingData, - getDatasetCollectionTrainingDetail, getTrainingDataDetail, getTrainingError, updateTrainingData -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/training'; import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; -import { type getTrainingDataDetailResponse } from '@/pages/api/core/dataset/training/getTrainingDataDetail'; +import { type GetTrainingDataDetailResponse } from '@fastgpt/global/openapi/core/dataset/training/api'; import MyTextarea from '@/components/common/Textarea/MyTextarea'; import { TrainingProcess } from '@/web/core/dataset/constants'; import { useForm } from 'react-hook-form'; -import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail'; +import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import MyImage from '@/components/MyImage'; @@ -47,7 +47,11 @@ enum TrainingStatus { Error = 'Error' } -const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailResponse }) => { +const ProgressView = ({ + trainingDetail +}: { + trainingDetail: GetCollectionTrainingDetailResponseType; +}) => { const { t } = useTranslation(); const isQA = trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.qa; @@ -301,7 +305,7 @@ const ErrorView = ({ [TrainingModeEnum.auto]: t('dataset:process.Auto_Index') }; - const [editChunk, setEditChunk] = useState(); + const [editChunk, setEditChunk] = useState(); const { data: errorList, @@ -449,7 +453,7 @@ const EditView = ({ onSave }: { loading: boolean; - editChunk: getTrainingDataDetailResponse; + editChunk: GetTrainingDataDetailResponse; onCancel: () => void; onSave: (data: { q: string; a?: string }) => void; }) => { diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx index 71bf210438..3659fa1975 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx @@ -19,7 +19,7 @@ import { delDatasetCollectionById, putDatasetCollectionById, postLinkCollectionSync -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/collection'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; diff --git a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx index f1cfb7ea37..19c54e8e0d 100644 --- a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx @@ -1,10 +1,7 @@ import React, { useState, useMemo } from 'react'; import { Box, Card, IconButton, Flex, Button, useTheme, Image } from '@chakra-ui/react'; -import { - getDatasetDataList, - delOneDatasetDataById, - getDatasetCollectionById -} from '@/web/core/dataset/api'; +import { getDatasetCollectionById } from '@/web/core/dataset/api/collection'; +import { getDatasetDataList, delOneDatasetDataById } from '@/web/core/dataset/api/data'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useTranslation } from 'next-i18next'; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx index 4fa4aae8c7..7539614208 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx @@ -9,7 +9,7 @@ import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; -import { getPreviewChunks } from '@/web/core/dataset/api'; +import { getPreviewChunks } from '@/web/core/dataset/api/file'; import { type ImportSourceItemType } from '@/web/core/dataset/type'; import { getPreviewSourceReadType } from '../utils'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx index 2cd3c9ca90..71a9814b82 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx @@ -27,12 +27,12 @@ import { postCreateDatasetLinkCollection, postCreateDatasetTextCollection, postReTrainingDatasetFileCollection -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/collection'; import MyTag from '@fastgpt/web/components/common/Tag/index'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { DatasetImportContext, type ImportFormType } from '../Context'; -import { type ApiCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import { type ApiCreateDatasetCollectionParams } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; const Upload = () => { const { t } = useTranslation(); diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx index 22073b51e7..2ff40b390e 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx @@ -6,7 +6,10 @@ import Loading from '@fastgpt/web/components/common/MyLoading'; import { Box, Button, Checkbox, Flex } from '@chakra-ui/react'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { getApiDatasetFileList, getApiDatasetFileListExistId } from '@/web/core/dataset/api'; +import { + getApiDatasetFileList, + getApiDatasetFileListExistId +} from '@/web/core/dataset/api/apiDataset'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { useTranslation } from 'next-i18next'; import { type ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx index 30d51b5fe1..4488908bd0 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx @@ -13,7 +13,7 @@ import { getErrText } from '@fastgpt/global/common/error/utils'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; import { getFileIcon } from '@fastgpt/global/common/file/icon'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { getUploadDatasetFilePresignedUrl } from '@/web/common/file/api'; +import { getUploadDatasetFilePresignedUrl } from '@/web/core/dataset/api/file'; import { putFileToS3 } from '@fastgpt/web/common/file/utils'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess')); diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx index 30a9540885..76f8fbea87 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx @@ -6,7 +6,7 @@ import dynamic from 'next/dynamic'; import DataProcess from '../commonProgress/DataProcess'; import { useRouter } from 'next/router'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { getDatasetCollectionById } from '@/web/core/dataset/api'; +import { getDatasetCollectionById } from '@/web/core/dataset/api/collection'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; import { Box } from '@chakra-ui/react'; diff --git a/projects/app/src/pageComponents/dataset/detail/Info/index.tsx b/projects/app/src/pageComponents/dataset/detail/Info/index.tsx index 24999e6126..ab6bf704b1 100644 --- a/projects/app/src/pageComponents/dataset/detail/Info/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Info/index.tsx @@ -8,7 +8,7 @@ import { useTranslation } from 'next-i18next'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import AIModelSelector from '@/components/Select/AIModelSelector'; -import { postRebuildEmbedding } from '@/web/core/dataset/api'; +import { postRebuildEmbedding } from '@/web/core/dataset/api/training'; import type { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.schema'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; diff --git a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx index e13f55eb02..e1f501330e 100644 --- a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx @@ -2,12 +2,12 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack, Image } from '@chakra-ui/react'; import type { UseFormRegister } from 'react-hook-form'; import { useFieldArray, useForm } from 'react-hook-form'; +import { getDatasetCollectionById } from '@/web/core/dataset/api/collection'; import { postInsertData2Dataset, putDatasetDataById, - getDatasetCollectionById, getDatasetDataItemById -} from '@/web/core/dataset/api'; +} from '@/web/core/dataset/api/data'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyModal from '@fastgpt/web/components/common/MyModal'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; diff --git a/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx b/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx index 7d06929704..012af728f6 100644 --- a/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx @@ -2,7 +2,7 @@ import React, { useMemo } from 'react'; import { Box, Flex, Button } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { getDatasetCollectionById } from '@/web/core/dataset/api'; +import { getDatasetCollectionById } from '@/web/core/dataset/api/collection'; import { useRouter } from 'next/router'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; @@ -92,7 +92,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => { } ] : []), - ...(DatasetCollectionDataProcessModeMap[collection.trainingType] + ...(collection.trainingType && DatasetCollectionDataProcessModeMap[collection.trainingType] ? [ { label: t('dataset:collection.training_type'), diff --git a/projects/app/src/pages/api/admin/initv4101.ts b/projects/app/src/pages/api/admin/initv4101.ts deleted file mode 100644 index 81fe2507b1..0000000000 --- a/projects/app/src/pages/api/admin/initv4101.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { retryFn } from '@fastgpt/global/common/system/utils'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { websiteSyncQueue } from './initv494'; -import { - upsertDatasetSyncJobScheduler, - addDatasetSyncJob -} from '@fastgpt/service/core/dataset/datasetSync'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { addHours } from 'date-fns'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4101']); - -const clearAllWebsiteSyncJobs = async () => { - logger.info('start clear all websiteSync jobs'); - - try { - // 1. get all job status statistics - const repeatableJobs = await websiteSyncQueue.getJobSchedulers(); - - // 2. delete all schedulers - for (const scheduler of repeatableJobs) { - try { - await websiteSyncQueue.removeJobScheduler(scheduler.name); - logger.info(`delete scheduler: ${scheduler.name}`); - } catch (error) { - logger.error(`delete scheduler failed (${scheduler.name}):`, { error: error }); - } - } - - // 3. clear all jobs in queue (including all status jobs) - await websiteSyncQueue.drain(true); // true parameter means delete delayed jobs - - logger.info('websiteSync queue clear completed'); - } catch (error) { - logger.error('error when clearing websiteSync queue:', { error: error }); - throw error; - } -}; - -const initDatasetSyncData = async () => { - // 1. find all autoSync datasets - const datasets = await MongoDataset.find({ autoSync: true }).lean(); - logger.info(`find ${datasets.length} datasets need auto sync`); - - let addedJobsCount = 0; - let addedSchedulersCount = 0; - - // 2. add autoSync datasets to DatasetSync queue - for (const dataset of datasets) { - try { - const datasetId = String(dataset._id); - - // add immediate execution task - await addDatasetSyncJob({ datasetId }); - addedJobsCount++; - - // add scheduler - const time = addHours(new Date(), Math.floor(Math.random() * 5) + 1); - await retryFn(() => upsertDatasetSyncJobScheduler({ datasetId }, time.getTime())); - addedSchedulersCount++; - logger.info(`add DatasetSync scheduler: ${datasetId}`); - } catch (error) { - logger.error(`error when processing dataset (${dataset._id}):`, { error: error }); - } - } - - // 3. clear all websiteSync jobs - const clearJobsResult = await clearAllWebsiteSyncJobs(); - - // 4. remove nextSyncTime field in database - logger.info('remove nextSyncTime field in database'); - await retryFn(() => - MongoDatasetCollection.updateMany( - { - teamId: { $in: datasets.map((dataset) => dataset.teamId) }, - datasetId: { $in: datasets.map((dataset) => dataset._id) } - }, - { - $unset: { - nextSyncTime: 1 - } - } - ) - ); - - const result = { - autoSyncDatasets: datasets.length, - addedJobs: addedJobsCount, - addedSchedulers: addedSchedulersCount - }; - - logger.info('migrate completed statistics:', result); - return result; -}; -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - const result = await initDatasetSyncData(); - - return { - success: true, - result - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4120.ts b/projects/app/src/pages/api/admin/initv4120.ts deleted file mode 100644 index 5cba0c6b17..0000000000 --- a/projects/app/src/pages/api/admin/initv4120.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { AppReadChatLogRoleVal } from '@fastgpt/global/support/permission/app/constant'; -import { AppPermission } from '@fastgpt/global/support/permission/app/controller'; -import type { AnyBulkWriteOperation } from '@fastgpt/service/common/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - // 初始化 app 权限：所有有 write 的都加上 readChatLog role - const rps = await MongoResourcePermission.find({ - resourceType: 'app' - }).lean(); - - const ops: AnyBulkWriteOperation[] = []; - - for (const rp of rps) { - const per = new AppPermission({ role: rp.permission }); - if (per.hasManagePer) { - per.addRole(AppReadChatLogRoleVal); - ops.push({ - updateOne: { - filter: { _id: rp._id }, - update: { $set: { permission: per.role } } - } - }); - } - } - - const result = await MongoResourcePermission.bulkWrite(ops); - - return { - success: true, - result - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4121.ts b/projects/app/src/pages/api/admin/initv4121.ts deleted file mode 100644 index 4e592e9d18..0000000000 --- a/projects/app/src/pages/api/admin/initv4121.ts +++ /dev/null @@ -1,173 +0,0 @@ -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; -import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; -import { MongoAppChatLog } from '@fastgpt/service/core/app/logs/chatLogsSchema'; -import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; -import { getLogger } from '@fastgpt/service/common/logger'; -import type { ChatSchemaType } from '@fastgpt/global/core/chat/type'; -import { surrenderProcess } from '@fastgpt/service/common/system/tools'; -const logger = getLogger(['initv4121']); - -export type SyncAppChatLogQuery = {}; - -export type SyncAppChatLogBody = { - batchSize?: number; -}; - -export type SyncAppChatLogResponse = {}; - -/* - 将 chats 表全部扫一遍，来获取统计数据 -*/ -async function handler( - req: ApiRequestProps, - res: ApiResponseType -) { - await authCert({ req, authRoot: true }); - - const { batchSize = 10 } = req.body; - - logger.info('开始同步AppChatLog数据...'); - logger.info(`批处理大小: ${batchSize}`); - - let success = 0; - const total = await MongoChat.countDocuments({}); - logger.info(`总共需要处理的chat记录数: ${total}`); - - res.json({ - data: '同步任务已开始，可在日志中看到进度' - }); - - while (true) { - logger.info(`对话同步处理进度: ${success}/${total}`); - - try { - const chats = await MongoChat.find({ - initStatistics: { $exists: false } - }) - .sort({ _id: -1 }) - .limit(batchSize) - .lean(); - - if (chats.length === 0) break; - - const result = await Promise.allSettled(chats.map((chat) => processChatRecord(chat))); - success += result.filter((r) => r.status === 'fulfilled').length; - } catch (error) { - logger.error('处理chat记录失败', { error }); - } - } - - logger.info('同步对话完成'); -} - -async function processChatRecord(chat: ChatSchemaType) { - async function calculateChatItemStats() { - const chatItems = await MongoChatItem.find({ appId: chat.appId, chatId: chat.chatId }) - .limit(1000) - .lean(); - - let chatItemCount = chatItems.length; - let errorCount = 0; - let totalPoints = 0; - let goodFeedbackCount = 0; - let badFeedbackCount = 0; - let totalResponseTime = 0; - - for (const item of chatItems) { - await surrenderProcess(); - - const itemData = item as any; - - if (itemData.userGoodFeedback && itemData.userGoodFeedback.trim() !== '') { - goodFeedbackCount++; - } - if (itemData.userBadFeedback && itemData.userBadFeedback.trim() !== '') { - badFeedbackCount++; - } - - if (itemData.durationSeconds) { - totalResponseTime += itemData.durationSeconds; - } else if ( - itemData[DispatchNodeResponseKeyEnum.nodeResponse] && - Array.isArray(itemData[DispatchNodeResponseKeyEnum.nodeResponse]) - ) { - for (const response of itemData[DispatchNodeResponseKeyEnum.nodeResponse]) { - if (response.runningTime) { - totalResponseTime += response.runningTime / 1000; - } - } - } - - if ( - itemData[DispatchNodeResponseKeyEnum.nodeResponse] && - Array.isArray(itemData[DispatchNodeResponseKeyEnum.nodeResponse]) - ) { - for (const response of itemData[DispatchNodeResponseKeyEnum.nodeResponse]) { - if (response.errorText) { - errorCount++; - break; - } - - if (response.totalPoints) { - totalPoints += response.totalPoints; - } - } - } - } - - return { - chatItemCount, - errorCount, - totalPoints, - goodFeedbackCount, - badFeedbackCount, - totalResponseTime - }; - } - - async function checkIsFirstChat(): Promise { - const earliestChat = await MongoChat.findOne( - { - appId: chat.appId, - tmbId: chat.tmbId, - ...(chat.outLinkUid && { outLinkUid: chat.outLinkUid }) - }, - '_id' - ).lean(); - - return earliestChat?._id.toString() === chat._id.toString(); - } - - const chatItemStats = await calculateChatItemStats(); - const isFirstChat = await checkIsFirstChat(); - - const chatLogData = { - appId: chat.appId, - teamId: chat.teamId, - chatId: chat.chatId, - userId: String(chat.outLinkUid || chat.tmbId), - source: chat.source, - sourceName: chat.sourceName, - createTime: chat.createTime, - updateTime: chat.updateTime, - chatItemCount: chatItemStats.chatItemCount, - errorCount: chatItemStats.errorCount, - totalPoints: chatItemStats.totalPoints, - goodFeedbackCount: chatItemStats.goodFeedbackCount, - badFeedbackCount: chatItemStats.badFeedbackCount, - totalResponseTime: chatItemStats.totalResponseTime, - isFirstChat - }; - - await MongoAppChatLog.updateOne( - { teamId: chat.teamId, appId: chat.appId, chatId: chat.chatId }, - { $set: chatLogData }, - { upsert: true } - ); - await MongoChat.updateOne({ _id: chat._id }, { $set: { initStatistics: true } }); -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4124.ts b/projects/app/src/pages/api/admin/initv4124.ts deleted file mode 100644 index b9f3b931b2..0000000000 --- a/projects/app/src/pages/api/admin/initv4124.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { batchRun } from '@fastgpt/global/common/system/utils'; -import { OwnerRoleVal, PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; -import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; - -export type SyncAppChatLogQuery = {}; - -export type SyncAppChatLogBody = { - batchSize?: number; -}; - -export type SyncAppChatLogResponse = {}; - -/** - * 初始化脚本 v4.12.4 - * 对系统内所有资源 App 和 dataset 添加 tmbId 为自己 owner 的协作者，权限为 OwnerRoleVal - */ -async function handler( - req: ApiRequestProps, - res: ApiResponseType -) { - await authCert({ req, authRoot: true }); - - // find all resources - const [apps, datasets, tmbs] = await Promise.all([ - MongoApp.find({}, '_id teamId tmbId').lean(), - MongoDataset.find({}, '_id teamId tmbId').lean(), - MongoTeamMember.find({ role: 'owner' }, '_id teamId').lean() - ]); - - for (let i = 0; i < apps.length; i += 10000) { - const appList = apps.slice(i, i + 10000); - await MongoResourcePermission.bulkWrite( - appList.map((app) => ({ - updateOne: { - filter: { - resourceId: app._id, - resourceType: PerResourceTypeEnum.app, - teamId: app.teamId, - tmbId: app.tmbId - }, - update: { - permission: OwnerRoleVal - }, - upsert: true - } - })) - ); - } - - for (let i = 0; i < datasets.length; i += 10000) { - const datasetList = datasets.slice(i, i + 10000); - await MongoResourcePermission.bulkWrite( - datasetList.map((dataset) => ({ - updateOne: { - filter: { - resourceId: dataset._id, - resourceType: PerResourceTypeEnum.dataset, - teamId: dataset.teamId, - tmbId: dataset.tmbId - }, - update: { - permission: OwnerRoleVal - }, - upsert: true - } - })) - ); - } - - for (let i = 0; i < tmbs.length; i += 10000) { - const tmbList = tmbs.slice(i, i + 10000); - await MongoResourcePermission.bulkWrite( - tmbList.map((team) => ({ - deleteOne: { - filter: { - resourceType: PerResourceTypeEnum.team, - teamId: team.teamId, - tmbId: team._id - } - } - })) - ); - } - - return { - message: 'Success' - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv471.ts b/projects/app/src/pages/api/admin/initv471.ts deleted file mode 100644 index 525539e234..0000000000 --- a/projects/app/src/pages/api/admin/initv471.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { PgClient } from '@fastgpt/service/common/vectorDB/pg/controller'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv471']); - -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await authCert({ req, authRoot: true }); - - // 删除索引 - await PgClient.query(`DROP INDEX IF EXISTS team_dataset_index;`); - await PgClient.query(`DROP INDEX IF EXISTS team_collection_index;`); - await PgClient.query(`DROP INDEX IF EXISTS team_id_index;`); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - logger.error('Migration v471 failed', { error }); - - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/pages/api/admin/initv481.ts b/projects/app/src/pages/api/admin/initv481.ts deleted file mode 100644 index fbb787298e..0000000000 --- a/projects/app/src/pages/api/admin/initv481.ts +++ /dev/null @@ -1,249 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { NextAPI } from '@/service/middleware/entry'; -import { connectionMongo } from '@fastgpt/service/common/mongo'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv481']); - -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - // 重命名 dataset.trainigns -> dataset_trainings - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'dataset.trainings' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('dataset.trainings'); - const targetCol = connectionMongo.connection.db.collection('dataset_trainings'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'dataset_trainings 中有数据，无法自动将 dataset.trainings 迁移到 dataset_trainings，请手动操作' - ); - } else { - await sourceCol.rename('dataset_trainings', { dropTarget: true }); - logger.info('success rename dataset.trainings -> dataset_trainings'); - } - } - } catch (error) { - logger.error('error： rename dataset.trainings -> dataset_trainings', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'dataset.collections' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('dataset.collections'); - const targetCol = connectionMongo.connection.db.collection('dataset_collections'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'dataset_collections 中有数据，无法自动将 dataset.collections 迁移到 dataset_collections，请手动操作' - ); - } else { - await sourceCol.rename('dataset_collections', { dropTarget: true }); - logger.info('success rename dataset.collections -> dataset_collections'); - } - } - } catch (error) { - logger.error('error： rename dataset.collections -> dataset_collections', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'dataset.datas' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('dataset.datas'); - const targetCol = connectionMongo.connection.db.collection('dataset_datas'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'dataset_datas 中有数据，无法自动将 dataset.datas 迁移到 dataset_datas，请手动操作' - ); - } else { - await sourceCol.rename('dataset_datas', { dropTarget: true }); - logger.info('success rename dataset.datas -> dataset_datas'); - } - } - } catch (error) { - logger.error('error： rename dataset.datas -> dataset_datas', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'app.versions' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('app.versions'); - const targetCol = connectionMongo.connection.db.collection('app_versions'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'app_versions 中有数据，无法自动将 app.versions 迁移到 app_versions，请手动操作' - ); - } else { - await sourceCol.rename('app_versions', { dropTarget: true }); - logger.info('success rename app.versions -> app_versions'); - } - } - } catch (error) { - logger.error('error： rename app.versions -> app_versions', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'buffer.rawtexts' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('buffer.rawtexts'); - const targetCol = connectionMongo.connection.db.collection('buffer_rawtexts'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'buffer_rawtexts 中有数据，无法自动将 buffer.rawtexts 迁移到 buffer_rawtexts，请手动操作' - ); - } else { - await sourceCol.rename('buffer_rawtexts', { dropTarget: true }); - logger.info('success rename buffer.rawtexts -> buffer_rawtexts'); - } - } - } catch (error) { - logger.error('error： rename buffer.rawtext -> buffer_rawtext', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'buffer.tts' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('buffer.tts'); - const targetCol = connectionMongo.connection.db.collection('buffer_tts'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info('buffer_tts 中有数据，无法自动将 buffer.tts 迁移到 buffer_tts，请手动操作'); - } else { - await sourceCol.rename('buffer_tts', { dropTarget: true }); - logger.info('success rename buffer.tts -> buffer_tts'); - } - } - } catch (error) { - logger.error('error： rename buffer.tts -> buffer_tts', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'team.members' }) - .toArray(); - - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('team.members'); - const targetCol = connectionMongo.connection.db.collection('team_members'); - - if ((await targetCol.countDocuments()) > 1) { - // 除了root - logger.info('team_members 中有数据，无法自动将 team.tts 迁移到 team_members，请手动操作'); - } else { - await sourceCol.rename('team_members', { dropTarget: true }); - logger.info('success rename team.members -> team_members'); - } - } - } catch (error) { - logger.error('error： rename team.members -> team_members', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'team.tags' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('team.tags'); - const targetCol = connectionMongo.connection.db.collection('team_tags'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info('team_tags 中有数据，无法自动将 team.tags 迁移到 team_tags，请手动操作'); - } else { - await sourceCol.rename('team_tags', { dropTarget: true }); - logger.info('success rename team.tags -> team_tags'); - } - } - } catch (error) { - logger.error('error： rename team.tags -> team_tags', { error }); - } - - try { - if (!connectionMongo.connection.db) { - return jsonRes(res, { - message: '数据库连接失败' - }); - } - const collections = await connectionMongo.connection.db - .listCollections({ name: 'team.subscriptions' }) - .toArray(); - if (collections.length > 0) { - const sourceCol = connectionMongo.connection.db.collection('team.subscriptions'); - const targetCol = connectionMongo.connection.db.collection('team_subscriptions'); - - if ((await targetCol.countDocuments()) > 0) { - logger.info( - 'team_subscriptions 中有数据，无法自动将 team.subscriptions 迁移到 team_subscriptions，请手动操作' - ); - } else { - await sourceCol.rename('team_subscriptions', { dropTarget: true }); - logger.info('success rename team.subscriptions -> team_subscriptions'); - } - } - } catch (error) { - logger.error('error： rename team.subscriptions -> team_subscriptions', { error }); - } - - jsonRes(res, { - message: 'success' - }); -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4810.ts b/projects/app/src/pages/api/admin/initv4810.ts deleted file mode 100644 index cb4701a57e..0000000000 --- a/projects/app/src/pages/api/admin/initv4810.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema'; -import { FastGPTProUrl } from '@fastgpt/service/common/system/constants'; -import { POST } from '@fastgpt/service/common/api/plusRequest'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4810']); - -/* 初始化发布的版本 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await authCert({ req, authRoot: true }); - - await MongoAppVersion.updateMany( - {}, - { - $set: { - isPublish: true - } - } - ); - - if (FastGPTProUrl) { - await POST('/admin/init/4810'); - } - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - logger.error('Migration v4810 failed', { error }); - - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/pages/api/admin/initv4815.ts b/projects/app/src/pages/api/admin/initv4815.ts deleted file mode 100644 index 80434f21ac..0000000000 --- a/projects/app/src/pages/api/admin/initv4815.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { NextAPI } from '@/service/middleware/entry'; -import { MongoApp } from '@fastgpt/service/core/app/schema'; - -/* 初始化发布的版本 */ -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - // scheduledTriggerConfig为 null 的，都转成 unExist - return MongoApp.updateMany( - { - $or: [ - { scheduledTriggerConfig: { $eq: null } }, - { 'scheduledTriggerConfig.cronString': { $eq: '' } } - ] - }, - { - $unset: { - scheduledTriggerConfig: '', - scheduledTriggerNextTime: '' - } - } - ); -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4817.ts b/projects/app/src/pages/api/admin/initv4817.ts deleted file mode 100644 index 132497c301..0000000000 --- a/projects/app/src/pages/api/admin/initv4817.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4817']); - -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - const users = await MongoUser.find( - { openaiAccount: { $exists: true, $ne: null } }, - '_id openaiAccount' - ); - - logger.info(`共 ${users.length} 个用户需要更新`); - let count = 0; - for (const user of users) { - await mongoSessionRun(async (session) => { - await MongoTeam.updateOne( - { ownerId: user._id }, - { - $set: { openaiAccount: (user as any).openaiAccount } - }, - { session } - ); - - // @ts-ignore - user.openaiAccount = undefined; - await user.save({ session }); - }); - count++; - logger.info(`已更新 ${count} 个用户`); - } - - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4818.ts b/projects/app/src/pages/api/admin/initv4818.ts deleted file mode 100644 index 4d105f9e78..0000000000 --- a/projects/app/src/pages/api/admin/initv4818.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index'; -import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4818']); - -/* - 简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。 - 无缝迁移： - 1. 先用 4.8.18-tmp 版本，会同时有 MongoDatasetData 和 MongoDatasetDataText 两个表和索引，依然是 MongoDatasetData 生效。会同步更新两张表数据。 - 2. 执行升级脚本，不要删除 MongoDatasetData 里的数据。 - 3. 切换正式版镜像，让 MongoDatasetDataText 生效。 - 4. 删除 MongoDatasetData 里的索引和多余字段。（4819 再删 - 5. 移动 User 表中的 avatar 字段到 TeamMember 表中。 -*/ -let success = 0; -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - const batchSize = req.body.batchSize || 500; - success = 0; - - const start = Date.now(); - await initData(batchSize); - // await restore(); - logger.info('Migration init data loaded', { durationMs: Date.now() - start }); - - success = 0; - - // batchUpdateFields(); - - return { success: true }; -} - -export default NextAPI(handler); - -const restore = async () => { - try { - const data = await MongoDatasetData.findOne({ fullTextToken: { $exists: false } }); - if (!data) return; - - data.fullTextToken = await jiebaSplit({ text: `${data.q}\n${data.a}`.trim() }); - await data.save(); - - success++; - logger.info('Migration progress', { success }); - - await restore(); - } catch (error) { - logger.error('Failed to initialize migration data', { error }); - await delay(500); - await restore(); - } -}; - -const initData = async (batchSize: number) => { - while (true) { - try { - // 找到没有初始化的数据 - const dataList = await MongoDatasetData.find( - { - initFullText: { $exists: false } - }, - '_id teamId datasetId collectionId fullTextToken' - ) - .limit(batchSize) - .lean(); - - if (dataList.length === 0) break; - - try { - await MongoDatasetDataText.insertMany( - dataList.map((item) => ({ - teamId: item.teamId, - datasetId: item.datasetId, - collectionId: item.collectionId, - dataId: item._id, - fullTextToken: item.fullTextToken - })), - { ordered: false, lean: true } - ); - } catch (error: any) { - if (error.code === 11000) { - logger.info('Duplicate key error'); - } else { - throw error; - } - } - - // 把成功插入的新数据的 dataId 更新为已初始化 - await MongoDatasetData.updateMany( - { _id: { $in: dataList.map((item) => item._id) } }, - // FullText tmp - // { $set: { initFullText: true } } - { $set: { initFullText: true }, $unset: { fullTextToken: 1 } } - ); - - success += dataList.length; - logger.info('Migration progress', { success }); - - // await initData(batchSize); - } catch (error: any) { - logger.error('Failed to migrate tags data', { error }); - await delay(500); - // await initData(batchSize); - } - } -}; - -// const batchUpdateFields = async (batchSize = 2000) => { -// // Find documents that still have these fields -// const documents = await MongoDatasetData.find({ initFullText: { $exists: true } }, '_id') -// .limit(batchSize) -// .lean(); - -// if (documents.length === 0) return; - -// // Update in batches -// await MongoDatasetData.updateMany( -// { _id: { $in: documents.map((doc) => doc._id) } }, -// { -// $unset: { -// initFullText: 1 -// // fullTextToken: 1 -// } -// } -// ); - -// success += documents.length; -// logger.info('Delete success:', success); -// await batchUpdateFields(batchSize); -// }; diff --git a/projects/app/src/pages/api/admin/initv4819.ts b/projects/app/src/pages/api/admin/initv4819.ts deleted file mode 100644 index 70f790cb66..0000000000 --- a/projects/app/src/pages/api/admin/initv4819.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4819']); - -/* - 简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。 - 无缝迁移： - 1. 移动 User 表中的 avatar 字段到 TeamMember 表中。 -*/ -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - await moveUserAvatar(); - return { success: true }; -} - -export default NextAPI(handler); - -const moveUserAvatar = async () => { - try { - const users = await MongoUser.find({}, '_id avatar'); - logger.info('Start avatar migration', { totalUsers: users.length }); - let success = 0; - for await (const user of users) { - // @ts-ignore - if (!user.avatar) continue; - try { - await mongoSessionRun(async (session) => { - await MongoTeamMember.updateMany( - { - userId: user._id - }, - { - $set: { - avatar: (user as any).avatar // 删除 avatar 字段, 因为 Type 改了，所以这里不能直接写 user.avatar - } - }, - { session } - ); - // @ts-ignore - user.avatar = undefined; - await user.save({ session }); - }); - success++; - logger.info('Avatar migration progress', { success }); - } catch (error) { - logger.error('Failed to migrate app permission records', { error }); - } - } - } catch (error) { - logger.error('Failed to migrate app permission records', { error }); - } -}; diff --git a/projects/app/src/pages/api/admin/initv4820.ts b/projects/app/src/pages/api/admin/initv4820.ts deleted file mode 100644 index 19fa71a708..0000000000 --- a/projects/app/src/pages/api/admin/initv4820.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { readConfigData } from '@/service/common/system'; -import { NextAPI } from '@/service/middleware/entry'; -import { - getFastGPTConfigFromDB, - updateFastGPTConfigBuffer -} from '@fastgpt/service/common/system/config/controller'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import json5 from 'json5'; -import { type FastGPTConfigFileType } from '@fastgpt/global/common/system/types'; -import { MongoSystemModel } from '@fastgpt/service/core/ai/config/schema'; -import { loadSystemModels } from '@fastgpt/service/core/ai/config/utils'; -import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4820']); - -/* - 简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。 - 无缝迁移： - 1. 移动 User 表中的 avatar 字段到 TeamMember 表中。 -*/ -async function handler(req: NextApiRequest, res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - // load config - const [{ fastgptConfig: dbConfig }, fileConfig] = await Promise.all([ - getFastGPTConfigFromDB(), - readConfigData('config.json') - ]); - const fileRes = json5.parse(fileConfig) as FastGPTConfigFileType; - - const llmModels = dbConfig.llmModels || fileRes.llmModels || []; - const vectorModels = dbConfig.vectorModels || fileRes.vectorModels || []; - const reRankModels = dbConfig.reRankModels || fileRes.reRankModels || []; - const audioSpeechModels = dbConfig.audioSpeechModels || fileRes.audioSpeechModels || []; - const whisperModel = dbConfig.whisperModel || fileRes.whisperModel; - - const list = [ - ...llmModels.map((item) => ({ - ...item, - type: ModelTypeEnum.llm - })), - ...vectorModels.map((item) => ({ - ...item, - type: ModelTypeEnum.embedding - })), - ...reRankModels.map((item) => ({ - ...item, - type: ModelTypeEnum.rerank - })), - ...audioSpeechModels.map((item) => ({ - ...item, - type: ModelTypeEnum.tts - })), - { - ...whisperModel, - type: ModelTypeEnum.stt - } - ]; - - for await (const item of list) { - try { - await MongoSystemModel.updateOne( - { model: item.model }, - { $set: { model: item.model, metadata: { ...item, isActive: true } } }, - { upsert: true } - ); - } catch (error) { - logger.error('Failed to migrate account points data', { error }); - } - } - - await loadSystemModels(true); - await updateFastGPTConfigBuffer(); - - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4822.ts b/projects/app/src/pages/api/admin/initv4822.ts deleted file mode 100644 index bd8cf37e13..0000000000 --- a/projects/app/src/pages/api/admin/initv4822.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4822']); - -/* - * 复制 Team 表中的 notificationAccount 到 User 表的 contact 中 - */ -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - const users = await MongoUser.find(); - const teams = await MongoTeam.find(); - - logger.info('Start bill migration', { totalUsers: users.length }); - let success = 0; - for await (const user of users) { - try { - const team = teams.find((team) => String(team.ownerId) === String(user._id)); - if (team && !user.contact) { - user.contact = team.notificationAccount; - } - await user.save(); - logger.info('Bill migration progress', { success: ++success }); - } catch (error) { - logger.error('Failed to migrate user bill records', { error }); - } - } - - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4823.ts b/projects/app/src/pages/api/admin/initv4823.ts deleted file mode 100644 index c26e4ac344..0000000000 --- a/projects/app/src/pages/api/admin/initv4823.ts +++ /dev/null @@ -1,208 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { addHours } from 'date-fns'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { delay, retryFn } from '@fastgpt/global/common/system/utils'; -import { delCollection } from '@fastgpt/service/core/dataset/collection/controller'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { type DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorDB/controller'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4823']); - -// 删了库，没删集合 -const checkInvalidCollection = async () => { - const batchSize = 1000; - - let skip = 0; - let success = 0; - while (true) { - try { - const collections = await MongoDatasetCollection.find( - {}, - '_id teamId datasetId fileId metadata' - ) - .limit(batchSize) - .skip(skip) - .lean(); - if (collections.length === 0) break; - - const datasetMap: Record = {}; - - // 相同 datasetId 的集合放到一起 - for await (const collection of collections) { - const datasetId = String(collection.datasetId); - const val = datasetMap[datasetId]; - if (val) { - val.push(collection); - } else { - datasetMap[datasetId] = [collection]; - } - } - - const datasetIds = Object.keys(datasetMap); - for await (const datasetId of datasetIds) { - try { - const val = datasetMap[datasetId]; - if (!val) { - continue; - } - - await retryFn(async () => { - const datasetExists = await MongoDataset.findById(datasetId, '_id').lean(); - if (!datasetExists) { - logger.info('清理无效的知识库集合', { datasetId }); - await mongoSessionRun(async (session) => { - return await delCollection({ - collections: val, - delImg: true, - delFile: true, - session - }); - }); - } - }); - } catch (error) { - logger.error('Failed to clean invalid dataset records', { error }); - } - } - - success += batchSize; - skip += batchSize; - logger.info(`检测集合完成：${success}`); - } catch (error) { - logger.error('Failed to clean invalid dataset records', { error }); - await delay(1000); - } - } -}; - -// 删了集合，没删 data -const checkInvalidData = async () => { - try { - const datas = (await MongoDatasetData.aggregate([ - { - $group: { - _id: '$collectionId', - teamId: { $first: '$teamId' }, - datasetId: { $first: '$datasetId' }, - collectionId: { $first: '$collectionId' } - } - } - ])) as { - _id: string; - teamId: string; - datasetId: string; - collectionId: string; - }[]; - logger.info('Total data collections length', { total: datas.length }); - // 批量获取集合 - const collections = await MongoDatasetCollection.find({}, '_id').lean(); - logger.info('Total collection length', { total: collections.length }); - const collectionMap: Record = {}; - for await (const collection of collections) { - collectionMap[collection._id] = collection; - } - // 逐一删除无效的集合内容 - for await (const data of datas) { - try { - const col = collectionMap[data.collectionId]; - if (!col) { - logger.info('清理无效的知识库集合内容', { collectionId: data.collectionId }); - await retryFn(async () => { - await MongoDatasetTraining.deleteMany({ - teamId: data.teamId, - datasetId: data.datasetId, - collectionId: data.collectionId - }); - await MongoDatasetDataText.deleteMany({ - teamId: data.teamId, - datasetId: data.datasetId, - collectionId: data.collectionId - }); - await deleteDatasetDataVector({ - teamId: data.teamId, - datasetIds: [data.datasetId], - collectionIds: [data.collectionId] - }); - await MongoDatasetData.deleteMany({ - teamId: data.teamId, - datasetId: data.datasetId, - collectionId: data.collectionId - }); - }); - } - } catch (error) { - logger.error('Failed to clean invalid dataset records', { error }); - } - } - - logger.info(`检测集合完成`); - } catch (error) { - logger.error('checkInvalidData error', { error: error }); - } -}; - -// 删了data，没删 data_text -const checkInvalidDataText = async () => { - try { - // 获取所有索引层的 dataId - const dataTexts = await MongoDatasetDataText.find({}, 'dataId').lean(); - const dataIds = dataTexts.map((item) => String(item.dataId)); - logger.info('Total data_text dataIds', { total: dataIds.length }); - - // 获取数据层的 dataId - const datas = await MongoDatasetData.find({}, '_id').lean(); - const datasSet = new Set(datas.map((item) => String(item._id))); - logger.info('Total data length', { total: datas.length }); - - // 存在索引层，不存在数据层的 dataId，说明数据已经被删了 - const unExistsSet = dataIds.filter((id) => !datasSet.has(id)); - logger.info('Total unExists dataIds', { total: unExistsSet.length }); - await MongoDatasetDataText.deleteMany({ - dataId: { $in: unExistsSet } - }); - } catch (error) { - logger.error('checkInvalidDataText error', { error: error }); - } -}; - -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await authCert({ req, authRoot: true }); - const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number }; - - (async () => { - try { - // 360天 ~ 2小时前 - const endTime = addHours(new Date(), start); - const startTime = addHours(new Date(), end); - logger.info('清理无效的集合'); - await checkInvalidCollection(); - logger.info('清理无效的数据'); - await checkInvalidData(); - logger.info('清理无效的data_text'); - await checkInvalidDataText(); - } catch (error) { - logger.info('执行脏数据清理任务出错了'); - } - })(); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - logger.error('Failed to clean invalid dataset records', { error }); - - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/pages/api/admin/initv486.ts b/projects/app/src/pages/api/admin/initv486.ts deleted file mode 100644 index 40efef6f8e..0000000000 --- a/projects/app/src/pages/api/admin/initv486.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv486']); - -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await authCert({ req, authRoot: true }); - - await MongoApp.updateMany( - {}, - { - $set: { - inheritPermission: true - } - } - ); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - logger.error('Migration v486 failed', { error }); - - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/pages/api/admin/initv488.ts b/projects/app/src/pages/api/admin/initv488.ts deleted file mode 100644 index 5bbc7a628c..0000000000 --- a/projects/app/src/pages/api/admin/initv488.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { DataSetDefaultRoleVal } from '@fastgpt/global/support/permission/dataset/constant'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv488']); - -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await authCert({ req, authRoot: true }); - - await MongoDataset.updateMany( - { - inheritPermission: { $exists: false } - }, - { - $set: { - inheritPermission: true - } - } - ); - await MongoDataset.updateMany( - { - defaultPermission: { $exists: false } - }, - { - $set: { - defaultPermission: DataSetDefaultRoleVal - } - } - ); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - logger.error('Migration v488 failed', { error }); - - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/pages/api/admin/initv490.ts b/projects/app/src/pages/api/admin/initv490.ts deleted file mode 100644 index c261ab5d1c..0000000000 --- a/projects/app/src/pages/api/admin/initv490.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants'; -import { PgClient } from '@fastgpt/service/common/vectorDB/pg/controller'; -import { PG_ADDRESS } from '@fastgpt/service/common/vectorDB/constants'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv490']); - -// 所有 trainingType=auto 的 collection，都改成 trainingType=chunk -const updateCollections = async () => { - await MongoDatasetCollection.updateMany( - { - trainingType: DatasetCollectionDataProcessModeEnum.auto - }, - { - $set: { - trainingType: DatasetCollectionDataProcessModeEnum.chunk, - autoIndexes: true - } - } - ); -}; -const updateData = async () => { - await MongoDatasetData.updateMany({}, [ - { - $set: { - indexes: { - $map: { - input: '$indexes', - as: 'index', - in: { - $mergeObjects: [ - '$$index', - { - type: { - $cond: { - if: { $eq: ['$$index.defaultIndex', true] }, - then: DatasetDataIndexTypeEnum.default, - else: DatasetDataIndexTypeEnum.custom - } - } - } - ] - } - } - } - } - } - ]); -}; -const upgradePgVector = async () => { - if (!PG_ADDRESS) return; - await PgClient.query(` - ALTER EXTENSION vector UPDATE; - `); -}; - -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - logger.info('升级 PG vector 插件'); - await upgradePgVector(); - - logger.info('变更所有 collection 的 trainingType 为 chunk'); - await updateCollections(); - - logger.info( - "更新所有 data 的 index, autoIndex=true 的，增加type='default'，其他的增加 type='custom'" - ); - await updateData(); - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv491.ts b/projects/app/src/pages/api/admin/initv491.ts deleted file mode 100644 index a43980176c..0000000000 --- a/projects/app/src/pages/api/admin/initv491.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { jiebaSplit } from '@fastgpt/service/common/string/jieba'; -import { getLogger } from '@fastgpt/service/common/logger'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { type DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type'; -import type { AnyBulkWriteOperation } from '@fastgpt/service/common/mongo'; -const logger = getLogger(['initv491']); - -const updateData = async () => { - let success = 0; - - while (true) { - try { - const time = Date.now(); - const data = await MongoDatasetData.find({ - initJieba: { $exists: false }, - updateTime: { $lte: time } // 只需要取旧的数据 - }) - .limit(1000) - .lean(); - if (data.length === 0) { - logger.info('更新分词完成'); - break; - } - - const dataTextOps: AnyBulkWriteOperation[] = []; - const datasetDataIds: string[] = []; - - // 先进行分词处理 - for await (const item of data) { - const text = `${item.q} ${item.a}`.trim(); - try { - const tokens = await jiebaSplit({ text }); - dataTextOps.push({ - updateOne: { - filter: { dataId: item._id }, - update: { $set: { fullTextToken: tokens } } - } - }); - datasetDataIds.push(item._id); - } catch (error) { - logger.error(`分词处理错误: ${item._id}`, { error }); - } - } - - await mongoSessionRun(async (session) => { - if (dataTextOps.length > 0) { - await MongoDatasetDataText.bulkWrite(dataTextOps, { session, ordered: true }); - } - if (datasetDataIds.length > 0) { - await MongoDatasetData.updateMany( - { _id: { $in: datasetDataIds } }, - { $set: { initJieba: true } }, - { - session - } - ); - } - }); - - success += dataTextOps.length; - logger.info(`成功 ${success}`); - } catch (error) { - logger.error('更新所有旧的 jieba 分词失败', { error }); - await delay(1000); - } - } -}; - -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - logger.info('更新所有旧的 jieba 分词'); - updateData(); - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv4911.ts b/projects/app/src/pages/api/admin/initv4911.ts deleted file mode 100644 index 9fbc5f0b6d..0000000000 --- a/projects/app/src/pages/api/admin/initv4911.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type NextApiRequest, type NextApiResponse } from 'next'; - -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv4911']); - -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - logger.info('更新所有 API 知识库'); - - const datasets = await MongoDataset.find({ - type: { - $in: [DatasetTypeEnum.apiDataset, DatasetTypeEnum.feishu, DatasetTypeEnum.yuque] - } - }).lean(); - - for (const dataset of datasets) { - logger.info('Migrating API dataset', { datasetId: dataset._id }); - await MongoDataset.updateOne( - { _id: dataset._id }, - { - $set: { - apiDatasetServer: { - ...(dataset.apiServer && { apiServer: dataset.apiServer }), - ...(dataset.feishuServer && { feishuServer: dataset.feishuServer }), - ...(dataset.yuqueServer && { yuqueServer: dataset.yuqueServer }) - } - } - } - ); - } - - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/initv494.ts b/projects/app/src/pages/api/admin/initv494.ts deleted file mode 100644 index 964d60137d..0000000000 --- a/projects/app/src/pages/api/admin/initv494.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { retryFn } from '@fastgpt/global/common/system/utils'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { addHours } from 'date-fns'; -import { type NextApiRequest, type NextApiResponse } from 'next'; - -import { type Processor } from '@fastgpt/service/common/bullmq'; -import { getQueue, getWorker, QueueNames } from '@fastgpt/service/common/bullmq'; -import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv494']); - -const initWebsiteSyncData = async () => { - // find out all website dataset - const datasets = await MongoDataset.find({ type: DatasetTypeEnum.websiteDataset }).lean(); - - logger.info('更新站点同步的定时器'); - // Add scheduler for all website dataset - await Promise.all( - datasets.map((dataset) => { - if (dataset.autoSync) { - // 随机生成一个往后 1～24 小时的时间 - const time = addHours(new Date(), Math.floor(Math.random() * 23) + 1); - return retryFn(() => - upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) }, time.getTime()) - ); - } - }) - ); - - logger.info('移除站点同步集合的定时器'); - // Remove all nextSyncTime - await retryFn(() => - MongoDatasetCollection.updateMany( - { - teamId: datasets.map((dataset) => dataset.teamId), - datasetId: datasets.map((dataset) => dataset._id) - }, - { - $unset: { - nextSyncTime: 1 - } - } - ) - ); -}; -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - - await initWebsiteSyncData(); - - return { success: true }; -} - -export default NextAPI(handler); - -export type WebsiteSyncJobData = { - datasetId: string; -}; - -export const websiteSyncQueue = getQueue(QueueNames.websiteSync, { - defaultJobOptions: { - attempts: 3, // retry 3 times - backoff: { - type: 'exponential', - delay: 1000 // delay 1 second between retries - } - } -}); -export const getWebsiteSyncWorker = (processor: Processor) => { - return getWorker(QueueNames.websiteSync, processor, { - removeOnFail: { - age: 15 * 24 * 60 * 60, // Keep up to 15 days - count: 1000 // Keep up to 1000 jobs - }, - concurrency: 1 // Set worker to process only 1 job at a time - }); -}; - -export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => { - const datasetId = String(data.datasetId); - // deduplication: make sure only 1 job - return websiteSyncQueue.add(datasetId, data, { deduplication: { id: datasetId } }); -}; - -export const getWebsiteSyncDatasetStatus = async (datasetId: string) => { - const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId); - if (!jobId) { - return { - status: DatasetStatusEnum.active, - errorMsg: undefined - }; - } - const job = await websiteSyncQueue.getJob(jobId); - if (!job) { - return { - status: DatasetStatusEnum.active, - errorMsg: undefined - }; - } - - const jobState = await job.getState(); - - if (jobState === 'failed' || jobState === 'unknown') { - return { - status: DatasetStatusEnum.error, - errorMsg: job.failedReason - }; - } - if (['waiting-children', 'waiting'].includes(jobState)) { - return { - status: DatasetStatusEnum.waiting, - errorMsg: undefined - }; - } - if (jobState === 'active') { - return { - status: DatasetStatusEnum.syncing, - errorMsg: undefined - }; - } - - return { - status: DatasetStatusEnum.active, - errorMsg: undefined - }; -}; - -// Scheduler setting -const repeatDuration = 24 * 60 * 60 * 1000; // every day -export const upsertWebsiteSyncJobScheduler = (data: WebsiteSyncJobData, startDate?: number) => { - const datasetId = String(data.datasetId); - - return websiteSyncQueue.upsertJobScheduler( - datasetId, - { - every: repeatDuration, - startDate: startDate || new Date().getTime() + repeatDuration // First run tomorrow - }, - { - name: datasetId, - data - } - ); -}; - -export const getWebsiteSyncJobScheduler = (datasetId: string) => { - return websiteSyncQueue.getJobScheduler(String(datasetId)); -}; - -export const removeWebsiteSyncJobScheduler = (datasetId: string) => { - return websiteSyncQueue.removeJobScheduler(String(datasetId)); -}; diff --git a/projects/app/src/pages/api/admin/initv495.ts b/projects/app/src/pages/api/admin/initv495.ts deleted file mode 100644 index 45592f0625..0000000000 --- a/projects/app/src/pages/api/admin/initv495.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type NextApiRequest, type NextApiResponse } from 'next'; -import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; -import { TeamPermission } from '@fastgpt/global/support/permission/user/controller'; -import { - TeamApikeyCreatePermissionVal, - TeamAppCreatePermissionVal, - TeamDatasetCreatePermissionVal -} from '@fastgpt/global/support/permission/user/constant'; -import { retryFn } from '@fastgpt/global/common/system/utils'; -import { getLogger } from '@fastgpt/service/common/logger'; -const logger = getLogger(['initv495']); - -async function handler(req: NextApiRequest, _res: NextApiResponse) { - await authCert({ req, authRoot: true }); - // 更新团队权限： - // 目前所有有 TeamWritePermission 的，都需要添加三个新的权限。 - - const rps = await MongoResourcePermission.find({ - resourceType: 'team', - teamId: { $exists: true }, - resourceId: null - }); - - for await (const rp of rps) { - const per = rp.permission; - logger.info('Collection training type migration progress', { progress: per }); - if (per & 0b010) { - // has 0b010 - rp.permission = - per | - TeamAppCreatePermissionVal | - TeamDatasetCreatePermissionVal | - TeamApikeyCreatePermissionVal; - - try { - await retryFn(async () => { - await rp.save(); - }); - } catch (error) { - logger.error('更新权限异常', { error: error }); - } - } - } - - return { success: true }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/admin/resetMilvus.ts b/projects/app/src/pages/api/admin/resetMilvus.ts deleted file mode 100644 index 20b6deef4d..0000000000 --- a/projects/app/src/pages/api/admin/resetMilvus.ts +++ /dev/null @@ -1,141 +0,0 @@ -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; -import { NextAPI } from '@/service/middleware/entry'; -import { MilvusCtrl } from '@fastgpt/service/common/vectorDB/milvus/index'; -import { DatasetVectorTableName } from '@fastgpt/service/common/vectorDB/constants'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { type DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { startTrainingQueue } from '@/service/core/dataset/training/utils'; -import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; -const logger = getLogger(LogCategories.SYSTEM); - -export type resetMilvusQuery = {}; - -export type resetMilvusBody = {}; - -export type resetMilvusResponse = {}; - -async function handler( - req: ApiRequestProps, - res: ApiResponseType -): Promise { - await authCert({ req, authRoot: true }); - - // 删除 milvus DatasetVectorTableName 表 - const milvus = new MilvusCtrl(); - const client = await milvus.getClient(); - await client.dropCollection({ - collection_name: DatasetVectorTableName - }); - await milvus.init(); - - const datasets = await MongoDataset.find({}, '_id name teamId tmbId vectorModel').lean(); - - let dataLength = 0; - const rebuild = async (dataset: DatasetSchemaType, retry = 3) => { - try { - return mongoSessionRun(async (session) => { - // 更新数据状态进入重建 - const data = await MongoDatasetData.updateMany( - { - teamId: dataset.teamId, - datasetId: dataset._id - }, - { - $set: { - rebuilding: true - } - }, - { - session - } - ); - dataLength += data.matchedCount; - - // 插入数据进入训练库 - const max = global.systemEnv?.vectorMaxProcess || 10; - const arr = new Array(max * 2).fill(0); - - for await (const _ of arr) { - try { - const hasNext = await mongoSessionRun(async (session) => { - // get next dataset.data - const data = await MongoDatasetData.findOneAndUpdate( - { - rebuilding: true - }, - { - $unset: { - rebuilding: null - }, - updateTime: new Date() - }, - { - session - } - ).select({ - _id: 1, - collectionId: 1, - teamId: 1, - tmbId: 1, - datasetId: 1 - }); - - if (data) { - await MongoDatasetTraining.create( - [ - { - teamId: dataset.teamId, - tmbId: dataset.tmbId, - datasetId: dataset._id, - collectionId: data.collectionId, - mode: TrainingModeEnum.chunk, - model: dataset.vectorModel, - dataId: data._id - } - ], - { - session, - ordered: true - } - ); - } - - return !!data; - }); - - if (!hasNext) { - break; - } - } catch (error) { - logger.error('Failed to reset vector record', { error }); - } - } - }); - } catch (error) { - logger.error('Reset Milvus migration failed', { error }); - await delay(500); - if (retry > 0) { - return rebuild(dataset, retry - 1); - } - } - }; - - // 重置所有集合进入 rebuild 状态 - (async () => { - for await (const dataset of datasets) { - await rebuild(dataset); - } - startTrainingQueue(); - logger.info('Milvus reset completed', { totalResetLength: dataLength }); - })(); - - return {}; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/common/file/presignAvatarPostUrl.ts b/projects/app/src/pages/api/common/file/presignAvatarPostUrl.ts index 1d4a641f4a..9ddbddf8cf 100644 --- a/projects/app/src/pages/api/common/file/presignAvatarPostUrl.ts +++ b/projects/app/src/pages/api/common/file/presignAvatarPostUrl.ts @@ -1,8 +1,8 @@ import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; -import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; export type updateAvatarQuery = {}; @@ -11,7 +11,7 @@ export type updateAvatarBody = { autoExpired?: boolean; }; -export type updateAvatarResponse = CreatePostPresignedUrlResult; +export type updateAvatarResponse = CreatePostPresignedUrlResponseType; async function handler( req: ApiRequestProps, diff --git a/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts b/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts index 824251a164..8e72891688 100644 --- a/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts +++ b/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts @@ -1,6 +1,6 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; -import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; import { authUserPer } from '@fastgpt/service/support/permission/user/auth'; import { TeamDatasetCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant'; import { getFileS3Key } from '@fastgpt/service/common/s3/utils'; @@ -15,7 +15,7 @@ export type PresignTempFilePostUrlParams = { async function handler( req: ApiRequestProps -): Promise { +): Promise { const { filename } = req.body; const { teamId, tmbId } = await authUserPer({ diff --git a/projects/app/src/pages/api/core/agentSkills/debugChat.ts b/projects/app/src/pages/api/core/agentSkills/debugChat.ts index 8d308ada6f..e9ebb42f9b 100644 --- a/projects/app/src/pages/api/core/agentSkills/debugChat.ts +++ b/projects/app/src/pages/api/core/agentSkills/debugChat.ts @@ -18,7 +18,7 @@ import { import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { NextAPI } from '@/service/middleware/entry'; import { GPTMessages2Chats } from '@fastgpt/global/core/chat/adapt'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { getLastInteractiveValue, textAdaptGptResponse diff --git a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts index 65b4fbe4a1..a2774ea2b4 100644 --- a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts +++ b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts @@ -20,7 +20,7 @@ import { type CreateQuestionGuideResponseType } from '@fastgpt/global/openapi/core/ai/agent/api'; import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; -import { type ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import { type ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; async function handler( req: ApiRequestProps, diff --git a/projects/app/src/pages/api/core/ai/optimizePrompt.ts b/projects/app/src/pages/api/core/ai/optimizePrompt.ts index 9520d22cfa..382f8f85b6 100644 --- a/projects/app/src/pages/api/core/ai/optimizePrompt.ts +++ b/projects/app/src/pages/api/core/ai/optimizePrompt.ts @@ -3,7 +3,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { responseWrite } from '@fastgpt/service/common/response'; import { sseErrRes } from '@fastgpt/service/common/response'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { formatModelChars2Points } from '@fastgpt/service/support/wallet/usage/utils'; import { createUsage } from '@fastgpt/service/support/wallet/usage/controller'; diff --git a/projects/app/src/pages/api/core/ai/token.ts b/projects/app/src/pages/api/core/ai/token.ts index ae57e7b68e..40b793860f 100644 --- a/projects/app/src/pages/api/core/ai/token.ts +++ b/projects/app/src/pages/api/core/ai/token.ts @@ -1,7 +1,7 @@ import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { type ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import { type ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { countGptMessagesTokens } from '@fastgpt/service/common/string/tiktoken'; export type tokenQuery = {}; diff --git a/projects/app/src/pages/api/core/app/folder/path.ts b/projects/app/src/pages/api/core/app/folder/path.ts index 5fe5fc1a7b..6807544b08 100644 --- a/projects/app/src/pages/api/core/app/folder/path.ts +++ b/projects/app/src/pages/api/core/app/folder/path.ts @@ -13,7 +13,7 @@ async function handler( req: NextApiRequest, res: NextApiResponse ): Promise { - const { sourceId: appId, type } = req.query as GetPathProps; + const { sourceId: appId, type = 'current' } = req.query as GetPathProps; if (!appId) { return []; diff --git a/projects/app/src/pages/api/core/app/tool/path.ts b/projects/app/src/pages/api/core/app/tool/path.ts index 1593f88a60..c7b06609a7 100644 --- a/projects/app/src/pages/api/core/app/tool/path.ts +++ b/projects/app/src/pages/api/core/app/tool/path.ts @@ -18,7 +18,7 @@ async function handler( req: ApiRequestProps, res: ApiResponseType ): Promise { - const { sourceId: pluginId, type } = req.query; + const { sourceId: pluginId, type = 'current' } = req.query; const lang = getLocale(req); if (!pluginId) return []; diff --git a/projects/app/src/pages/api/core/chat/chatTest.ts b/projects/app/src/pages/api/core/chat/chatTest.ts index b01bde9a8a..906d28b93a 100644 --- a/projects/app/src/pages/api/core/chat/chatTest.ts +++ b/projects/app/src/pages/api/core/chat/chatTest.ts @@ -10,7 +10,6 @@ import type { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat import { authApp } from '@fastgpt/service/support/permission/app/auth'; import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; import { getRunningUserInfoByTmbId } from '@fastgpt/service/support/user/team/utils'; -import type { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge'; import { concatHistories, getChatTitleFromChatMessage, @@ -24,8 +23,6 @@ import { } from '@fastgpt/service/core/app/tool/workflowTool/utils'; import { NextAPI } from '@/service/middleware/entry'; import { GPTMessages2Chats } from '@fastgpt/global/core/chat/adapt'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; -import type { AppChatConfigType } from '@fastgpt/global/core/app/type'; import { getLastInteractiveValue, getMaxHistoryLimitFromNodes, @@ -35,7 +32,6 @@ import { storeNodes2RuntimeNodes, textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils'; -import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node'; import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatch/utils'; import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants'; import { getWorkflowToolInputsFromStoreNodes } from '@fastgpt/global/core/app/tool/workflowTool/utils'; @@ -50,18 +46,7 @@ import { LimitTypeEnum, teamFrequencyLimit } from '@fastgpt/service/common/api/f import { getIpFromRequest } from '@fastgpt/service/common/geo'; import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils'; import { UserError } from '@fastgpt/global/common/error/utils'; - -export type Props = { - messages: ChatCompletionMessageParam[]; - responseChatItemId: string; - nodes: StoreNodeItemType[]; - edges: StoreEdgeItemType[]; - variables: Record; - appId: string; - appName: string; - chatId: string; - chatConfig: AppChatConfigType; -}; +import { ChatTestPropsSchema } from '@fastgpt/global/openapi/core/chat/completion/api'; async function handler(req: NextApiRequest, res: NextApiResponse) { let { @@ -74,15 +59,8 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { appId, chatConfig, chatId - } = req.body as Props; + } = ChatTestPropsSchema.parse(req.body); try { - if (!Array.isArray(nodes)) { - throw new Error('Nodes is not array'); - } - if (!Array.isArray(edges)) { - throw new Error('Edges is not array'); - } - const originIp = getIpFromRequest(req); const chatMessages = GPTMessages2Chats({ messages }); diff --git a/projects/app/src/pages/api/core/chat/file/presignChatFilePostUrl.ts b/projects/app/src/pages/api/core/chat/file/presignChatFilePostUrl.ts index 7eedded3af..e0b5271611 100644 --- a/projects/app/src/pages/api/core/chat/file/presignChatFilePostUrl.ts +++ b/projects/app/src/pages/api/core/chat/file/presignChatFilePostUrl.ts @@ -1,6 +1,6 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; -import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat'; import { authChatCrud } from '@/service/support/permission/auth/chat'; import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils'; @@ -8,7 +8,7 @@ import { addSeconds } from 'date-fns'; import { PresignChatFilePostUrlSchema } from '@fastgpt/global/openapi/core/chat/file/api'; import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils'; -async function handler(req: ApiRequestProps): Promise { +async function handler(req: ApiRequestProps): Promise { const { filename, appId, chatId, outLinkAuthData } = PresignChatFilePostUrlSchema.parse(req.body); const { teamId, uid } = await authChatCrud({ diff --git a/projects/app/src/pages/api/core/chat/helperBot/getFilePresign.ts b/projects/app/src/pages/api/core/chat/helperBot/getFilePresign.ts index 8fe8ba0390..d2440e205f 100644 --- a/projects/app/src/pages/api/core/chat/helperBot/getFilePresign.ts +++ b/projects/app/src/pages/api/core/chat/helperBot/getFilePresign.ts @@ -1,8 +1,7 @@ import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; import type { GetHelperBotFilePresignParamsType } from '@fastgpt/global/openapi/core/chat/helperBot/api'; -import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; -import { authHelperBotChatCrud } from '@/service/support/permission/auth/chat'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { getS3HelperBotSource } from '../../../../../../../../packages/service/common/s3/sources/helperbot/index'; import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils'; @@ -12,7 +11,7 @@ export type getFilePresignQuery = {}; export type getFilePresignBody = GetHelperBotFilePresignParamsType; -export type getFilePresignResponse = CreatePostPresignedUrlResult; +export type getFilePresignResponse = CreatePostPresignedUrlResponseType; const authUploadLimit = (tmbId: string) => { if (!global.feConfigs.uploadFileMaxAmount) return; diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts b/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts index c6161608dd..0b778f8de1 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts @@ -1,37 +1,41 @@ import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; import { NextAPI } from '@/service/middleware/entry'; -import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; -import { type NextApiRequest } from 'next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import type { - ApiDatasetServerType, - APIFileItemType -} from '@fastgpt/global/core/dataset/apiDataset/type'; +import { + GetApiDatasetCatalogBodySchema, + GetApiDatasetCatalogResponseSchema, + type GetApiDatasetCatalogBody, + type GetApiDatasetCatalogResponse +} from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; -export type GetApiDatasetCataLogProps = { - parentId?: ParentIdType; - apiDatasetServer?: ApiDatasetServerType; -}; - -export type GetApiDatasetCataLogResponse = APIFileItemType[]; - -async function handler(req: NextApiRequest) { - let { searchKey = '', parentId = null, apiDatasetServer } = req.body; +async function handler( + req: ApiRequestProps +): Promise { + const { + parentId, + searchKey = '', + apiDatasetServer + } = GetApiDatasetCatalogBodySchema.parse(req.body); await authCert({ req, authToken: true }); // Remove basePath from apiDatasetServer - Object.values(apiDatasetServer).forEach((server: any) => { - if (server.basePath) { - delete server.basePath; - } - }); + if (apiDatasetServer) { + Object.values(apiDatasetServer).forEach((server) => { + if (server && 'basePath' in server && server.basePath) { + delete server.basePath; + } + }); + } const data = await ( await getApiDatasetRequest(apiDatasetServer) - ).listFiles({ parentId, searchKey }); + ).listFiles({ parentId, searchKey: searchKey }); - return data?.filter((item: APIFileItemType) => item.hasChild === true) || []; + const folders = data?.filter((item) => item.hasChild === true) ?? []; + + return GetApiDatasetCatalogResponseSchema.parse(folders); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts b/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts index 9965c2b17e..1d597a87af 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts @@ -1,25 +1,17 @@ import { NextAPI } from '@/service/middleware/entry'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; -import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; -import type { - ApiDatasetDetailResponse, - ApiDatasetServerType -} from '@fastgpt/global/core/dataset/apiDataset/type'; +import type { ApiDatasetDetailResponse } from '@fastgpt/global/core/dataset/apiDataset/type'; import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant'; - -export type GetApiDatasetPathQuery = {}; - -export type GetApiDatasetPathBody = { - datasetId?: string; - parentId?: ParentIdType; - apiDatasetServer?: ApiDatasetServerType; -}; - -export type GetApiDatasetPathResponse = string; +import { + GetApiDatasetPathNamesBodySchema, + GetApiDatasetPathNamesResponseSchema, + type GetApiDatasetPathNamesBody, + type GetApiDatasetPathNamesResponse +} from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; const getFullPath = async ( currentId: string, @@ -40,11 +32,15 @@ const getFullPath = async ( }; async function handler( - req: ApiRequestProps, - res: ApiResponseType -): Promise { - const { datasetId, parentId } = req.body; - if (!parentId) return ''; + req: ApiRequestProps +): Promise { + const { + datasetId, + parentId, + apiDatasetServer: bodyApiDatasetServer + } = GetApiDatasetPathNamesBodySchema.parse(req.body); + + if (!parentId) return GetApiDatasetPathNamesResponseSchema.parse(''); const apiDatasetServer = await (async () => { if (datasetId) { @@ -60,7 +56,7 @@ async function handler( } else { await authCert({ req, authToken: true }); - return req.body.apiDatasetServer; + return bodyApiDatasetServer; } })(); @@ -70,7 +66,9 @@ async function handler( return Promise.reject(DatasetErrEnum.noApiServer); } - return await getFullPath(parentId, apiDataset.getFileDetail); + const fullPath = await getFullPath(parentId, apiDataset.getFileDetail); + + return GetApiDatasetPathNamesResponseSchema.parse(fullPath); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/list.ts b/projects/app/src/pages/api/core/dataset/apiDataset/list.ts index 3fc793995b..e306a57ddd 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/list.ts @@ -1,18 +1,19 @@ import { NextAPI } from '@/service/middleware/entry'; -import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; -import { type NextApiRequest } from 'next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + GetApiDatasetFileListBodySchema, + GetApiDatasetFileListResponseSchema, + type GetApiDatasetFileListBody, + type GetApiDatasetFileListResponse +} from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; -export type GetApiDatasetFileListProps = { - searchKey?: string; - parentId?: ParentIdType; - datasetId: string; -}; - -async function handler(req: NextApiRequest) { - let { searchKey = '', parentId = null, datasetId } = req.body; +async function handler( + req: ApiRequestProps +): Promise { + const { datasetId, searchKey = '', parentId } = GetApiDatasetFileListBodySchema.parse(req.body); const { dataset } = await authDataset({ req, @@ -22,7 +23,11 @@ async function handler(req: NextApiRequest) { per: ReadPermissionVal }); - return (await getApiDatasetRequest(dataset.apiDatasetServer)).listFiles({ searchKey, parentId }); + const files = await ( + await getApiDatasetRequest(dataset.apiDatasetServer) + ).listFiles({ searchKey: searchKey, parentId }); + + return GetApiDatasetFileListResponseSchema.parse(files); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/listExistId.ts b/projects/app/src/pages/api/core/dataset/apiDataset/listExistId.ts index 18b8904206..6730a95974 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/listExistId.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/listExistId.ts @@ -1,22 +1,19 @@ -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; - -export type listExistIdQuery = { - datasetId: string; -}; - -export type listExistIdBody = {}; - -export type listExistIdResponse = string[]; +import { + GetApiDatasetFileListExistIdQuerySchema, + GetApiDatasetFileListExistIdResponseSchema, + type GetApiDatasetFileListExistIdQuery, + type GetApiDatasetFileListExistIdResponse +} from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; async function handler( - req: ApiRequestProps, - res: ApiResponseType -): Promise { - const { datasetId } = req.query; + req: ApiRequestProps +): Promise { + const { datasetId } = GetApiDatasetFileListExistIdQuerySchema.parse(req.query); const { dataset } = await authDataset({ req, @@ -34,7 +31,9 @@ async function handler( '_id apiFileId' ).lean(); - return collections.map((col) => col.apiFileId).filter(Boolean) as string[]; + const existIds = collections.map((col) => col.apiFileId).filter(Boolean) as string[]; + + return GetApiDatasetFileListExistIdResponseSchema.parse(existIds); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/create.ts b/projects/app/src/pages/api/core/dataset/collection/create.ts index 89ce1a14bb..b9fcf95f6c 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create.ts @@ -1,5 +1,3 @@ -import type { NextApiRequest } from 'next'; -import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; import { NextAPI } from '@/service/middleware/entry'; @@ -7,9 +5,15 @@ import { WritePermissionVal } from '@fastgpt/global/support/permission/constant' import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + CreateCollectionBodySchema, + CreateCollectionResponseSchema, + type CreateCollectionResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; -async function handler(req: NextApiRequest) { - const body = req.body as CreateDatasetCollectionParams; +async function handler(req: ApiRequestProps): Promise { + const body = CreateCollectionBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -38,7 +42,7 @@ async function handler(req: NextApiRequest) { }); })(); - return _id; + return CreateCollectionResponseSchema.parse(_id); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts b/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts index d257058e64..4941060f89 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts @@ -1,14 +1,13 @@ -import type { ApiDatasetCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; - import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; import { createApiDatasetCollection } from './apiCollectionV2'; +import { CreateApiCollectionBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; -async function handler(req: ApiRequestProps) { - const { apiFileId, ...body } = req.body; +async function handler(req: ApiRequestProps) { + const { apiFileId, ...body } = CreateApiCollectionBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, diff --git a/projects/app/src/pages/api/core/dataset/collection/create/apiCollectionV2.ts b/projects/app/src/pages/api/core/dataset/collection/create/apiCollectionV2.ts index a664fe8ea8..5b6b81ce04 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/apiCollectionV2.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/apiCollectionV2.ts @@ -1,4 +1,7 @@ -import type { ApiDatasetCreateDatasetCollectionV2Params } from '@fastgpt/global/core/dataset/api'; +import { + CreateApiCollectionV2BodySchema, + type CreateApiCollectionV2BodyType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { createCollectionAndInsertData, @@ -17,17 +20,19 @@ import { type DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { RootCollectionId } from '@fastgpt/global/core/dataset/collection/constants'; import type { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller'; -async function handler(req: ApiRequestProps) { +async function handler(req: ApiRequestProps) { + const body = CreateApiCollectionV2BodySchema.parse(req.body); + const { teamId, tmbId, dataset } = await authDataset({ req, authToken: true, authApiKey: true, - datasetId: req.body.datasetId, + datasetId: body.datasetId, per: WritePermissionVal }); return createApiDatasetCollection({ - ...req.body, + ...body, teamId, tmbId, dataset @@ -43,7 +48,7 @@ export const createApiDatasetCollection = async ({ tmbId, dataset, ...body -}: ApiDatasetCreateDatasetCollectionV2Params & { +}: CreateApiCollectionV2BodyType & { teamId: string; tmbId: string; dataset: DatasetSchemaType & { diff --git a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts index a3398863a4..036d1c3ed1 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts @@ -13,15 +13,10 @@ import { i18nT } from '@fastgpt/web/i18n/utils'; import { isCSVFile } from '@fastgpt/global/common/file/utils'; import { multer } from '@fastgpt/service/common/file/multer'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; +import { CreateBackupCollectionFormSchema } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; const logger = getLogger(LogCategories.MODULE.DATASET.COLLECTION); -export type backupQuery = {}; - -export type backupBody = {}; - -export type backupResponse = {}; - -async function handler(req: ApiRequestProps) { +async function handler(req: ApiRequestProps) { const filepaths: string[] = []; try { @@ -31,6 +26,7 @@ async function handler(req: ApiRequestProps) { }); filepaths.push(result.fileMetadata.path); const filename = decodeURIComponent(result.fileMetadata.originalname); + const { datasetId, parentId } = CreateBackupCollectionFormSchema.parse(result.data); if (!isCSVFile(filename)) { return Promise.reject('File must be a CSV file'); @@ -41,7 +37,7 @@ async function handler(req: ApiRequestProps) { authToken: true, authApiKey: true, per: WritePermissionVal, - datasetId: result.data.datasetId + datasetId }); const { rawText } = await readRawTextByLocalFile({ @@ -71,6 +67,7 @@ async function handler(req: ApiRequestProps) { teamId, tmbId, datasetId: dataset._id, + parentId, name: filename, type: DatasetCollectionTypeEnum.file, fileId, diff --git a/projects/app/src/pages/api/core/dataset/collection/create/fileId.ts b/projects/app/src/pages/api/core/dataset/collection/create/fileId.ts index f3e1bacc60..9347fae992 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/fileId.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/fileId.ts @@ -1,19 +1,19 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; -import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import { + CreateCollectionByFileIdBodySchema, + type CreateCollectionWithResultResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type CreateCollectionResponse } from '@/global/core/dataset/api'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; -async function handler( - req: ApiRequestProps -): CreateCollectionResponse { - const { fileId, customPdfParse, ...body } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { fileId, customPdfParse, ...body } = CreateCollectionByFileIdBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -32,7 +32,7 @@ async function handler( return Promise.reject(CommonErrEnum.fileNotFound); } - const { collectionId, insertResults } = await createCollectionAndInsertData({ + return createCollectionAndInsertData({ dataset, createCollectionParams: { ...body, @@ -44,11 +44,6 @@ async function handler( customPdfParse } }); - - return { - collectionId, - results: insertResults - }; } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/images.ts b/projects/app/src/pages/api/core/dataset/collection/create/images.ts index 011a431a86..f703cfdd28 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/images.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/images.ts @@ -1,5 +1,8 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; -import type { ImageCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import { + CreateImageCollectionFormSchema, + type CreateCollectionWithResultResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionTypeEnum, @@ -8,7 +11,6 @@ import { import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import type { CreateCollectionResponse } from '@/global/core/dataset/api'; import { i18nT } from '@fastgpt/web/i18n/utils'; import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils'; import { addDays, addSeconds } from 'date-fns'; @@ -18,9 +20,7 @@ import { getFileS3Key, uploadImage2S3Bucket } from '@fastgpt/service/common/s3/u import { multer } from '@fastgpt/service/common/file/multer'; import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils'; -async function handler( - req: ApiRequestProps -): CreateCollectionResponse { +async function handler(req: ApiRequestProps): Promise { const filepaths: string[] = []; try { @@ -29,7 +29,9 @@ async function handler( maxFileSize: global.feConfigs.uploadFileMaxSize }); filepaths.push(...result.fileMetadata.map((item) => item.path)); - const { parentId, datasetId, collectionName } = result.data; + const { parentId, datasetId, collectionName } = CreateImageCollectionFormSchema.parse( + result.data + ); const { dataset, teamId, tmbId } = await authDataset({ datasetId, @@ -65,7 +67,7 @@ async function handler( }) ); - const { collectionId, insertResults } = await createCollectionAndInsertData({ + return createCollectionAndInsertData({ dataset, imageIds, createCollectionParams: { @@ -78,11 +80,6 @@ async function handler( trainingType: DatasetCollectionDataProcessModeEnum.imageParse } }); - - return { - collectionId, - results: insertResults - }; } catch (error) { return Promise.reject(error); } finally { diff --git a/projects/app/src/pages/api/core/dataset/collection/create/link.ts b/projects/app/src/pages/api/core/dataset/collection/create/link.ts index 5ce1934138..fbc7b6887e 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/link.ts @@ -1,14 +1,16 @@ -import type { NextApiRequest } from 'next'; -import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type CreateCollectionResponse } from '@/global/core/dataset/api'; +import { type ApiRequestProps } from '@fastgpt/service/type/next'; +import { + CreateLinkCollectionBodySchema, + type CreateCollectionWithResultResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; -async function handler(req: NextApiRequest): CreateCollectionResponse { - const { link, ...body } = req.body as LinkCreateDatasetCollectionParams; +async function handler(req: ApiRequestProps): Promise { + const { link, ...body } = CreateLinkCollectionBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -18,7 +20,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { per: WritePermissionVal }); - const { collectionId, insertResults } = await createCollectionAndInsertData({ + return createCollectionAndInsertData({ dataset, createCollectionParams: { ...body, @@ -33,11 +35,6 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { rawLink: link } }); - - return { - collectionId, - results: insertResults - }; } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts index e422f170f2..c0f057abbd 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts @@ -1,14 +1,17 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { + CreateCollectionByLocalFileBodySchema, + type CreateCollectionWithResultResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { NextAPI } from '@/service/middleware/entry'; +import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type CreateCollectionResponse } from '@/global/core/dataset/api'; import { multer } from '@fastgpt/service/common/file/multer'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; -async function handler(req: NextApiRequest): CreateCollectionResponse { +async function handler(req: ApiRequestProps): Promise { const filepaths: string[] = []; try { @@ -26,7 +29,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { datasetId: result.data.datasetId }); - const { fileMetadata, collectionMetadata, ...collectionData } = result.data; + const collectionData = CreateCollectionByLocalFileBodySchema.parse(result.data); const collectionName = decodeURIComponent(result.fileMetadata.originalname); const fileId = await getS3DatasetSource().upload({ @@ -36,7 +39,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { filename: collectionName }); - const { collectionId, insertResults } = await createCollectionAndInsertData({ + return await createCollectionAndInsertData({ dataset, createCollectionParams: { ...collectionData, @@ -47,13 +50,11 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { type: DatasetCollectionTypeEnum.file, fileId, metadata: { - ...collectionMetadata, + ...collectionData.metadata, relatedImgId: fileId } } }); - - return { collectionId, results: insertResults }; } catch (error) { return Promise.reject(error); } finally { diff --git a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts index 88e2955a0a..22f176ec87 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts @@ -1,37 +1,30 @@ -import { type reTrainingDatasetFileCollectionParams } from '@fastgpt/global/core/dataset/api'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { delCollection } from '@fastgpt/service/core/dataset/collection/controller'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; -import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; +import { + ReTrainingCollectionBodySchema, + ReTrainingCollectionResponseSchema, + type ReTrainingCollectionResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; -type RetrainingCollectionResponse = { - collectionId: string; -}; +async function handler(req: ApiRequestProps): Promise { + const { collectionId: inputCollectionId, ...data } = ReTrainingCollectionBodySchema.parse( + req.body + ); -// 获取集合并处理 -async function handler( - req: ApiRequestProps -): Promise { - const { collectionId, ...data } = req.body; - - if (!collectionId) { - return Promise.reject(CommonErrEnum.missingParams); - } - - // 凭证校验 const { collection, teamId, tmbId } = await authDatasetCollection({ req, authToken: true, authApiKey: true, - collectionId: collectionId, + collectionId: inputCollectionId, per: WritePermissionVal }); @@ -69,7 +62,7 @@ async function handler( }); })(); - return { collectionId }; + return ReTrainingCollectionResponseSchema.parse({ collectionId }); }); } diff --git a/projects/app/src/pages/api/core/dataset/collection/create/template.ts b/projects/app/src/pages/api/core/dataset/collection/create/template.ts index 10e786d8a5..8f2c2040aa 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/template.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/template.ts @@ -13,15 +13,10 @@ import { i18nT } from '@fastgpt/web/i18n/utils'; import { isCSVFile } from '@fastgpt/global/common/file/utils'; import { multer } from '@fastgpt/service/common/file/multer'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; +import { CreateTemplateCollectionFormSchema } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; const logger = getLogger(LogCategories.MODULE.DATASET.COLLECTION); -export type templateImportQuery = {}; - -export type templateImportBody = { datasetId: string }; - -export type templateImportResponse = {}; - -async function handler(req: ApiRequestProps) { +async function handler(req: ApiRequestProps) { const filepaths: string[] = []; try { @@ -31,6 +26,7 @@ async function handler(req: ApiRequestProps { + const { name, text, ...body } = CreateTextCollectionBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -20,7 +23,6 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { per: WritePermissionVal }); - // 1. Create file from text const filename = `${name}.txt`; const s3DatasetSource = getS3DatasetSource(); const key = await s3DatasetSource.upload({ @@ -30,7 +32,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { contentType: 'text/plain; charset=UTF-8' }); - const { collectionId, insertResults } = await createCollectionAndInsertData({ + const res = await createCollectionAndInsertData({ dataset, createCollectionParams: { ...body, @@ -41,13 +43,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { name: filename } }); - await removeS3TTL({ key, bucketName: 'private' }); - - return { - collectionId, - results: insertResults - }; + return res; } export const config = { diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 3840659940..abb14216fe 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -9,14 +9,14 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + DeleteCollectionBodySchema, + DeleteCollectionQuerySchema +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -export type DelCollectionBody = { - collectionIds: string[]; -}; - -async function handler(req: ApiRequestProps) { - const id = req.query.id; - const { collectionIds } = req.body; +async function handler(req: ApiRequestProps) { + const { id } = DeleteCollectionQuerySchema.parse(req.query); + const { collectionIds } = DeleteCollectionBodySchema.parse(req.body); const deletedIds = id ? [id] : collectionIds; diff --git a/projects/app/src/pages/api/core/dataset/collection/detail.ts b/projects/app/src/pages/api/core/dataset/collection/detail.ts index 2e266e8182..3d17e42888 100644 --- a/projects/app/src/pages/api/core/dataset/collection/detail.ts +++ b/projects/app/src/pages/api/core/dataset/collection/detail.ts @@ -1,26 +1,25 @@ /* Get one dataset collection detail */ -import type { NextApiRequest } from 'next'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils'; import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type'; -import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; import { getVectorCount } from '@fastgpt/service/common/vectorDB/controller'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import type { GetCollectionDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { + GetCollectionDetailQuerySchema, + GetCollectionDetailResponseSchema +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -async function handler(req: NextApiRequest): Promise { - const { id } = req.query as { id: string }; - - if (!id) { - return Promise.reject(CommonErrEnum.missingParams); - } +async function handler(req: ApiRequestProps): Promise { + const { id } = GetCollectionDetailQuerySchema.parse(req.query); // 凭证校验 const { collection, permission } = await authDatasetCollection({ @@ -55,7 +54,7 @@ async function handler(req: NextApiRequest): Promise ) ]); - return { + return GetCollectionDetailResponseSchema.parse({ ...collection, indexAmount: indexAmount ?? 0, ...getCollectionSourceData(collection), @@ -66,7 +65,7 @@ async function handler(req: NextApiRequest): Promise permission, file, errorCount - }; + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 980ff7ad17..f72fdeb18e 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -1,7 +1,8 @@ +/** @deprecated */ import type { NextApiRequest } from 'next'; import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; import { Types } from '@fastgpt/service/common/mongo'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; diff --git a/projects/app/src/pages/api/core/dataset/collection/listV2.ts b/projects/app/src/pages/api/core/dataset/collection/listV2.ts index 2d67728d3a..914f9043e9 100644 --- a/projects/app/src/pages/api/core/dataset/collection/listV2.ts +++ b/projects/app/src/pages/api/core/dataset/collection/listV2.ts @@ -1,7 +1,4 @@ -import type { NextApiRequest } from 'next'; import { Types } from '@fastgpt/service/common/mongo'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; -import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; @@ -9,26 +6,32 @@ import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; -import { type PaginationResponse } from '@fastgpt/global/openapi/api'; -import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; import { type DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { replaceRegChars } from '@fastgpt/global/common/string/tools'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + ListCollectionV2BodySchema, + ListCollectionV2ResponseSchema, + type ListCollectionV2ResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -async function handler( - req: NextApiRequest -): Promise> { +async function handler(req: ApiRequestProps): Promise { let { datasetId, - parentId = null, - searchText = '', - selectFolder = false, - filterTags = [], - simple = false - } = req.body as GetDatasetCollectionsProps; - let { pageSize, offset } = parsePaginationRequest(req); - pageSize = Math.min(pageSize, 100); + parentId, + searchText, + selectFolder, + filterTags, + simple, + pageSize: rawPageSize, + offset: rawOffset, + pageNum: rawPageNum + } = ListCollectionV2BodySchema.parse(req.body); + let pageSize = Math.min(Number(rawPageSize ?? 10), 100); + let offset = + rawOffset !== undefined ? Number(rawOffset) : (Number(rawPageNum ?? 1) - 1) * pageSize; searchText = searchText?.replace(/'/g, ''); // auth dataset and get my role @@ -83,7 +86,7 @@ async function handler( .limit(pageSize) .lean(); - return { + return ListCollectionV2ResponseSchema.parse({ list: await Promise.all( collections.map(async (item) => ({ ...item, @@ -92,14 +95,13 @@ async function handler( tags: item.tags }), dataAmount: 0, - indexAmount: 0, trainingAmount: 0, hasError: false, permission })) ), total: await MongoDatasetCollection.countDocuments(match) - }; + }); } const [collections, total]: [DatasetCollectionSchemaType[], number] = await Promise.all([ @@ -177,10 +179,7 @@ async function handler( ); // count collections - return { - list, - total - }; + return ListCollectionV2ResponseSchema.parse({ list, total }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/paths.ts b/projects/app/src/pages/api/core/dataset/collection/paths.ts index 09dddf164e..a6b0061890 100644 --- a/projects/app/src/pages/api/core/dataset/collection/paths.ts +++ b/projects/app/src/pages/api/core/dataset/collection/paths.ts @@ -1,4 +1,3 @@ -import type { NextApiRequest } from 'next'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import type { @@ -7,26 +6,30 @@ import type { } from '@fastgpt/global/common/parentFolder/type'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { NextAPI } from '@/service/middleware/entry'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + GetCollectionPathsQuerySchema, + GetCollectionPathsResponseSchema, + type GetCollectionPathsResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -export async function handler(req: NextApiRequest) { - const { parentId } = req.query as { parentId: string }; +export async function handler(req: ApiRequestProps): Promise { + const { sourceId } = GetCollectionPathsQuerySchema.parse(req.query); - if (!parentId) { + if (!sourceId) { return []; } await authDatasetCollection({ req, authToken: true, - collectionId: parentId, + collectionId: sourceId, per: ReadPermissionVal }); - const paths = await getDatasetCollectionPaths({ - parentId - }); + const paths = await getDatasetCollectionPaths({ parentId: sourceId }); - return paths; + return GetCollectionPathsResponseSchema.parse(paths); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/read.ts b/projects/app/src/pages/api/core/dataset/collection/read.ts index 3b4787aafe..0745bcbb9a 100644 --- a/projects/app/src/pages/api/core/dataset/collection/read.ts +++ b/projects/app/src/pages/api/core/dataset/collection/read.ts @@ -3,34 +3,21 @@ import { NextAPI } from '@/service/middleware/entry'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; +import { + ReadCollectionSourceBodySchema, + ReadCollectionSourceResponseSchema, + type ReadCollectionSourceResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -export type readCollectionSourceQuery = {}; - -export type readCollectionSourceBody = { - collectionId: string; - - appId?: string; - chatId?: string; - chatItemDataId?: string; -} & OutLinkChatAuthProps; - -export type readCollectionSourceResponse = { - type: 'url'; - value: string; -}; - -async function handler( - req: ApiRequestProps -): Promise { +async function handler(req: ApiRequestProps): Promise { const { collectionId, appId, chatId, chatItemDataId, shareId, outLinkUid, teamId, teamToken } = - req.body; + ReadCollectionSourceBodySchema.parse(req.body); const { collection } = await (async () => { if (!appId || !chatId || !chatItemDataId) { @@ -107,10 +94,7 @@ async function handler( return ''; })(); - return { - type: 'url', - value: sourceUrl - }; + return ReadCollectionSourceResponseSchema.parse({ type: 'url', value: sourceUrl }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/scrollList.ts b/projects/app/src/pages/api/core/dataset/collection/scrollList.ts index 8079192c70..d0b5bd668d 100644 --- a/projects/app/src/pages/api/core/dataset/collection/scrollList.ts +++ b/projects/app/src/pages/api/core/dataset/collection/scrollList.ts @@ -8,7 +8,7 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { type PaginationResponse } from '@fastgpt/global/openapi/api'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; import { replaceRegChars } from '@fastgpt/global/common/string/tools'; import { ScrollCollectionsBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api'; diff --git a/projects/app/src/pages/api/core/dataset/collection/sync.ts b/projects/app/src/pages/api/core/dataset/collection/sync.ts index 413637ed6f..8b004238cd 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync.ts @@ -1,11 +1,15 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { syncCollection } from '@fastgpt/service/core/dataset/collection/utils'; +import { + SyncCollectionBodySchema, + SyncCollectionResponseSchema, + type SyncCollectionResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; -/* +/* Collection sync 1. Check collection type: link, api dataset collection 2. Get collection and raw text @@ -13,16 +17,8 @@ import { syncCollection } from '@fastgpt/service/core/dataset/collection/utils'; 4. Create new collection 5. Delete old collection */ -export type CollectionSyncBody = { - collectionId: string; -}; - -async function handler(req: ApiRequestProps) { - const { collectionId } = req.body; - - if (!collectionId) { - return Promise.reject(CommonErrEnum.missingParams); - } +async function handler(req: ApiRequestProps): Promise { + const { collectionId } = SyncCollectionBodySchema.parse(req.body); const { collection } = await authDatasetCollection({ req, @@ -32,7 +28,7 @@ async function handler(req: ApiRequestProps) { per: WritePermissionVal }); - return syncCollection(collection); + return SyncCollectionResponseSchema.parse(await syncCollection(collection)); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts b/projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts index d54059ce94..393ff51ad3 100644 --- a/projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts +++ b/projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts @@ -1,32 +1,19 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import type { +import { DatasetCollectionDataProcessModeEnum, - TrainingModeEnum + type TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { Types } from '@fastgpt/service/common/mongo'; - -type getTrainingDetailParams = { - collectionId: string; -}; - -export type getTrainingDetailResponse = { - trainingType: DatasetCollectionDataProcessModeEnum; - advancedTraining: { - customPdfParse: boolean; - imageIndex: boolean; - autoIndexes: boolean; - }; - queuedCounts: Record; - trainingCounts: Record; - errorCounts: Record; - trainedCount: number; -}; +import { + GetCollectionTrainingDetailQuerySchema, + GetCollectionTrainingDetailResponseSchema, + type GetCollectionTrainingDetailResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; const defaultCounts: Record = { parse: 0, @@ -37,10 +24,8 @@ const defaultCounts: Record = { imageParse: 0 }; -async function handler( - req: ApiRequestProps<{}, getTrainingDetailParams> -): Promise { - const { collectionId } = req.query; +async function handler(req: ApiRequestProps): Promise { + const { collectionId } = GetCollectionTrainingDetailQuerySchema.parse(req.query); const { collection } = await authDatasetCollection({ req, @@ -139,19 +124,18 @@ async function handler( { ...defaultCounts } ); - return { + return GetCollectionTrainingDetailResponseSchema.parse({ trainingType: collection.trainingType, advancedTraining: { customPdfParse: !!collection.customPdfParse, imageIndex: !!collection.imageIndex, autoIndexes: !!collection.autoIndexes }, - queuedCounts, trainingCounts, errorCounts, trainedCount - }; + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/delete.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts index bf664a74a4..9e37b615a8 100644 --- a/projects/app/src/pages/api/core/dataset/data/delete.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -1,21 +1,15 @@ -import type { NextApiRequest } from 'next'; import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth'; import { deleteDatasetData } from '@/service/core/dataset/data/controller'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; +import { type ApiRequestProps } from '@fastgpt/service/type/next'; +import { DeleteDatasetDataQuerySchema } from '@fastgpt/global/openapi/core/dataset/data/api'; -async function handler(req: NextApiRequest) { - const { id: dataId } = req.query as { - id: string; - }; - - if (!dataId) { - Promise.reject(CommonErrEnum.missingParams); - } +async function handler(req: ApiRequestProps) { + const { id: dataId } = DeleteDatasetDataQuerySchema.parse(req.query); // 凭证校验 const { datasetData, tmbId, teamId, collection } = await authDatasetData({ diff --git a/projects/app/src/pages/api/core/dataset/data/detail.ts b/projects/app/src/pages/api/core/dataset/data/detail.ts index 89bfeb6345..15cc53075e 100644 --- a/projects/app/src/pages/api/core/dataset/data/detail.ts +++ b/projects/app/src/pages/api/core/dataset/data/detail.ts @@ -2,24 +2,14 @@ import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth'; import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + GetDatasetDataDetailQuerySchema, + GetDatasetDataDetailResponseSchema, + type GetDatasetDataDetailResponse +} from '@fastgpt/global/openapi/core/dataset/data/api'; -export type Response = { - id: string; - q: string; - a: string; - imageId?: string; - source: string; -}; - -async function handler( - req: ApiRequestProps< - {}, - { - id: string; - } - > -) { - const { id: dataId } = req.query; +async function handler(req: ApiRequestProps): Promise { + const { id: dataId } = GetDatasetDataDetailQuerySchema.parse(req.query); const { datasetData } = await authDatasetData({ req, @@ -29,7 +19,7 @@ async function handler( per: ReadPermissionVal }); - return datasetData; + return GetDatasetDataDetailResponseSchema.parse(datasetData); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts b/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts index af4389ec05..4b8b8bfd02 100644 --- a/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts +++ b/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts @@ -1,41 +1,28 @@ import { NextAPI } from '@/service/middleware/entry'; -import { type DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth'; -import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat'; import { i18nT } from '@fastgpt/web/i18n/utils'; import { formatDatasetDataValue } from '@fastgpt/service/core/dataset/data/controller'; import { UserError } from '@fastgpt/global/common/error/utils'; +import { + GetQuoteDataBodySchema, + GetQuoteDataResponseSchema, + type GetQuoteDataResponse +} from '@fastgpt/global/openapi/core/dataset/data/api'; -export type GetQuoteDataResponse = { - collection: DatasetCollectionSchemaType; - q: string; - a?: string; -}; - -export type GetQuoteDataProps = - | { - id: string; - } - | ({ - id: string; - appId: string; - chatId: string; - chatItemDataId: string; - } & OutLinkChatAuthProps); - -async function handler(req: ApiRequestProps): Promise { - const { id: dataId } = req.body; +async function handler(req: ApiRequestProps): Promise { + const body = GetQuoteDataBodySchema.parse(req.body); + const { id: dataId } = body; // Auth const { collection, q, a } = await (async () => { - if ('chatId' in req.body) { - const { appId, chatId, shareId, outLinkUid, teamId, teamToken, chatItemDataId } = req.body; + if ('chatId' in body) { + const { appId, chatId, shareId, outLinkUid, teamId, teamToken, chatItemDataId } = body; await authChatCrud({ req, authToken: true, @@ -105,11 +92,11 @@ async function handler(req: ApiRequestProps): Promise { + const { collectionId, q, a, indexes } = InsertDataBodySchema.parse(req.body); // 凭证校验 const { teamId, tmbId, collection } = await authDatasetCollection({ @@ -102,7 +96,8 @@ async function handler(req: NextApiRequest) { } }); })(); - return insertId; + + return InsertDataResponseSchema.parse(insertId); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/insertImages.ts b/projects/app/src/pages/api/core/dataset/data/insertImages.ts index f3ac6d0caf..7915c56f0e 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertImages.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertImages.ts @@ -15,18 +15,12 @@ import fs from 'node:fs'; import { getFileS3Key, uploadImage2S3Bucket } from '@fastgpt/service/common/s3/utils'; import { multer } from '@fastgpt/service/common/file/multer'; import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils'; +import { + InsertImagesBodySchema, + type InsertImagesResponse +} from '@fastgpt/global/openapi/core/dataset/data/api'; -export type insertImagesQuery = {}; - -export type insertImagesBody = { - collectionId: string; -}; - -export type insertImagesResponse = {}; - -async function handler( - req: ApiRequestProps -): Promise { +async function handler(req: ApiRequestProps): Promise { const filepaths: string[] = []; try { @@ -35,7 +29,7 @@ async function handler( maxFileSize: global.feConfigs.uploadFileMaxSize }); filepaths.push(...result.fileMetadata.map((item) => item.path)); - const { collectionId } = result.data; + const { collectionId } = InsertImagesBodySchema.parse(result.data); const { collection, teamId, tmbId } = await authDatasetCollection({ collectionId, diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index c211d8a6fe..bb5a4308a0 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -1,60 +1,5 @@ -import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { replaceRegChars } from '@fastgpt/global/common/string/tools'; +/** @deprecated Use /core/dataset/data/v2/list instead */ import { NextAPI } from '@/service/middleware/entry'; -import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; -import { type ApiRequestProps } from '@fastgpt/service/type/next'; -import { type DatasetDataListItemType } from '@/global/core/dataset/type'; -import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; -import { type PaginationResponse } from '@fastgpt/global/openapi/api'; - -export type GetDatasetDataListProps = { - searchText?: string; - collectionId: string; -}; - -async function handler( - req: ApiRequestProps -): Promise> { - let { searchText = '', collectionId } = req.body; - let { offset, pageSize } = parsePaginationRequest(req); - - pageSize = Math.min(pageSize, 30); - - // 凭证校验 - const { teamId, collection } = await authDatasetCollection({ - req, - authToken: true, - authApiKey: true, - collectionId, - per: ReadPermissionVal - }); - - const queryReg = new RegExp(`${replaceRegChars(searchText)}`, 'i'); - const match = { - teamId, - datasetId: collection.datasetId, - collectionId, - ...(searchText.trim() - ? { - $or: [{ q: queryReg }, { a: queryReg }] - } - : {}) - }; - - const [list, total] = await Promise.all([ - MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') - .sort({ chunkIndex: 1, updateTime: -1 }) - .skip(offset) - .limit(pageSize) - .lean(), - MongoDatasetData.countDocuments(match) - ]); - - return { - list, - total - }; -} +import handler from './v2/list'; export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index 2a190f03b0..1a3a9ecf55 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -1,6 +1,4 @@ /* push data to training queue */ -import type { NextApiResponse } from 'next'; -import type { PushDatasetDataProps } from '@fastgpt/global/core/dataset/api'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { checkDatasetIndexLimit } from '@fastgpt/service/support/permission/teamLimit'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; @@ -9,21 +7,23 @@ import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { getTrainingModeByCollection } from '@fastgpt/service/core/dataset/collection/utils'; import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import { + PushDataBodySchema, + type PushDataResponseType +} from '@fastgpt/global/openapi/core/dataset/data/api'; +import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'; +import { getEmbeddingModel } from '@fastgpt/service/core/ai/model'; +import { getLLMModel } from '@fastgpt/service/core/ai/model'; +import { getVlmModel } from '@fastgpt/service/core/ai/model'; +import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller'; +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -async function handler(req: ApiRequestProps, res: NextApiResponse) { - const body = req.body; - // Adapter 4.9.0 +async function handler(req: ApiRequestProps): Promise { + const body = PushDataBodySchema.parse(req.body); + // Adapter 4.9.0: support legacy trainingMode field body.trainingType = body.trainingType || body.trainingMode; - const { collectionId, data } = body; - - if (!collectionId || !Array.isArray(data)) { - throw new Error('collectionId or data is empty'); - } - - if (data.length > 200) { - throw new Error('Data is too long, max 200'); - } + const { collectionId, billId, data } = body; // 凭证校验 const { teamId, tmbId, collection } = await authDatasetCollection({ @@ -42,15 +42,34 @@ async function handler(req: ApiRequestProps, res: NextApiR insertLen: predictDataLimitLength(mode, data) }); - return pushDataListToTrainingQueue({ - ...body, - mode, // Use collection's training mode - teamId, - tmbId, - datasetId: collection.datasetId, - vectorModel: collection.dataset.vectorModel, - agentModel: collection.dataset.agentModel, - vlmModel: collection.dataset.vlmModel + return mongoSessionRun(async (session) => { + const traingUsageId = await (async () => { + if (billId) return billId; + const { usageId: newUsageId } = await createTrainingUsage({ + teamId, + tmbId, + appName: collection.name, + billSource: UsageSourceEnum.training, + vectorModel: getEmbeddingModel(collection.dataset.vectorModel)?.name, + agentModel: getLLMModel(collection.dataset.agentModel)?.name, + vllmModel: getVlmModel(collection.dataset.vlmModel)?.name, + session + }); + return newUsageId; + })(); + + return pushDataListToTrainingQueue({ + ...body, + session, + billId: traingUsageId, + mode, // Use collection's training mode + teamId, + tmbId, + datasetId: collection.datasetId, + vectorModel: collection.dataset.vectorModel, + agentModel: collection.dataset.agentModel, + vlmModel: collection.dataset.vlmModel + }); }); } diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index f44bd9cd5a..d2bb6bc8c2 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -1,6 +1,5 @@ import { updateData2Dataset } from '@/service/core/dataset/data/controller'; import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push'; -import { type UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth'; @@ -8,9 +7,10 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; +import { UpdateDatasetDataBodySchema } from '@fastgpt/global/openapi/core/dataset/data/api'; -async function handler(req: ApiRequestProps) { - const { dataId, q, a, indexes = [] } = req.body; +async function handler(req: ApiRequestProps) { + const { dataId, q, a, indexes = [] } = UpdateDatasetDataBodySchema.parse(req.body); // auth data permission const { diff --git a/projects/app/src/pages/api/core/dataset/data/v2/list.ts b/projects/app/src/pages/api/core/dataset/data/v2/list.ts index 52ddbee389..5482a069e4 100644 --- a/projects/app/src/pages/api/core/dataset/data/v2/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/v2/list.ts @@ -4,8 +4,6 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools'; import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import type { ApiRequestProps } from '@fastgpt/service/type/next'; -import type { DatasetDataListItemType } from '@/global/core/dataset/type'; -import type { PaginationProps, PaginationResponse } from '@fastgpt/global/openapi/api'; import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; import { MongoDatasetImageSchema } from '@fastgpt/service/core/dataset/image/schema'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; @@ -13,17 +11,14 @@ import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { addHours } from 'date-fns'; import { jwtSignS3ObjectKey, isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; import { replaceS3KeyToPreviewUrl } from '@fastgpt/service/core/dataset/utils'; +import { + GetDatasetDataListBodySchema, + GetDatasetDataListResponseSchema, + type GetDatasetDataListResponse +} from '@fastgpt/global/openapi/core/dataset/data/api'; -export type GetDatasetDataListProps = PaginationProps & { - searchText?: string; - collectionId: string; -}; -export type GetDatasetDataListRes = PaginationResponse; - -async function handler( - req: ApiRequestProps -): Promise { - let { searchText = '', collectionId } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { searchText = '', collectionId } = GetDatasetDataListBodySchema.parse(req.body); let { offset, pageSize } = parsePaginationRequest(req); pageSize = Math.min(pageSize, 30); @@ -49,8 +44,8 @@ async function handler( }; const [list, total] = await Promise.all([ - MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex imageId teamId') - .sort({ chunkIndex: 1, _id: 1 }) + MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex imageId') + .sort({ chunkIndex: 1, _id: -1 }) .skip(offset) .limit(pageSize) .lean(), @@ -89,24 +84,26 @@ async function handler( } } - return { - list: await Promise.all( - list.map(async (item) => { - const imageSize = item.imageId ? imageSizeMap.get(String(item.imageId)) : undefined; - const imagePreviewUrl = - item.imageId && isS3ObjectKey(item.imageId, 'dataset') - ? jwtSignS3ObjectKey(item.imageId, addHours(new Date(), 1)) - : undefined; + const formatList = await Promise.all( + list.map(async (item) => { + const imageSize = item.imageId ? imageSizeMap.get(String(item.imageId)) : undefined; + const imagePreviewUrl = + item.imageId && isS3ObjectKey(item.imageId, 'dataset') + ? jwtSignS3ObjectKey(item.imageId, addHours(new Date(), 1)) + : undefined; - return { - ...item, - imageSize, - imagePreviewUrl - }; - }) - ), - total - }; + return { + ...item, + imageSize, + imagePreviewUrl + }; + }) + ); + + return GetDatasetDataListResponseSchema.parse({ + total, + list: formatList + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts index 45c9274202..35e77b6a9e 100644 --- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts +++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts @@ -1,7 +1,7 @@ import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read'; import { NextAPI } from '@/service/middleware/entry'; -import { type ApiRequestProps } from '@fastgpt/service/type/next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { OwnerPermissionVal, WritePermissionVal @@ -14,47 +14,28 @@ import { } from '@fastgpt/global/core/dataset/training/utils'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; -import type { ChunkSettingsType } from '@fastgpt/global/core/dataset/type'; import { replaceS3KeyToPreviewUrl } from '@fastgpt/service/core/dataset/utils'; import { addDays } from 'date-fns'; - -export type PostPreviewFilesChunksProps = ChunkSettingsType & { - datasetId: string; - type: DatasetSourceReadTypeEnum; - sourceId: string; - - customPdfParse?: boolean; - - // Chunk settings - overlapRatio: number; - - // Read params - selector?: string; - externalFileId?: string; -}; -export type PreviewChunksResponse = { - chunks: { - q: string; - a: string; - }[]; - total: number; -}; +import { + GetPreviewChunksBodySchema, + GetPreviewChunksResponseSchema, + type GetPreviewChunksBody, + type GetPreviewChunksResponse +} from '@fastgpt/global/openapi/core/dataset/file/api'; async function handler( - req: ApiRequestProps -): Promise { - let { + req: ApiRequestProps +): Promise { + const { type, sourceId, customPdfParse = false, - overlapRatio, selector, datasetId, externalFileId, - ...chunkSettings - } = req.body; + } = GetPreviewChunksBodySchema.parse(req.body); if (!sourceId) { throw new Error('sourceId is empty'); @@ -118,9 +99,10 @@ async function handler( a: replaceS3KeyToPreviewUrl(chunk.a, addDays(new Date(), 1)) })); - return { + return GetPreviewChunksResponseSchema.parse({ chunks: chunksWithJWT, total: chunks.length - }; + }); } + export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/file/presignDatasetFilePostUrl.ts b/projects/app/src/pages/api/core/dataset/file/presignDatasetFilePostUrl.ts index dca7c01733..a4006b4618 100644 --- a/projects/app/src/pages/api/core/dataset/file/presignDatasetFilePostUrl.ts +++ b/projects/app/src/pages/api/core/dataset/file/presignDatasetFilePostUrl.ts @@ -1,18 +1,22 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; -import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils'; import { addSeconds } from 'date-fns'; -import type { PresignDatasetFilePostUrlParams } from '@fastgpt/global/core/dataset/v2/api'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils'; +import { + PresignDatasetFilePostUrlBodySchema, + PresignDatasetFilePostUrlResponseSchema, + type PresignDatasetFilePostUrlBody, + type PresignDatasetFilePostUrlResponse +} from '@fastgpt/global/openapi/core/dataset/file/api'; async function handler( - req: ApiRequestProps -): Promise { - const { filename, datasetId } = req.body; + req: ApiRequestProps +): Promise { + const { filename, datasetId } = PresignDatasetFilePostUrlBodySchema.parse(req.body); const { teamId, userId } = await authDataset({ datasetId, @@ -29,11 +33,13 @@ async function handler( expiredTime: addSeconds(new Date(), 30) // 30s }); - return await getS3DatasetSource().createUploadDatasetFileURL({ + const result = await getS3DatasetSource().createUploadDatasetFileURL({ datasetId, filename, maxFileSize: planStatus.standard?.maxUploadFileSize || global.feConfigs.uploadFileMaxSize }); + + return PresignDatasetFilePostUrlResponseSchema.parse(result); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/getPermission.ts b/projects/app/src/pages/api/core/dataset/getPermission.ts similarity index 62% rename from projects/app/src/pages/api/core/dataset/data/getPermission.ts rename to projects/app/src/pages/api/core/dataset/getPermission.ts index b0a014a8cd..fc6cbdb6c9 100644 --- a/projects/app/src/pages/api/core/dataset/data/getPermission.ts +++ b/projects/app/src/pages/api/core/dataset/getPermission.ts @@ -1,26 +1,16 @@ -import type { NextApiRequest } from 'next'; import { NextAPI } from '@/service/middleware/entry'; +import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; +import { + GetDatasetPermissionQuerySchema, + GetDatasetPermissionResponseSchema, + type GetDatasetPermissionResponse +} from '@fastgpt/global/openapi/core/dataset/api'; -export type GetQuotePermissionResponse = - | { - datasetName: string; - permission: { - hasWritePer: boolean; - hasReadPer: boolean; - }; - } - | undefined; - -async function handler(req: NextApiRequest): Promise { - const { id: datasetId } = req.query as { - id?: string; - }; - if (!datasetId) { - return Promise.reject('datasetId is required'); - } +async function handler(req: ApiRequestProps): Promise { + const { id: datasetId } = GetDatasetPermissionQuerySchema.parse(req.query); try { const { permission, dataset } = await authDataset({ @@ -31,22 +21,22 @@ async function handler(req: NextApiRequest): Promise per: ReadPermissionVal }); - return { + return GetDatasetPermissionResponseSchema.parse({ datasetName: dataset.name, permission: { hasReadPer: permission.hasReadPer, hasWritePer: permission.hasWritePer } - }; + }); } catch (error) { if (error === DatasetErrEnum.unAuthDataset) { - return { + return GetDatasetPermissionResponseSchema.parse({ datasetName: '', permission: { hasWritePer: false, hasReadPer: false } - }; + }); } return Promise.reject(error); diff --git a/projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts b/projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts index c127b2f290..b3cce4f652 100644 --- a/projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts +++ b/projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts @@ -3,21 +3,14 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; +import { + DeleteTrainingDataBodySchema, + DeleteTrainingDataResponseSchema, + type DeleteTrainingDataResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; -export type deleteTrainingDataBody = { - datasetId: string; - collectionId: string; - dataId: string; -}; - -export type deleteTrainingDataQuery = {}; - -export type deleteTrainingDataResponse = {}; - -async function handler( - req: ApiRequestProps -): Promise { - const { datasetId, collectionId, dataId } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { datasetId, collectionId, dataId } = DeleteTrainingDataBodySchema.parse(req.body); const { teamId } = await authDatasetCollection({ req, @@ -33,7 +26,7 @@ async function handler( _id: dataId }); - return {}; + return DeleteTrainingDataResponseSchema.parse({}); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts index ba05dc6bff..7f2c44b01d 100644 --- a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts +++ b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts @@ -5,16 +5,14 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; +import { + GetDatasetTrainingQueueQuerySchema, + GetDatasetTrainingQueueResponseSchema, + type GetDatasetTrainingQueueResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; -export type getDatasetTrainingQueueResponse = { - rebuildingCount: number; - trainingCount: number; -}; - -async function handler( - req: ApiRequestProps -): Promise { - const { datasetId } = req.query; +async function handler(req: ApiRequestProps): Promise { + const { datasetId } = GetDatasetTrainingQueueQuerySchema.parse(req.query); const { teamId } = await authDataset({ req, @@ -39,10 +37,10 @@ async function handler( ) ]); - return { + return GetDatasetTrainingQueueResponseSchema.parse({ rebuildingCount, trainingCount - }; + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts b/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts index 3af74456e7..ef3b5bf13e 100644 --- a/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts +++ b/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts @@ -5,30 +5,14 @@ import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { isS3ObjectKey, jwtSignS3ObjectKey } from '@fastgpt/service/common/s3/utils'; import { addMinutes } from 'date-fns'; +import { + GetTrainingDataDetailBodySchema, + GetTrainingDataDetailResponseSchema, + type GetTrainingDataDetailResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; -export type getTrainingDataDetailQuery = {}; - -export type getTrainingDataDetailBody = { - datasetId: string; - collectionId: string; - dataId: string; -}; - -export type getTrainingDataDetailResponse = - | { - _id: string; - datasetId: string; - mode: string; - q?: string; - a?: string; - imagePreviewUrl?: string; - } - | undefined; - -async function handler( - req: ApiRequestProps -): Promise { - const { datasetId, collectionId, dataId } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { datasetId, collectionId, dataId } = GetTrainingDataDetailBodySchema.parse(req.body); const { teamId } = await authDatasetCollection({ req, @@ -41,10 +25,10 @@ async function handler( const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId }).lean(); if (!data) { - return undefined; + return GetTrainingDataDetailResponseSchema.parse(null); } - return { + return GetTrainingDataDetailResponseSchema.parse({ _id: data._id, datasetId: data.datasetId, mode: data.mode, @@ -54,7 +38,7 @@ async function handler( : undefined, q: data.q, a: data.a - }; + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/getTrainingError.ts b/projects/app/src/pages/api/core/dataset/training/getTrainingError.ts index 2ddc9f7662..c7a8c08e6f 100644 --- a/projects/app/src/pages/api/core/dataset/training/getTrainingError.ts +++ b/projects/app/src/pages/api/core/dataset/training/getTrainingError.ts @@ -1,21 +1,18 @@ import { NextAPI } from '@/service/middleware/entry'; -import { type DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; -import { type PaginationProps, type PaginationResponse } from '@fastgpt/global/openapi/api'; +import { + GetTrainingErrorBodySchema, + GetTrainingErrorResponseSchema, + type GetTrainingErrorResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; -export type getTrainingErrorBody = PaginationProps<{ - collectionId: string; -}>; - -export type getTrainingErrorResponse = PaginationResponse; - -async function handler(req: ApiRequestProps) { - const { collectionId } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { collectionId } = GetTrainingErrorBodySchema.parse(req.body); const { offset, pageSize } = parsePaginationRequest(req); const { collection } = await authDatasetCollection({ @@ -43,10 +40,10 @@ async function handler(req: ApiRequestProps) { MongoDatasetTraining.countDocuments(match, { ...readFromSecondary }) ]); - return { + return GetTrainingErrorResponseSchema.parse({ list: errorList, total - }; + }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts b/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts index 6a2ff0e58a..173e904015 100644 --- a/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts +++ b/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts @@ -10,16 +10,14 @@ import { getLLMModel, getEmbeddingModel, getVlmModel } from '@fastgpt/service/co import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { + RebuildEmbeddingBodySchema, + RebuildEmbeddingResponseSchema, + type RebuildEmbeddingResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; -export type rebuildEmbeddingBody = { - datasetId: string; - vectorModel: string; -}; - -export type Response = {}; - -async function handler(req: ApiRequestProps): Promise { - const { datasetId, vectorModel } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { datasetId, vectorModel } = RebuildEmbeddingBodySchema.parse(req.body); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -138,7 +136,7 @@ async function handler(req: ApiRequestProps): Promise -): Promise { - const { datasetId, collectionId, dataId, q, a, chunkIndex } = req.body; +async function handler(req: ApiRequestProps): Promise { + const { datasetId, collectionId, dataId, q, a, chunkIndex } = UpdateTrainingDataBodySchema.parse( + req.body + ); const { teamId } = await authDatasetCollection({ req, @@ -46,7 +38,7 @@ async function handler( lockTime: new Date('2000') } ); - return {}; + return UpdateTrainingDataResponseSchema.parse({}); } // Single data retry logic @@ -92,7 +84,7 @@ async function handler( ); } - return {}; + return UpdateTrainingDataResponseSchema.parse({}); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/workflow/optimizeCode.ts b/projects/app/src/pages/api/core/workflow/optimizeCode.ts index 204de6f979..6e13a8113d 100644 --- a/projects/app/src/pages/api/core/workflow/optimizeCode.ts +++ b/projects/app/src/pages/api/core/workflow/optimizeCode.ts @@ -1,5 +1,5 @@ import { NextAPI } from '@/service/middleware/entry'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'; import { responseWrite } from '@fastgpt/service/common/response'; diff --git a/projects/app/src/pages/api/support/user/team/update.ts b/projects/app/src/pages/api/support/user/team/update.ts index 5d9f1d647a..4c70bc19f2 100644 --- a/projects/app/src/pages/api/support/user/team/update.ts +++ b/projects/app/src/pages/api/support/user/team/update.ts @@ -1,22 +1,22 @@ -import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; -import { type UpdateTeamProps } from '@fastgpt/global/support/user/team/controller'; +import { + UpdateTeamBodySchema, + UpdateTeamResponseSchema, + type UpdateTeamResponseType +} from '@fastgpt/global/openapi/support/user/team/api'; import { authUserPer } from '@fastgpt/service/support/permission/user/auth'; import { updateTeam } from '@fastgpt/service/support/user/team/controller'; import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant'; -export type updateQuery = {}; - -export type updateBody = {}; - -export type updateResponse = {}; - -async function handler(req: ApiRequestProps, res: ApiResponseType) { - const body = req.body as UpdateTeamProps; +async function handler(req: ApiRequestProps): Promise { + const body = UpdateTeamBodySchema.parse(req.body); const { teamId } = await authUserPer({ req, authToken: true, per: ManagePermissionVal }); await updateTeam({ teamId, ...body }); + + return UpdateTeamResponseSchema.parse({}); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 82d495703f..c412db1827 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -6,10 +6,6 @@ import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; -import type { - ChatCompletionCreateParams, - ChatCompletionMessageParam -} from '@fastgpt/global/core/ai/type'; import { getWorkflowEntryNodeIds, getMaxHistoryLimitFromNodes, @@ -42,14 +38,13 @@ import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools'; import { getRunningUserInfoByTmbId } from '@fastgpt/service/support/user/team/utils'; import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant'; import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { type AppSchemaType } from '@fastgpt/global/core/app/type'; import { type AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api'; import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat'; -import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; import { type AIChatItemType, type UserChatItemType } from '@fastgpt/global/core/chat/type'; import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; - +import type { AuthResponseType } from '@fastgpt/global/openapi/core/chat/completion/api'; +import { CompletionsPropsSchema } from '@fastgpt/global/openapi/core/chat/completion/api'; import { NextAPI } from '@/service/middleware/entry'; import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; @@ -71,39 +66,6 @@ import { getIpFromRequest } from '@fastgpt/service/common/geo'; import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils'; const logger = getLogger(LogCategories.MODULE.CHAT.ITEM); -type FastGptWebChatProps = { - chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db - appId?: string; - customUid?: string; // non-undefined: will be the priority provider for the logger. - metadata?: Record; -}; - -export type Props = ChatCompletionCreateParams & - FastGptWebChatProps & - OutLinkChatAuthProps & { - messages: ChatCompletionMessageParam[]; - responseChatItemId?: string; - stream?: boolean; - detail?: boolean; - retainDatasetCite?: boolean; - showSkillReferences?: boolean; - variables: Record; // Global variables or plugin inputs - }; - -type AuthResponseType = { - teamId: string; - tmbId: string; - app: AppSchemaType; - showCite?: boolean; - showRunningStatus?: boolean; - showSkillReferences?: boolean; - authType: `${AuthUserTypeEnum}`; - apikey?: string; - responseAllData: boolean; - outLinkUserId?: string; - sourceName?: string; -}; - async function handler(req: NextApiRequest, res: NextApiResponse) { let { chatId, @@ -124,7 +86,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { variables = {}, responseChatItemId = getNanoid(), metadata - } = req.body as Props; + } = CompletionsPropsSchema.parse(req.body); const startTime = Date.now(); @@ -467,7 +429,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { res.json({ ...(detail ? { responseData: feResponseData, newVariables } : {}), error, - id: chatId || '', + id: saveChatId, model: '', usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 }, choices: [ diff --git a/projects/app/src/pages/api/v2/chat/completions.ts b/projects/app/src/pages/api/v2/chat/completions.ts index d9615ec4ad..aa823b90d9 100644 --- a/projects/app/src/pages/api/v2/chat/completions.ts +++ b/projects/app/src/pages/api/v2/chat/completions.ts @@ -6,10 +6,6 @@ import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; -import type { - ChatCompletionCreateParams, - ChatCompletionMessageParam -} from '@fastgpt/global/core/ai/type'; import { getWorkflowEntryNodeIds, getMaxHistoryLimitFromNodes, @@ -25,7 +21,6 @@ import { pushChatRecords, updateInteractiveChat } from '@fastgpt/service/core/chat/saveChat'; - import { responseWrite } from '@fastgpt/service/common/response'; import { authOutLinkChatStart } from '@/service/support/permission/auth/outLink'; import { recordAppUsage } from '@fastgpt/service/core/app/record/utils'; @@ -43,14 +38,11 @@ import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools'; import { getRunningUserInfoByTmbId } from '@fastgpt/service/support/user/team/utils'; import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant'; import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { type AppSchemaType } from '@fastgpt/global/core/app/type'; import { type AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api'; import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat'; -import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; import { type AIChatItemType, type UserChatItemType } from '@fastgpt/global/core/chat/type'; import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; - import { NextAPI } from '@/service/middleware/entry'; import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; @@ -60,7 +52,6 @@ import { updateWorkflowToolInputByVariables } from '@fastgpt/service/core/app/tool/workflowTool/utils'; import { getNanoid } from '@fastgpt/global/common/string/tools'; - import { rewriteNodeOutputByHistories } from '@fastgpt/global/core/workflow/runtime/utils'; import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatch/utils'; import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants'; @@ -71,41 +62,10 @@ import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; import { LimitTypeEnum, teamFrequencyLimit } from '@fastgpt/service/common/api/frequencyLimit'; import { getIpFromRequest } from '@fastgpt/service/common/geo'; import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils'; +import type { AuthResponseType } from '@fastgpt/global/openapi/core/chat/completion/api'; +import { CompletionsPropsSchema } from '@fastgpt/global/openapi/core/chat/completion/api'; const logger = getLogger(LogCategories.MODULE.CHAT.ITEM); -type FastGptWebChatProps = { - chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db - appId?: string; - customUid?: string; // non-undefined: will be the priority provider for the logger. - metadata?: Record; -}; - -export type Props = ChatCompletionCreateParams & - FastGptWebChatProps & - OutLinkChatAuthProps & { - messages: ChatCompletionMessageParam[]; - responseChatItemId?: string; - stream?: boolean; - detail?: boolean; - retainDatasetCite?: boolean; - showSkillReferences?: boolean; - variables: Record; // Global variables or plugin inputs - }; - -type AuthResponseType = { - teamId: string; - tmbId: string; - app: AppSchemaType; - showCite?: boolean; - showRunningStatus?: boolean; - showSkillReferences?: boolean; - authType: `${AuthUserTypeEnum}`; - apikey?: string; - responseAllData: boolean; - outLinkUserId?: string; - sourceName?: string; -}; - async function handler(req: NextApiRequest, res: NextApiResponse) { let { chatId, @@ -118,15 +78,15 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { teamId: spaceTeamId, teamToken, - stream = false, - detail = false, - retainDatasetCite = false, + stream, + detail, + retainDatasetCite, showSkillReferences, - messages = [], - variables = {}, - responseChatItemId = getNanoid(), + messages, + variables, + responseChatItemId, metadata - } = req.body as Props; + } = CompletionsPropsSchema.parse(req.body); const originIp = getIpFromRequest(req); diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 282a3ffc80..d5745db730 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -1,16 +1,13 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { - type CreateDatasetDataProps, - type PatchIndexesProps, - type UpdateDatasetDataProps -} from '@fastgpt/global/core/dataset/controller'; import { insertDatasetDataVector } from '@fastgpt/service/common/vectorDB/controller'; import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index'; import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorDB/controller'; import { pushCollectionUpdateJob } from '@fastgpt/service/core/dataset/collection/mq'; -import { - type DatasetDataIndexItemType, - type DatasetDataItemType +import type { + UpdateDatasetDataPropsType, + DatasetDataIndexItemType, + DatasetDataItemType, + CreateDatasetDataPropsType } from '@fastgpt/global/core/dataset/type'; import { getEmbeddingModel } from '@fastgpt/service/core/ai/model'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; @@ -178,7 +175,7 @@ export async function insertData2Dataset({ embeddingModel, imageDescMap, session -}: CreateDatasetDataProps & { +}: CreateDatasetDataPropsType & { embeddingModel: string; indexSize?: number; imageDescMap?: Record; @@ -276,6 +273,25 @@ export async function insertData2Dataset({ * 3. update mongo data(session run) * 4. delete old pg data */ +type PatchIndexesProps = + | { + type: 'create'; + index: Omit & { + dataId?: string; + }; + } + | { + type: 'update'; + index: DatasetDataIndexItemType; + } + | { + type: 'delete'; + index: DatasetDataIndexItemType; + } + | { + type: 'unChange'; + index: DatasetDataIndexItemType; + }; export async function updateData2Dataset({ dataId, q = '', @@ -284,7 +300,7 @@ export async function updateData2Dataset({ model, indexSize = 512, indexPrefix -}: UpdateDatasetDataProps & { model: string; indexSize?: number }) { +}: UpdateDatasetDataPropsType & { model: string; indexSize?: number }) { if (!Array.isArray(indexes)) { return Promise.reject('indexes is required'); } diff --git a/projects/app/src/service/core/dataset/queues/datasetParse.ts b/projects/app/src/service/core/dataset/queues/datasetParse.ts index 410c86067d..d1aedd578a 100644 --- a/projects/app/src/service/core/dataset/queues/datasetParse.ts +++ b/projects/app/src/service/core/dataset/queues/datasetParse.ts @@ -32,7 +32,7 @@ import { POST } from '@fastgpt/service/common/api/plusRequest'; import { pushLLMTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller'; import { UsageItemTypeEnum } from '@fastgpt/global/support/wallet/usage/constants'; -const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES); +const logger = getLogger(LogCategories.MODULE.DATASET.FILE_PARSE); const requestLLMPargraph = async ({ rawText, diff --git a/projects/app/src/service/core/dataset/queues/generateQA.ts b/projects/app/src/service/core/dataset/queues/generateQA.ts index 03b4e3f2c3..c43383b5eb 100644 --- a/projects/app/src/service/core/dataset/queues/generateQA.ts +++ b/projects/app/src/service/core/dataset/queues/generateQA.ts @@ -1,11 +1,11 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { pushLLMTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; -import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; +import type { PushDataChunkType } from '@fastgpt/global/openapi/core/dataset/data/api'; import { getLLMModel } from '@fastgpt/service/core/ai/model'; import { checkTeamAiPointsAndLock } from './utils'; import { addMinutes } from 'date-fns'; @@ -21,7 +21,7 @@ import { delay } from '@fastgpt/service/common/bullmq'; import { createLLMResponse } from '@fastgpt/service/core/ai/llm/request'; import { UsageItemTypeEnum } from '@fastgpt/global/support/wallet/usage/constants'; -const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES); +const logger = getLogger(LogCategories.MODULE.DATASET.QA); const reduceQueue = () => { global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; @@ -232,7 +232,7 @@ async function formatSplitText({ const regex = /Q\d+:(\s*)(.*)(\s*)A\d+:(\s*)([\s\S]*?)(?=Q\d|$)/g; // 匹配Q和A的正则表达式 const matches = answer.matchAll(regex); // 获取所有匹配到的结果 - const result: PushDatasetDataChunkProps[] = []; // 存储最终的结果 + const result: PushDataChunkType[] = []; // 存储最终的结果 for (const match of matches) { const q = match[2] || ''; const a = match[5] || ''; diff --git a/projects/app/src/service/core/dataset/queues/generateVector.ts b/projects/app/src/service/core/dataset/queues/generateVector.ts index bf392aa648..203814a74e 100644 --- a/projects/app/src/service/core/dataset/queues/generateVector.ts +++ b/projects/app/src/service/core/dataset/queues/generateVector.ts @@ -21,7 +21,7 @@ import type { import { retryFn } from '@fastgpt/global/common/system/utils'; import { delay } from '@fastgpt/service/common/bullmq'; -const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES); +const logger = getLogger(LogCategories.MODULE.DATASET.EMBEDDING); const reduceQueue = () => { global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0; diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index d0897798c5..01d18db7a0 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -3,30 +3,23 @@ import type { PresignChatFileGetUrlParams, PresignChatFilePostUrlParams } from '@fastgpt/global/openapi/core/chat/file/api'; -import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; export const getUploadAvatarPresignedUrl = (params: { filename: string; autoExpired?: boolean; }) => { - return POST('/common/file/presignAvatarPostUrl', params); + return POST('/common/file/presignAvatarPostUrl', params); }; export const getUploadChatFilePresignedUrl = (params: PresignChatFilePostUrlParams) => { - return POST('/core/chat/file/presignChatFilePostUrl', params); + return POST('/core/chat/file/presignChatFilePostUrl', params); }; export const getPresignedChatFileGetUrl = (params: PresignChatFileGetUrlParams) => { return POST('/core/chat/file/presignChatFileGetUrl', params); }; -export const getUploadDatasetFilePresignedUrl = (params: { - filename: string; - datasetId: string; -}) => { - return POST('/core/dataset/presignDatasetFilePostUrl', params); -}; - export const getUploadTempFilePresignedUrl = (params: { filename: string }) => { - return POST('/common/file/presignTempFilePostUrl', params); + return POST('/common/file/presignTempFilePostUrl', params); }; diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index 10a2c2b9b9..a31b4e5a0a 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -6,25 +6,9 @@ import type { import type { DatasetItemType, DatasetListItemType, - DatasetSimpleItemType, - DatasetTagType, - TagUsageType + DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type'; -import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; -import type { - AddTagsToCollectionsParams, - ApiDatasetCreateDatasetCollectionV2Params, - CreateDatasetCollectionParams, - CreateDatasetCollectionTagParams, - ExternalFileCreateDatasetCollectionParams, - FileIdCreateDatasetCollectionParams, - reTrainingDatasetFileCollectionParams, - LinkCreateDatasetCollectionParams, - PostDatasetSyncParams, - TextCreateDatasetCollectionParams, - UpdateDatasetCollectionTagParams -} from '@fastgpt/global/core/dataset/api'; -import type { InsertOneDatasetDataProps } from '@/global/core/dataset/api'; +import type { PostDatasetSyncParams } from '@fastgpt/global/openapi/core/dataset/api'; import type { CreateDatasetBody, CreateDatasetWithFilesBody, @@ -33,60 +17,9 @@ import type { UpdateDatasetBody, CreateDatasetFolderBody, SearchDatasetTestBody, - SearchDatasetTestResponse + SearchDatasetTestResponse, + GetDatasetPermissionResponse } from '@fastgpt/global/openapi/core/dataset/api'; -import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type'; -import type { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; -import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; -import type { getDatasetTrainingQueueResponse } from '@/pages/api/core/dataset/training/getDatasetTrainingQueue'; -import type { rebuildEmbeddingBody } from '@/pages/api/core/dataset/training/rebuildEmbedding'; -import type { - PostPreviewFilesChunksProps, - PreviewChunksResponse -} from '@/pages/api/core/dataset/file/getPreviewChunks'; -import type { - readCollectionSourceBody, - readCollectionSourceResponse -} from '@/pages/api/core/dataset/collection/read'; -import type { UpdateDatasetCollectionBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; -import type { - GetDatasetDataListProps, - GetDatasetDataListRes -} from '@/pages/api/core/dataset/data/v2/list'; -import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller'; -import type { PaginationProps, PaginationResponse } from '@fastgpt/global/openapi/api'; -import type { GetApiDatasetFileListProps } from '@/pages/api/core/dataset/apiDataset/list'; -import type { - listExistIdQuery, - listExistIdResponse -} from '@/pages/api/core/dataset/apiDataset/listExistId'; -import type { GetQuoteDataResponse } from '@/pages/api/core/dataset/data/getQuoteData'; -import type { GetQuotePermissionResponse } from '@/pages/api/core/dataset/data/getPermission'; -import type { updateTrainingDataBody } from '@/pages/api/core/dataset/training/updateTrainingData'; -import type { - getTrainingDataDetailBody, - getTrainingDataDetailResponse -} from '@/pages/api/core/dataset/training/getTrainingDataDetail'; -import type { deleteTrainingDataBody } from '@/pages/api/core/dataset/training/deleteTrainingData'; -import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail'; -import type { - getTrainingErrorBody, - getTrainingErrorResponse -} from '@/pages/api/core/dataset/training/getTrainingError'; -import type { APIFileItemType } from '@fastgpt/global/core/dataset/apiDataset/type'; -import type { GetQuoteDataProps } from '@/pages/api/core/dataset/data/getQuoteData'; -import type { - GetApiDatasetCataLogResponse, - GetApiDatasetCataLogProps -} from '@/pages/api/core/dataset/apiDataset/getCatalog'; -import type { - GetApiDatasetPathBody, - GetApiDatasetPathResponse -} from '@/pages/api/core/dataset/apiDataset/getPathNames'; -import type { DelCollectionBody } from '@/pages/api/core/dataset/collection/delete'; - -import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; /* ======================== dataset ======================= */ export const getDatasets = (data: GetDatasetListBody) => @@ -120,201 +53,15 @@ export const postDatasetSync = (data: PostDatasetSyncParams) => export const postCreateDatasetFolder = (data: CreateDatasetFolderBody) => POST(`/core/dataset/folder/create`, data); +export const getDatasetPermission = (id?: string) => + GET(`/core/dataset/getPermission`, { id }); + export const resumeInheritPer = (datasetId: string) => PUT(`/core/dataset/resumeInheritPermission`, { datasetId }); export const postChangeOwner = (data: { ownerId: string; datasetId: string }) => POST(`/proApi/core/dataset/changeOwner`, data); -/* ============================= collection ==================================== */ -export const postBackupDatasetCollection = ({ - file, - percentListen, - datasetId -}: { - file: File; - percentListen: (percent: number) => void; - datasetId: string; -}) => { - const formData = new FormData(); - formData.append('file', file, encodeURIComponent(file.name)); - formData.append('data', JSON.stringify({ datasetId })); - - return POST(`/core/dataset/collection/create/backup`, formData, { - timeout: 600000, - onUploadProgress: (e) => { - if (!e.total) return; - - const percent = Math.round((e.loaded / e.total) * 100); - percentListen?.(percent); - } - }); -}; -export const postTemplateDatasetCollection = ({ - file, - percentListen, - datasetId -}: { - file: File; - percentListen: (percent: number) => void; - datasetId: string; -}) => { - const formData = new FormData(); - formData.append('file', file, encodeURIComponent(file.name)); - formData.append('data', JSON.stringify({ datasetId })); - - return POST(`/core/dataset/collection/create/template`, formData, { - timeout: 600000, - onUploadProgress: (e) => { - if (!e.total) return; - - const percent = Math.round((e.loaded / e.total) * 100); - percentListen?.(percent); - } - }); -}; - /* =========== search test ============ */ export const postSearchText = (data: SearchDatasetTestBody) => POST(`/core/dataset/searchTest`, data); - -/* ============================= collections ==================================== */ -export const getDatasetCollections = (data: GetDatasetCollectionsProps) => - POST>(`/core/dataset/collection/listV2`, data); -export const getDatasetCollectionPathById = (parentId: ParentIdType) => - GET(`/core/dataset/collection/paths`, { parentId }); -export const getDatasetCollectionById = (id: string) => - GET(`/core/dataset/collection/detail`, { id }); -export const getDatasetCollectionTrainingDetail = (collectionId: string) => - GET(`/core/dataset/collection/trainingDetail`, { - collectionId - }); -export const postDatasetCollection = (data: CreateDatasetCollectionParams) => - POST(`/core/dataset/collection/create`, data); -export const postCreateDatasetFileCollection = (data: FileIdCreateDatasetCollectionParams) => - POST<{ collectionId: string }>(`/core/dataset/collection/create/fileId`, data, { - timeout: 360000 - }); -export const postReTrainingDatasetFileCollection = (data: reTrainingDatasetFileCollectionParams) => - POST<{ collectionId: string }>(`/core/dataset/collection/create/reTrainingCollection`, data, { - timeout: 360000 - }); -export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectionParams) => - POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data); -export const postCreateDatasetTextCollection = (data: TextCreateDatasetCollectionParams) => - POST<{ collectionId: string }>(`/core/dataset/collection/create/text`, data); - -export const postCreateDatasetExternalFileCollection = ( - data: ExternalFileCreateDatasetCollectionParams -) => - POST<{ collectionId: string }>(`/proApi/core/dataset/collection/create/externalFileUrl`, data, { - timeout: 360000 - }); -export const postCreateDatasetApiDatasetCollection = ( - data: ApiDatasetCreateDatasetCollectionV2Params -) => - POST(`/core/dataset/collection/create/apiCollectionV2`, data, { - timeout: 360000 - }); - -export const putDatasetCollectionById = (data: UpdateDatasetCollectionBodyType) => - POST(`/core/dataset/collection/update`, data); -export const delDatasetCollectionById = (params: DelCollectionBody) => - POST(`/core/dataset/collection/delete`, params); -export const postLinkCollectionSync = (collectionId: string) => - POST(`/core/dataset/collection/sync`, { - collectionId - }); - -/* =============================== tag ==================================== */ - -export const postCreateDatasetCollectionTag = (data: CreateDatasetCollectionTagParams) => - POST(`/proApi/core/dataset/tag/create`, data); -export const postAddTagsToCollections = (data: AddTagsToCollectionsParams) => - POST(`/proApi/core/dataset/tag/addToCollections`, data); -export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) => - DELETE(`/proApi/core/dataset/tag/delete`, data); -export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) => - POST(`/proApi/core/dataset/tag/update`, data); -export const getDatasetCollectionTags = ( - data: PaginationProps<{ - datasetId: string; - searchText?: string; - }> -) => POST>(`/proApi/core/dataset/tag/list`, data); -export const getTagUsage = (datasetId: string) => - GET(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`); -export const getAllTags = (datasetId: string) => - GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`); - -/* =============================== data ==================================== */ -/* get dataset list */ -export const getDatasetDataList = (data: GetDatasetDataListProps) => - POST(`/core/dataset/data/v2/list`, data); - -export const getDatasetDataPermission = (id?: string) => - GET(`/core/dataset/data/getPermission`, { id }); - -export const getDatasetDataItemById = (id: string) => - GET(`/core/dataset/data/detail`, { id }); - -/** - * insert one data to dataset (immediately insert) - */ -export const postInsertData2Dataset = (data: InsertOneDatasetDataProps) => - POST(`/core/dataset/data/insertData`, data); - -/** - * update one datasetData by id - */ -export const putDatasetDataById = (data: UpdateDatasetDataProps) => - PUT('/core/dataset/data/update', data); -/** - * 删除一条知识库数据 - */ -export const delOneDatasetDataById = (id: string) => - DELETE(`/core/dataset/data/delete`, { id }); - -// Get quote data -export const getQuoteData = (data: GetQuoteDataProps) => - POST(`/core/dataset/data/getQuoteData`, data); - -/* ================ training ==================== */ -export const postRebuildEmbedding = (data: rebuildEmbeddingBody) => - POST(`/core/dataset/training/rebuildEmbedding`, data); - -export const getDatasetTrainingQueue = (datasetId: string) => - GET(`/core/dataset/training/getDatasetTrainingQueue`, { - datasetId - }); - -export const getPreviewChunks = (data: PostPreviewFilesChunksProps) => - POST('/core/dataset/file/getPreviewChunks', data, { - maxQuantity: 1, - timeout: 600000 - }); - -export const deleteTrainingData = (data: deleteTrainingDataBody) => - POST(`/core/dataset/training/deleteTrainingData`, data); -export const updateTrainingData = (data: updateTrainingDataBody) => - PUT(`/core/dataset/training/updateTrainingData`, data); -export const getTrainingDataDetail = (data: getTrainingDataDetailBody) => - POST(`/core/dataset/training/getTrainingDataDetail`, data); -export const getTrainingError = (data: getTrainingErrorBody) => - POST(`/core/dataset/training/getTrainingError`, data); - -/* ================== read source ======================== */ -export const getCollectionSource = (data: readCollectionSourceBody) => - POST('/core/dataset/collection/read', data); - -/* ================== apiDataset ======================== */ -export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) => - POST('/core/dataset/apiDataset/list', data); -export const getApiDatasetFileListExistId = (data: listExistIdQuery) => - GET('/core/dataset/apiDataset/listExistId', data); - -export const getApiDatasetCatalog = (data: GetApiDatasetCataLogProps) => - POST('/core/dataset/apiDataset/getCatalog', data); - -export const getApiDatasetPaths = (data: GetApiDatasetPathBody) => - POST('/core/dataset/apiDataset/getPathNames', data); diff --git a/projects/app/src/web/core/dataset/api/apiDataset.ts b/projects/app/src/web/core/dataset/api/apiDataset.ts new file mode 100644 index 0000000000..b57252e3b1 --- /dev/null +++ b/projects/app/src/web/core/dataset/api/apiDataset.ts @@ -0,0 +1,23 @@ +import { GET, POST } from '@/web/common/api/request'; +import type { + GetApiDatasetCatalogBody, + GetApiDatasetCatalogResponse, + GetApiDatasetFileListBody, + GetApiDatasetFileListExistIdQuery, + GetApiDatasetFileListExistIdResponse, + GetApiDatasetFileListResponse, + GetApiDatasetPathNamesBody, + GetApiDatasetPathNamesResponse +} from '@fastgpt/global/openapi/core/dataset/apiDataset/api'; + +export const getApiDatasetFileList = (data: GetApiDatasetFileListBody) => + POST('/core/dataset/apiDataset/list', data); + +export const getApiDatasetFileListExistId = (data: GetApiDatasetFileListExistIdQuery) => + GET('/core/dataset/apiDataset/listExistId', data); + +export const getApiDatasetCatalog = (data: GetApiDatasetCatalogBody) => + POST('/core/dataset/apiDataset/getCatalog', data); + +export const getApiDatasetPaths = (data: GetApiDatasetPathNamesBody) => + POST('/core/dataset/apiDataset/getPathNames', data); diff --git a/projects/app/src/web/core/dataset/api/collection.ts b/projects/app/src/web/core/dataset/api/collection.ts new file mode 100644 index 0000000000..755103f2b1 --- /dev/null +++ b/projects/app/src/web/core/dataset/api/collection.ts @@ -0,0 +1,154 @@ +import { GET, POST, PUT, DELETE } from '@/web/common/api/request'; +import type { + ParentTreePathItemType, + ParentIdType +} from '@fastgpt/global/common/parentFolder/type'; +import type { + DatasetCollectionItemType, + DatasetTagType, + TagUsageType +} from '@fastgpt/global/core/dataset/type'; +import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; +import type { + CreateApiCollectionV2BodyType, + ExternalFileCreateDatasetCollectionParams, + CreateCollectionByFileIdBodyType, + ReTrainingCollectionBodyType, + CreateLinkCollectionBodyType, + CreateTextCollectionBodyType, + CreateCollectionBodyType +} from '@fastgpt/global/openapi/core/dataset/collection/createApi'; +import type { + AddTagsToCollectionsParams, + CreateDatasetCollectionTagParams, + UpdateDatasetCollectionTagParams +} from '@fastgpt/global/openapi/core/dataset/collection/tagApi'; +import type { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; +import type { + DatasetCollectionsListItemType, + DeleteCollectionBodyType, + ReadCollectionSourceBodyType, + ReadCollectionSourceResponseType, + UpdateDatasetCollectionBodyType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; +import type { PaginationProps, PaginationResponse } from '@fastgpt/global/openapi/api'; +import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api'; + +/* ============================= collections ==================================== */ +export const getDatasetCollections = (data: GetDatasetCollectionsProps) => + POST>(`/core/dataset/collection/listV2`, data); +export const getDatasetCollectionPathById = (sourceId: ParentIdType) => + GET(`/core/dataset/collection/paths`, { sourceId }); +export const getDatasetCollectionById = (id: string) => + GET(`/core/dataset/collection/detail`, { id }); +export const putDatasetCollectionById = (data: UpdateDatasetCollectionBodyType) => + POST(`/core/dataset/collection/update`, data); +export const delDatasetCollectionById = (params: DeleteCollectionBodyType) => + POST(`/core/dataset/collection/delete`, params); +export const postLinkCollectionSync = (collectionId: string) => + POST(`/core/dataset/collection/sync`, { + collectionId + }); + +export const getDatasetCollectionTrainingDetail = (collectionId: string) => + GET(`/core/dataset/collection/trainingDetail`, { + collectionId + }); + +/* ========================== collection create ========================== */ +export const postDatasetCollection = (data: CreateCollectionBodyType) => + POST(`/core/dataset/collection/create`, data); +export const postCreateDatasetFileCollection = (data: CreateCollectionByFileIdBodyType) => + POST<{ collectionId: string }>(`/core/dataset/collection/create/fileId`, data, { + timeout: 360000 + }); +export const postReTrainingDatasetFileCollection = (data: ReTrainingCollectionBodyType) => + POST<{ collectionId: string }>(`/core/dataset/collection/create/reTrainingCollection`, data, { + timeout: 360000 + }); +export const postCreateDatasetLinkCollection = (data: CreateLinkCollectionBodyType) => + POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data); +export const postCreateDatasetTextCollection = (data: CreateTextCollectionBodyType) => + POST<{ collectionId: string }>(`/core/dataset/collection/create/text`, data); +export const postCreateDatasetApiDatasetCollection = (data: CreateApiCollectionV2BodyType) => + POST(`/core/dataset/collection/create/apiCollectionV2`, data, { + timeout: 360000 + }); +/** @deprecated */ +export const postCreateDatasetExternalFileCollection = ( + data: ExternalFileCreateDatasetCollectionParams +) => + POST<{ collectionId: string }>(`/proApi/core/dataset/collection/create/externalFileUrl`, data, { + timeout: 360000 + }); + +export const postBackupDatasetCollection = ({ + file, + percentListen, + datasetId +}: { + file: File; + percentListen: (percent: number) => void; + datasetId: string; +}) => { + const formData = new FormData(); + formData.append('file', file, encodeURIComponent(file.name)); + formData.append('data', JSON.stringify({ datasetId })); + + return POST(`/core/dataset/collection/create/backup`, formData, { + timeout: 600000, + onUploadProgress: (e) => { + if (!e.total) return; + + const percent = Math.round((e.loaded / e.total) * 100); + percentListen?.(percent); + } + }); +}; +export const postTemplateDatasetCollection = ({ + file, + percentListen, + datasetId +}: { + file: File; + percentListen: (percent: number) => void; + datasetId: string; +}) => { + const formData = new FormData(); + formData.append('file', file, encodeURIComponent(file.name)); + formData.append('data', JSON.stringify({ datasetId })); + + return POST(`/core/dataset/collection/create/template`, formData, { + timeout: 600000, + onUploadProgress: (e) => { + if (!e.total) return; + + const percent = Math.round((e.loaded / e.total) * 100); + percentListen?.(percent); + } + }); +}; + +/* =============================== tag ==================================== */ +export const postCreateDatasetCollectionTag = (data: CreateDatasetCollectionTagParams) => + POST(`/proApi/core/dataset/tag/create`, data); +export const postAddTagsToCollections = (data: AddTagsToCollectionsParams) => + POST(`/proApi/core/dataset/tag/addToCollections`, data); +export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) => + DELETE(`/proApi/core/dataset/tag/delete`, data); +export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) => + POST(`/proApi/core/dataset/tag/update`, data); +export const getDatasetCollectionTags = ( + data: PaginationProps<{ + datasetId: string; + searchText?: string; + }> +) => POST>(`/proApi/core/dataset/tag/list`, data); +export const getTagUsage = (datasetId: string) => + GET(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`); +export const getAllTags = (datasetId: string) => + GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`); + +/* ================== read source ======================== */ +export const getCollectionSource = (data: ReadCollectionSourceBodyType) => + POST('/core/dataset/collection/read', data); diff --git a/projects/app/src/web/core/dataset/api/data.ts b/projects/app/src/web/core/dataset/api/data.ts new file mode 100644 index 0000000000..5f64cdae51 --- /dev/null +++ b/projects/app/src/web/core/dataset/api/data.ts @@ -0,0 +1,38 @@ +import { GET, POST, PUT, DELETE } from '@/web/common/api/request'; +import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; +import type { + GetDatasetDataListBody as GetDatasetDataListProps, + GetDatasetDataListResponse as GetDatasetDataListRes, + GetQuoteDataBody as GetQuoteDataProps, + GetQuoteDataResponse, + InsertDataBody, + UpdateDatasetDataBody +} from '@fastgpt/global/openapi/core/dataset/data/api'; + +export const getDatasetDataList = (data: GetDatasetDataListProps) => + POST(`/core/dataset/data/v2/list`, data); + +export const getDatasetDataItemById = (id: string) => + GET(`/core/dataset/data/detail`, { id }); + +/** + * insert one data to dataset (immediately insert) + */ +export const postInsertData2Dataset = (data: InsertDataBody) => + POST(`/core/dataset/data/insertData`, data); + +/** + * update one datasetData by id + */ +export const putDatasetDataById = (data: UpdateDatasetDataBody) => + PUT('/core/dataset/data/update', data); + +/** + * 删除一条知识库数据 + */ +export const delOneDatasetDataById = (id: string) => + DELETE(`/core/dataset/data/delete`, { id }); + +// Get quote data +export const getQuoteData = (data: GetQuoteDataProps) => + POST(`/core/dataset/data/getQuoteData`, data); diff --git a/projects/app/src/web/core/dataset/api/file.ts b/projects/app/src/web/core/dataset/api/file.ts new file mode 100644 index 0000000000..83646cd5d3 --- /dev/null +++ b/projects/app/src/web/core/dataset/api/file.ts @@ -0,0 +1,16 @@ +import { POST } from '@/web/common/api/request'; +import type { + GetPreviewChunksBody, + GetPreviewChunksResponse, + PresignDatasetFilePostUrlBody, + PresignDatasetFilePostUrlResponse +} from '@fastgpt/global/openapi/core/dataset/file/api'; + +export const getUploadDatasetFilePresignedUrl = (params: PresignDatasetFilePostUrlBody) => + POST('/core/dataset/file/presignDatasetFilePostUrl', params); + +export const getPreviewChunks = (data: GetPreviewChunksBody) => + POST('/core/dataset/file/getPreviewChunks', data, { + maxQuantity: 1, + timeout: 600000 + }); diff --git a/projects/app/src/web/core/dataset/api/training.ts b/projects/app/src/web/core/dataset/api/training.ts new file mode 100644 index 0000000000..356a4f15fb --- /dev/null +++ b/projects/app/src/web/core/dataset/api/training.ts @@ -0,0 +1,31 @@ +import { GET, POST, PUT } from '@/web/common/api/request'; +import type { + RebuildEmbeddingBody, + GetDatasetTrainingQueueResponse, + DeleteTrainingDataBody, + UpdateTrainingDataBody, + GetTrainingDataDetailBody, + GetTrainingDataDetailResponse, + GetTrainingErrorBody, + GetTrainingErrorResponse +} from '@fastgpt/global/openapi/core/dataset/training/api'; + +export const postRebuildEmbedding = (data: RebuildEmbeddingBody) => + POST(`/core/dataset/training/rebuildEmbedding`, data); + +export const getDatasetTrainingQueue = (datasetId: string) => + GET(`/core/dataset/training/getDatasetTrainingQueue`, { + datasetId + }); + +export const deleteTrainingData = (data: DeleteTrainingDataBody) => + POST(`/core/dataset/training/deleteTrainingData`, data); + +export const updateTrainingData = (data: UpdateTrainingDataBody) => + PUT(`/core/dataset/training/updateTrainingData`, data); + +export const getTrainingDataDetail = (data: GetTrainingDataDetailBody) => + POST(`/core/dataset/training/getTrainingDataDetail`, data); + +export const getTrainingError = (data: GetTrainingErrorBody) => + POST(`/core/dataset/training/getTrainingError`, data); diff --git a/projects/app/src/web/core/dataset/components/SelectCollections.tsx b/projects/app/src/web/core/dataset/components/SelectCollections.tsx index 3ba8cf960f..5624f1a854 100644 --- a/projects/app/src/web/core/dataset/components/SelectCollections.tsx +++ b/projects/app/src/web/core/dataset/components/SelectCollections.tsx @@ -2,7 +2,10 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import MyModal from '@fastgpt/web/components/common/MyModal'; import FolderPath from '@/components/common/folder/Path'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { getDatasetCollectionPathById, getDatasetCollections } from '@/web/core/dataset/api'; +import { + getDatasetCollectionPathById, + getDatasetCollections +} from '@/web/core/dataset/api/collection'; import { Box, Flex, ModalFooter, Button, useTheme, Grid, Card, ModalBody } from '@chakra-ui/react'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; diff --git a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx index a1afe5f35a..5cbb9b20a3 100644 --- a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx +++ b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx @@ -2,15 +2,13 @@ import { useQuery } from '@tanstack/react-query'; import { type Dispatch, type ReactNode, type SetStateAction, useState } from 'react'; import { useTranslation } from 'next-i18next'; import { createContext } from 'use-context-selector'; +import { getDatasetById, getDatasetPaths, putDatasetById } from '../api'; import { getAllTags, - getDatasetById, getDatasetCollectionTags, - getDatasetPaths, - getDatasetTrainingQueue, - postCreateDatasetCollectionTag, - putDatasetById -} from '../api'; + postCreateDatasetCollectionTag +} from '../api/collection'; +import { getDatasetTrainingQueue } from '../api/training'; import { defaultDatasetDetail } from '../constants'; import { type UpdateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; import { type DatasetItemType, type DatasetTagType } from '@fastgpt/global/core/dataset/type'; diff --git a/projects/app/src/web/core/dataset/hooks/readCollectionSource.ts b/projects/app/src/web/core/dataset/hooks/readCollectionSource.ts index 65b636e56c..8e32d5fe8e 100644 --- a/projects/app/src/web/core/dataset/hooks/readCollectionSource.ts +++ b/projects/app/src/web/core/dataset/hooks/readCollectionSource.ts @@ -1,9 +1,9 @@ import { useSystemStore } from '@/web/common/system/useSystemStore'; -import { getCollectionSource } from '@/web/core/dataset/api'; +import { getCollectionSource } from '@/web/core/dataset/api/collection'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useTranslation } from 'next-i18next'; -import type { readCollectionSourceBody } from '@/pages/api/core/dataset/collection/read'; +import type { ReadCollectionSourceBodyType as readCollectionSourceBody } from '@fastgpt/global/openapi/core/dataset/collection/api'; export function getCollectionSourceAndOpen( props: { collectionId: string } & readCollectionSourceBody diff --git a/projects/app/src/web/core/dataset/image/api.ts b/projects/app/src/web/core/dataset/image/api.ts index b437381dd5..e74029ebd4 100644 --- a/projects/app/src/web/core/dataset/image/api.ts +++ b/projects/app/src/web/core/dataset/image/api.ts @@ -1,5 +1,5 @@ import { POST } from '@/web/common/api/request'; -import type { ImageCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import type { ImageCreateDatasetCollectionParams } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; export const createImageDatasetCollection = async ({ files, diff --git a/projects/app/src/web/core/dataset/type.ts b/projects/app/src/web/core/dataset/type.ts index 65133dab32..2dc5e63734 100644 --- a/projects/app/src/web/core/dataset/type.ts +++ b/projects/app/src/web/core/dataset/type.ts @@ -1,4 +1,3 @@ -import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; import type { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import type { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants'; import type { UseFormReturn } from 'react-hook-form'; diff --git a/projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts b/projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts index e3ae925dd1..fcf5dba7f8 100644 --- a/projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts +++ b/projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts @@ -35,6 +35,7 @@ describe('delete training data test', () => { tmbId: root.tmbId, datasetId: dataset._id, collectionId: collection._id, + billId: 'test', mode: TrainingModeEnum.chunk }); diff --git a/projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts b/projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts index 2c3f672bc2..95b318a175 100644 --- a/projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts +++ b/projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts @@ -35,6 +35,7 @@ describe('get training data detail test', () => { tmbId: root.tmbId, datasetId: dataset._id, collectionId: collection._id, + billId: 'test', mode: TrainingModeEnum.chunk, q: 'test', a: 'test' diff --git a/projects/app/test/api/core/dataset/training/getTrainingError.test.ts b/projects/app/test/api/core/dataset/training/getTrainingError.test.ts index 66317d1b36..09847ea740 100644 --- a/projects/app/test/api/core/dataset/training/getTrainingError.test.ts +++ b/projects/app/test/api/core/dataset/training/getTrainingError.test.ts @@ -36,6 +36,7 @@ describe('training error list test', () => { tmbId: root.tmbId, datasetId: dataset._id, collectionId: collection._id, + billId: 'test', mode: TrainingModeEnum.chunk, errorMsg: 'test' })) diff --git a/projects/app/test/api/core/dataset/training/updateTrainingData.test.ts b/projects/app/test/api/core/dataset/training/updateTrainingData.test.ts index 06a99fcdca..e07690e83d 100644 --- a/projects/app/test/api/core/dataset/training/updateTrainingData.test.ts +++ b/projects/app/test/api/core/dataset/training/updateTrainingData.test.ts @@ -35,6 +35,7 @@ describe('update training data test', () => { tmbId: root.tmbId, datasetId: dataset._id, collectionId: collection._id, + billId: 'test', mode: TrainingModeEnum.chunk }); diff --git a/projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts b/projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts index b55c6b6621..81cad35972 100644 --- a/projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts +++ b/projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts @@ -4,6 +4,10 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +const datasetId = '507f1f77bcf86cd799439011'; +const collectionId = '507f1f77bcf86cd799439012'; +const dataId = '507f1f77bcf86cd799439013'; + vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({ MongoDatasetTraining: { findOne: vi.fn(), @@ -24,8 +28,8 @@ describe('updateTrainingData', () => { const req = { body: { - datasetId: 'dataset1', - collectionId: 'collection1' + datasetId, + collectionId } }; @@ -34,8 +38,8 @@ describe('updateTrainingData', () => { expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith( { teamId: 'team1', - datasetId: 'dataset1', - collectionId: 'collection1', + datasetId, + collectionId, errorMsg: { $exists: true, $ne: null } }, { @@ -57,9 +61,9 @@ describe('updateTrainingData', () => { const req = { body: { - datasetId: 'dataset1', - collectionId: 'collection1', - dataId: 'data1', + datasetId, + collectionId, + dataId, q: 'question', a: 'answer', chunkIndex: 1 @@ -71,8 +75,8 @@ describe('updateTrainingData', () => { expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith( { teamId: 'team1', - datasetId: 'dataset1', - _id: 'data1' + datasetId, + _id: dataId }, { $unset: { errorMsg: '' }, @@ -95,9 +99,9 @@ describe('updateTrainingData', () => { const req = { body: { - datasetId: 'dataset1', - collectionId: 'collection1', - dataId: 'data1', + datasetId, + collectionId, + dataId, q: 'question', a: 'answer', chunkIndex: 1 @@ -109,8 +113,8 @@ describe('updateTrainingData', () => { expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith( { teamId: 'team1', - datasetId: 'dataset1', - _id: 'data1' + datasetId, + _id: dataId }, { $unset: { errorMsg: '' }, @@ -132,9 +136,9 @@ describe('updateTrainingData', () => { const req = { body: { - datasetId: 'dataset1', - collectionId: 'collection1', - dataId: 'data1' + datasetId, + collectionId, + dataId } }; diff --git a/test/cases/global/core/chat/adapt.test.ts b/test/cases/global/core/chat/adapt.test.ts index dd6cd37e68..2dd23f557f 100644 --- a/test/cases/global/core/chat/adapt.test.ts +++ b/test/cases/global/core/chat/adapt.test.ts @@ -12,7 +12,7 @@ import { import { ChatRoleEnum, ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; describe('GPT2Chat mapping', () => { it('should map GPT roles to Chat roles correctly', () => { diff --git a/test/cases/global/core/dataset/v2/api.test.ts b/test/cases/global/core/dataset/v2/api.test.ts deleted file mode 100644 index 5e93c52f6e..0000000000 --- a/test/cases/global/core/dataset/v2/api.test.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { - PresignDatasetFileGetUrlSchema, - PresignDatasetFilePostUrlSchema, - ShortPreviewLinkSchema -} from '@fastgpt/global/core/dataset/v2/api'; - -describe('PresignDatasetFileGetUrlSchema', () => { - describe('key variant', () => { - it('should accept valid key starting with "dataset/"', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: 'dataset/test-file.pdf' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ key: 'dataset/test-file.pdf' }); - } - }); - - it('should accept key with preview option set to true', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: 'dataset/test-file.pdf', - preview: true - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ key: 'dataset/test-file.pdf', preview: true }); - } - }); - - it('should accept key with preview option set to false', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: 'dataset/test-file.pdf', - preview: false - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ key: 'dataset/test-file.pdf', preview: false }); - } - }); - - it('should decode URL-encoded key', () => { - const encodedKey = 'dataset/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6.pdf'; - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: encodedKey - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ key: 'dataset/中文文件.pdf' }); - } - }); - - it('should reject key not starting with "dataset/"', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: 'other/test-file.pdf' - }); - expect(result.success).toBe(false); - if (!result.success) { - expect(result.error.issues[0].message).toBe( - 'Invalid key format: must start with "dataset/"' - ); - } - }); - - it('should reject empty key', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: '' - }); - expect(result.success).toBe(false); - }); - - it('should reject missing key field', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - preview: true - }); - expect(result.success).toBe(false); - }); - }); - - describe('collectionId variant', () => { - it('should accept valid collectionId (24 hex characters)', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - collectionId: '68ee0bd23d17260b7829b137' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ collectionId: '68ee0bd23d17260b7829b137' }); - } - }); - - it('should accept collectionId as object with toString', () => { - const objectId = { - toString: () => '68ee0bd23d17260b7829b137' - }; - const result = PresignDatasetFileGetUrlSchema.safeParse({ - collectionId: objectId - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ collectionId: '68ee0bd23d17260b7829b137' }); - } - }); - - it('should reject invalid collectionId (wrong length)', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - collectionId: '123' - }); - expect(result.success).toBe(false); - }); - - it('should reject invalid collectionId (non-hex characters)', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - collectionId: 'zzzzzzzzzzzzzzzzzzzzzzzz' - }); - expect(result.success).toBe(false); - }); - - it('should reject empty collectionId', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({ - collectionId: '' - }); - expect(result.success).toBe(false); - }); - }); - - describe('union behavior', () => { - it('should reject empty object', () => { - const result = PresignDatasetFileGetUrlSchema.safeParse({}); - expect(result.success).toBe(false); - }); - - it('should reject object with both key and collectionId', () => { - // Union will match the first valid variant - const result = PresignDatasetFileGetUrlSchema.safeParse({ - key: 'dataset/test.pdf', - collectionId: '68ee0bd23d17260b7829b137' - }); - // This should succeed because it matches the first variant (key) - expect(result.success).toBe(true); - }); - }); -}); - -describe('PresignDatasetFilePostUrlSchema', () => { - it('should accept valid filename and datasetId', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: 'test-file.pdf', - datasetId: '68ee0bd23d17260b7829b137' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ - filename: 'test-file.pdf', - datasetId: '68ee0bd23d17260b7829b137' - }); - } - }); - - it('should accept datasetId as object with toString', () => { - const objectId = { - toString: () => '68ee0bd23d17260b7829b137' - }; - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: 'test-file.pdf', - datasetId: objectId - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ - filename: 'test-file.pdf', - datasetId: '68ee0bd23d17260b7829b137' - }); - } - }); - - it('should accept filename with special characters', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: '中文文件名.pdf', - datasetId: '68ee0bd23d17260b7829b137' - }); - expect(result.success).toBe(true); - }); - - it('should reject empty filename', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: '', - datasetId: '68ee0bd23d17260b7829b137' - }); - expect(result.success).toBe(false); - }); - - it('should reject missing filename', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - datasetId: '68ee0bd23d17260b7829b137' - }); - expect(result.success).toBe(false); - }); - - it('should reject invalid datasetId', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: 'test-file.pdf', - datasetId: 'invalid-id' - }); - expect(result.success).toBe(false); - }); - - it('should reject missing datasetId', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({ - filename: 'test-file.pdf' - }); - expect(result.success).toBe(false); - }); - - it('should reject empty object', () => { - const result = PresignDatasetFilePostUrlSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe('ShortPreviewLinkSchema', () => { - it('should accept valid k and transform to chat:temp_file: prefix', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: 'test-key' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ k: 'chat:temp_file:test-key' }); - } - }); - - it('should decode URL-encoded k value', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: '%E4%B8%AD%E6%96%87' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ k: 'chat:temp_file:中文' }); - } - }); - - it('should handle k with special characters', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: 'file%2Fpath%2Ftest.pdf' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ k: 'chat:temp_file:file/path/test.pdf' }); - } - }); - - it('should handle k with spaces encoded as %20', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: 'file%20name.pdf' - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data).toEqual({ k: 'chat:temp_file:file name.pdf' }); - } - }); - - it('should reject empty k', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: '' - }); - expect(result.success).toBe(false); - }); - - it('should reject missing k field', () => { - const result = ShortPreviewLinkSchema.safeParse({}); - expect(result.success).toBe(false); - }); - - it('should reject non-string k value', () => { - const result = ShortPreviewLinkSchema.safeParse({ - k: 123 - }); - expect(result.success).toBe(false); - }); -}); diff --git a/test/cases/service/core/ai/llm/request.test.ts b/test/cases/service/core/ai/llm/request.test.ts index d6177d71d7..5fc51bca92 100644 --- a/test/cases/service/core/ai/llm/request.test.ts +++ b/test/cases/service/core/ai/llm/request.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, - StreamChatType -} from '@fastgpt/global/core/ai/type'; + StreamResponseType +} from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; @@ -111,11 +111,11 @@ async function* createAsyncGenerator(items: T[]): AsyncGenerator { +const createMockStreamResponse = (chunks: any[]): StreamResponseType => { const generator = createAsyncGenerator(chunks); return Object.assign(generator, { controller: { abort: vi.fn() } - }) as unknown as StreamChatType; + }) as unknown as StreamResponseType; }; describe('createLLMResponse', () => { @@ -1006,7 +1006,7 @@ describe('createLLMResponse', () => { const mockStreamResponse = Object.assign(generator, { controller: { abort: vi.fn() } - }) as unknown as StreamChatType; + }) as unknown as StreamResponseType; const mockAI = { chat: { diff --git a/test/cases/service/core/ai/llm/toolCall.test.ts b/test/cases/service/core/ai/llm/toolCall.test.ts index 689d64ff54..1de2502d5a 100644 --- a/test/cases/service/core/ai/llm/toolCall.test.ts +++ b/test/cases/service/core/ai/llm/toolCall.test.ts @@ -2,7 +2,10 @@ import { parsePromptToolCall, promptToolCallMessageRewrite } from '@fastgpt/service/core/ai/llm/promptCall/index'; -import type { ChatCompletionMessageParam, ChatCompletionTool } from '@fastgpt/global/core/ai/type'; +import type { + ChatCompletionMessageParam, + ChatCompletionTool +} from '@fastgpt/global/core/ai/llm/type'; import { describe, expect, it } from 'vitest'; describe('parsePromptToolCall function tests', () => { diff --git a/test/cases/service/core/ai/llm/utils.test.ts b/test/cases/service/core/ai/llm/utils.test.ts index 8dea0c1cd0..b44d035eb4 100644 --- a/test/cases/service/core/ai/llm/utils.test.ts +++ b/test/cases/service/core/ai/llm/utils.test.ts @@ -2,7 +2,7 @@ import { loadRequestMessages, filterGPTMessageByMaxContext } from '@fastgpt/service/core/ai/llm/utils'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; +import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { describe, expect, it, vi, beforeEach } from 'vitest'; diff --git a/test/cases/service/core/ai/utils.test.ts b/test/cases/service/core/ai/utils.test.ts index 4f29265637..8c69d3088d 100644 --- a/test/cases/service/core/ai/utils.test.ts +++ b/test/cases/service/core/ai/utils.test.ts @@ -6,7 +6,7 @@ import { computedTemperature, parseReasoningContent } from '@fastgpt/service/core/ai/utils'; -import type { CompletionFinishReason } from '@fastgpt/global/core/ai/type'; +import type { CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; const mockModel = (maxResponse: number, maxTemperature?: number) => diff --git a/test/mocks/core/ai/llm.ts b/test/mocks/core/ai/llm.ts index c72278d32c..729ba3d1b9 100644 --- a/test/mocks/core/ai/llm.ts +++ b/test/mocks/core/ai/llm.ts @@ -1,5 +1,5 @@ import { vi } from 'vitest'; -import type { ChatCompletion } from '@fastgpt/global/core/ai/type'; +import type { UnStreamResponseType } from '@fastgpt/global/core/ai/llm/type'; /** * Mock LLM response utilities for testing @@ -15,7 +15,7 @@ export const createMockCompleteResponseWithReason = (options?: { finishReason?: 'stop' | 'length' | 'content_filter'; promptTokens?: number; completionTokens?: number; -}): ChatCompletion => { +}): UnStreamResponseType => { const { content = 'This is the answer to your question.', reasoningContent = 'First, I need to analyze the question...', @@ -48,7 +48,7 @@ export const createMockCompleteResponseWithReason = (options?: { total_tokens: promptTokens + completionTokens }, system_fingerprint: 'fp_test' - } as ChatCompletion; + } as UnStreamResponseType; }; /** @@ -64,7 +64,7 @@ export const createMockCompleteResponseWithTool = (options?: { finishReason?: 'tool_calls' | 'stop'; promptTokens?: number; completionTokens?: number; -}): ChatCompletion => { +}): UnStreamResponseType => { const { toolCalls = [ { @@ -110,7 +110,7 @@ export const createMockCompleteResponseWithTool = (options?: { total_tokens: promptTokens + completionTokens }, system_fingerprint: 'fp_test' - } as ChatCompletion; + } as UnStreamResponseType; }; /** @@ -118,7 +118,7 @@ export const createMockCompleteResponseWithTool = (options?: { * Can be configured to return different types of responses based on test needs */ export const mockCreateChatCompletion = vi.fn( - async (body: any, options?: any): Promise => { + async (body: any, options?: any): Promise => { // Default: return response with text if (body.tools && body.tools.length > 0) { return createMockCompleteResponseWithTool();