From 7494ce8453a3ea8daf08708a32044eb96432132b Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Sun, 8 Jun 2025 18:46:36 +0800 Subject: [PATCH] perf: async read file (#4978) * perf: async read file * package * doc --- packages/service/core/ai/config/utils.ts | 2 +- .../service/worker/readFile/extension/csv.ts | 2 +- .../service/worker/readFile/extension/html.ts | 2 +- .../worker/readFile/extension/rawText.ts | 5 ++- packages/templates/register.ts | 4 +-- pnpm-lock.yaml | 8 ++--- projects/app/package.json | 2 +- projects/app/public/docs/versionIntro.md | 33 ++++++++++--------- .../dashboard/apps/CreateModal.tsx | 10 +++--- 9 files changed, 36 insertions(+), 32 deletions(-) diff --git a/packages/service/core/ai/config/utils.ts b/packages/service/core/ai/config/utils.ts index 122b42bc8..b7a67d46c 100644 --- a/packages/service/core/ai/config/utils.ts +++ b/packages/service/core/ai/config/utils.ts @@ -103,7 +103,7 @@ export const loadSystemModels = async (init = false) => { // Load system model from local const modelsPath = getModelConfigBaseUrl(); - const providerList = fs.readdirSync(modelsPath) as string[]; + const providerList = await fs.promises.readdir(modelsPath); await Promise.all( providerList.map(async (name) => { const fileContent = (await import(`./provider/${name}`))?.default as { diff --git a/packages/service/worker/readFile/extension/csv.ts b/packages/service/worker/readFile/extension/csv.ts index 67e08d854..887247c13 100644 --- a/packages/service/worker/readFile/extension/csv.ts +++ b/packages/service/worker/readFile/extension/csv.ts @@ -4,7 +4,7 @@ import { readFileRawText } from './rawText'; // 加载源文件内容 export const readCsvRawText = async (params: ReadRawTextByBuffer): Promise => { - const { rawText } = readFileRawText(params); + const { rawText } = await readFileRawText(params); const csvArr = Papa.parse(rawText).data as string[][]; diff --git a/packages/service/worker/readFile/extension/html.ts b/packages/service/worker/readFile/extension/html.ts index 5dbad8ae5..c64ef5e4d 100644 --- a/packages/service/worker/readFile/extension/html.ts +++ b/packages/service/worker/readFile/extension/html.ts @@ -3,7 +3,7 @@ import { readFileRawText } from './rawText'; import { html2md } from '../../htmlStr2Md/utils'; export const readHtmlRawText = async (params: ReadRawTextByBuffer): Promise => { - const { rawText: html } = readFileRawText(params); + const { rawText: html } = await readFileRawText(params); const { rawText, imageList } = html2md(html); diff --git a/packages/service/worker/readFile/extension/rawText.ts b/packages/service/worker/readFile/extension/rawText.ts index 6baa06e56..45571b948 100644 --- a/packages/service/worker/readFile/extension/rawText.ts +++ b/packages/service/worker/readFile/extension/rawText.ts @@ -18,7 +18,10 @@ const rawEncodingList = [ ]; // 加载源文件内容 -export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => { +export const readFileRawText = async ({ + buffer, + encoding +}: ReadRawTextByBuffer): Promise => { const content = (() => { try { if (rawEncodingList.includes(encoding)) { diff --git a/packages/templates/register.ts b/packages/templates/register.ts index b8ab72abb..a828e2e20 100644 --- a/packages/templates/register.ts +++ b/packages/templates/register.ts @@ -14,11 +14,11 @@ const getTemplateNameList = () => { ); const templatesPath = path.join(path.dirname(filePath), 'src'); - return fs.readdirSync(templatesPath) as string[]; + return fs.promises.readdir(templatesPath); }; const getFileTemplates = async (): Promise => { - const templateNames = getTemplateNameList(); + const templateNames = await getTemplateNameList(); return Promise.all( templateNames.map>(async (name) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3b1ab7fc9..0ae6c51be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15142,7 +15142,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0): + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0))(eslint@8.56.0): dependencies: debug: 3.2.7 optionalDependencies: @@ -15153,7 +15153,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: @@ -15175,7 +15175,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0))(eslint@8.56.0) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -15204,7 +15204,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 diff --git a/projects/app/package.json b/projects/app/package.json index d0bb7b2d5..b66ad1e12 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -1,6 +1,6 @@ { "name": "app", - "version": "4.9.10", + "version": "4.9.11", "private": false, "scripts": { "dev": "next dev", diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index c7aeb68e7..d480f406c 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,21 +1,22 @@ -### FastGPT V4.9.0 更新说明 +### FastGPT V4.9.11 更新说明 -#### 弃用 & 兼容 +## 🚀 新增内容 -1. 弃用 - 之前私有化部署的自定义文件解析方案，请同步更新到最新的配置方案。[点击查看 PDF 增强解析配置](/docs/development/configuration/#使用-doc2x-解析-pdf-文件) -2. 弃用 - 弃用旧版本地文件上传 API：/api/core/dataset/collection/create/file（以前仅商业版可用的 API，该接口已放切换成：/api/core/dataset/collection/create/localFile） -3. 停止维护，即将弃用 - 外部文件库相关 API，可通过 API 文件库替代。 -4. API更新 - 上传文件至知识库、创建连接集合、API 文件库、推送分块数据等带有 `trainingType` 字段的接口，`trainingType`字段未来仅支持`chunk`和`QA`两种模式。增强索引模式将设置单独字段：`autoIndexes`，目前仍有适配旧版`trainingType=auto`代码，但请尽快变更成新接口类型。具体可见：[知识库 OpenAPI 文档](/docs/development/openapi/dataset.md) +1. 支持图片知识库。 +2. 工作流中增加节点搜索功能。 +3. 工作流中，子流程版本控制，可选择“保持最新版本”，无需手动更新。 +4. 增加更多审计操作日志。 +5. 知识库增加文档解析异步队列，导入文档时，无需等待文档解析完毕才进行导入。 -#### 功能更新 +## ⚙️ 优化 -1. 新增 - PDF 增强解析，可以识别图片、公式、扫描件，并将内容转化成 Markdown 格式。 -2. 新增 - 支持对文档中的图片链接，进行图片索引，提高图片内容的检索精度。 -3. 新增 - 语义检索增加迭代搜索，减少漏检。 -4. 优化 - 知识库数据不再限制索引数量，可无限自定义。同时可自动更新输入文本的索引，不影响自定义索引。 -5. 优化 - Markdown 解析，增加链接后中文标点符号检测，增加空格。 -6. 优化 - Prompt 模式工具调用，支持思考模型。同时优化其格式检测，减少空输出的概率。 -7. 优化 - 优化文件读取代码，极大提高大文件读取速度。50M PDF 读取时间提高 3 倍。 -8. 优化 - HTTP Body 适配，增加对字符串对象的适配。 -9. 修复 - 批量运行时，全局变量未进一步传递到下一次运行中，导致最终变量更新错误。 +1. 原文缓存改用 gridfs 存储，提高上限。 +## 🐛 修复 + +1. 工作流中，管理员声明的全局系统工具，无法进行版本管理。 +2. 工具调用节点前，有交互节点时，上下文异常。 +3. 修复备份导入，小于 1000 字时，无法分块问题。 +4. 自定义 PDF 解析，无法保存 base64 图片。 +5. 非流请求，未进行 CITE 标记替换。 +6. Python 沙盒存在隐藏风险。 \ No newline at end of file diff --git a/projects/app/src/pageComponents/dashboard/apps/CreateModal.tsx b/projects/app/src/pageComponents/dashboard/apps/CreateModal.tsx index c0fd23f91..55f3761c8 100644 --- a/projects/app/src/pageComponents/dashboard/apps/CreateModal.tsx +++ b/projects/app/src/pageComponents/dashboard/apps/CreateModal.tsx @@ -305,21 +305,21 @@ const CreateModal = ({ onClose, type }: { type: CreateAppType; onClose: () => vo w={'560px'} h={'260px'} bg={'myGray.50'} - {...register('curlContent')} + {...register('curlContent', { required: true })} /> )} - {!isTemplateMode && ( + + {currentCreateType === 'curl' && ( )} -