diff --git a/docSite/assets/imgs/getfile_id.png b/docSite/assets/imgs/getfile_id.png new file mode 100644 index 000000000..3995d5669 Binary files /dev/null and b/docSite/assets/imgs/getfile_id.png differ diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index 325996e54..c75eba7c9 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -77,7 +77,7 @@ weight: 520 "price": 0, "prompt": "" }, - "CQModel": { // 问题分类模型 + "CQModel": { // Classify Question: 问题分类模型 "model": "gpt-3.5-turbo-16k", "functionCall": true, "name": "GPT35-16k", @@ -85,7 +85,7 @@ weight: 520 "price": 0, "prompt": "" }, - "QGModel": { // 生成下一步指引模型 + "QGModel": { // Question Generation: 生成下一步指引模型 "model": "gpt-3.5-turbo", "name": "GPT35-4k", "maxToken": 4000, diff --git a/docSite/content/docs/development/openApi.md b/docSite/content/docs/development/openApi.md index df64f55b2..f93d18d11 100644 --- a/docSite/content/docs/development/openApi.md +++ b/docSite/content/docs/development/openApi.md @@ -225,9 +225,10 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2 此部分 API 需使用全局通用的 API Key。 {{% /alert %}} -### 如何获取知识库ID（kbId） +| 如何获取知识库ID（kbId） | 如何获取文件ID（file_id） | +| --------------------- | --------------------- | +| ![](/imgs/getKbId.png) | ![](/imgs/getfile_id.png) | -![](/imgs/getKbId.png) ### 知识库添加数据 @@ -248,6 +249,8 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa         {             "a": "test",             "q": "1111", + "file_id": "关联的文件ID/URL/manual/mark", + "source": "来源名称"，         },         {             "a": "test2", @@ -271,7 +274,8 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa "data": [ { "q": "生成索引的内容，index 模式下最大 tokens 为3000，建议不超过 1000", - "a": "预期回答/补充" + "a": "预期回答/补充", + "file_id": "如果推送数据到手动录入，这里可以留空; 如果希望关联到某个文件中，需要填写对应文件的ID; 如果希望加入到手动标注中，可设置为: mark", }, { "q": "生成索引的内容，qa 模式下最大 tokens 为10000，建议 8000 左右", @@ -292,7 +296,16 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa "code": 200, "statusText": "", "data": { - "insertLen": 1 // 最终插入成功的数量，可能因为超出 tokens 或者插入异常，index 可以重复插入，会自动去重 + "insertLen": 1, // 最终插入成功的数量 + "overToken": [], // 超出 token 的 + "fileIdInvalid": [ // file_id 无效的 + { + "a": "飞飞dsaf飞", + "q": "测试是32否收到", + "file_id": "32dwe" + } + ], + "error": [] // 其他错误 } } ``` diff --git a/docSite/content/docs/installation/upgrading/447.md b/docSite/content/docs/installation/upgrading/447.md index 8160cbef0..03bc72d43 100644 --- a/docSite/content/docs/installation/upgrading/447.md +++ b/docSite/content/docs/installation/upgrading/447.md @@ -26,4 +26,6 @@ curl --location --request POST 'https://{{host}}/api/admin/initv447' \ ### Fast GPT V4.4.7 1. 优化了数据库文件 crud。 -2. 兼容链接读取，作为 source。 \ No newline at end of file +2. 兼容链接读取，作为 source。 +3. 区分手动录入和标注，可追数据至某个文件。 +4. 升级 openai sdk。 \ No newline at end of file diff --git a/packages/core/ai/config.ts b/packages/core/ai/config.ts index cc9c94e2f..11df30451 100644 --- a/packages/core/ai/config.ts +++ b/packages/core/ai/config.ts @@ -1,28 +1,16 @@ import { UserModelSchema } from '../user/type'; -import { Configuration, OpenAIApi } from 'openai'; +import OpenAI from 'openai'; export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'; export const baseUrl = process.env.ONEAPI_URL || openaiBaseUrl; export const systemAIChatKey = process.env.CHAT_API_KEY || ''; -export const getAIChatApi = (props?: UserModelSchema['openaiAccount']) => { - return new OpenAIApi( - new Configuration({ - basePath: props?.baseUrl || baseUrl, - apiKey: props?.key || systemAIChatKey - }) - ); -}; - -/* openai axios config */ -export const axiosConfig = (props?: UserModelSchema['openaiAccount']) => { - return { - baseURL: props?.baseUrl || baseUrl, // 此处仅对非 npm 模块有效 - httpsAgent: global.httpsAgent, - headers: { - Authorization: `Bearer ${props?.key || systemAIChatKey}`, - auth: process.env.OPENAI_BASE_URL_AUTH || '' - } - }; +export const getAIApi = (props?: UserModelSchema['openaiAccount'], timeout = 6000) => { + return new OpenAI({ + apiKey: props?.key || systemAIChatKey, + baseURL: props?.baseUrl || baseUrl, + httpAgent: global.httpsAgent, + timeout + }); }; diff --git a/packages/core/ai/constant.ts b/packages/core/ai/constant.ts index 2c6b9bee5..4773d0895 100644 --- a/packages/core/ai/constant.ts +++ b/packages/core/ai/constant.ts @@ -1 +1,6 @@ -export { ChatCompletionRequestMessageRoleEnum } from 'openai'; +export enum ChatCompletionRequestMessageRoleEnum { + 'System' = 'system', + 'User' = 'user', + 'Assistant' = 'assistant', + 'Function' = 'function' +} diff --git a/packages/core/ai/functions/createQuestionGuide.ts b/packages/core/ai/functions/createQuestionGuide.ts index d7a667756..3201c34e2 100644 --- a/packages/core/ai/functions/createQuestionGuide.ts +++ b/packages/core/ai/functions/createQuestionGuide.ts @@ -1,5 +1,5 @@ import { ChatCompletionRequestMessage } from '../type'; -import { getAIChatApi } from '../config'; +import { getAIApi } from '../config'; export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`; @@ -10,8 +10,8 @@ export async function createQuestionGuide({ messages: ChatCompletionRequestMessage[]; model: string; }) { - const chatAPI = getAIChatApi(); - const { data } = await chatAPI.createChatCompletion({ + const ai = getAIApi(); + const data = await ai.chat.completions.create({ model: model, temperature: 0, max_tokens: 200, diff --git a/packages/core/ai/type.d.ts b/packages/core/ai/type.d.ts index 7fa86c127..89a5a42d5 100644 --- a/packages/core/ai/type.d.ts +++ b/packages/core/ai/type.d.ts @@ -1 +1,6 @@ -export type { CreateChatCompletionRequest, ChatCompletionRequestMessage } from 'openai'; +import OpenAI from 'openai'; +export type ChatCompletionRequestMessage = OpenAI.Chat.CreateChatCompletionRequestMessage; +export type ChatCompletion = OpenAI.Chat.ChatCompletion; +export type CreateChatCompletionRequest = OpenAI.Chat.ChatCompletionCreateParams; + +export type StreamChatType = Stream; diff --git a/packages/core/package.json b/packages/core/package.json index 0329f7d68..5013eac09 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -2,10 +2,11 @@ "name": "@fastgpt/core", "version": "1.0.0", "dependencies": { - "openai": "^3.3.0", - "tunnel": "^0.0.6", "@fastgpt/common": "workspace:*", - "@fastgpt/support": "workspace:*" + "@fastgpt/support": "workspace:*", + "encoding": "^0.1.13", + "openai": "^4.11.1", + "tunnel": "^0.0.6" }, "devDependencies": { "@types/tunnel": "^0.0.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d4511557..e15f8565c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,9 +37,12 @@ importers: '@fastgpt/support': specifier: workspace:* version: link:../support + encoding: + specifier: ^0.1.13 + version: registry.npmmirror.com/encoding@0.1.13 openai: - specifier: ^3.3.0 - version: registry.npmmirror.com/openai@3.3.0 + specifier: ^4.11.1 + version: registry.npmmirror.com/openai@4.11.1(encoding@0.1.13) tunnel: specifier: ^0.0.6 version: registry.npmmirror.com/tunnel@0.0.6 @@ -5584,6 +5587,15 @@ packages: '@types/express': registry.npmmirror.com/@types/express@4.17.17 dev: true + registry.npmmirror.com/@types/node-fetch@2.6.6: + resolution: {integrity: sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node-fetch/-/node-fetch-2.6.6.tgz} + name: '@types/node-fetch' + version: 2.6.6 + dependencies: + '@types/node': registry.npmmirror.com/@types/node@18.14.0 + form-data: registry.npmmirror.com/form-data@4.0.0 + dev: false + registry.npmmirror.com/@types/node@18.14.0: resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-18.14.0.tgz} name: '@types/node' @@ -5825,6 +5837,15 @@ packages: version: 2.0.6 dev: false + registry.npmmirror.com/abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abort-controller/-/abort-controller-3.0.0.tgz} + name: abort-controller + version: 3.0.0 + engines: {node: '>=6.5'} + dependencies: + event-target-shim: registry.npmmirror.com/event-target-shim@5.0.1 + dev: false + registry.npmmirror.com/acorn-jsx@5.3.2(acorn@8.10.0): resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz} id: registry.npmmirror.com/acorn-jsx/5.3.2 @@ -5855,6 +5876,15 @@ packages: - supports-color dev: false + registry.npmmirror.com/agentkeepalive@4.5.0: + resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz} + name: agentkeepalive + version: 4.5.0 + engines: {node: '>= 8.0.0'} + dependencies: + humanize-ms: registry.npmmirror.com/humanize-ms@1.2.1 + dev: false + registry.npmmirror.com/aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/aggregate-error/-/aggregate-error-3.1.0.tgz} name: aggregate-error @@ -6115,16 +6145,6 @@ packages: engines: {node: '>=4'} dev: true - registry.npmmirror.com/axios@0.26.1: - resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/axios/-/axios-0.26.1.tgz} - name: axios - version: 0.26.1 - dependencies: - follow-redirects: registry.npmmirror.com/follow-redirects@1.15.3 - transitivePeerDependencies: - - debug - dev: false - registry.npmmirror.com/axios@1.3.3: resolution: {integrity: sha512-eYq77dYIFS77AQlhzEL937yUBSepBfPIe8FcgEDN35vMNZKMrs81pgnyrQpwfy4NF4b4XWX1Zgx7yX+25w8QJA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/axios/-/axios-1.3.3.tgz} name: axios @@ -6213,6 +6233,12 @@ packages: version: 1.0.2 dev: true + registry.npmmirror.com/base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/base-64/-/base-64-0.1.0.tgz} + name: base-64 + version: 0.1.0 + dev: false + registry.npmmirror.com/base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/base64-js/-/base64-js-1.5.1.tgz} name: base64-js @@ -6418,6 +6444,12 @@ packages: version: 1.1.4 dev: false + registry.npmmirror.com/charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/charenc/-/charenc-0.0.2.tgz} + name: charenc + version: 0.0.2 + dev: false + registry.npmmirror.com/chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz} name: chokidar @@ -6706,6 +6738,12 @@ packages: which: registry.npmmirror.com/which@2.0.2 dev: true + registry.npmmirror.com/crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/crypt/-/crypt-0.0.2.tgz} + name: crypt + version: 0.0.2 + dev: false + registry.npmmirror.com/crypto@1.0.1: resolution: {integrity: sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/crypto/-/crypto-1.0.1.tgz} name: crypto @@ -7284,6 +7322,15 @@ packages: engines: {node: '>=0.3.1'} dev: false + registry.npmmirror.com/digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/digest-fetch/-/digest-fetch-1.3.0.tgz} + name: digest-fetch + version: 1.3.0 + dependencies: + base-64: registry.npmmirror.com/base-64@0.1.0 + md5: registry.npmmirror.com/md5@2.3.0 + dev: false + registry.npmmirror.com/dingbat-to-unicode@1.0.1: resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz} name: dingbat-to-unicode @@ -7446,6 +7493,14 @@ packages: version: 2.0.0 dev: false + registry.npmmirror.com/encoding@0.1.13: + resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/encoding/-/encoding-0.1.13.tgz} + name: encoding + version: 0.1.13 + dependencies: + iconv-lite: registry.npmmirror.com/iconv-lite@0.6.3 + dev: false + registry.npmmirror.com/enhanced-resolve@5.15.0: resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz} name: enhanced-resolve @@ -7933,6 +7988,13 @@ packages: engines: {node: '>=0.10.0'} dev: true + registry.npmmirror.com/event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/event-target-shim/-/event-target-shim-5.0.1.tgz} + name: event-target-shim + version: 5.0.1 + engines: {node: '>=6'} + dev: false + registry.npmmirror.com/execa@7.1.1: resolution: {integrity: sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/execa/-/execa-7.1.1.tgz} name: execa @@ -8105,6 +8167,12 @@ packages: is-callable: registry.npmmirror.com/is-callable@1.2.7 dev: true + registry.npmmirror.com/form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz} + name: form-data-encoder + version: 1.7.2 + dev: false + registry.npmmirror.com/form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/form-data/-/form-data-4.0.0.tgz} name: form-data @@ -8123,6 +8191,16 @@ packages: engines: {node: '>=0.4.x'} dev: false + registry.npmmirror.com/formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/formdata-node/-/formdata-node-4.4.1.tgz} + name: formdata-node + version: 4.4.1 + engines: {node: '>= 12.20'} + dependencies: + node-domexception: registry.npmmirror.com/node-domexception@1.0.0 + web-streams-polyfill: registry.npmmirror.com/web-streams-polyfill@4.0.0-beta.3 + dev: false + registry.npmmirror.com/formidable@2.1.1: resolution: {integrity: sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/formidable/-/formidable-2.1.1.tgz} name: formidable @@ -8565,6 +8643,14 @@ packages: engines: {node: '>=14.18.0'} dev: true + registry.npmmirror.com/humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/humanize-ms/-/humanize-ms-1.2.1.tgz} + name: humanize-ms + version: 1.2.1 + dependencies: + ms: registry.npmmirror.com/ms@2.1.3 + dev: false + registry.npmmirror.com/husky@8.0.3: resolution: {integrity: sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/husky/-/husky-8.0.3.tgz} name: husky @@ -8785,6 +8871,12 @@ packages: has-tostringtag: registry.npmmirror.com/has-tostringtag@1.0.0 dev: true + registry.npmmirror.com/is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz} + name: is-buffer + version: 1.1.6 + dev: false + registry.npmmirror.com/is-buffer@2.0.5: resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-buffer/-/is-buffer-2.0.5.tgz} name: is-buffer @@ -9516,6 +9608,16 @@ packages: version: 3.0.3 dev: false + registry.npmmirror.com/md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz} + name: md5 + version: 2.3.0 + dependencies: + charenc: registry.npmmirror.com/charenc@0.0.2 + crypt: registry.npmmirror.com/crypt@0.0.2 + is-buffer: registry.npmmirror.com/is-buffer@1.1.6 + dev: false + registry.npmmirror.com/mdast-util-definitions@5.1.2: resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mdast-util-definitions/-/mdast-util-definitions-5.1.2.tgz} name: mdast-util-definitions @@ -10381,6 +10483,29 @@ packages: next: registry.npmmirror.com/next@13.5.2(@babel/core@7.22.20)(react-dom@18.2.0)(react@18.2.0)(sass@1.58.3) dev: false + registry.npmmirror.com/node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-domexception/-/node-domexception-1.0.0.tgz} + name: node-domexception + version: 1.0.0 + engines: {node: '>=10.5.0'} + dev: false + + registry.npmmirror.com/node-fetch@2.7.0(encoding@0.1.13): + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-fetch/-/node-fetch-2.7.0.tgz} + id: registry.npmmirror.com/node-fetch/2.7.0 + name: node-fetch + version: 2.7.0 + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + encoding: registry.npmmirror.com/encoding@0.1.13 + whatwg-url: registry.npmmirror.com/whatwg-url@5.0.0 + dev: false + registry.npmmirror.com/node-releases@2.0.13: resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz} name: node-releases @@ -10543,15 +10668,23 @@ packages: mimic-fn: registry.npmmirror.com/mimic-fn@4.0.0 dev: true - registry.npmmirror.com/openai@3.3.0: - resolution: {integrity: sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/openai/-/openai-3.3.0.tgz} + registry.npmmirror.com/openai@4.11.1(encoding@0.1.13): + resolution: {integrity: sha512-GU0HQWbejXuVAQlDjxIE8pohqnjptFDIm32aPlNT1H9ucMz1VJJD0DaTJRQsagNaJ97awWjjVLEG7zCM6sm4SA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/openai/-/openai-4.11.1.tgz} + id: registry.npmmirror.com/openai/4.11.1 name: openai - version: 3.3.0 + version: 4.11.1 + hasBin: true dependencies: - axios: registry.npmmirror.com/axios@0.26.1 - form-data: registry.npmmirror.com/form-data@4.0.0 + '@types/node': registry.npmmirror.com/@types/node@18.14.0 + '@types/node-fetch': registry.npmmirror.com/@types/node-fetch@2.6.6 + abort-controller: registry.npmmirror.com/abort-controller@3.0.0 + agentkeepalive: registry.npmmirror.com/agentkeepalive@4.5.0 + digest-fetch: registry.npmmirror.com/digest-fetch@1.3.0 + form-data-encoder: registry.npmmirror.com/form-data-encoder@1.7.2 + formdata-node: registry.npmmirror.com/formdata-node@4.4.1 + node-fetch: registry.npmmirror.com/node-fetch@2.7.0(encoding@0.1.13) transitivePeerDependencies: - - debug + - encoding dev: false registry.npmmirror.com/option@0.2.4: @@ -12157,6 +12290,12 @@ packages: url-parse: registry.npmmirror.com/url-parse@1.5.10 dev: false + registry.npmmirror.com/tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz} + name: tr46 + version: 0.0.3 + dev: false + registry.npmmirror.com/tr46@3.0.0: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-3.0.0.tgz} name: tr46 @@ -12662,12 +12801,25 @@ packages: version: 2.0.1 dev: false + registry.npmmirror.com/web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz} + name: web-streams-polyfill + version: 4.0.0-beta.3 + engines: {node: '>= 14'} + dev: false + registry.npmmirror.com/web-worker@1.2.0: resolution: {integrity: sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/web-worker/-/web-worker-1.2.0.tgz} name: web-worker version: 1.2.0 dev: false + registry.npmmirror.com/webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz} + name: webidl-conversions + version: 3.0.1 + dev: false + registry.npmmirror.com/webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz} name: webidl-conversions @@ -12711,6 +12863,15 @@ packages: webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0 dev: false + registry.npmmirror.com/whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz} + name: whatwg-url + version: 5.0.0 + dependencies: + tr46: registry.npmmirror.com/tr46@0.0.3 + webidl-conversions: registry.npmmirror.com/webidl-conversions@3.0.1 + dev: false + registry.npmmirror.com/which-boxed-primitive@1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz} name: which-boxed-primitive diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index aacab4f75..d8efed71d 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,9 +1,6 @@ -### Fast GPT V4.4.6 +### Fast GPT V4.4.7 -1. 高级编排新增模块 - 应用调用 -2. 新增 - 必要连接校验 -3. 新增 - 下一步指引选项，可以通过模型生成 3 个预测问题。 -4. 新增 - 分享链接 hook 身份校验。 -5. [使用文档](https://doc.fastgpt.run/docs/intro/) -6. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow) -7. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/) +1. 优化数据集管理，区分手动录入和标注，可追数据至某个文件，保留链接读取的原始链接。 +2. [使用文档](https://doc.fastgpt.run/docs/intro/) +3. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow) +4. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/) diff --git a/projects/app/public/icon/human.svg b/projects/app/public/icon/human.svg new file mode 100644 index 000000000..9be629cd0 --- /dev/null +++ b/projects/app/public/icon/human.svg @@ -0,0 +1,187 @@ + + + + diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index eb4811079..744c16c70 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -131,6 +131,7 @@ "Rename Success": "Rename Success", "Search": "Search", "Status": "Status", + "Unknow": "Unknow", "Update Successful": "Update Successful", "export": "" }, diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 2c12ca555..15f6476c5 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -131,6 +131,7 @@ "Rename Success": "重命名成功", "Search": "搜索", "Status": "状态", + "Unknow": "未知", "Update Successful": "更新成功", "export": "" }, diff --git a/projects/app/src/api/app.ts b/projects/app/src/api/app.ts index 355464dbf..609412cf7 100644 --- a/projects/app/src/api/app.ts +++ b/projects/app/src/api/app.ts @@ -2,9 +2,9 @@ import { GET, POST, DELETE, PUT } from './request'; import type { AppSchema } from '@/types/mongoSchema'; import type { AppListItemType, AppUpdateParams } from '@/types/app'; import { RequestPaging } from '../types/index'; -import type { Props as CreateAppProps } from '@/pages/api/app/create'; import { addDays } from 'date-fns'; import { GetAppChatLogsParams } from './request/app'; +import type { CreateAppParams } from '@/types/app'; /** * 获取模型列表 @@ -14,7 +14,7 @@ export const getMyModels = () => GET('/app/myApps'); /** * 创建一个模型 */ -export const postCreateApp = (data: CreateAppProps) => POST('/app/create', data); +export const postCreateApp = (data: CreateAppParams) => POST('/app/create', data); /** * 根据 ID 删除模型 diff --git a/projects/app/src/components/ChatBox/QuoteModal.tsx b/projects/app/src/components/ChatBox/QuoteModal.tsx index 12723e44f..a486e1008 100644 --- a/projects/app/src/components/ChatBox/QuoteModal.tsx +++ b/projects/app/src/components/ChatBox/QuoteModal.tsx @@ -1,5 +1,5 @@ import React, { useCallback, useMemo, useState } from 'react'; -import { ModalBody, Box, useTheme } from '@chakra-ui/react'; +import { ModalBody, Box, useTheme, Flex, Progress } from '@chakra-ui/react'; import { getDatasetDataItemById } from '@/api/core/dataset/data'; import { useLoading } from '@/hooks/useLoading'; import { useToast } from '@/hooks/useToast'; @@ -8,22 +8,19 @@ import { QuoteItemType } from '@/types/chat'; import MyIcon from '@/components/Icon'; import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal'; import MyModal from '../MyModal'; -import type { PgDataItemType } from '@/types/core/dataset/data'; +import { useTranslation } from 'react-i18next'; import { useRouter } from 'next/router'; -type SearchType = PgDataItemType & { - kb_id?: string; -}; - const QuoteModal = ({ onUpdateQuote, rawSearch = [], onClose }: { onUpdateQuote: (quoteId: string, sourceText?: string) => Promise; - rawSearch: SearchType[]; + rawSearch: QuoteItemType[]; onClose: () => void; }) => { + const { t } = useTranslation(); const theme = useTheme(); const router = useRouter(); const { toast } = useToast(); @@ -36,7 +33,7 @@ const QuoteModal = ({ * click edit, get new kbDataItem */ const onclickEdit = useCallback( - async (item: SearchType) => { + async (item: QuoteItemType) => { if (!item.id) return; try { setIsLoading(true); @@ -95,9 +92,30 @@ const QuoteModal = ({ _hover={{ '& .edit': { display: 'flex' } }} overflow={'hidden'} > - {item.source && !isShare && ( - + {!isShare && ( + + + + {item.score && ( + <> + + {item.score.toFixed(4)} + + )} + )} + {item.q} {item.a} {item.id && !isShare && ( diff --git a/projects/app/src/components/Layout/index.tsx b/projects/app/src/components/Layout/index.tsx index d10743d7c..640076ec5 100644 --- a/projects/app/src/components/Layout/index.tsx +++ b/projects/app/src/components/Layout/index.tsx @@ -102,7 +102,7 @@ const Layout = ({ children }: { children: JSX.Element }) => { )} - + ); }; diff --git a/projects/app/src/components/Loading/index.tsx b/projects/app/src/components/Loading/index.tsx index ac3676340..c790e9cd1 100644 --- a/projects/app/src/components/Loading/index.tsx +++ b/projects/app/src/components/Loading/index.tsx @@ -4,16 +4,18 @@ import { Spinner, Flex, Box } from '@chakra-ui/react'; const Loading = ({ fixed = true, text = '', - bg = 'rgba(255,255,255,0.5)' + bg = 'rgba(255,255,255,0.5)', + zIndex = 1000 }: { fixed?: boolean; text?: string; bg?: string; + zIndex?: number; }) => { return ( { ({ loading, fixed = true, - text = '' + text = '', + zIndex }: { loading?: boolean; fixed?: boolean; text?: string; + zIndex?: number; }): JSX.Element | null => { - return isLoading || loading ? : null; + return isLoading || loading ? ( + + ) : null; }, [isLoading] ); diff --git a/projects/app/src/pages/api/admin/initv447.ts b/projects/app/src/pages/api/admin/initv447.ts index 5d0ef3d18..0eb519c60 100644 --- a/projects/app/src/pages/api/admin/initv447.ts +++ b/projects/app/src/pages/api/admin/initv447.ts @@ -15,16 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await connectToDatabase(); await authUser({ req, authRoot: true }); - console.log('add index'); - await PgClient.query( - ` - ALTER TABLE modeldata - ALTER COLUMN source TYPE VARCHAR(256), - ALTER COLUMN file_id TYPE VARCHAR(256); - CREATE INDEX IF NOT EXISTS modelData_fileId_index ON modeldata (file_id); - ` - ); - console.log('index success'); console.log('count rows'); // 去重获取 fileId const { rows } = await PgClient.query(`SELECT DISTINCT file_id @@ -36,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await init(rows.slice(i, i + limit), initFileIds); console.log(i); } - console.log('filter success'); - console.log('start update'); for (let i = 0; i < initFileIds.length; i++) { await PgClient.query(`UPDATE ${PgDatasetTableName} @@ -49,9 +37,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const { rows: emptyIds } = await PgClient.query( `SELECT id FROM ${PgDatasetTableName} WHERE file_id IS NULL OR file_id=''` ); + console.log('filter success'); console.log(emptyIds.length); await delay(5000); + console.log('start update'); async function start(start: number) { for (let i = start; i < emptyIds.length; i += limit) { @@ -65,12 +55,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) start(i); } - // await PgClient.query( - // `UPDATE ${PgDatasetTableName} - // SET file_id = '${DatasetSpecialIdEnum.manual}' - // WHERE file_id IS NULL OR file_id = ''` - // ); - console.log('update success'); jsonRes(res, { diff --git a/projects/app/src/pages/api/app/create.ts b/projects/app/src/pages/api/app/create.ts index 40a7d8b03..39efacef5 100644 --- a/projects/app/src/pages/api/app/create.ts +++ b/projects/app/src/pages/api/app/create.ts @@ -4,17 +4,17 @@ import { jsonRes } from '@/service/response'; import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { App } from '@/service/models/app'; -import { AppModuleItemType } from '@/types/app'; - -export type Props = { - name: string; - avatar?: string; - modules: AppModuleItemType[]; -}; +import type { CreateAppParams } from '@/types/app'; +import { AppTypeEnum } from '@/constants/app'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - const { name, avatar, modules } = req.body as Props; + const { + name = 'APP', + avatar, + type = AppTypeEnum.advanced, + modules + } = req.body as CreateAppParams; if (!name || !Array.isArray(modules)) { throw new Error('缺少参数'); @@ -38,7 +38,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< avatar, name, userId, - modules + modules, + type }); jsonRes(res, { diff --git a/projects/app/src/pages/api/app/update.ts b/projects/app/src/pages/api/app/update.ts index ed887ed12..297b99ee0 100644 --- a/projects/app/src/pages/api/app/update.ts +++ b/projects/app/src/pages/api/app/update.ts @@ -9,7 +9,7 @@ import { authApp } from '@/service/utils/auth'; /* 获取我的模型 */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - const { name, avatar, type, chat, share, intro, modules } = req.body as AppUpdateParams; + const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams; const { appId } = req.query as { appId: string }; if (!appId) { @@ -37,7 +37,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< type, avatar, intro, - chat, ...(share && { 'share.isShare': share.isShare, 'share.isShareDetail': share.isShareDetail diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index dace05c99..1f10b914d 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -13,6 +13,7 @@ import { getVectorModel } from '@/service/utils/data'; import { getVector } from '@/pages/api/openapi/plugin/vector'; import { DatasetDataItemType } from '@/types/core/dataset/data'; import { countPromptTokens } from '@/utils/common/tiktoken'; +import { authFileIdValid } from '@/service/dataset/auth'; export type Props = { kbId: string; @@ -72,6 +73,8 @@ export async function getVectorAndInsertDataset( return Promise.reject('已经存在完全一致的数据'); } + await authFileIdValid(data.file_id); + const { vectors } = await getVector({ model: kb.vectorModel, input: [q], diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index a8e21a968..bcd1707a9 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -1,16 +1,17 @@ +/* push data to training queue */ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { connectToDatabase, TrainingData, KB } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { authKb } from '@/service/utils/auth'; import { withNextCors } from '@/service/utils/tools'; -import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin'; +import { TrainingModeEnum } from '@/constants/plugin'; import { startQueue } from '@/service/utils/tools'; -import { PgClient } from '@/service/pg'; import { getVectorModel } from '@/service/utils/data'; import { DatasetDataItemType } from '@/types/core/dataset/data'; import { countPromptTokens } from '@/utils/common/tiktoken'; import type { PushDataProps, PushDataResponse } from '@/api/core/dataset/data.d'; +import { authFileIdValid } from '@/service/dataset/auth'; const modeMap = { [TrainingModeEnum.index]: true, @@ -80,69 +81,49 @@ export async function pushDataToKb({ [TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8 }; - // 过滤重复的 qa 内容 + // filter repeat or equal content const set = new Set(); - const filterData: DatasetDataItemType[] = []; + const filterResult: Record = { + success: [], + overToken: [], + fileIdInvalid: [], + error: [] + }; - data.forEach((item) => { - if (!item.q) return; + await Promise.all( + data.map(async (item) => { + if (!item.q) { + filterResult.error.push(item); + return; + } - const text = item.q + item.a; + const text = item.q + item.a; - // count q token - const token = countPromptTokens(item.q, 'system'); + // count q token + const token = countPromptTokens(item.q, 'system'); - if (token > modeMaxToken[mode]) { - return; - } + if (token > modeMaxToken[mode]) { + filterResult.overToken.push(item); + return; + } - if (!set.has(text)) { - filterData.push(item); - set.add(text); - } - }); + try { + await authFileIdValid(item.file_id); + } catch (error) { + filterResult.fileIdInvalid.push(item); + return; + } - // 数据库去重 - const insertData = ( - await Promise.allSettled( - filterData.map(async (data) => { - let { q, a } = data; - if (mode !== TrainingModeEnum.index) { - return Promise.resolve(data); - } - - if (!q) { - return Promise.reject('q为空'); - } - - q = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"'); - a = a.replace(/\\n/g, '\n').trim().replace(/'/g, '"'); - - // Exactly the same data, not push - try { - const { rows } = await PgClient.query(` - SELECT COUNT(*) > 0 AS exists - FROM ${PgDatasetTableName} - WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' - `); - const exists = rows[0]?.exists || false; - - if (exists) { - return Promise.reject('已经存在'); - } - } catch (error) { - console.log(error); - } - return Promise.resolve(data); - }) - ) - ) - .filter((item) => item.status === 'fulfilled') - .map((item: any) => item.value); + if (!set.has(text)) { + filterResult.success.push(item); + set.add(text); + } + }) + ); // 插入记录 const insertRes = await TrainingData.insertMany( - insertData.map((item) => ({ + filterResult.success.map((item) => ({ ...item, userId, kbId, @@ -154,9 +135,11 @@ export async function pushDataToKb({ ); insertRes.length > 0 && startQueue(); + delete filterResult.success; return { - insertLen: insertRes.length + insertLen: insertRes.length, + ...filterResult }; } diff --git a/projects/app/src/pages/api/core/dataset/file/list.ts b/projects/app/src/pages/api/core/dataset/file/list.ts index 49037fa61..abef540e7 100644 --- a/projects/app/src/pages/api/core/dataset/file/list.ts +++ b/projects/app/src/pages/api/core/dataset/file/list.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; import { connectToDatabase, TrainingData } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { GridFSStorage } from '@/service/lib/gridfs'; -import { PgClient } from '@/service/pg'; +import { PgClient, updateDataFileId } from '@/service/pg'; import { PgDatasetTableName } from '@/constants/plugin'; import { FileStatusEnum } from '@/constants/dataset'; import { strIsLink } from '@fastgpt/common/tools/str'; @@ -35,8 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< .join(' ')} ${searchText ? `AND source ILIKE '%${searchText}%'` : ''}`; - const [{ rows }, { rowCount: total }] = await Promise.all([ - PgClient.query(`SELECT file_id, COUNT(*) AS count + let [{ rows }, { rowCount: total }] = await Promise.all([ + PgClient.query<{ file_id: string; count: number }>(`SELECT file_id, COUNT(*) AS count FROM ${PgDatasetTableName} where ${pgWhere} GROUP BY file_id @@ -49,6 +49,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< `) ]); + // If fileId is invalid, reset it to manual + await Promise.all( + rows.map((row) => { + if (!strIsLink(row.file_id) && row.file_id.length !== 24) { + return updateDataFileId({ + oldFileId: row.file_id, + userId, + newFileId: DatasetSpecialIdEnum.manual + }); + } + }) + ); + // just filter link or fileData + rows = rows.filter((row) => strIsLink(row.file_id) || row.file_id.length === 24); + // find files const gridFs = new GridFSStorage('dataset', userId); const collection = gridFs.Collection(); @@ -96,6 +111,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const data = await Promise.all([ getSpecialData(), ...rows.map(async (row) => { + if (!row.file_id) return null; // link data if (strIsLink(row.file_id)) { const { rows } = await PgClient.select(PgDatasetTableName, { diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts index b62d1c434..288d3b028 100644 --- a/projects/app/src/pages/api/core/dataset/searchTest.ts +++ b/projects/app/src/pages/api/core/dataset/searchTest.ts @@ -47,7 +47,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex data: response?.[2]?.rows || [] }); } catch (err) { - console.log(err); jsonRes(res, { code: 500, error: err diff --git a/projects/app/src/pages/api/openapi/plugin/vector.ts b/projects/app/src/pages/api/openapi/plugin/vector.ts index e973f8b99..95583c761 100644 --- a/projects/app/src/pages/api/openapi/plugin/vector.ts +++ b/projects/app/src/pages/api/openapi/plugin/vector.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { authBalanceByUid, authUser } from '@/service/utils/auth'; import { withNextCors } from '@/service/utils/tools'; -import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config'; +import { getAIApi } from '@fastgpt/core/ai/config'; import { pushGenerateVectorBill } from '@/service/common/bill/push'; type Props = { @@ -54,29 +54,31 @@ export async function getVector({ } // 获取 chatAPI - const chatAPI = getAIChatApi(); + const ai = getAIApi(); // 把输入的内容转成向量 - const result = await chatAPI - .createEmbedding( + const result = await ai.embeddings + .create( { model, input }, { - timeout: 60000, - ...axiosConfig() + timeout: 60000 } ) .then(async (res) => { - if (!res.data?.data?.[0]?.embedding) { - console.log(res.data); + if (!res.data) { + return Promise.reject('Embedding API 404'); + } + if (!res?.data?.[0]?.embedding) { + console.log(res?.data); // @ts-ignore return Promise.reject(res.data?.err?.message || 'Embedding API Error'); } return { - tokenLen: res.data.usage.total_tokens || 0, - vectors: await Promise.all(res.data.data.map((item) => unityDimensional(item.embedding))) + tokenLen: res.usage.total_tokens || 0, + vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding))) }; }); diff --git a/projects/app/src/pages/api/user/account/update.ts b/projects/app/src/pages/api/user/account/update.ts index d7a0094d6..8a5164d61 100644 --- a/projects/app/src/pages/api/user/account/update.ts +++ b/projects/app/src/pages/api/user/account/update.ts @@ -5,7 +5,7 @@ import { User } from '@/service/models/user'; import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { UserUpdateParams } from '@/types/user'; -import { axiosConfig, getAIChatApi, openaiBaseUrl } from '@fastgpt/core/ai/config'; +import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config'; /* update user info */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -22,20 +22,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const baseUrl = openaiAccount?.baseUrl || openaiBaseUrl; openaiAccount.baseUrl = baseUrl; - const chatAPI = getAIChatApi(openaiAccount); + const ai = getAIApi(openaiAccount); - const response = await chatAPI.createChatCompletion( - { - model: 'gpt-3.5-turbo', - max_tokens: 1, - messages: [{ role: 'user', content: 'hi' }] - }, - { - ...axiosConfig(openaiAccount) - } - ); - if (response?.data?.choices?.[0]?.message?.content === undefined) { - throw new Error(JSON.stringify(response?.data)); + const response = await ai.chat.completions.create({ + model: 'gpt-3.5-turbo', + max_tokens: 1, + messages: [{ role: 'user', content: 'hi' }] + }); + if (response?.choices?.[0]?.message?.content === undefined) { + throw new Error('Key response is empty'); } } diff --git a/projects/app/src/pages/app/detail/index.tsx b/projects/app/src/pages/app/detail/index.tsx index 31d01a1e1..4b0e79701 100644 --- a/projects/app/src/pages/app/detail/index.tsx +++ b/projects/app/src/pages/app/detail/index.tsx @@ -6,6 +6,7 @@ import dynamic from 'next/dynamic'; import { defaultApp } from '@/constants/model'; import { useToast } from '@/hooks/useToast'; import { useQuery } from '@tanstack/react-query'; +import { feConfigs } from '@/store/static'; import Tabs from '@/components/Tabs'; import SideTabs from '@/components/SideTabs'; @@ -52,7 +53,9 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => { const tabList = useMemo( () => [ { label: '简易配置', id: TabEnum.basicEdit, icon: 'overviewLight' }, - { label: '高级编排', id: TabEnum.adEdit, icon: 'settingLight' }, + ...(feConfigs?.hide_app_flow + ? [] + : [{ label: '高级编排', id: TabEnum.adEdit, icon: 'settingLight' }]), { label: '外部使用', id: TabEnum.outLink, icon: 'shareLight' }, { label: '对话日志', id: TabEnum.logs, icon: 'logsLight' }, { label: '立即对话', id: TabEnum.startChat, icon: 'chat' } diff --git a/projects/app/src/pages/app/list/component/CreateModal.tsx b/projects/app/src/pages/app/list/component/CreateModal.tsx index 426cb19e1..7590a2c03 100644 --- a/projects/app/src/pages/app/list/component/CreateModal.tsx +++ b/projects/app/src/pages/app/list/component/CreateModal.tsx @@ -21,6 +21,7 @@ import { useRouter } from 'next/router'; import { appTemplates } from '@/constants/flow/ModuleTemplate'; import { useGlobalStore } from '@/store/global'; import { useRequest } from '@/hooks/useRequest'; +import { feConfigs } from '@/store/static'; import Avatar from '@/components/Avatar'; import MyTooltip from '@/components/MyTooltip'; import MyModal from '@/components/MyModal'; @@ -74,10 +75,15 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: ( const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: FormType) => { + const template = appTemplates.find((item) => item.id === data.templateId); + if (!template) { + return Promise.reject('模板不存在'); + } return postCreateApp({ avatar: data.avatar, name: data.name, - modules: appTemplates.find((item) => item.id === data.templateId)?.modules || [] + type: template.type, + modules: template.modules || [] }); }, onSuccess(id: string) { @@ -118,48 +124,52 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: ( })} /> - - 从模板中选择 - - - {appTemplates.map((item) => ( - { - setValue('templateId', item.id); - setRefresh((state) => !state); - }} + {!feConfigs?.hide_app_flow && ( + <> + + 从模板中选择 + + - - - - {item.name} - - - - {item.intro} - - - ))} - + {appTemplates.map((item) => ( + { + setValue('templateId', item.id); + setRefresh((state) => !state); + }} + > + + + + {item.name} + + + + {item.intro} + + + ))} + + + )} diff --git a/projects/app/src/pages/kb/detail/components/InputDataModal.tsx b/projects/app/src/pages/kb/detail/components/InputDataModal.tsx index f02515b0a..f31544189 100644 --- a/projects/app/src/pages/kb/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/kb/detail/components/InputDataModal.tsx @@ -263,6 +263,10 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp const { setLoading } = useGlobalStore(); const hasFile = useMemo(() => fileId && !datasetSpecialIds.includes(fileId), [fileId]); + const formatName = useMemo( + () => (filename.startsWith('kb') ? t(filename) : filename), + [filename, t] + ); return ( @@ -293,7 +297,7 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp : {})} {...props} > - {t(filename)} + {formatName} ); diff --git a/projects/app/src/service/dataset/auth.ts b/projects/app/src/service/dataset/auth.ts new file mode 100644 index 000000000..6e62819a7 --- /dev/null +++ b/projects/app/src/service/dataset/auth.ts @@ -0,0 +1,22 @@ +import { isSpecialFileId } from '@fastgpt/core/dataset/utils'; +import { GridFSStorage } from '../lib/gridfs'; +import { Types } from 'mongoose'; + +export async function authFileIdValid(fileId?: string) { + if (!fileId) return true; + if (isSpecialFileId(fileId)) return true; + try { + // find file + const gridFs = new GridFSStorage('dataset', ''); + const collection = gridFs.Collection(); + const file = await collection.findOne( + { _id: new Types.ObjectId(fileId) }, + { projection: { _id: 1 } } + ); + if (!file) { + return Promise.reject('Invalid fileId'); + } + } catch (error) { + return Promise.reject('Invalid fileId'); + } +} diff --git a/projects/app/src/service/errorCode.ts b/projects/app/src/service/errorCode.ts index f631f4e7d..89da25b99 100644 --- a/projects/app/src/service/errorCode.ts +++ b/projects/app/src/service/errorCode.ts @@ -17,19 +17,6 @@ export const TOKEN_ERROR_CODE: Record = { 403: '登录状态无效,请重新登录' }; -export const openaiError: Record = { - context_length_exceeded: '内容超长了，请重置对话', - Unauthorized: 'API-KEY 不合法', - rate_limit_reached: 'API被限制，请稍后再试', - 'Bad Request': 'Bad Request~ 可能内容太多了', - 'Bad Gateway': '网关异常，请重试' -}; -export const openaiAccountError: Record = { - insufficient_quota: 'API 余额不足', - invalid_api_key: 'openai 账号异常', - account_deactivated: '账号已停用', - invalid_request_error: '无效请求' -}; export const proxyError: Record = { ECONNABORTED: true, ECONNRESET: true diff --git a/projects/app/src/service/events/generateQA.ts b/projects/app/src/service/events/generateQA.ts index 52317a9d0..6b6cad882 100644 --- a/projects/app/src/service/events/generateQA.ts +++ b/projects/app/src/service/events/generateQA.ts @@ -4,7 +4,7 @@ import { TrainingModeEnum } from '@/constants/plugin'; import { ERROR_ENUM } from '../errorCode'; import { sendInform } from '@/pages/api/user/inform/send'; import { authBalanceByUid } from '../utils/auth'; -import { axiosConfig, getAIChatApi } from '@fastgpt/core/ai/config'; +import { getAIApi } from '@fastgpt/core/ai/config'; import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type'; import { addLog } from '../utils/tools'; import { splitText2Chunks } from '@/utils/file'; @@ -58,8 +58,6 @@ export async function generateQA(): Promise { const startTime = Date.now(); - const chatAPI = getAIChatApi(); - // request LLM to get QA const text = data.q; const messages: ChatCompletionRequestMessage[] = [ @@ -73,19 +71,13 @@ export async function generateQA(): Promise { }) } ]; - - const { data: chatResponse } = await chatAPI.createChatCompletion( - { - model: global.qaModel.model, - temperature: 0.01, - messages, - stream: false - }, - { - timeout: 480000, - ...axiosConfig() - } - ); + const ai = getAIApi(undefined, 480000); + const chatResponse = await ai.chat.completions.create({ + model: global.qaModel.model, + temperature: 0.01, + messages, + stream: false + }); const answer = chatResponse.choices?.[0].message?.content; const totalTokens = chatResponse.usage?.total_tokens || 0; diff --git a/projects/app/src/service/models/user.ts b/projects/app/src/service/models/user.ts index 64fb0b92b..da54e7796 100644 --- a/projects/app/src/service/models/user.ts +++ b/projects/app/src/service/models/user.ts @@ -23,7 +23,7 @@ const UserSchema = new Schema({ }, avatar: { type: String, - default: '/icon/human.png' + default: '/icon/human.svg' }, balance: { type: Number, diff --git a/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts b/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts index 7e2a0ebca..919d51452 100644 --- a/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts +++ b/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts @@ -2,7 +2,7 @@ import { adaptChat2GptMessages } from '@/utils/common/adapt/message'; import { ChatContextFilter } from '@/service/common/tiktoken'; import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat'; import { ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat'; -import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config'; +import { getAIApi } from '@fastgpt/core/ai/config'; import type { ClassifyQuestionAgentItemType } from '@/types/app'; import { SystemInputEnum } from '@/constants/app'; import { SpecialInputKeyEnum } from '@/constants/flow'; @@ -105,27 +105,22 @@ async function functionCall({ required: ['type'] } }; - const chatAPI = getAIChatApi(user.openaiAccount); + const ai = getAIApi(user.openaiAccount); - const response = await chatAPI.createChatCompletion( - { - model: cqModel.model, - temperature: 0, - messages: [...adaptMessages], - function_call: { name: agentFunName }, - functions: [agentFunction] - }, - { - ...axiosConfig(user.openaiAccount) - } - ); + const response = await ai.chat.completions.create({ + model: cqModel.model, + temperature: 0, + messages: [...adaptMessages], + function_call: { name: agentFunName }, + functions: [agentFunction] + }); try { - const arg = JSON.parse(response.data.choices?.[0]?.message?.function_call?.arguments || ''); + const arg = JSON.parse(response.choices?.[0]?.message?.function_call?.arguments || ''); return { arg, - tokens: response.data.usage?.total_tokens || 0 + tokens: response.usage?.total_tokens || 0 }; } catch (error) { console.log('Your model may not support function_call'); @@ -155,20 +150,14 @@ Human:${userChatInput}` } ]; - const chatAPI = getAIChatApi(user.openaiAccount); + const ai = getAIApi(user.openaiAccount, 480000); - const { data } = await chatAPI.createChatCompletion( - { - model: extractModel.model, - temperature: 0.01, - messages: adaptChat2GptMessages({ messages, reserveId: false }), - stream: false - }, - { - timeout: 480000, - ...axiosConfig(user.openaiAccount) - } - ); + const data = await ai.chat.completions.create({ + model: extractModel.model, + temperature: 0.01, + messages: adaptChat2GptMessages({ messages, reserveId: false }), + stream: false + }); const answer = data.choices?.[0].message?.content || ''; const totalTokens = data.usage?.total_tokens || 0; diff --git a/projects/app/src/service/moduleDispatch/agent/extract.ts b/projects/app/src/service/moduleDispatch/agent/extract.ts index 06ad86026..40044488a 100644 --- a/projects/app/src/service/moduleDispatch/agent/extract.ts +++ b/projects/app/src/service/moduleDispatch/agent/extract.ts @@ -2,7 +2,7 @@ import { adaptChat2GptMessages } from '@/utils/common/adapt/message'; import { ChatContextFilter } from '@/service/common/tiktoken'; import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat'; import { ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat'; -import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config'; +import { getAIApi } from '@fastgpt/core/ai/config'; import type { ContextExtractAgentItemType } from '@/types/app'; import { ContextExtractEnum } from '@/constants/flow/flowField'; import { FlowModuleTypeEnum } from '@/constants/flow'; @@ -126,30 +126,25 @@ async function functionCall({ } }; - const chatAPI = getAIChatApi(user.openaiAccount); + const ai = getAIApi(user.openaiAccount); - const response = await chatAPI.createChatCompletion( - { - model: extractModel.model, - temperature: 0, - messages: [...adaptMessages], - function_call: { name: agentFunName }, - functions: [agentFunction] - }, - { - ...axiosConfig(user.openaiAccount) - } - ); + const response = await ai.chat.completions.create({ + model: extractModel.model, + temperature: 0, + messages: [...adaptMessages], + function_call: { name: agentFunName }, + functions: [agentFunction] + }); const arg: Record = (() => { try { - return JSON.parse(response.data.choices?.[0]?.message?.function_call?.arguments || '{}'); + return JSON.parse(response.choices?.[0]?.message?.function_call?.arguments || '{}'); } catch (error) { return {}; } })(); - const tokens = response.data.usage?.total_tokens || 0; + const tokens = response.usage?.total_tokens || 0; return { tokens, arg @@ -181,20 +176,14 @@ Human: ${content}` } ]; - const chatAPI = getAIChatApi(user.openaiAccount); + const ai = getAIApi(user.openaiAccount, 480000); - const { data } = await chatAPI.createChatCompletion( - { - model: extractModel.model, - temperature: 0.01, - messages: adaptChat2GptMessages({ messages, reserveId: false }), - stream: false - }, - { - timeout: 480000, - ...axiosConfig(user.openaiAccount) - } - ); + const data = await ai.chat.completions.create({ + model: extractModel.model, + temperature: 0.01, + messages: adaptChat2GptMessages({ messages, reserveId: false }), + stream: false + }); const answer = data.choices?.[0].message?.content || ''; const totalTokens = data.usage?.total_tokens || 0; diff --git a/projects/app/src/service/moduleDispatch/chat/oneapi.ts b/projects/app/src/service/moduleDispatch/chat/oneapi.ts index 50e0d8c22..75ec9aff6 100644 --- a/projects/app/src/service/moduleDispatch/chat/oneapi.ts +++ b/projects/app/src/service/moduleDispatch/chat/oneapi.ts @@ -3,9 +3,9 @@ import { ChatContextFilter } from '@/service/common/tiktoken'; import type { ChatItemType, QuoteItemType } from '@/types/chat'; import type { ChatHistoryItemResType } from '@/types/chat'; import { ChatRoleEnum, sseResponseEventEnum } from '@/constants/chat'; -import { SSEParseData, parseStreamChunk } from '@/utils/sse'; import { textAdaptGptResponse } from '@/utils/adapt'; -import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config'; +import { getAIApi } from '@fastgpt/core/ai/config'; +import type { ChatCompletion, StreamChatType } from '@fastgpt/core/ai/type'; import { TaskResponseKeyEnum } from '@/constants/chat'; import { getChatModel } from '@/service/utils/data'; import { countModelPrice } from '@/service/common/bill/push'; @@ -20,9 +20,7 @@ import type { AIChatProps } from '@/types/core/aiChat'; import { replaceVariable } from '@/utils/common/tools/text'; import { FlowModuleTypeEnum } from '@/constants/flow'; import type { ModuleDispatchProps } from '@/types/core/chat/type'; -import { Readable } from 'stream'; import { responseWrite, responseWriteController } from '@/service/common/stream'; -import { addLog } from '@/service/utils/tools'; export type ChatProps = ModuleDispatchProps< AIChatProps & { @@ -106,32 +104,25 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise { if (stream) { @@ -139,7 +130,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise((resolve, reject) => { - const stream = response.data as Readable; - let answer = ''; - const parseData = new SSEParseData(); - - const write = responseWriteController({ - res, - readStream: stream - }); - - stream.on('data', (data) => { - if (res.closed) { - stream.destroy(); - return resolve({ answer }); - } - - const parse = parseStreamChunk(data); - parse.forEach((item) => { - const { data } = parseData.parse(item); - if (!data || data === '[DONE]') return; - - const content: string = data?.choices?.[0]?.delta?.content || ''; - if (data.error) { - addLog.error(`SSE response`, data.error); - } else { - answer += content; - - responseWrite({ - write, - event: detail ? sseResponseEventEnum.answer : undefined, - data: textAdaptGptResponse({ - text: content - }) - }); - } - }); - }); - stream.on('end', () => { - resolve({ answer }); - }); - stream.on('close', () => { - resolve({ answer }); - }); - stream.on('error', (err) => { - reject(err); - }); + const write = responseWriteController({ + res, + readStream: stream }); + let answer = ''; + + for await (const part of stream) { + if (res.closed) { + stream.controller?.abort(); + break; + } + const content = part.choices[0]?.delta?.content || ''; + answer += content; + + responseWrite({ + write, + event: detail ? sseResponseEventEnum.answer : undefined, + data: textAdaptGptResponse({ + text: content + }) + }); + } + + if (!answer) { + return Promise.reject('Chat API is error or undefined'); + } + + return { answer }; } function getHistoryPreview(completeMessages: ChatItemType[]) { diff --git a/projects/app/src/service/moduleDispatch/kb/search.ts b/projects/app/src/service/moduleDispatch/kb/search.ts index adf336ca4..bdcec1d62 100644 --- a/projects/app/src/service/moduleDispatch/kb/search.ts +++ b/projects/app/src/service/moduleDispatch/kb/search.ts @@ -46,7 +46,9 @@ export async function dispatchKBSearch(props: Record): Promise '[${ + vectors[0] + }]') * -1 AS score from ${PgDatasetTableName} where kb_id IN (${kbList .map((item) => `'${item.kbId}'`) .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${ vectors[0] diff --git a/projects/app/src/service/pg.ts b/projects/app/src/service/pg.ts index fe40fd35f..a6eb33fda 100644 --- a/projects/app/src/service/pg.ts +++ b/projects/app/src/service/pg.ts @@ -3,6 +3,7 @@ import type { QueryResultRow } from 'pg'; import { PgDatasetTableName } from '@/constants/plugin'; import { addLog } from './utils/tools'; import type { DatasetDataItemType } from '@/types/core/dataset/data'; +import { DatasetSpecialIdEnum, datasetSpecialIdMap } from '@fastgpt/core/dataset/constant'; export const connectPg = async (): Promise => { if (global.pgClient) { @@ -179,8 +180,13 @@ export const insertData2Dataset = ({ values: data.map((item) => [ { key: 'user_id', value: userId }, { key: 'kb_id', value: kbId }, - { key: 'source', value: item.source?.slice(0, 200)?.trim() || '' }, - { key: 'file_id', value: item.file_id?.slice(0, 200)?.trim() || '' }, + { + key: 'source', + value: + item.source?.slice(0, 200)?.trim() || + datasetSpecialIdMap[DatasetSpecialIdEnum.manual].sourceName + }, + { key: 'file_id', value: item.file_id?.slice(0, 200)?.trim() || DatasetSpecialIdEnum.manual }, { key: 'q', value: item.q.replace(/'/g, '"') }, { key: 'a', value: item.a.replace(/'/g, '"') }, { key: 'vector', value: `[${item.vector}]` } @@ -188,6 +194,25 @@ export const insertData2Dataset = ({ }); }; +/** + * Update data file_id + */ +export const updateDataFileId = async ({ + oldFileId, + userId, + newFileId = DatasetSpecialIdEnum.manual +}: { + oldFileId: string; + userId: string; + newFileId?: string; +}) => { + await PgClient.update(PgDatasetTableName, { + where: [['file_id', oldFileId], 'AND', ['user_id', userId]], + values: [{ key: 'file_id', value: newFileId }] + }); + return newFileId; +}; + export async function initPg() { try { await connectPg(); @@ -203,10 +228,6 @@ export async function initPg() { q TEXT NOT NULL, a TEXT ); - CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgDatasetTableName} USING HASH (user_id); - CREATE INDEX IF NOT EXISTS modelData_kb_id_index ON ${PgDatasetTableName} (kb_id); - CREATE INDEX IF NOT EXISTS modelData_fileId_index ON ${PgDatasetTableName} (file_id); - CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgDatasetTableName} (md5(q), md5(a), user_id, kb_id); `); console.log('init pg successful'); } catch (error) { diff --git a/projects/app/src/service/response.ts b/projects/app/src/service/response.ts index 7b4d20c63..bfe5c3d94 100644 --- a/projects/app/src/service/response.ts +++ b/projects/app/src/service/response.ts @@ -1,12 +1,6 @@ import { sseResponseEventEnum } from '@/constants/chat'; import { NextApiResponse } from 'next'; -import { - openaiError, - openaiAccountError, - proxyError, - ERROR_RESPONSE, - ERROR_ENUM -} from './errorCode'; +import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode'; import { clearCookie, sseResponse, addLog } from './utils/tools'; export interface ResponseType { @@ -47,10 +41,8 @@ export const jsonRes = ( msg = '网络连接异常'; } else if (error?.response?.data?.error?.message) { msg = error?.response?.data?.error?.message; - } else if (openaiAccountError[error?.response?.data?.error?.code]) { - msg = openaiAccountError[error?.response?.data?.error?.code]; - } else if (openaiError[error?.response?.statusText]) { - msg = openaiError[error.response.statusText]; + } else if (error?.error?.message) { + msg = error?.error?.message; } addLog.error(`response error: ${msg}`, error); @@ -88,10 +80,8 @@ export const sseErrRes = (res: NextApiResponse, error: any) => { msg = '网络连接异常'; } else if (error?.response?.data?.error?.message) { msg = error?.response?.data?.error?.message; - } else if (openaiAccountError[error?.response?.data?.error?.code]) { - msg = openaiAccountError[error?.response?.data?.error?.code]; - } else if (openaiError[error?.response?.statusText]) { - msg = openaiError[error.response.statusText]; + } else if (error?.error?.message) { + msg = error?.error?.message; } addLog.error(`sse error: ${msg}`, error); diff --git a/projects/app/src/types/app.d.ts b/projects/app/src/types/app.d.ts index d29c23d3f..dbb752e3f 100644 --- a/projects/app/src/types/app.d.ts +++ b/projects/app/src/types/app.d.ts @@ -22,12 +22,17 @@ export type AppListItemType = { intro: string; }; +export type CreateAppParams = { + name?: string; + avatar?: string; + type?: `${AppTypeEnum}`; + modules: AppSchema['modules']; +}; export interface AppUpdateParams { name?: string; type?: `${AppTypeEnum}`; avatar?: string; intro?: string; - chat?: AppSchema['chat']; share?: AppSchema['share']; modules?: AppSchema['modules']; } diff --git a/projects/app/src/types/chat.d.ts b/projects/app/src/types/chat.d.ts index 5d1e3f89f..2b8e51387 100644 --- a/projects/app/src/types/chat.d.ts +++ b/projects/app/src/types/chat.d.ts @@ -45,6 +45,7 @@ export type ShareChatType = InitShareChatResponse & { export type QuoteItemType = PgDataItemType & { kb_id: string; + score?: number; }; // response data diff --git a/projects/app/src/types/core/chat/type.d.ts b/projects/app/src/types/core/chat/type.d.ts index 5f1e702ef..4c98e2031 100644 --- a/projects/app/src/types/core/chat/type.d.ts +++ b/projects/app/src/types/core/chat/type.d.ts @@ -3,7 +3,7 @@ import type { NextApiResponse } from 'next'; import { RunningModuleItemType } from '@/types/app'; import { UserModelSchema } from '@/types/mongoSchema'; -export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string }; +export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string; content: string }; // module dispatch props type export type ModuleDispatchProps = { diff --git a/projects/app/src/types/index.d.ts b/projects/app/src/types/index.d.ts index 66008d469..594b09da8 100644 --- a/projects/app/src/types/index.d.ts +++ b/projects/app/src/types/index.d.ts @@ -29,6 +29,7 @@ export type FeConfigsType = { show_pay?: boolean; show_openai_account?: boolean; show_promotion?: boolean; + hide_app_flow?: boolean; openAPIUrl?: string; systemTitle?: string; authorText?: string; diff --git a/projects/app/src/utils/adapt.ts b/projects/app/src/utils/adapt.ts index 433424918..2bb16e7f1 100644 --- a/projects/app/src/utils/adapt.ts +++ b/projects/app/src/utils/adapt.ts @@ -25,7 +25,7 @@ export const adaptBill = (bill: BillSchema): UserBillType => { }; export const gptMessage2ChatType = (messages: MessageItemType[]): ChatItemType[] => { - const roleMap: Record<`${ChatCompletionRequestMessageRoleEnum}`, `${ChatRoleEnum}`> = { + const roleMap = { [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI, [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human, [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System, diff --git a/projects/app/src/utils/common/tools/text.ts b/projects/app/src/utils/common/tools/text.ts index 2f329a621..3d7c6630f 100644 --- a/projects/app/src/utils/common/tools/text.ts +++ b/projects/app/src/utils/common/tools/text.ts @@ -1,7 +1,7 @@ /* replace {{variable}} to value */ -export function replaceVariable(text: string, obj: Record) { +export function replaceVariable(text: string, obj: Record) { for (const key in obj) { const val = obj[key]; if (typeof val !== 'string') continue; diff --git a/projects/app/src/utils/file.ts b/projects/app/src/utils/file.ts index f6139e800..8b72add79 100644 --- a/projects/app/src/utils/file.ts +++ b/projects/app/src/utils/file.ts @@ -11,7 +11,14 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe const overlapLen = Math.floor(maxLen * 0.25); // Overlap length try { - const splitTexts = text.split(/(?<=[。！？；.!?;\n])/g); + const tempMarker = 'SPLIT_HERE'; + text = text.replace(/\n{3,}/g, '\n'); + text = text.replace(/\s/g, ' '); + text = text.replace('\n\n', ''); + const splitTexts = text + .replace(/([。！？；]|\.\s|!\s|\?\s|;\s|\n)/g, `$1${tempMarker}`) + .split(tempMarker) + .filter((part) => part); const chunks: string[] = []; let preChunk = ''; diff --git a/projects/app/src/utils/web/core/dataset.ts b/projects/app/src/utils/web/core/dataset.ts index 16f936632..727c69738 100644 --- a/projects/app/src/utils/web/core/dataset.ts +++ b/projects/app/src/utils/web/core/dataset.ts @@ -9,7 +9,7 @@ export async function chunksUpload({ mode, chunks, prompt, - rate = 50, + rate = 150, onUploading }: { kbId: string;