diff --git a/.vscode/settings.json b/.vscode/settings.json index 2260a5cea..933fb1112 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,7 +5,6 @@ "prettier.prettierPath": "./node_modules/prettier", "i18n-ally.localesPaths": [ "projects/app/public/locales", - "projects/home/public/locales" ], "i18n-ally.enabledParsers": ["json"], "i18n-ally.keystyle": "nested", diff --git a/README.md b/README.md index 96fdd130c..51d885f46 100644 --- a/README.md +++ b/README.md @@ -48,19 +48,20 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b # -## 💡 功能 +## 💡 RoadMap -`1` 强大的可视化编排，轻松构建 AI 应用 +`1` 应用编排能力 - [x] 提供简易模式，无需操作编排 - [x] 对话下一步指引 - [x] 工作流编排 - - [x] 对话下一步指引 - [x] 源文件引用追踪 - [x] 模块封装，实现多级复用 + - [x] 混合检索 & 重排 + - [ ] 自查询规划 - [ ] 嵌入 [Laf](https://github.com/labring/laf)，实现在线编写 HTTP 模块 - - [ ] 混合检索 & 重排 + - [ ] 插件封装功能 -`2` 丰富的知识库预处理 +`2` 知识库能力 - [x] 多库复用，混用 - [x] chunk 记录修改和删除 - [x] 支持知识库单独设置向量模型 @@ -68,22 +69,26 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b - [x] 支持手动输入，直接分段，QA 拆分导入 - [x] 支持 pdf、word、txt、md 等常用文件，支持 url 读取、CSV 批量导入 - [ ] 支持 HTML、csv、PPT、Excel 导入 + - [ ] 支持文件阅读器 - [ ] 支持差异性文件同步 - [ ] 更多的数据预处理方案 -`3` 多种效果测试渠道 +`3` 应用调试能力 - [x] 知识库单点搜索测试 - [x] 对话时反馈引用并可修改与删除 - [x] 完整上下文呈现 - [x] 完整模块中间值呈现 + - [ ] 高级编排 DeBug 模式 -`4` OpenAPI +`4` OpenAPI 接口 - [x] completions 接口 (对齐 GPT 接口) - [ ] 知识库 CRUD + - [ ] 对话 CRUD -`5` 运营功能 +`5` 运营能力 - [x] 免登录分享窗口 - [x] Iframe 一键嵌入 + - [x] 聊天窗口嵌入支持自定义 Icon，默认打开，拖拽等功能 - [x] 统一查阅对话记录，并对数据进行标注 diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index c4af23116..fb7202491 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -307,3 +307,27 @@ weight: 708 } } ``` + +## 特殊模型 + +### ReRank 接入 + +请使用 4.6.6-alpha 以上版本，配置文件中的 `reRankModels` 为重排模型，虽然是数组，不过目前仅有第1个生效。 + +1. [部署 ReRank 模型](/docs/development/custom-models/reranker/) +1. 找到 FastGPT 的配置文件中的 `reRankModels`， 4.6.6 以前是 `ReRankModels`。 +2. 修改对应的值：（记得去掉注释） + +```json +{ + "reRankModels": [ + { + "model": "bge-reranker-base", // 随意 + "name": "检索重排-base", // 随意 + "inputPrice": 0, + "requestUrl": "{{host}}/api/v1/rerank", + "requestAuth": "安全凭证，已自动补 Bearer" + } + ] +} +``` diff --git a/docSite/content/docs/development/custom-models/reranker.md b/docSite/content/docs/development/custom-models/reranker.md new file mode 100644 index 000000000..a10470292 --- /dev/null +++ b/docSite/content/docs/development/custom-models/reranker.md @@ -0,0 +1,64 @@ +--- +title: '接入 ReRank 重排模型' +description: '接入 ReRank 重排模型' +icon: 'sort' +draft: false +toc: true +weight: 910 +--- + +## 推荐配置 + +推荐配置如下： + +{{< table "table-hover table-striped-columns" >}} +| 类型 | 内存 | 显存 | 硬盘空间 | 启动命令 | +|------|---------|---------|----------|--------------------------| +| base | >=4GB | >=3GB | >=8GB | python app.py | +{{< /table >}} + +## 部署 + +### 环境要求 + +- Python 3.10.11 +- CUDA 11.7 +- 科学上网环境 + +### 源码部署 + +1. 根据上面的环境配置配置好环境，具体教程自行 GPT； +2. 下载 [python 文件](app.py) +3. 在命令行输入命令 `pip install -r requirments.txt`； +4. 按照[https://huggingface.co/BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base)下载模型仓库到app.py同级目录 +5. 添加环境变量 `export ACCESS_TOKEN=XXXXXX` 配置 token，这里的 token 只是加一层验证，防止接口被人盗用，默认值为 `ACCESS_TOKEN` ； +6. 执行命令 `python app.py`。 + +然后等待模型下载，直到模型加载完毕为止。如果出现报错先问 GPT。 + +启动成功后应该会显示如下地址： + +![](/imgs/chatglm2.png) + +> 这里的 `http://0.0.0.0:6006` 就是连接地址。 + +### docker 部署 + ++ 镜像名: `luanshaotong/reranker:v0.1` ++ 端口号: 6006 ++ 大小：约8GB + +**设置安全凭证（即oneapi中的渠道密钥）** +``` +ACCESS_TOKEN=mytoken +``` + +**运行命令示例** + +```sh +docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken luanshaotong/reranker:v0.1 +``` + +## 接入 FastGPT + +参考 [ReRank模型接入](/docs/development/configuration/#rerank-接入)，host 变量为部署的域名。 \ No newline at end of file diff --git a/docSite/content/docs/development/openapi/dataset.md b/docSite/content/docs/development/openapi/dataset.md index 61a4dffa3..87b995f5b 100644 --- a/docSite/content/docs/development/openapi/dataset.md +++ b/docSite/content/docs/development/openapi/dataset.md @@ -157,7 +157,7 @@ A2: ## 搜索测试 -{{< tabs tabTotal="2" >}} +{{< tabs tabTotal="3" >}} {{< tab tabName="请求示例" >}} {{< markdownify >}} @@ -168,18 +168,33 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe --data-raw '{ "datasetId": "知识库的ID", "text": "导演是谁", - "rarank": true, - "limit": 20 + "limit": 5000, + "similarity": 0, + "searchMode": "embedding", + "usingReRank": false }' ``` {{< /markdownify >}} {{< /tab >}} +{{< tab tabName="参数说明" >}} +{{< markdownify >}} + +- datasetId - 知识库ID +- text - 需要测试的文本 +- limit - 最大 tokens 数量 +- similarity - 最低相关度（0~1，可选） +- searchMode - 搜索模式：embedding | fullTextRecall | mixedRecall +- usingReRank - 使用重排 + +{{< /markdownify >}} +{{< /tab >}} + {{< tab tabName="响应示例" >}} {{< markdownify >}} -返回 top k 结果， limit 为预估条数，会按每条数据 800 tokens 的长度进行预估，20条也就是返回 16000 tokens 长度的数据，最多测试 30000 tokens 的数据。 +返回 top k 结果， limit 为最大 Tokens 数量，最多 20000 tokens。 ```bash { @@ -190,15 +205,6 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe "id": "65599c54a5c814fb803363cb", "q": "你是谁", "a": "我是FastGPT助手", - "indexes": [ - { - "defaultIndex": true, - "type": "qa", - "dataId": "3645952", - "text": "你是谁\n我是FastGPT助手", - "_id": "65599c5588271af95b019862" - } - ], "datasetId": "6554684f7f9ed18a39a4d15c", "collectionId": "6556cd795e4b663e770bb66d", "sourceName": "GBT 15104-2021 装饰单板贴面人造板.pdf", diff --git a/docSite/content/docs/development/qa.md b/docSite/content/docs/development/qa.md index 169a9159f..b5178ab98 100644 --- a/docSite/content/docs/development/qa.md +++ b/docSite/content/docs/development/qa.md @@ -7,12 +7,29 @@ draft: false images: [] --- +## 错误排查方式 + +遇到问题先按下面方式排查。 + +1. `docker ps -a` 查看所有容器运行状态，检查是否全部 running，如有异常，尝试`docker logs 容器名`查看对应日志。 +2. 不懂 docker 不要瞎改端口，只需要改`OPENAI_BASE_URL`和`CHAT_API_KEY`即可。 +3. 容器都运行正常的，`docker logs 容器名` 查看报错日志 +4. 无法解决时，可以找找[Issue](https://github.com/labring/FastGPT/issues)，或新提 Issue，私有部署错误，务必提供详细的日志，否则很难排查。 + + ## 通用问题 ### insufficient_user_quota user quota is not enough OneAPI 账号的余额不足，默认 root 用户只有 200 刀，可以手动修改。 +### xxx渠道找不到 + +OneAPI 中没有配置该模型渠道。 + +### 页面中可以正常回复，API 报错 + +页面中是用 stream=true 模式，所以API也需要设置 stream=true 来进行测试。部分模型接口（国产居多）非 Stream 的兼容有点垃圾。 ## Docker 部署常见问题 @@ -66,11 +83,3 @@ PG 数据库没有连接上/初始化失败，可以查看日志。FastGPT 会 mongo连接失败，检查 1. mongo 服务有没有起来(有些 cpu 不支持 AVX，无法用 mongo5，需要换成 mongo4.x，可以dockerhub找个最新的4.x，修改镜像版本，重新运行） 2. 环境变量（账号密码，注意host和port） - - -### 错误排查方式 - -遇到问题先按下面方式排查。 - -1. `docker ps -a` 查看所有容器运行状态，检查是否全部 running，如有异常，尝试`docker logs 容器名`查看对应日志。 -2. 不懂 docker 不要瞎改端口，只需要改`OPENAI_BASE_URL`和`CHAT_API_KEY`即可。 diff --git a/docSite/content/docs/development/upgrading/466.md b/docSite/content/docs/development/upgrading/466.md index 8659b536f..89aac365d 100644 --- a/docSite/content/docs/development/upgrading/466.md +++ b/docSite/content/docs/development/upgrading/466.md @@ -13,10 +13,9 @@ weight: 830 为了减少代码重复度，我们对配置文件做了一些修改：[点击查看最新的配置文件](/docs/development/configuration/) +## V4.6.6 更新说明 - -## V4.6.6 即将更新 - -1. UI 优化，未来将逐步替换新的UI设计。 - - +1. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。 +2. 优化 - 问题分类提示词，id引导。测试国产商用 api 模型（百度阿里智谱讯飞）使用 Prompt 模式均可分类。 +3. UI 优化，未来将逐步替换新的UI设计。 +4. 查看 [FastGPT 2024 RoadMap](https://github.com/labring/FastGPT?tab=readme-ov-file#-%E5%9C%A8%E7%BA%BF%E4%BD%BF%E7%94%A8) \ No newline at end of file diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constant.ts index c9ff7f806..aba4b0b12 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constant.ts @@ -148,5 +148,19 @@ export enum SearchScoreTypeEnum { reRank = 'reRank', rrf = 'rrf' } +export const SearchScoreTypeMap = { + [SearchScoreTypeEnum.embedding]: { + label: 'core.dataset.search.score.embedding' + }, + [SearchScoreTypeEnum.fullText]: { + label: 'core.dataset.search.score.fullText' + }, + [SearchScoreTypeEnum.reRank]: { + label: 'core.dataset.search.score.reRank' + }, + [SearchScoreTypeEnum.rrf]: { + label: 'core.dataset.search.score.rrf' + } +}; export const FolderAvatarSrc = '/imgs/files/folder.svg'; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 3e679fa2c..c2b088ec7 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -162,7 +162,10 @@ export type DatasetFileSchema = { }; /* ============= search =============== */ -export type SearchDataResponseItemType = Omit & { +export type SearchDataResponseItemType = Omit< + DatasetDataItemType, + 'indexes' | 'isOwner' | 'canWrite' +> & { score: { type: `${SearchScoreTypeEnum}`; value: number; index: number }[]; // score: number; }; diff --git a/packages/global/core/module/template/input.ts b/packages/global/core/module/template/input.ts index a3e243ecc..fd747b0fd 100644 --- a/packages/global/core/module/template/input.ts +++ b/packages/global/core/module/template/input.ts @@ -27,7 +27,7 @@ export const Input_Template_History: FlowNodeInputItemType = { export const Input_Template_UserChatInput: FlowNodeInputItemType = { key: ModuleInputKeyEnum.userChatInput, - type: FlowNodeInputTypeEnum.target, + type: FlowNodeInputTypeEnum.hidden, label: 'core.module.input.label.user question', required: true, valueType: ModuleIOValueTypeEnum.string, diff --git a/packages/global/core/module/template/output.ts b/packages/global/core/module/template/output.ts index 95f65a8e9..22ad1763a 100644 --- a/packages/global/core/module/template/output.ts +++ b/packages/global/core/module/template/output.ts @@ -3,6 +3,14 @@ import { ModuleOutputKeyEnum } from '../constants'; import { FlowNodeOutputTypeEnum } from '../node/constant'; import { ModuleIOValueTypeEnum } from '../constants'; +export const Output_Template_UserChatInput: FlowNodeOutputItemType = { + key: ModuleOutputKeyEnum.userChatInput, + label: 'core.module.input.label.user question', + type: FlowNodeOutputTypeEnum.hidden, + valueType: ModuleIOValueTypeEnum.string, + targets: [] +}; + export const Output_Template_Finish: FlowNodeOutputItemType = { key: ModuleOutputKeyEnum.finish, label: 'core.module.output.label.running done', diff --git a/packages/global/core/module/template/system/aiChat.ts b/packages/global/core/module/template/system/aiChat.ts index 24a304b1c..363c1dd74 100644 --- a/packages/global/core/module/template/system/aiChat.ts +++ b/packages/global/core/module/template/system/aiChat.ts @@ -16,7 +16,7 @@ import { Input_Template_UserChatInput } from '../input'; import { chatNodeSystemPromptTip } from '../tip'; -import { Output_Template_Finish } from '../output'; +import { Output_Template_Finish, Output_Template_UserChatInput } from '../output'; export const AiChatModule: FlowModuleTemplateType = { id: FlowNodeTypeEnum.chatNode, @@ -131,6 +131,7 @@ export const AiChatModule: FlowModuleTemplateType = { Input_Template_UserChatInput ], outputs: [ + Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.history, label: '新的上下文', diff --git a/packages/global/core/module/template/system/classifyQuestion.ts b/packages/global/core/module/template/system/classifyQuestion.ts index 8479c6a08..a31e3f07d 100644 --- a/packages/global/core/module/template/system/classifyQuestion.ts +++ b/packages/global/core/module/template/system/classifyQuestion.ts @@ -10,6 +10,7 @@ import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; +import { Output_Template_UserChatInput } from '../output'; export const ClassifyQuestionModule: FlowModuleTemplateType = { id: FlowNodeTypeEnum.classifyQuestion, @@ -72,6 +73,7 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { } ], outputs: [ + Output_Template_UserChatInput, // custom output { key: 'wqre', diff --git a/packages/global/core/module/template/system/coreferenceResolution.ts b/packages/global/core/module/template/system/coreferenceResolution.ts index aed23d475..651d38b86 100644 --- a/packages/global/core/module/template/system/coreferenceResolution.ts +++ b/packages/global/core/module/template/system/coreferenceResolution.ts @@ -15,6 +15,7 @@ import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; +import { Output_Template_UserChatInput } from '../output'; export const AiCFR: FlowModuleTemplateType = { id: FlowNodeTypeEnum.chatNode, @@ -50,6 +51,7 @@ export const AiCFR: FlowModuleTemplateType = { Input_Template_UserChatInput ], outputs: [ + Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.text, label: 'core.module.output.label.cfr result', diff --git a/packages/global/core/module/template/system/datasetSearch.ts b/packages/global/core/module/template/system/datasetSearch.ts index 50c20ff14..e6ed8c8bf 100644 --- a/packages/global/core/module/template/system/datasetSearch.ts +++ b/packages/global/core/module/template/system/datasetSearch.ts @@ -11,7 +11,7 @@ import { ModuleTemplateTypeEnum } from '../../constants'; import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; -import { Output_Template_Finish } from '../output'; +import { Output_Template_Finish, Output_Template_UserChatInput } from '../output'; import { DatasetSearchModeEnum } from '../../../dataset/constant'; export const DatasetSearchModule: FlowModuleTemplateType = { @@ -90,6 +90,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = { Input_Template_UserChatInput ], outputs: [ + Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.datasetIsEmpty, label: '搜索结果为空', diff --git a/packages/global/core/module/template/system/runApp.ts b/packages/global/core/module/template/system/runApp.ts index 128c2040d..ce740dfdd 100644 --- a/packages/global/core/module/template/system/runApp.ts +++ b/packages/global/core/module/template/system/runApp.ts @@ -15,7 +15,7 @@ import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; -import { Output_Template_Finish } from '../output'; +import { Output_Template_Finish, Output_Template_UserChatInput } from '../output'; export const RunAppModule: FlowModuleTemplateType = { id: FlowNodeTypeEnum.runApp, @@ -41,6 +41,7 @@ export const RunAppModule: FlowModuleTemplateType = { Input_Template_UserChatInput ], outputs: [ + Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.history, label: '新的上下文', diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index a2a51cbac..0828c31f7 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -24,10 +24,9 @@ export async function initPg() { ); `); - await PgClient.query(` - CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} - USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64); - `); + await PgClient.query( + `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);` + ); console.log('init pg successful'); } catch (error) { diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index e950aecec..beed3af23 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -80,8 +80,8 @@ const ChatItemSchema = new Schema({ }); try { + ChatItemSchema.index({ dataId: -1 }); ChatItemSchema.index({ time: -1 }); - ChatItemSchema.index({ userId: 1 }); ChatItemSchema.index({ appId: 1 }); ChatItemSchema.index({ chatId: 1 }); ChatItemSchema.index({ userGoodFeedback: 1 }); diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 3453c040b..4fb75c801 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -86,6 +86,8 @@ export const getSameRawTextCollection = async ({ datasetId: string; hashRawText?: string; }) => { + if (!hashRawText) return undefined; + const collection = await MongoDatasetCollection.findOne({ datasetId, hashRawText diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index 201481aad..ea9214ad8 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,4 +1,4 @@ -### Fast GPT V4.6.5 +### Fast GPT V4.6.6 1. 新增 - [问题补全模块](https://doc.fastgpt.in/docs/workflow/modules/coreferenceresolution/) 2. 新增 - [文本编辑模块](https://doc.fastgpt.in/docs/workflow/modules/text_editor/) diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 6068b1ff6..45346ea65 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -296,6 +296,7 @@ "Select Image": "Select Image", "Send Message": "Send Message", "Speaking": "I'm listening...", + "Start Chat": "Start Chat", "Stop Speak": "Stop Speak", "Type a message": "Input problem", "error": { @@ -379,6 +380,7 @@ "Name": "Name", "Quote Length": "Quote Length", "Read Dataset": "Read Dataset", + "Search score tip": "{{scoreText}}Here are the rankings and scores:\n----\n{{detailScore}}", "Set Empty Result Tip": ",Response empty text", "Set Website Config": "Configuring Website", "Similarity": "Similarity", @@ -435,6 +437,7 @@ "Too Long": "Content is too long", "Total Amount": "{{total}} Chunks", "data is deleted": "Data is deleted", + "get data error": "Get data error", "id": "Data ID" }, "error": { @@ -482,6 +485,7 @@ "link": "Link", "search": { "Dataset Search Params": "Dataset Search Params", + "Embedding score": "Embedding score", "Empty result response": "Empty Response", "Empty result response Tips": "If you fill in the content, if no suitable content is found, you will directly reply to the content.", "Max Tokens": "Max Tokens", @@ -489,8 +493,14 @@ "Min Similarity": "Min Similarity", "Min Similarity Tips": "The similarity of different index models is different, please use the search test to select the appropriate value", "Params Setting": "Params Setting", + "Rank": "Rank", + "Rank Tip": "Ranking in all data", "ReRank": "ReRank", "ReRank desc": "The rearrangement model is used for secondary ranking to enhance the overall ranking.", + "Read score": "Read score", + "Rerank score": "ReRank score", + "Score": "Score", + "Search type": "Type", "Top K": "Top K", "mode": { "embedding": "Vector search", @@ -500,6 +510,12 @@ "mixedRecall": "Mixedrecall", "mixedRecall desc": "Returns the combined results of vector and full-text searches, sorted using the RRF algorithm." }, + "score": { + "embedding": "Embedding", + "fullText": "Full text", + "reRank": "ReRank", + "rrf": "RRF Merge" + }, "search mode": "Search Mode" }, "status": { @@ -507,11 +523,15 @@ "syncing": "Syncing" }, "test": { + "Batch test": "Batch test", + "Batch test Placeholder": "Select one csv file", "Search Test": "Search Test", "Test": "Start", + "Test File": "", "Test Result": "Results", "Test Text": "Text", "Test Text Placeholder": "Enter the text you want to test", + "Test params": "Search Params", "delete test history": "Delete the test result", "test history": "Test History", "test result placeholder": "The test results will be presented here", diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index dfcb8c2f6..a77c40120 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -296,6 +296,7 @@ "Select Image": "选择图片", "Send Message": "发送", "Speaking": "我在听，请说...", + "Start Chat": "开始对话", "Stop Speak": "停止录音", "Type a message": "输入问题", "error": { @@ -379,6 +380,7 @@ "Name": "知识库名称", "Quote Length": "引用内容长度", "Read Dataset": "查看知识库详情", + "Search score tip": "{{scoreText}}下面是详细排名和得分情况:\n----\n{{detailScore}}", "Set Empty Result Tip": ",未搜索到内容时回复指定内容", "Set Website Config": "开始配置网站信息", "Similarity": "相关度", @@ -435,6 +437,7 @@ "Too Long": "总长度超长了", "Total Amount": "{{total}} 组", "data is deleted": "该数据已被删除", + "get data error": "获取数据异常", "id": "数据ID" }, "error": { @@ -482,6 +485,7 @@ "link": "链接", "search": { "Dataset Search Params": "搜索参数", + "Embedding score": "语意检索得分", "Empty result response": "空搜索回复", "Empty result response Tips": "若填写该内容，没有搜索到合适内容时，将直接回复填写的内容。", "Max Tokens": "引用上限", @@ -489,8 +493,14 @@ "Min Similarity": "最低相关度", "Min Similarity Tips": "不同索引模型的相关度有区别，请通过搜索测试来选择合适的数值，使用 ReRank 时，相关度可能会很低。", "Params Setting": "搜索参数设置", + "Rank": "排名", + "Rank Tip": "在所有数据中的排名", "ReRank": "结果重排", "ReRank desc": "使用重排模型来进行二次排序，可增强综合排名。", + "Read score": "查看得分", + "Rerank score": "结果重排得分", + "Score": "得分", + "Search type": "类型", "Top K": "单次搜索上限", "mode": { "embedding": "语义检索", @@ -500,6 +510,12 @@ "mixedRecall": "混合检索", "mixedRecall desc": "使用向量检索与全文检索的综合结果返回，使用RRF算法进行排序。" }, + "score": { + "embedding": "语义检索", + "fullText": "全文检索", + "reRank": "结果重排", + "rrf": "RRF 合并" + }, "search mode": "搜索模式" }, "status": { @@ -507,11 +523,15 @@ "syncing": "同步中" }, "test": { + "Batch test": "批量测试", + "Batch test Placeholder": "选择一个 Csv 文件", "Search Test": "搜索测试", "Test": "测试", + "Test File": "批量测试", "Test Result": "测试结果", - "Test Text": "测试文本", + "Test Text": "单个文本测试", "Test Text Placeholder": "输入需要测试的文本", + "Test params": "测试参数", "delete test history": "删除该测试结果", "test history": "测试历史", "test result placeholder": "测试结果将在这里展示", @@ -847,7 +867,7 @@ "Response Detail": "返回详情", "Response Detail tips": "是否需要返回详情（引用内容，调用时间等，不会返回预设提示词和完整上下文）", "token auth": "身份验证", - "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会想指定服务器发送一个请求，进行身份校验", + "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会向指定服务器发送一个请求，进行身份校验", "token auth use cases": "查看身份验证使用说明" }, "permission": { diff --git a/projects/app/src/components/ChatBox/ContextModal.tsx b/projects/app/src/components/ChatBox/ContextModal.tsx index 1ad96c2ac..903984373 100644 --- a/projects/app/src/components/ChatBox/ContextModal.tsx +++ b/projects/app/src/components/ChatBox/ContextModal.tsx @@ -32,7 +32,7 @@ const ContextModal = ({ void; onClose: () => void; }) => { @@ -28,6 +32,8 @@ const FeedbackModal = ({ appId, chatId, chatItemId, + shareId, + outLinkUid, userBadFeedback: val }); }, diff --git a/projects/app/src/components/ChatBox/QuoteModal.tsx b/projects/app/src/components/ChatBox/QuoteModal.tsx index c177ab0e5..b058943e8 100644 --- a/projects/app/src/components/ChatBox/QuoteModal.tsx +++ b/projects/app/src/components/ChatBox/QuoteModal.tsx @@ -1,31 +1,38 @@ -import React, { useCallback, useState } from 'react'; -import { ModalBody, Box, useTheme, Flex, Progress, Link } from '@chakra-ui/react'; -import { getDatasetDataItemById } from '@/web/core/dataset/api'; -import { useLoading } from '@/web/common/hooks/useLoading'; -import { useToast } from '@/web/common/hooks/useToast'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import MyIcon from '@/components/Icon'; -import InputDataModal, { - RawSourceText, - type InputDataType -} from '@/pages/dataset/detail/components/InputDataModal'; +import React, { useMemo, useState } from 'react'; +import { ModalBody, Box, useTheme } from '@chakra-ui/react'; + import MyModal from '../MyModal'; import { useTranslation } from 'next-i18next'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; -import MyTooltip from '../MyTooltip'; -import NextLink from 'next/link'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; +import QuoteItem from '../core/dataset/QuoteItem'; +import { RawSourceText } from '@/pages/dataset/detail/components/InputDataModal'; const QuoteModal = ({ rawSearch = [], onClose, - isShare + isShare, + metadata }: { rawSearch: SearchDataResponseItemType[]; onClose: () => void; isShare: boolean; + metadata?: { + collectionId: string; + sourceId?: string; + sourceName: string; + }; }) => { const { t } = useTranslation(); + const filterResults = useMemo( + () => + metadata + ? rawSearch.filter( + (item) => + item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId + ) + : rawSearch, + [metadata, rawSearch] + ); return ( <> @@ -35,18 +42,22 @@ const QuoteModal = ({ h={['90vh', '80vh']} isCentered minW={['90vw', '600px']} - iconSrc="/imgs/modal/quote.svg" + iconSrc={!!metadata ? undefined : '/imgs/modal/quote.svg'} title={ - {t('core.chat.Quote Amount', { amount: rawSearch.length })} - + {metadata ? ( + + ) : ( + <>{t('core.chat.Quote Amount', { amount: rawSearch.length })} + )} + {t('core.chat.quote.Quote Tip')} } > - - + + @@ -62,38 +73,7 @@ export const QuoteList = React.memo(function QuoteList({ rawSearch: SearchDataResponseItemType[]; isShare: boolean; }) { - const { t } = useTranslation(); - const { isPc } = useSystemStore(); const theme = useTheme(); - const { toast } = useToast(); - const { setIsLoading, Loading } = useLoading(); - const [editInputData, setEditInputData] = useState(); - - /** - * click edit, get new DataItem - */ - const onclickEdit = useCallback( - async (item: InputDataType) => { - if (!item.id) return; - try { - setIsLoading(true); - const data = await getDatasetDataItemById(item.id); - - if (!data) { - throw new Error('该数据已被删除'); - } - - setEditInputData(data); - } catch (err) { - toast({ - status: 'warning', - title: getErrText(err) - }); - } - setIsLoading(false); - }, - [setIsLoading, toast] - ); return ( <> @@ -102,113 +82,15 @@ export const QuoteList = React.memo(function QuoteList({ key={i} flex={'1 0 0'} p={2} - borderRadius={'lg'} + borderRadius={'sm'} border={theme.borders.base} _notLast={{ mb: 2 }} - position={'relative'} - overflow={'hidden'} _hover={{ '& .hover-data': { display: 'flex' } }} bg={i % 2 === 0 ? 'white' : 'myWhite.500'} > - - - - {!isShare && ( - - {t('core.dataset.Go Dataset')} - - - )} - - - {item.q} - {item.a} - {!isShare && ( - - {isPc && ( - - - # {item.id} - - - )} - - - - {item.q.length + (item.a?.length || 0)} - - - {/* {!isShare && item.score && ( - - - - - {item.score.toFixed(4)} - - - )} */} - - {item.id && ( - - - onclickEdit(item)} - /> - - - )} - - )} + ))} - {editInputData && editInputData.id && ( - setEditInputData(undefined)} - onSuccess={() => { - console.log('更新引用成功'); - }} - onDelete={() => { - console.log('删除引用成功'); - }} - defaultValue={editInputData} - collectionId={editInputData.collectionId} - /> - )} - ); }); diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx index 308176073..62e7cc410 100644 --- a/projects/app/src/components/ChatBox/ResponseTags.tsx +++ b/projects/app/src/components/ChatBox/ResponseTags.tsx @@ -11,8 +11,6 @@ import MyTooltip from '../MyTooltip'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; import ChatBoxDivider from '@/components/core/chat/Divider'; -import MyIcon from '../Icon'; -import { getFileAndOpen } from '@/web/core/dataset/utils'; import { strIsLink } from '@fastgpt/global/common/string/tools'; const QuoteModal = dynamic(() => import('./QuoteModal'), { ssr: false }); @@ -29,7 +27,14 @@ const ResponseTags = ({ const theme = useTheme(); const { isPc } = useSystemStore(); const { t } = useTranslation(); - const [quoteModalData, setQuoteModalData] = useState(); + const [quoteModalData, setQuoteModalData] = useState<{ + rawSearch: SearchDataResponseItemType[]; + metadata?: { + collectionId: string; + sourceId?: string; + sourceName: string; + }; + }>(); const [contextModalData, setContextModalData] = useState(); const { isOpen: isOpenWholeModal, @@ -52,8 +57,8 @@ const ResponseTags = ({ .filter(Boolean) as SearchDataResponseItemType[]; const sourceList = quoteList.reduce( (acc: Record, cur) => { - if (!acc[cur.sourceName]) { - acc[cur.sourceName] = [cur]; + if (!acc[cur.collectionId]) { + acc[cur.collectionId] = [cur]; } return acc; }, @@ -70,7 +75,8 @@ const ResponseTags = ({ sourceName: item.sourceName, sourceId: item.sourceId, icon: getSourceNameIcon({ sourceId: item.sourceId, sourceName: item.sourceName }), - canReadQuote: !isShare || strIsLink(item.sourceId) + canReadQuote: !isShare || strIsLink(item.sourceId), + collectionId: item.collectionId })), historyPreview: chatData?.historyPreview, runningTime: +responseData.reduce((sum, item) => sum + (item.runningTime || 0), 0).toFixed(2) @@ -89,88 +95,52 @@ const ResponseTags = ({ {sourceList.map((item) => ( - - {''} - - {item.sourceName} - - - + { + e.stopPropagation(); + setQuoteModalData({ + rawSearch: quoteList, + metadata: { + collectionId: item.collectionId, + sourceId: item.sourceId, + sourceName: item.sourceName + } + }); + }} > - - { - e.stopPropagation(); - setQuoteModalData(quoteList); - }} - /> - - {item.sourceId && item.canReadQuote && ( - - { - e.stopPropagation(); - - if (!item.sourceId) return; - await getFileAndOpen(item.sourceId); - }} - /> - - )} - - + {''} + + {item.sourceName} + + + ))} )} - + {quoteList.length > 0 && ( setQuoteModalData(quoteList)} + onClick={() => setQuoteModalData({ rawSearch: quoteList })} > {quoteList.length}条引用 @@ -213,7 +183,7 @@ const ResponseTags = ({ {!!quoteModalData && ( setQuoteModalData(undefined)} /> diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx index 63081b618..323e00147 100644 --- a/projects/app/src/components/ChatBox/index.tsx +++ b/projects/app/src/components/ChatBox/index.tsx @@ -632,14 +632,13 @@ const ChatBox = ( leftIcon={} size={'sm'} maxW={'100px'} - borderRadius={'lg'} onClick={handleSubmit((data) => { onUpdateVariable?.(data); setVariables(data); setVariableInputFinish(true); })} > - {'开始对话'} + {t('core.chat.Start Chat')} )} @@ -952,6 +951,8 @@ const ChatBox = ( appId={appId} chatId={chatId} chatItemId={feedbackId} + shareId={shareId} + outLinkUid={outLinkUid} onClose={() => setFeedbackId(undefined)} onSuccess={(content: string) => { setChatHistory((state) => @@ -1142,7 +1143,7 @@ function ChatAvatar({ src, type }: { src?: string; type: 'Human' | 'AI' }) { w={['28px', '34px']} h={['28px', '34px']} p={'2px'} - borderRadius={'lg'} + borderRadius={'sm'} border={theme.borders.base} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} bg={type === 'Human' ? 'white' : 'primary.50'} @@ -1208,7 +1209,7 @@ function ChatController({ cursor: 'pointer', p: 1, bg: 'white', - borderRadius: 'lg', + borderRadius: 'md', boxShadow: '0 0 5px rgba(0,0,0,0.1)', border: theme.borders.base, mr: 3 diff --git a/projects/app/src/components/Markdown/img/Image.tsx b/projects/app/src/components/Markdown/img/Image.tsx index ef2c898f0..8b81bf2aa 100644 --- a/projects/app/src/components/Markdown/img/Image.tsx +++ b/projects/app/src/components/Markdown/img/Image.tsx @@ -9,7 +9,6 @@ import { Skeleton, useDisclosure } from '@chakra-ui/react'; -import MyModal from '@/components/MyModal'; const MdImage = ({ src }: { src?: string }) => { const [isLoading, setIsLoading] = useState(true); @@ -34,6 +33,7 @@ const MdImage = ({ src }: { src?: string }) => { cursor={succeed ? 'pointer' : 'default'} loading="eager" objectFit={'contain'} + referrerPolicy="no-referrer" onLoad={() => { setIsLoading(false); setSucceed(true); @@ -53,6 +53,7 @@ const MdImage = ({ src }: { src?: string }) => { alt={''} w={'100%'} maxH={'80vh'} + referrerPolicy="no-referrer" fallbackSrc={'/imgs/errImg.png'} fallbackStrategy={'onError'} objectFit={'contain'} diff --git a/projects/app/src/components/Tag/index.tsx b/projects/app/src/components/Tag/index.tsx index 2f647b95e..aa497b35f 100644 --- a/projects/app/src/components/Tag/index.tsx +++ b/projects/app/src/components/Tag/index.tsx @@ -39,7 +39,7 @@ const Tag = ({ children, colorSchema = 'blue', ...props }: Props) => { px={2} lineHeight={1} py={1} - borderRadius={'md'} + borderRadius={'sm'} fontSize={'xs'} alignItems={'center'} {...theme} diff --git a/projects/app/src/components/common/Textarea/PromptTextarea/index.tsx b/projects/app/src/components/common/Textarea/PromptTextarea/index.tsx index c101b4f51..1f0710403 100644 --- a/projects/app/src/components/common/Textarea/PromptTextarea/index.tsx +++ b/projects/app/src/components/common/Textarea/PromptTextarea/index.tsx @@ -77,7 +77,7 @@ const Editor = React.memo(function Editor({ return ( -