diff --git a/docSite/content/docs/commercial/intro.md b/docSite/content/docs/commercial/intro.md index 53fc2ed21..cf9733d48 100644 --- a/docSite/content/docs/commercial/intro.md +++ b/docSite/content/docs/commercial/intro.md @@ -44,15 +44,19 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 **特有服务** {{< table "table-hover table-striped-columns" >}} -| 部署方式 | 特有服务 | 上线时长 | 价格 | +| 部署方式 | 特有服务 | 上线时长 | 标品价格 | | ---- | ---- | ---- | ---- | | Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 3000元起/月（3个月起）
或
30000元起/年 | | 自有服务器-单机版 | 1. 6个版本的升级服务。 | 14天内 | 60000元/套（不限时长） | -| 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）| +| 自有服务器-高可用版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）| {{< /table >}} {{% alert icon="🤖 " context="success" %}} -6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。 +- 6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。 +- 全托管版本适合技术人员紧缺的团队，仅需关注业务推动，无需关心服务是否正常运行。 +- 单机版和高可用版可以完全部署在自己服务器中。 +- 单机版适合中小团队对内提供服务，需要自己维护数据库备份等。 +- 高可用版适合对外提供在线服务，包含可视化监控、多副本、负载均衡、数据库自动备份等生产环境的基础设施。 {{% /alert %}} diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index 054204ff0..d082645d6 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -277,7 +277,7 @@ weight: 708 "maxContext": 1600, "maxResponse": 4000, "inputPrice": 0, - "outputPrice": 0, + "outputPrice": 0 } ], "vectorModels": [ // 向量模型 diff --git a/docSite/content/docs/development/openapi/dataset.md b/docSite/content/docs/development/openapi/dataset.md index bcad43f57..efa70009c 100644 --- a/docSite/content/docs/development/openapi/dataset.md +++ b/docSite/content/docs/development/openapi/dataset.md @@ -51,7 +51,7 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/data/pus --header 'Content-Type: application/json' \ --data-raw '{     "collectionId": "64663f451ba1676dbdef0499", - "mode": "chunk", + "trainingMode": "chunk", "prompt": "可选。qa 拆分引导词，chunk 模式下忽略", "billId": "可选。如果有这个值，本次的数据会被聚合到一个订单中，这个值可以重复使用。可以参考 [创建训练订单] 获取该值。",     "data": [ @@ -220,3 +220,1156 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe {{< /tab >}} {{< /tabs >}} + + +# 更多接口 + +目前未整理，简陋导出： + +## POST 知识库搜索测试 + +POST /core/dataset/searchTest + +> Body Parameters + +```json +{ + "datasetId": "656c2ccff7f114064daa72f6", + "text": "导演是谁", + "limit": 1500, + "searchMode": "embedding", + "usingReRank": true, + "similarity": 0.5 +} +``` + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|Authorization|header|string| no |none| +|body|body|object| no |none| +|» datasetId|body|string| yes |none| +|» text|body|string| yes |none| +|» limit|body|integer| no |none| +|» searchMode|body|[search mode](#schemasearch%20mode)| yes |none| +|» usingReRank|body|boolean| no |none| +|» similarity|body|[similary](#schemasimilary)| no |none| + +> Response Examples + +> 成功 + +```json +{ + "code": 200, + "statusText": "", + "message": "", + "data": { + "list": [ + { + "id": "65962b23f5fac58e46330dfd", + "q": "# 快速了解 FastGPT\nFastGPT 的能力与优势\n\nFastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的问答场景！\n\n🤖\n\nFastGPT 在线使用：[https://fastgpt.in](https://fastgpt.in)\n\n| | |\n| --- | --- |\n| ![](https://doc.fastgpt.in/imgs/intro1.png) | ![](https://doc.fastgpt.in/imgs/intro2.png) |\n| ![](https://doc.fastgpt.in/imgs/intro3.png) | ![](https://doc.fastgpt.in/imgs/intro4.png) |\n\n", + "a": "", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8036568760871887, + "index": 20 + }, + { + "type": "fullText", + "value": 1.168349443855932, + "index": 2 + }, + { + "type": "reRank", + "value": 0.9870296135626316, + "index": 0 + }, + { + "type": "rrf", + "value": 0.04366449476962486, + "index": 0 + } + ] + }, + { + "id": "65962b24f5fac58e46330dff", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 2. 简单易用的可视化界面\nFastGPT 采用直观的可视化界面设计，为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤，可以轻松完成 AI 客服的创建和训练流程。\n\n![](https://doc.fastgpt.in/imgs/ability5.png)\n\n", + "a": "", + "chunkIndex": 2, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8152669668197632, + "index": 3 + }, + { + "type": "fullText", + "value": 1.0511363636363635, + "index": 8 + }, + { + "type": "reRank", + "value": 0.9287972729281414, + "index": 14 + }, + { + "type": "rrf", + "value": 0.04265696347031964, + "index": 1 + } + ] + }, + { + "id": "65962b25f5fac58e46330e00", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 3. 自动数据预处理\n提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径，其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割，节省手动训练时间，提升效能。\n\n![](https://doc.fastgpt.in/imgs/ability2.png)\n\n", + "a": "", + "chunkIndex": 3, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8158369064331055, + "index": 2 + }, + { + "type": "fullText", + "value": 1.014030612244898, + "index": 20 + }, + { + "type": "reRank", + "value": 0.9064876908461501, + "index": 17 + }, + { + "type": "rrf", + "value": 0.04045823457588163, + "index": 2 + } + ] + }, + { + "id": "65a7e1e8fc13bdf20fd46d41", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 5. 强大的 API 集成\nFastGPT 对外的 API 接口对齐了 OpenAI 官方接口，可以直接接入现有的 GPT 应用，也可以轻松集成到企业微信、公众号、飞书等平台。\n\n![](https://doc.fastgpt.in/imgs/ability4.png)", + "a": "", + "chunkIndex": 66, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e1d4fc13bdf20fd46abe", + "sourceName": "dataset - 2024-01-04T151625.388.csv", + "sourceId": "65a7e1d2fc13bdf20fd46abc", + "score": [ + { + "type": "embedding", + "value": 0.803692102432251, + "index": 18 + }, + { + "type": "fullText", + "value": 1.0511363636363635, + "index": 7 + }, + { + "type": "reRank", + "value": 0.9177460552422909, + "index": 15 + }, + { + "type": "rrf", + "value": 0.03970501147383226, + "index": 3 + } + ] + }, + { + "id": "65a7be319d96e21823f69c9b", + "q": "FastGPT Flow 的工作流设计方案提供了哪些操作？", + "a": "FastGPT Flow 的工作流设计方案提供了数据预处理、各类 AI 应用设置、调试测试及结果反馈等操作。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8283981680870056, + "index": 0 + }, + { + "type": "reRank", + "value": 0.9620363047907355, + "index": 4 + }, + { + "type": "rrf", + "value": 0.03177805800756621, + "index": 4 + } + ] + }, + { + "id": "65a7be389d96e21823f69d58", + "q": "FastGPT Flow 的实验室预约示例中使用了哪些参数？", + "a": "FastGPT Flow 的实验室预约示例中使用了姓名、时间和实验室名称等参数。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8143455386161804, + "index": 9 + }, + { + "type": "reRank", + "value": 0.9806919138043485, + "index": 1 + }, + { + "type": "rrf", + "value": 0.0304147465437788, + "index": 5 + } + ] + }, + { + "id": "65a7be309d96e21823f69c78", + "q": "FastGPT Flow 是什么？", + "a": "FastGPT Flow 是一款基于大型语言模型的知识库问答系统，通过引入 Flow 可视化工作流编排技术，提供了一个即插即用的解决方案。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8115077018737793, + "index": 11 + }, + { + "type": "reRank", + "value": 0.9686195704870232, + "index": 3 + }, + { + "type": "rrf", + "value": 0.029513888888888888, + "index": 6 + } + ] + }, + { + "id": "65a7be389d96e21823f69d5e", + "q": "FastGPT Flow 的实验室预约示例中的代码实现了哪些功能？", + "a": "FastGPT Flow 的实验室预约示例中的代码实现了预约实验室、修改预约、查询预约和取消预约等功能。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8166953921318054, + "index": 1 + }, + { + "type": "reRank", + "value": 0.8350804533361768, + "index": 20 + }, + { + "type": "rrf", + "value": 0.028474711270410194, + "index": 8 + } + ] + }, + { + "id": "65a7be389d96e21823f69d4f", + "q": "FastGPT Flow 的联网搜索示例中使用了哪些参数？", + "a": "FastGPT Flow 的联网搜索示例中使用了搜索关键词、Google 搜索的 API 密钥和自定义搜索引擎 ID。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8025297522544861, + "index": 21 + }, + { + "type": "reRank", + "value": 0.9730876959261983, + "index": 2 + }, + { + "type": "rrf", + "value": 0.028068137824235385, + "index": 10 + } + ] + }, + { + "id": "65a7e1e8fc13bdf20fd46d55", + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e1d4fc13bdf20fd46abe", + "sourceName": "dataset - 2024-01-04T151625.388.csv", + "sourceId": "65a7e1d2fc13bdf20fd46abc", + "q": "# 快速了解 FastGPT\n## FastGPT 特点\n1. **项目开源**\n \n FastGPT 遵循附加条件 Apache License 2.0 开源协议，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。\n \n2. **独特的 QA 结构**\n \n 针对客服问答场景设计的 QA 结构，提高在大量数据场景中的问答准确性。\n \n3. **可视化工作流**\n \n 通过 Flow 模块展示了从问题输入到模型输出的完整流程，便于调试和设计复杂流程。\n \n4. **无限扩展**\n \n 基于 API 进行扩展，无需修改 FastGPT 源码，也可快速接入现有的程序中。\n \n5. **便于调试**\n \n 提供搜索测试、引用修改、完整对话预览等多种调试途径。\n \n6. **支持多种模型**\n \n 支持 GPT、Claude、文心一言等多种 LLM 模型，未来也将支持自定义的向量模型。", + "a": "", + "chunkIndex": 67, + "score": [ + { + "type": "fullText", + "value": 1.0340073529411764, + "index": 12 + }, + { + "type": "reRank", + "value": 0.9542227274192233, + "index": 9 + }, + { + "type": "rrf", + "value": 0.027272727272727275, + "index": 11 + } + ] + }, + { + "id": "65a7be319d96e21823f69c8f", + "q": "FastGPT Flow 的工作流设计中，模块之间如何进行组合和组装？", + "a": "FastGPT Flow 允许用户在核心工作流模块中进行自由组合和组装，从而衍生出一个新的模块。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8098832368850708, + "index": 13 + }, + { + "type": "reRank", + "value": 0.9478657435317039, + "index": 12 + }, + { + "type": "rrf", + "value": 0.027212143650499815, + "index": 12 + } + ] + }, + { + "id": "65a7be359d96e21823f69ce0", + "q": "FastGPT Flow 的模块的输入和输出如何连接？", + "a": "FastGPT Flow 的模块的输入和输出通过连接点进行连接，连接点的颜色代表了不同的数据类型。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8060981035232544, + "index": 16 + }, + { + "type": "reRank", + "value": 0.9530133603823691, + "index": 10 + }, + { + "type": "rrf", + "value": 0.027071520029266508, + "index": 13 + } + ] + }, + { + "id": "65a7be319d96e21823f69c98", + "q": "FastGPT Flow 的工作流设计方案能够满足哪些问答场景？", + "a": "FastGPT Flow 的工作流设计方案能够满足基本的 AI 知识库问答需求，并适应各种复杂的问答场景，例如联网搜索、数据库操作、数据实时更新、消息通知等。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.814436137676239, + "index": 8 + }, + { + "type": "reRank", + "value": 0.8814109034236719, + "index": 19 + }, + { + "type": "rrf", + "value": 0.026992753623188405, + "index": 16 + } + ] + }, + { + "id": "65a7e058fc13bdf20fd46577", + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e01efc13bdf20fd45815", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7e01dfc13bdf20fd457f3", + "q": "FastGPT Flow 工作流设计112312 3123213123 232321312 21312 23一、介绍FastGPT 作为一款基于大型语言模型（LLM）的知识库问答系统，旨在为用户提供一个即插即用的解决方案。它集成了数据处理、模型调用等多项功能，通过引入 Flow 可视化工作流编排技术，进一步增强了对复杂问答场景的支持能力。本文将重点介绍 FastGPT Flow工作流的设计方案和应用优势。\nFastGPT Flow 工 作 流 采 用 了 React Flow 框 架 作 为 UI 底 座 ， 结 合 自 研 的 FlowController 实现工作流的运行。FastGPT 使用 Flow 模块为用户呈现了一个直观、可视化的界面，从而简化了 AI 应用工作流程的设计和管理方式。React Flow 的应用使得用户能够以图形化的方式组织和编排工作流，这不仅使得工作流的创建过程更为直观，同时也为用户提供了强大且灵活的工作流编辑器。在 FastGPT Flow 工作流设计中，核心工作流模块包括用户引导、问题输入、知识库检索、AI 文本生成、问题分类、结构化内容提取、指定回复、应用调用和 HTTP 扩展，并允许用户在这类模块中进行自由组合和组装，从而衍生出一个新的模块。", + "a": "", + "chunkIndex": 0, + "score": [ + { + "type": "fullText", + "value": 1.0229779411764706, + "index": 15 + }, + { + "type": "reRank", + "value": 0.9577545043363116, + "index": 8 + }, + { + "type": "rrf", + "value": 0.026992753623188405, + "index": 17 + } + ] + } + ], + "duration": "2.978s", + "searchMode": "mixedRecall", + "limit": 1500, + "similarity": 0.1, + "usingReRank": true, + "usingSimilarityFilter": true + } +} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +HTTP Status Code **200** + +|Name|Type|Required|Restrictions|Title|description| +|---|---|---|---|---|---| +|» code|integer|true|none||none| +|» statusText|string|true|none||none| +|» message|string|true|none||none| +|» data|object|true|none||none| +|»» list|[object]|true|none||none| +|»»» id|string|true|none||none| +|»»» q|string|true|none||none| +|»»» a|string|true|none||none| +|»»» chunkIndex|integer|true|none||none| +|»»» datasetId|string|true|none||none| +|»»» collectionId|string|true|none||none| +|»»» sourceName|string|true|none||none| +|»»» sourceId|string|true|none||none| +|»»» score|[object]|true|none||none| +|»»»» type|string|true|none||none| +|»»»» value|number|true|none||none| +|»»»» index|integer|true|none||none| +|»» duration|string|true|none||none| +|»» searchMode|string|true|none||none| +|»» limit|integer|true|none||none| +|»» similarity|number|true|none||none| +|»» usingReRank|boolean|true|none||none| +|»» usingSimilarityFilter|boolean|true|none||none| + +# openapi/知识库/知识库crud + +## GET 获取知识库列表 + +GET /core/dataset/list + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|parentId|query|string| no |父级的ID| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取知识库详情 + +GET /core/dataset/detail + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|id|query|string| no |知识库id| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# openapi/知识库/集合crud + +## POST 获取知识库集合列表 + +POST /core/dataset/collection/list + +> Body Parameters + +```json +{ + "pageNum": 1, + "pageSize": 10, + "datasetId": "6597ca43e26f2a90a1501414", + "parentId": null, + "searchText": "", + "simple": true +} +``` + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|Authorization|header|string| no |none| +|body|body|object| no |none| +|» pageNum|body|integer| no |none| +|» pageSize|body|integer| no |none| +|» datasetId|body|string| yes |none| +|» parentId|body|null| no |none| +|» searchText|body|string| no |none| +|» simple|body|boolean| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取集合详情 + +GET /core/dataset/collection/detail + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|id|query|string| no |知识库id| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## PUT 更新集合 + +PUT /core/dataset/collection/update + +> Body Parameters + +```json +{ + "id": "6597ce094e10ee661f0891c8", + "parentId": null, + "name": "222" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» id|body|string| yes ||none| +|» parentId|body|null| no | 父级的id|none| +|» name|body|string| no | 名称|none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建空集合（文件夹或者一个空集合） + +POST /core/dataset/collection/create + +> Body Parameters + +```json +{ + "datasetId": "6597ca43e26f2a90a1501414", + "parentId": null, + "name": "集合名", + "type": "folder", + "metadata": {} +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| yes ||none| +|» parentId|body|null| no ||none| +|» name|body|string| yes ||none| +|» type|body|[collection type](#schemacollection%20type)| yes ||none| +|» metadata|body|object| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建文本集合 + +POST /core/dataset/collection/create/text + +> Body Parameters + +```json +{ + "text": "xxxxxxxxxxxxxx", + "datasetId": "6593e137231a2be9c5603ba7", + "parentId": null, + "name": "测试", + "trainingType": "qa", + "chunkSize": 8000, + "chunkSplitter": "", + "qaPrompt": "", + "metadata": {} +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| no ||none| +|» parentId|body|null| no ||none| +|» name|body|string| yes ||none| +|» text|body|string| yes | 原文本|none| +|» trainingType|body|[training type](#schematraining%20type)| yes ||none| +|» chunkSize|body|integer| no | 分块大小|none| +|» chunkSplitter|body|string| no | 自定义最高优先级的分段符号|none| +|» qaPrompt|body|string| no ||none| +|» metadata|body|object| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建网络链接集合 + +POST /core/dataset/collection/create/link + +> Body Parameters + +```json +{ + "link": "https://doc.fastgpt.in/docs/course/quick-start/", + "datasetId": "6593e137231a2be9c5603ba7", + "parentId": null, + "trainingType": "chunk", + "chunkSize": 512, + "chunkSplitter": "", + "qaPrompt": "", + "metadata": { + "webPageSelector": ".docs-content" + } +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| yes ||none| +|» parentId|body|null| no ||none| +|» link|body|string| yes ||none| +|» trainingType|body|[training type](#schematraining%20type)| yes ||none| +|» chunkSize|body|integer| no ||none| +|» chunkSplitter|body|string| no ||none| +|» qaPrompt|body|string| no ||none| +|» metadata|body|object| no ||none| +|»» webPageSelector|body|string| no | web选择器|none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## DELETE 删除一个集合 + +DELETE /core/dataset/collection/delete + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| no ||知识库id| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# openapi/知识库/数据crud + +## POST 获取数据列表 + +POST /core/dataset/data/list + +> Body Parameters + +```json +{ + "pageNum": 1, + "pageSize": 10, + "collectionId": "65a8d2700d70d3de0bf09186", + "searchText": "" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» pageNum|body|integer| yes ||none| +|» pageSize|body|integer| yes ||none| +|» searchText|body|string| yes ||none| +|» collectionId|body|string| yes ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取数据详情 + +GET /core/dataset/data/detail + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| yes ||none| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## DELETE 删除一条数据 + +DELETE /core/dataset/data/delete + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| no ||none| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## PUT 更新数据 + +PUT /core/dataset/data/update + +> Body Parameters + +```json +{ + "id": "6597ce094e10ee661f0891c8", + "parentId": null, + "name": "222" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» id|body|string| yes ||none| +|» q|body|string| yes ||none| +|» a|body|string| no ||none| +|» indexes|body|[[数据自定义向量](#schema%e6%95%b0%e6%8d%ae%e8%87%aa%e5%ae%9a%e4%b9%89%e5%90%91%e9%87%8f)]| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 知识库插入记录（批量插入） + +POST /core/dataset/data/pushData + +> Body Parameters + +```json +{ + "collectionId": "string", + "data": [ + { + "a": "string", + "q": "string", + "chunkIndex": 1 + } + ], + "trainingMode": "string", + "promot": "string", + "billId": "" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» collectionId|body|string| yes ||none| +|» data|body|[object]| yes ||none| +|»» a|body|string| no ||none| +|»» q|body|string| no ||none| +|»» chunkIndex|body|integer| no ||none| +|» trainingMode|body|[training type](#schematraining%20type)| no ||none| +|» promot|body|string| no ||none| +|» billId|body|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# Data Schema + +

similary

+ + + + + + +```json +1 + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|integer|false|none||none| + + + + + + + + +```json +"embedding" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|embedding| +|*anonymous*|fullTextRecall| +|*anonymous*|mixedRecall| + +

training type

+ + + + + + +```json +"chunk" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|chunk| +|*anonymous*|qa| + +

collection type

+ + + + + + +```json +"folder" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|folder| +|*anonymous*|virtual| +|*anonymous*|link| +|*anonymous*|file| + +

数据自定义向量

+ + + + + + +```json +{ + "defaultIndex": true, + "type": "string", + "text": "string" +} + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|defaultIndex|boolean|false|none||是否为默认| +|type|string|true|none||none| +|text|string|true|none||索引文本| + diff --git a/docSite/content/docs/development/qa.md b/docSite/content/docs/development/qa.md index 00bd08b36..3784c3337 100644 --- a/docSite/content/docs/development/qa.md +++ b/docSite/content/docs/development/qa.md @@ -25,7 +25,7 @@ OneAPI 账号的余额不足，默认 root 用户只有 200 刀，可以手动 ### xxx渠道找不到 -OneAPI 中没有配置该模型渠道。 +OneAPI 中没有配置该模型渠道。或者是修改了配置文件中一部分的模型，但没有全部修改。 ### 页面中可以正常回复，API 报错 @@ -35,6 +35,15 @@ OneAPI 中没有配置该模型渠道。 OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 stop 然后 rm 掉，最后再 up -d 运行一次）。可以`exec`进入容器，`env`查看环境变量是否生效。 +### 其他模型没法进行问题分类/内容提取 + +需要给其他模型配置`toolChoice=false`，就会默认走提示词模式。目前内置提示词仅针对了商业模型API进行测试，国内外的商业模型基本都可用。 + +### 页面崩溃 + +1. 关闭翻译 +2. 检查配置文件是否正常加载，如果没有正常加载会导致缺失系统信息，在某些操作下会导致空指针。 + ## Docker 部署常见问题 ### 如何更新？ diff --git a/docSite/content/docs/development/upgrading/467.md b/docSite/content/docs/development/upgrading/467.md new file mode 100644 index 000000000..c1ae38654 --- /dev/null +++ b/docSite/content/docs/development/upgrading/467.md @@ -0,0 +1,33 @@ +--- +title: 'V4.6.7（需要初始化）' +description: 'FastGPT V4.6.7' +icon: 'upgrade' +draft: false +toc: true +weight: 829 +--- + +## 1。执行初始化 API + +发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名) + +1. https://xxxxx/api/admin/initv464 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv467' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +初始化说明： +1. 将 images 重新关联到数据集（不初始化也问题不大，就是可能会留下永久脏数据） + + +## V4.6.7 更新说明 + +1. 修改了知识库UI及新的导入交互方式。 +2. 优化知识库和对话的数据索引。 +3. 知识库 openAPI，支持通过 API 操作知识库。（文档待补充） +4. 新增 - 输入框变量提示。输入 { 号后将会获得可用变量提示。根据社区针对高级编排的反馈，我们计划于 2 月份的版本中，优化变量内容，支持模块的局部变量以及更多全局变量写入。 +5. 修复 - API 对话时，chatId 冲突问题。 +6. 修复 - Iframe 嵌入网页可能导致的 window.onLoad 冲突。 \ No newline at end of file diff --git a/docSite/content/docs/use-cases/datasetEngine.md b/docSite/content/docs/use-cases/datasetEngine.md index 6a1f16152..26f672381 100644 --- a/docSite/content/docs/use-cases/datasetEngine.md +++ b/docSite/content/docs/use-cases/datasetEngine.md @@ -25,7 +25,9 @@ FastGPT 采用了 RAG 中的 Embedding 方案构建知识库，要使用好 Fast FastGPT 采用了 `PostgresSQL` 的 `PG Vector` 插件作为向量检索器，索引为`HNSW`。且`PostgresSQL`仅用于向量检索，`MongoDB`用于其他数据的存取。 -在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量，以及一个`data_id`用于在`MongoDB`中寻找对应的映射值。多个`index`可以对应一组`data_id`，也就是说，一组向量可以对应多组数据。在进行检索时，相同数据会进行合并。 +在`MongoDB`的`dataset.datas`表中，会存储向量原数据的信息，同时有一个`indexes`字段，会记录其对应的向量ID，这是一个数组，也就是说，一组向量可以对应多组数据。 + +在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量。在检索时，会先召回向量，再根据向量的ID，去`MongoDB`中寻找原数据内容，如果对应了同一组原数据，则进行合并，向量得分取最高得分。 ![](/imgs/datasetSetting1.png) diff --git a/packages/global/common/file/icon.ts b/packages/global/common/file/icon.ts index aeee3cca1..9e1444f64 100644 --- a/packages/global/common/file/icon.ts +++ b/packages/global/common/file/icon.ts @@ -1,9 +1,9 @@ export const fileImgs = [ - { suffix: 'pdf', src: '/imgs/files/pdf.svg' }, - { suffix: 'csv', src: '/imgs/files/csv.svg' }, - { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' }, - { suffix: 'txt', src: '/imgs/files/txt.svg' }, - { suffix: 'md', src: '/imgs/files/markdown.svg' } + { suffix: 'pdf', src: 'file/fill/pdf' }, + { suffix: 'csv', src: 'file/fill/csv' }, + { suffix: '(doc|docs)', src: 'file/fill/doc' }, + { suffix: 'txt', src: 'file/fill/txt' }, + { suffix: 'md', src: 'file/fill/markdown' } // { suffix: '.', src: '/imgs/files/file.svg' } ]; diff --git a/packages/global/common/file/image/constants.ts b/packages/global/common/file/image/constants.ts index 7136f8da4..643d6c21b 100644 --- a/packages/global/common/file/image/constants.ts +++ b/packages/global/common/file/image/constants.ts @@ -9,7 +9,7 @@ export enum MongoImageTypeEnum { teamAvatar = 'teamAvatar', chatImage = 'chatImage', - docImage = 'docImage' + collectionImage = 'collectionImage' } export const mongoImageTypeMap = { [MongoImageTypeEnum.systemAvatar]: { @@ -41,8 +41,8 @@ export const mongoImageTypeMap = { label: 'common.file.type.chatImage', unique: false }, - [MongoImageTypeEnum.docImage]: { - label: 'common.file.type.docImage', + [MongoImageTypeEnum.collectionImage]: { + label: 'common.file.type.collectionImage', unique: false } }; diff --git a/packages/global/common/file/image/type.d.ts b/packages/global/common/file/image/type.d.ts index 79228d3d5..9bb4d28cb 100644 --- a/packages/global/common/file/image/type.d.ts +++ b/packages/global/common/file/image/type.d.ts @@ -1,11 +1,14 @@ import { MongoImageTypeEnum } from './constants'; export type MongoImageSchemaType = { + _id: string; teamId: string; binary: Buffer; createTime: Date; expiredTime?: Date; type: `${MongoImageTypeEnum}`; - metadata?: { fileId?: string }; + metadata?: { + relatedId?: string; // This id is associated with a set of images + }; }; diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index 45f13aaec..5fb21ae57 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -13,13 +13,12 @@ export const splitText2Chunks = (props: { chunkLen: number; overlapRatio?: number; customReg?: string[]; - countTokens?: boolean; }): { chunks: string[]; - tokens: number; + chars: number; overlapRatio?: number; } => { - let { text = '', chunkLen, overlapRatio = 0.2, customReg = [], countTokens = true } = props; + let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props; const splitMarker = 'SPLIT_HERE_SPLIT_HERE'; const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER'; const overlapLen = Math.round(chunkLen * overlapRatio); @@ -240,13 +239,11 @@ export const splitText2Chunks = (props: { mdTitle: '' }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block - const tokens = countTokens - ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0) - : 0; + const chars = chunks.reduce((sum, chunk) => sum + chunk.length, 0); return { chunks, - tokens + chars }; } catch (err) { throw new Error(getErrText(err)); diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index 2dd489f0a..57e759962 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -55,6 +55,8 @@ export type FastGPTFeConfigsType = { datasetStoreFreeSize?: number; datasetStorePrice?: number; }; + + uploadFileMaxSize?: number; }; export type SystemEnvType = { diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts index 8e29426c0..7b00561e4 100644 --- a/packages/global/core/app/type.d.ts +++ b/packages/global/core/app/type.d.ts @@ -4,7 +4,7 @@ import { PermissionTypeEnum } from '../../support/permission/constant'; import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d'; import { VariableInputEnum } from '../module/constants'; import { SelectedDatasetType } from '../module/api'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export interface AppSchema { _id: string; diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index 48f22335a..3b4bbcfa2 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -4,7 +4,7 @@ import { ModuleOutputKeyEnum, ModuleInputKeyEnum } from '../module/constants'; import type { FlowNodeInputItemType } from '../module/node/type.d'; import { getGuideModule, splitGuideModule } from '../module/utils'; import { ModuleItemType } from '../module/type.d'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => { return { diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts index 8c8cb9c88..05b421c10 100644 --- a/packages/global/core/chat/type.d.ts +++ b/packages/global/core/chat/type.d.ts @@ -4,7 +4,7 @@ import { ChatRoleEnum, ChatSourceEnum, ChatStatusEnum } from './constants'; import { FlowNodeTypeEnum } from '../module/node/constant'; import { ModuleOutputKeyEnum } from '../module/constants'; import { AppSchema } from '../app/type'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export type ChatSchema = { _id: string; @@ -92,6 +92,7 @@ export type moduleDispatchResType = { runningTime?: number; inputTokens?: number; outputTokens?: number; + charsLength?: number; model?: string; query?: string; contextTotalLen?: number; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 0b3f18697..43a15f75a 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -1,5 +1,5 @@ import { DatasetDataIndexItemType, DatasetSchemaType } from './type'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constants'; import type { LLMModelItemType } from '../ai/model.d'; /* ================= dataset ===================== */ @@ -17,27 +17,25 @@ export type DatasetUpdateBody = { /* ================= collection ===================== */ export type DatasetCollectionChunkMetadataType = { + parentId?: string; trainingType?: `${TrainingModeEnum}`; chunkSize?: number; chunkSplitter?: string; qaPrompt?: string; + metadata?: Record; }; export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; - parentId?: string; name: string; type: `${DatasetCollectionTypeEnum}`; fileId?: string; rawLink?: string; rawTextLength?: number; hashRawText?: string; - metadata?: Record; }; export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; - parentId?: string; - metadata?: Record; }; export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { name: string; @@ -45,16 +43,24 @@ export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams }; export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { link: string; - chunkSplitter?: string; +}; +export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + name: string; + rawTextLength: number; + hashRawText: string; + trainingType: `${TrainingModeEnum}`; + chunkSize: number; + chunkSplitter: string; + qaPrompt: string; + + fileMetadata?: Record; + collectionMetadata?: Record; }; /* ================= data ===================== */ export type PgSearchRawType = { id: string; - team_id: string; - tmb_id: string; collection_id: string; - data_id: string; score: number; }; export type PushDatasetDataChunkProps = { diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constants.ts similarity index 91% rename from packages/global/core/dataset/constant.ts rename to packages/global/core/dataset/constants.ts index 8c6aaf38d..005fcf155 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constants.ts @@ -6,7 +6,7 @@ export enum DatasetTypeEnum { } export const DatasetTypeMap = { [DatasetTypeEnum.folder]: { - icon: 'core/dataset/folderDataset', + icon: 'common/folderFill', label: 'core.dataset.Folder Dataset', collectionLabel: 'common.Folder' }, @@ -104,10 +104,12 @@ export enum TrainingModeEnum { export const TrainingTypeMap = { [TrainingModeEnum.chunk]: { - label: 'core.dataset.training.type chunk' + label: 'core.dataset.training.Chunk mode', + tooltip: 'core.dataset.import.Chunk Split Tip' }, [TrainingModeEnum.qa]: { - label: 'core.dataset.training.type qa' + label: 'core.dataset.training.QA mode', + tooltip: 'core.dataset.import.QA Import Tip' } }; @@ -168,4 +170,8 @@ export const SearchScoreTypeMap = { } }; -export const FolderAvatarSrc = '/imgs/files/folder.svg'; +export const FolderIcon = 'file/fill/folder'; +export const FolderImgUrl = '/imgs/files/folder.svg'; + +export const CustomCollectionIcon = 'common/linkBlue'; +export const LinkCollectionIcon = 'common/linkBlue'; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 8e99af6dd..bac51bd41 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -8,7 +8,7 @@ import { DatasetTypeEnum, SearchScoreTypeEnum, TrainingModeEnum -} from './constant'; +} from './constants'; /* schema */ export type DatasetSchemaType = { @@ -55,6 +55,8 @@ export type DatasetCollectionSchemaType = { hashRawText?: string; metadata?: { webPageSelector?: string; + relatedImgId?: string; // The id of the associated image collections + [key: string]: any; }; }; diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts index 577d009b4..e8626e1c8 100644 --- a/packages/global/core/dataset/utils.ts +++ b/packages/global/core/dataset/utils.ts @@ -1,4 +1,4 @@ -import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constants'; import { getFileIcon } from '../../common/file/icon'; import { strIsLink } from '../../common/string/tools'; @@ -7,18 +7,13 @@ export function getCollectionIcon( name = '' ) { if (type === DatasetCollectionTypeEnum.folder) { - return '/imgs/files/folder.svg'; + return 'common/folderFill'; } if (type === DatasetCollectionTypeEnum.link) { - return '/imgs/files/link.svg'; + return 'common/linkBlue'; } if (type === DatasetCollectionTypeEnum.virtual) { - if (name === '手动录入') { - return '/imgs/files/manual.svg'; - } else if (name === '手动标注') { - return '/imgs/files/mark.svg'; - } - return '/imgs/files/collection.svg'; + return 'file/fill/manual'; } return getFileIcon(name); } @@ -30,19 +25,14 @@ export function getSourceNameIcon({ sourceId?: string; }) { if (strIsLink(sourceId)) { - return '/imgs/files/link.svg'; + return 'common/linkBlue'; } const fileIcon = getFileIcon(sourceName, ''); if (fileIcon) { return fileIcon; } - if (sourceName === '手动录入') { - return '/imgs/files/manual.svg'; - } else if (sourceName === '手动标注') { - return '/imgs/files/mark.svg'; - } - return '/imgs/files/collection.svg'; + return 'file/fill/manual'; } export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: string }) { diff --git a/packages/global/core/module/constants.ts b/packages/global/core/module/constants.ts index a9bf7db72..003f8006a 100644 --- a/packages/global/core/module/constants.ts +++ b/packages/global/core/module/constants.ts @@ -113,5 +113,16 @@ export enum VariableInputEnum { textarea = 'textarea', select = 'select' } +export const variableMap = { + [VariableInputEnum.input]: { + icon: 'core/app/variable/input' + }, + [VariableInputEnum.textarea]: { + icon: 'core/app/variable/textarea' + }, + [VariableInputEnum.select]: { + icon: 'core/app/variable/select' + } +}; export const DYNAMIC_INPUT_KEY = 'DYNAMIC_INPUT_KEY'; diff --git a/packages/global/core/module/node/constant.ts b/packages/global/core/module/node/constant.ts index 17db8dd87..312719193 100644 --- a/packages/global/core/module/node/constant.ts +++ b/packages/global/core/module/node/constant.ts @@ -54,10 +54,9 @@ export enum FlowNodeTypeEnum { pluginModule = 'pluginModule', pluginInput = 'pluginInput', pluginOutput = 'pluginOutput', - cfr = 'cfr', + cfr = 'cfr' // abandon - variable = 'variable' } export const EDGE_TYPE = 'default'; diff --git a/packages/global/core/module/template/system/aiChat.ts b/packages/global/core/module/template/system/aiChat.ts index 363c1dd74..3c548df1e 100644 --- a/packages/global/core/module/template/system/aiChat.ts +++ b/packages/global/core/module/template/system/aiChat.ts @@ -23,15 +23,15 @@ export const AiChatModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.textAnswer, flowType: FlowNodeTypeEnum.chatNode, avatar: '/imgs/module/AI.png', - name: 'AI 对话', - intro: 'AI 大模型对话', + name: 'core.module.template.Ai chat', + intro: 'core.module.template.Ai chat intro', showStatus: true, inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectChatModel, - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, @@ -41,42 +41,31 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatTemperature, type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window - label: '温度', + label: '', value: 0, valueType: ModuleIOValueTypeEnum.number, min: 0, max: 10, step: 1, - markList: [ - { label: '严谨', value: 0 }, - { label: '发散', value: 10 } - ], showTargetInApp: false, showTargetInPlugin: false }, { key: ModuleInputKeyEnum.aiChatMaxToken, type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window - label: '回复上限', + label: '', value: 2000, valueType: ModuleIOValueTypeEnum.number, min: 100, max: 4000, step: 50, - markList: [ - { label: '100', value: 100 }, - { - label: `${4000}`, - value: 4000 - } - ], showTargetInApp: false, showTargetInPlugin: false }, { key: ModuleInputKeyEnum.aiChatIsResponseText, type: FlowNodeInputTypeEnum.hidden, - label: '返回AI内容', + label: '', value: true, valueType: ModuleIOValueTypeEnum.boolean, showTargetInApp: false, @@ -85,7 +74,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatQuoteTemplate, type: FlowNodeInputTypeEnum.hidden, - label: '引用内容模板', + label: '', valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, showTargetInPlugin: false @@ -93,7 +82,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatQuotePrompt, type: FlowNodeInputTypeEnum.hidden, - label: '引用内容提示词', + label: '', valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, showTargetInPlugin: false @@ -110,7 +99,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiSystemPrompt, type: FlowNodeInputTypeEnum.textarea, - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: ModuleIOValueTypeEnum.string, description: chatNodeSystemPromptTip, @@ -122,8 +111,8 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatDatasetQuote, type: FlowNodeInputTypeEnum.target, - label: '引用内容', - description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]", + label: 'core.module.input.label.Quote', + description: 'core.module.input.description.Quote', valueType: ModuleIOValueTypeEnum.datasetQuote, showTargetInApp: true, showTargetInPlugin: true @@ -134,16 +123,16 @@ export const AiChatModule: FlowModuleTemplateType = { Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.history, - label: '新的上下文', - description: '将本次回复内容拼接上历史记录，作为新的上下文返回', + label: 'core.module.output.label.New context', + description: 'core.module.output.description.New context', valueType: ModuleIOValueTypeEnum.chatHistory, type: FlowNodeOutputTypeEnum.source, targets: [] }, { key: ModuleOutputKeyEnum.answerText, - label: 'AI回复内容', - description: '将在 stream 回复完毕后触发', + label: 'core.module.output.label.Ai response content', + description: 'core.module.output.description.Ai response content', valueType: ModuleIOValueTypeEnum.string, type: FlowNodeOutputTypeEnum.source, targets: [] diff --git a/packages/global/core/module/template/system/assignedAnswer.ts b/packages/global/core/module/template/system/assignedAnswer.ts index cc48fa066..fb2a3d431 100644 --- a/packages/global/core/module/template/system/assignedAnswer.ts +++ b/packages/global/core/module/template/system/assignedAnswer.ts @@ -9,19 +9,17 @@ export const AssignedAnswerModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.textAnswer, flowType: FlowNodeTypeEnum.answerNode, avatar: '/imgs/module/reply.png', - name: '指定回复', - intro: '该模块可以直接回复一段指定的内容。常用于引导、提示', + name: 'core.module.template.Assigned reply', + intro: 'core.module.template.Assigned reply intro', inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.answerText, type: FlowNodeInputTypeEnum.textarea, valueType: ModuleIOValueTypeEnum.any, - label: '回复的内容', - description: - '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串', - placeholder: - '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串', + label: 'core.module.input.label.Response content', + description: 'core.module.input.description.Response content', + placeholder: 'core.module.input.description.Response content', showTargetInApp: true, showTargetInPlugin: true } diff --git a/packages/global/core/module/template/system/classifyQuestion.ts b/packages/global/core/module/template/system/classifyQuestion.ts index a31e3f07d..4e0d1a141 100644 --- a/packages/global/core/module/template/system/classifyQuestion.ts +++ b/packages/global/core/module/template/system/classifyQuestion.ts @@ -17,12 +17,8 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.classifyQuestion, avatar: '/imgs/module/cq.png', - name: '问题分类', - intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子： -类型1: 打招呼 -类型2: 关于商品“使用”问题 -类型3: 关于商品“购买”问题 -类型4: 其他问题`, + name: 'core.module.template.Classify question', + intro: `core.module.template.Classify question intro`, showStatus: true, inputs: [ Input_Template_Switch, @@ -30,7 +26,7 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectCQModel, valueType: ModuleIOValueTypeEnum.string, - label: '分类模型', + label: 'core.module.input.label.Classify model', required: true, showTargetInApp: false, showTargetInPlugin: false @@ -39,11 +35,9 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiSystemPrompt, type: FlowNodeInputTypeEnum.textarea, valueType: ModuleIOValueTypeEnum.string, - label: '背景知识', - description: - '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。', - placeholder: - '例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。', + label: 'core.module.input.label.Background', + description: 'core.module.input.description.Background', + placeholder: 'core.module.input.placeholder.Classify background', showTargetInApp: true, showTargetInPlugin: true }, diff --git a/packages/global/core/module/template/system/contextExtract.ts b/packages/global/core/module/template/system/contextExtract.ts index 83b2ae87b..5616ffa40 100644 --- a/packages/global/core/module/template/system/contextExtract.ts +++ b/packages/global/core/module/template/system/contextExtract.ts @@ -17,8 +17,8 @@ export const ContextExtractModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.contentExtract, avatar: '/imgs/module/extract.png', - name: '文本内容提取', - intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等', + name: 'core.module.template.Extract field', + intro: 'core.module.template.Extract field intro', showStatus: true, inputs: [ Input_Template_Switch, @@ -26,7 +26,7 @@ export const ContextExtractModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectExtractModel, valueType: ModuleIOValueTypeEnum.string, - label: '提取模型', + label: 'core.module.input.label.LLM', required: true, showTargetInApp: false, showTargetInPlugin: false diff --git a/packages/global/core/module/template/system/datasetSearch.ts b/packages/global/core/module/template/system/datasetSearch.ts index e6ed8c8bf..c06c03727 100644 --- a/packages/global/core/module/template/system/datasetSearch.ts +++ b/packages/global/core/module/template/system/datasetSearch.ts @@ -12,22 +12,22 @@ import { } from '../../constants'; import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; import { Output_Template_Finish, Output_Template_UserChatInput } from '../output'; -import { DatasetSearchModeEnum } from '../../../dataset/constant'; +import { DatasetSearchModeEnum } from '../../../dataset/constants'; export const DatasetSearchModule: FlowModuleTemplateType = { id: FlowNodeTypeEnum.datasetSearchNode, templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.datasetSearchNode, avatar: '/imgs/module/db.png', - name: '知识库搜索', - intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。', + name: 'core.module.template.Dataset search', + intro: 'core.module.template.Dataset search intro', showStatus: true, inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.datasetSelectList, type: FlowNodeInputTypeEnum.selectDataset, - label: '关联的知识库', + label: 'core.module.input.label.Select dataset', value: [], valueType: ModuleIOValueTypeEnum.selectDataset, list: [], @@ -38,7 +38,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.datasetSimilarity, type: FlowNodeInputTypeEnum.hidden, - label: '最低相关性', + label: '', value: 0.4, valueType: ModuleIOValueTypeEnum.number, min: 0, @@ -54,8 +54,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.datasetLimit, type: FlowNodeInputTypeEnum.hidden, - label: '引用上限', - description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens', + label: '', value: 1500, valueType: ModuleIOValueTypeEnum.number, showTargetInApp: false, @@ -93,23 +92,22 @@ export const DatasetSearchModule: FlowModuleTemplateType = { Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.datasetIsEmpty, - label: '搜索结果为空', + label: 'core.module.output.label.Search result empty', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.boolean, targets: [] }, { key: ModuleOutputKeyEnum.datasetUnEmpty, - label: '搜索结果不为空', + label: 'core.module.output.label.Search result not empty', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.boolean, targets: [] }, { key: ModuleOutputKeyEnum.datasetQuoteQA, - label: '引用内容', - description: - '始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器', + label: 'core.module.output.label.Quote', + description: 'core.module.output.label.Quote intro', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.datasetQuote, targets: [] diff --git a/packages/global/core/module/template/system/http.ts b/packages/global/core/module/template/system/http.ts index 22d6ada02..4397a4936 100644 --- a/packages/global/core/module/template/system/http.ts +++ b/packages/global/core/module/template/system/http.ts @@ -17,8 +17,8 @@ export const HttpModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.externalCall, flowType: FlowNodeTypeEnum.httpRequest, avatar: '/imgs/module/http.png', - name: 'HTTP模块', - intro: '可以发出一个 HTTP POST 请求，实现更为复杂的操作（联网搜索、数据库查询等）', + name: 'core.module.template.Http request', + intro: 'core.module.template.Http request intro', showStatus: true, inputs: [ Input_Template_Switch, diff --git a/packages/global/core/module/template/system/runApp.ts b/packages/global/core/module/template/system/runApp.ts index ce740dfdd..ef4cdcecc 100644 --- a/packages/global/core/module/template/system/runApp.ts +++ b/packages/global/core/module/template/system/runApp.ts @@ -22,8 +22,8 @@ export const RunAppModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.externalCall, flowType: FlowNodeTypeEnum.runApp, avatar: '/imgs/module/app.png', - name: '应用调用', - intro: '可以选择一个其他应用进行调用', + name: 'core.module.template.Running app', + intro: 'core.module.template.Running app intro', showStatus: true, inputs: [ Input_Template_Switch, diff --git a/packages/global/core/module/template/system/runPlugin.ts b/packages/global/core/module/template/system/runPlugin.ts index 51b563b52..710a0fa9a 100644 --- a/packages/global/core/module/template/system/runPlugin.ts +++ b/packages/global/core/module/template/system/runPlugin.ts @@ -8,7 +8,7 @@ export const RunPluginModule: FlowModuleTemplateType = { flowType: FlowNodeTypeEnum.pluginModule, avatar: '/imgs/module/custom.png', intro: '', - name: '自定义模块', + name: '', showStatus: false, inputs: [], // [{key:'pluginId'},...] outputs: [] diff --git a/packages/global/core/module/template/system/userGuide.ts b/packages/global/core/module/template/system/userGuide.ts index 1401cbbf4..95c63ff68 100644 --- a/packages/global/core/module/template/system/userGuide.ts +++ b/packages/global/core/module/template/system/userGuide.ts @@ -8,14 +8,14 @@ export const UserGuideModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.userGuide, flowType: FlowNodeTypeEnum.userGuide, avatar: '/imgs/module/userGuide.png', - name: '用户引导', + name: 'core.module.template.User guide', intro: userGuideTip, inputs: [ { key: ModuleInputKeyEnum.welcomeText, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.string, - label: '开场白', + label: 'core.app.Welcome Text', showTargetInApp: false, showTargetInPlugin: false }, @@ -23,7 +23,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.variables, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.any, - label: '对话框变量', + label: 'core.module.Variable', value: [], showTargetInApp: false, showTargetInPlugin: false @@ -32,7 +32,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.questionGuide, valueType: ModuleIOValueTypeEnum.boolean, type: FlowNodeInputTypeEnum.switch, - label: '问题引导', + label: '', showTargetInApp: false, showTargetInPlugin: false }, @@ -40,7 +40,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.tts, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.any, - label: '语音播报', + label: '', showTargetInApp: false, showTargetInPlugin: false } diff --git a/packages/global/core/module/template/system/userInput.ts b/packages/global/core/module/template/system/userInput.ts index 3005bf33c..289943e03 100644 --- a/packages/global/core/module/template/system/userInput.ts +++ b/packages/global/core/module/template/system/userInput.ts @@ -16,14 +16,14 @@ export const UserInputModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.systemInput, flowType: FlowNodeTypeEnum.questionInput, avatar: '/imgs/module/userChatInput.png', - name: '用户问题(入口)', - intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。', + name: 'core.module.template.Chat entrance', + intro: 'core.module.template.Chat entrance intro', inputs: [ { key: ModuleInputKeyEnum.userChatInput, type: FlowNodeInputTypeEnum.systemInput, valueType: ModuleIOValueTypeEnum.string, - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false } @@ -31,7 +31,7 @@ export const UserInputModule: FlowModuleTemplateType = { outputs: [ { key: ModuleOutputKeyEnum.userChatInput, - label: '用户问题', + label: 'core.module.input.label.user question', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.string, targets: [] diff --git a/packages/global/core/module/template/tip.ts b/packages/global/core/module/template/tip.ts index e64ea65e2..2aa9dd405 100644 --- a/packages/global/core/module/template/tip.ts +++ b/packages/global/core/module/template/tip.ts @@ -1,7 +1,4 @@ -export const chatNodeSystemPromptTip = - '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}'; -export const userGuideTip = '可以在对话前设置引导语，设置全局变量，设置下一步指引'; -export const welcomeTextTip = - '每次对话开始前，发送一个初始内容。支持标准 Markdown 语法，可使用的额外标记:\n[快捷按键]: 用户点击后可以直接发送该问题'; -export const variableTip = - '可以在对话开始前，要求用户填写一些内容作为本轮对话的特定变量。该模块位于开场引导之后。\n变量可以通过 {{变量key}} 的形式注入到其他模块 string 类型的输入中，例如：提示词、限定词等'; +export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip'; +export const userGuideTip = 'core.app.tip.userGuideTip'; +export const welcomeTextTip = 'core.app.tip.welcomeTextTip'; +export const variableTip = 'core.app.tip.variableTip'; diff --git a/packages/global/core/module/utils.ts b/packages/global/core/module/utils.ts index 476e0e2d2..a066df945 100644 --- a/packages/global/core/module/utils.ts +++ b/packages/global/core/module/utils.ts @@ -1,5 +1,5 @@ import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from './node/constant'; -import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from './constants'; +import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, variableMap } from './constants'; import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type'; import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type'; import { Input_Template_Switch } from './template/input'; @@ -94,3 +94,12 @@ export function plugin2ModuleIO( : [] }; } + +export const formatVariablesIcon = ( + variables: VariableItemType[] +): (VariableItemType & { icon: string })[] => { + return variables.map((item) => ({ + ...item, + icon: variableMap[item.type]?.icon + })); +}; diff --git a/packages/global/support/user/api.d.ts b/packages/global/support/user/api.d.ts index dd256039d..5f7d8e212 100644 --- a/packages/global/support/user/api.d.ts +++ b/packages/global/support/user/api.d.ts @@ -3,7 +3,6 @@ import { OAuthEnum } from './constant'; export type PostLoginProps = { username: string; password: string; - tmbId?: string; }; export type OauthLoginProps = { diff --git a/packages/global/support/user/type.d.ts b/packages/global/support/user/type.d.ts index b8f7a64a7..3cb978810 100644 --- a/packages/global/support/user/type.d.ts +++ b/packages/global/support/user/type.d.ts @@ -13,6 +13,7 @@ export type UserModelSchema = { createTime: number; timezone: string; status: `${UserStatusEnum}`; + lastLoginTmbId?: string; openaiAccount?: { key: string; baseUrl: string; diff --git a/packages/global/support/wallet/bill/api.d.ts b/packages/global/support/wallet/bill/api.d.ts index 6314ab164..dcc421ecf 100644 --- a/packages/global/support/wallet/bill/api.d.ts +++ b/packages/global/support/wallet/bill/api.d.ts @@ -3,8 +3,7 @@ import { BillListItemCountType, BillListItemType } from './type'; export type CreateTrainingBillProps = { name: string; - vectorModel?: string; - agentModel?: string; + datasetId: string; }; export type ConcatBillProps = BillListItemCountType & { diff --git a/packages/global/support/wallet/bill/type.d.ts b/packages/global/support/wallet/bill/type.d.ts index 5b0606c8a..b80d7afef 100644 --- a/packages/global/support/wallet/bill/type.d.ts +++ b/packages/global/support/wallet/bill/type.d.ts @@ -4,9 +4,8 @@ import { BillSourceEnum } from './constants'; export type BillListItemCountType = { inputTokens?: number; outputTokens?: number; - textLen?: number; + charsLength?: number; duration?: number; - dataLen?: number; // abandon tokenLen?: number; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index 84a05d631..729c9fb02 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -3,9 +3,10 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import fsp from 'fs/promises'; import fs from 'fs'; import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type'; -import { delImgByFileIdList } from '../image/controller'; +import { MongoFileSchema } from './schema'; export function getGFSCollection(bucket: `${BucketNameEnum}`) { + MongoFileSchema; return connectionMongo.connection.db.collection(`${bucket}.files`); } export function getGridBucket(bucket: `${BucketNameEnum}`) { @@ -21,6 +22,7 @@ export async function uploadFile({ tmbId, path, filename, + contentType, metadata = {} }: { bucketName: `${BucketNameEnum}`; @@ -28,6 +30,7 @@ export async function uploadFile({ tmbId: string; path: string; filename: string; + contentType?: string; metadata?: Record; }) { if (!path) return Promise.reject(`filePath is empty`); @@ -44,7 +47,7 @@ export async function uploadFile({ const stream = bucket.openUploadStream(filename, { metadata, - contentType: metadata?.contentType + contentType }); // save to gridfs @@ -96,40 +99,6 @@ export async function delFileByFileIdList({ } } } -// delete file by metadata(datasetId) -export async function delFileByMetadata({ - bucketName, - datasetId -}: { - bucketName: `${BucketNameEnum}`; - datasetId?: string; -}) { - const bucket = getGridBucket(bucketName); - - const files = await bucket - .find( - { - ...(datasetId && { 'metadata.datasetId': datasetId }) - }, - { - projection: { - _id: 1 - } - } - ) - .toArray(); - - const idList = files.map((item) => String(item._id)); - - // delete img - await delImgByFileIdList(idList); - - // delete file - await delFileByFileIdList({ - bucketName, - fileIdList: idList - }); -} export async function getDownloadStream({ bucketName, diff --git a/packages/service/common/file/gridfs/schema.ts b/packages/service/common/file/gridfs/schema.ts new file mode 100644 index 000000000..457447d7f --- /dev/null +++ b/packages/service/common/file/gridfs/schema.ts @@ -0,0 +1,15 @@ +import { connectionMongo, type Model } from '../../mongo'; +const { Schema, model, models } = connectionMongo; + +const FileSchema = new Schema({}); + +try { + FileSchema.index({ 'metadata.teamId': 1 }); + FileSchema.index({ 'metadata.uploadDate': -1 }); +} catch (error) { + console.log(error); +} + +export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema); + +MongoFileSchema.syncIndexes(); diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index c8da371c8..ecb079666 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -46,8 +46,8 @@ export async function readMongoImg({ id }: { id: string }) { return data?.binary; } -export async function delImgByFileIdList(fileIds: string[]) { +export async function delImgByRelatedId(relateIds: string[]) { return MongoImage.deleteMany({ - 'metadata.fileId': { $in: fileIds.map((item) => String(item)) } + 'metadata.relatedId': { $in: relateIds.map((id) => String(id)) } }); } diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index 0b4ff832a..baa799ac9 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -35,6 +35,8 @@ try { ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); ImageSchema.index({ type: 1 }); ImageSchema.index({ teamId: 1 }); + ImageSchema.index({ createTime: 1 }); + ImageSchema.index({ 'metadata.relatedId': 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/common/file/load/pdf.ts b/packages/service/common/file/load/pdf.ts deleted file mode 100644 index 4cbb4673d..000000000 --- a/packages/service/common/file/load/pdf.ts +++ /dev/null @@ -1,68 +0,0 @@ -import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'; -// @ts-ignore -import('pdfjs-dist/legacy/build/pdf.worker.min.mjs'); -import { ReadFileParams } from './type'; - -type TokenType = { - str: string; - dir: string; - width: number; - height: number; - transform: number[]; - fontName: string; - hasEOL: boolean; -}; - -export const readPdfFile = async ({ path }: ReadFileParams) => { - const readPDFPage = async (doc: any, pageNo: number) => { - const page = await doc.getPage(pageNo); - const tokenizedText = await page.getTextContent(); - - const viewport = page.getViewport({ scale: 1 }); - const pageHeight = viewport.height; - const headerThreshold = pageHeight * 0.95; - const footerThreshold = pageHeight * 0.05; - - const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => { - return ( - !token.transform || - (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold) - ); - }); - - // concat empty string 'hasEOL' - for (let i = 0; i < pageTexts.length; i++) { - const item = pageTexts[i]; - if (item.str === '' && pageTexts[i - 1]) { - pageTexts[i - 1].hasEOL = item.hasEOL; - pageTexts.splice(i, 1); - i--; - } - } - - page.cleanup(); - - return pageTexts - .map((token) => { - const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str); - - return paragraphEnd ? `${token.str}\n` : token.str; - }) - .join(''); - }; - - const loadingTask = pdfjs.getDocument(path); - const doc = await loadingTask.promise; - - const pageTextPromises = []; - for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) { - pageTextPromises.push(readPDFPage(doc, pageNo)); - } - const pageTexts = await Promise.all(pageTextPromises); - - loadingTask.destroy(); - - return { - rawText: pageTexts.join('') - }; -}; diff --git a/packages/service/common/file/load/type.d.ts b/packages/service/common/file/load/type.d.ts deleted file mode 100644 index ffb9f3ee0..000000000 --- a/packages/service/common/file/load/type.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -export type ReadFileParams = { - preview: boolean; - teamId: string; - path: string; - metadata?: Record; -}; - -export type ReadFileResponse = { - rawText: string; -}; - -export type ReadFileBufferItemType = ReadFileParams & { - rawText: string; -}; - -declare global { - var readFileBuffers: ReadFileBufferItemType[]; -} diff --git a/packages/service/common/file/load/utils.ts b/packages/service/common/file/load/utils.ts deleted file mode 100644 index 760bc610e..000000000 --- a/packages/service/common/file/load/utils.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { readPdfFile } from './pdf'; -import { readDocFle } from './word'; -import { ReadFileBufferItemType, ReadFileParams } from './type'; - -global.readFileBuffers = global.readFileBuffers || []; - -const bufferMaxSize = 200; - -export const pushFileReadBuffer = (params: ReadFileBufferItemType) => { - global.readFileBuffers.push(params); - - if (global.readFileBuffers.length > bufferMaxSize) { - global.readFileBuffers.shift(); - } -}; -export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) => - global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId); - -export const readFileContent = async (params: ReadFileParams) => { - const { path } = params; - - const buffer = getReadFileBuffer(params); - - if (buffer) { - return buffer; - } - - const extension = path?.split('.')?.pop()?.toLowerCase() || ''; - - const { rawText } = await (async () => { - switch (extension) { - case 'pdf': - return readPdfFile(params); - case 'docx': - return readDocFle(params); - default: - return Promise.reject('Only support .pdf, .docx'); - } - })(); - - pushFileReadBuffer({ - ...params, - rawText - }); - - return { - ...params, - rawText - }; -}; diff --git a/packages/service/common/file/load/word.ts b/packages/service/common/file/load/word.ts deleted file mode 100644 index 8842a6fd9..000000000 --- a/packages/service/common/file/load/word.ts +++ /dev/null @@ -1,22 +0,0 @@ -import mammoth from 'mammoth'; -import { htmlToMarkdown } from '../../string/markdown'; -import { ReadFileParams } from './type'; -/** - * read docx to markdown - */ -export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => { - try { - const { value: html } = await mammoth.convertToHtml({ - path - }); - - const md = await htmlToMarkdown(html); - - return { - rawText: md - }; - } catch (error) { - console.log('error doc read:', error); - return Promise.reject('Can not read doc file, please convert to PDF'); - } -}; diff --git a/packages/service/common/file/multer.ts b/packages/service/common/file/multer.ts index 62ff53313..90f378923 100644 --- a/packages/service/common/file/multer.ts +++ b/packages/service/common/file/multer.ts @@ -3,7 +3,6 @@ import multer from 'multer'; import path from 'path'; import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants'; import { getNanoid } from '@fastgpt/global/common/string/tools'; -import { tmpFileDirPath } from './constants'; type FileType = { fieldname: string; @@ -15,8 +14,6 @@ type FileType = { size: number; }; -const expiredTime = 30 * 60 * 1000; - export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { maxSize *= 1024 * 1024; class UploadModel { @@ -31,15 +28,16 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { // }, filename: async (req, file, cb) => { const { ext } = path.parse(decodeURIComponent(file.originalname)); - cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`); + cb(null, `${getNanoid(32)}${ext}`); } }) - }).any(); + }).single('file'); async doUpload>(req: NextApiRequest, res: NextApiResponse) { return new Promise<{ - files: FileType[]; - metadata: T; + file: FileType; + metadata: Record; + data: T; bucketName?: `${BucketNameEnum}`; }>((resolve, reject) => { // @ts-ignore @@ -54,20 +52,28 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { return reject('BucketName is invalid'); } + // @ts-ignore + const file = req.file as FileType; + resolve({ - ...req.body, - files: - // @ts-ignore - req.files?.map((file) => ({ - ...file, - originalname: decodeURIComponent(file.originalname) - })) || [], + file: { + ...file, + originalname: decodeURIComponent(file.originalname) + }, + bucketName, metadata: (() => { if (!req.body?.metadata) return {}; try { return JSON.parse(req.body.metadata); } catch (error) { - console.log(error); + return {}; + } + })(), + data: (() => { + if (!req.body?.data) return {}; + try { + return JSON.parse(req.body.data); + } catch (error) { return {}; } })() diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts index 641aed8f9..f3214056f 100644 --- a/packages/service/common/file/utils.ts +++ b/packages/service/common/file/utils.ts @@ -1,5 +1,4 @@ import fs from 'fs'; -import { tmpFileDirPath } from './constants'; export const removeFilesByPaths = (paths: string[]) => { paths.forEach((path) => { @@ -10,24 +9,3 @@ export const removeFilesByPaths = (paths: string[]) => { }); }); }; - -/* cron job. check expired tmp files */ -export const checkExpiredTmpFiles = () => { - // get all file name - const files = fs.readdirSync(tmpFileDirPath).map((name) => { - const timestampStr = name.split('-')[0]; - const expiredTimestamp = timestampStr ? Number(timestampStr) : 0; - - return { - filename: name, - expiredTimestamp, - path: `${tmpFileDirPath}/${name}` - }; - }); - - // count expiredFiles - const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now()); - - // remove expiredFiles - removeFilesByPaths(expiredFiles.map((item) => item.path)); -}; diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 722e77c46..5bb56495b 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -64,41 +64,39 @@ export const urlsFetch = async ({ }: UrlFetchParams): Promise => { urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url)); - const response = ( - await Promise.all( - urlList.map(async (url) => { - try { - const fetchRes = await axios.get(url, { - timeout: 30000 - }); + const response = await Promise.all( + urlList.map(async (url) => { + try { + const fetchRes = await axios.get(url, { + timeout: 30000 + }); - const $ = cheerio.load(fetchRes.data); - const { title, html, usedSelector } = cheerioToHtml({ - fetchUrl: url, - $, - selector - }); - const md = await htmlToMarkdown(html); + const $ = cheerio.load(fetchRes.data); + const { title, html, usedSelector } = cheerioToHtml({ + fetchUrl: url, + $, + selector + }); + const md = await htmlToMarkdown(html); - return { - url, - title, - content: md, - selector: usedSelector - }; - } catch (error) { - console.log(error, 'fetch error'); + return { + url, + title, + content: md, + selector: usedSelector + }; + } catch (error) { + console.log(error, 'fetch error'); - return { - url, - title: '', - content: '', - selector: '' - }; - } - }) - ) - ).filter((item) => item.content); + return { + url, + title: '', + content: '', + selector: '' + }; + } + }) + ); return response; }; diff --git a/packages/service/common/vectorStore/controller.d.ts b/packages/service/common/vectorStore/controller.d.ts index 671230299..c395939b4 100644 --- a/packages/service/common/vectorStore/controller.d.ts +++ b/packages/service/common/vectorStore/controller.d.ts @@ -1,21 +1,19 @@ export type DeleteDatasetVectorProps = { + teamId: string; + id?: string; datasetIds?: string[]; collectionIds?: string[]; - - collectionId?: string; - dataIds?: string[]; + idList?: string[]; }; export type InsertVectorProps = { teamId: string; - tmbId: string; datasetId: string; collectionId: string; - dataId: string; }; export type EmbeddingRecallProps = { - similarity?: number; datasetIds: string[]; + similarity?: number; }; diff --git a/packages/service/common/vectorStore/controller.ts b/packages/service/common/vectorStore/controller.ts index 9d9ef5660..423c4093c 100644 --- a/packages/service/common/vectorStore/controller.ts +++ b/packages/service/common/vectorStore/controller.ts @@ -10,6 +10,7 @@ const getVectorObj = () => { export const initVectorStore = getVectorObj().init; export const deleteDatasetDataVector = getVectorObj().delete; export const recallFromVectorStore = getVectorObj().recall; +export const checkVectorDataExist = getVectorObj().checkDataExist; export const getVectorDataByTime = getVectorObj().getVectorDataByTime; export const getVectorCountByTeamId = getVectorObj().getVectorCountByTeamId; @@ -21,7 +22,7 @@ export const insertDatasetDataVector = async ({ query: string; model: string; }) => { - const { vectors, tokens } = await getVectorsByText({ + const { vectors, charsLength } = await getVectorsByText({ model, input: query }); @@ -31,32 +32,27 @@ export const insertDatasetDataVector = async ({ }); return { - tokens, + charsLength, insertId }; }; export const updateDatasetDataVector = async ({ id, - query, - model -}: { + ...props +}: InsertVectorProps & { id: string; query: string; model: string; }) => { - // get vector - const { vectors, tokens } = await getVectorsByText({ - model, - input: query + // insert new vector + const { charsLength, insertId } = await insertDatasetDataVector(props); + + // delete old vector + await deleteDatasetDataVector({ + teamId: props.teamId, + id }); - await getVectorObj().update({ - id, - vectors - }); - - return { - tokens - }; + return { charsLength, insertId }; }; diff --git a/packages/service/common/vectorStore/pg/class.ts b/packages/service/common/vectorStore/pg/class.ts index 3685bf41f..11a8480a9 100644 --- a/packages/service/common/vectorStore/pg/class.ts +++ b/packages/service/common/vectorStore/pg/class.ts @@ -1,20 +1,20 @@ import { initPg, insertDatasetDataVector, - updateDatasetDataVector, deleteDatasetDataVector, embeddingRecall, getVectorDataByTime, - getVectorCountByTeamId + getVectorCountByTeamId, + checkDataExist } from './controller'; export class PgVector { constructor() {} init = initPg; insert = insertDatasetDataVector; - update = updateDatasetDataVector; delete = deleteDatasetDataVector; recall = embeddingRecall; + checkDataExist = checkDataExist; getVectorCountByTeamId = getVectorCountByTeamId; getVectorDataByTime = getVectorDataByTime; } diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index a2e044d6f..f91da1efb 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -4,7 +4,7 @@ import { delay } from '@fastgpt/global/common/system/utils'; import { PgClient, connectPg } from './index'; import { PgSearchRawType } from '@fastgpt/global/core/dataset/api'; import { EmbeddingRecallItemType } from '../type'; -import { DeleteDatasetVectorProps, EmbeddingRecallProps } from '../controller.d'; +import { DeleteDatasetVectorProps, EmbeddingRecallProps, InsertVectorProps } from '../controller.d'; import dayjs from 'dayjs'; export async function initPg() { @@ -16,11 +16,9 @@ export async function initPg() { id BIGSERIAL PRIMARY KEY, vector VECTOR(1536) NOT NULL, team_id VARCHAR(50) NOT NULL, - tmb_id VARCHAR(50) NOT NULL, dataset_id VARCHAR(50) NOT NULL, collection_id VARCHAR(50) NOT NULL, - data_id VARCHAR(50) NOT NULL, - createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP + createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); `); @@ -34,26 +32,21 @@ export async function initPg() { } } -export const insertDatasetDataVector = async (props: { - teamId: string; - tmbId: string; - datasetId: string; - collectionId: string; - dataId: string; - vectors: number[][]; - retry?: number; -}): Promise<{ insertId: string }> => { - const { dataId, teamId, tmbId, datasetId, collectionId, vectors, retry = 3 } = props; +export const insertDatasetDataVector = async ( + props: InsertVectorProps & { + vectors: number[][]; + retry?: number; + } +): Promise<{ insertId: string }> => { + const { teamId, datasetId, collectionId, vectors, retry = 3 } = props; try { const { rows } = await PgClient.insert(PgDatasetTableName, { values: [ [ { key: 'vector', value: `[${vectors[0]}]` }, { key: 'team_id', value: String(teamId) }, - { key: 'tmb_id', value: String(tmbId) }, { key: 'dataset_id', value: datasetId }, - { key: 'collection_id', value: collectionId }, - { key: 'data_id', value: String(dataId) } + { key: 'collection_id', value: collectionId } ] ] }); @@ -72,48 +65,33 @@ export const insertDatasetDataVector = async (props: { } }; -export const updateDatasetDataVector = async (props: { - id: string; - vectors: number[][]; - retry?: number; -}): Promise => { - const { id, vectors, retry = 2 } = props; - try { - // update pg - await PgClient.update(PgDatasetTableName, { - where: [['id', id]], - values: [{ key: 'vector', value: `[${vectors[0]}]` }] - }); - } catch (error) { - if (retry <= 0) { - return Promise.reject(error); - } - await delay(500); - return updateDatasetDataVector({ - ...props, - retry: retry - 1 - }); - } -}; - export const deleteDatasetDataVector = async ( props: DeleteDatasetVectorProps & { retry?: number; } ): Promise => { - const { id, datasetIds, collectionIds, collectionId, dataIds, retry = 2 } = props; + const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props; + + const teamIdWhere = `team_id='${String(teamId)}' AND`; const where = await (() => { - if (id) return `id=${id}`; - if (datasetIds) return `dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})`; - if (collectionIds) { - return `collection_id IN (${collectionIds.map((id) => `'${String(id)}'`).join(',')})`; - } - if (collectionId && dataIds) { - return `collection_id='${String(collectionId)}' and data_id IN (${dataIds + if (id) return `${teamIdWhere} id=${id}`; + + if (datasetIds) { + return `${teamIdWhere} dataset_id IN (${datasetIds .map((id) => `'${String(id)}'`) .join(',')})`; } + + if (collectionIds) { + return `${teamIdWhere} collection_id IN (${collectionIds + .map((id) => `'${String(id)}'`) + .join(',')})`; + } + + if (idList) { + return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`; + } return Promise.reject('deleteDatasetData: no where'); })(); @@ -142,13 +120,13 @@ export const embeddingRecall = async ( ): Promise<{ results: EmbeddingRecallItemType[]; }> => { - const { vectors, limit, similarity = 0, datasetIds, retry = 2 } = props; + const { datasetIds, vectors, limit, similarity = 0, retry = 2 } = props; try { const results: any = await PgClient.query( `BEGIN; SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100}; - select id, collection_id, data_id, (vector <#> '[${vectors[0]}]') * -1 AS score + select id, collection_id, (vector <#> '[${vectors[0]}]') * -1 AS score from ${PgDatasetTableName} where dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} @@ -158,21 +136,10 @@ export const embeddingRecall = async ( const rows = results?.[2]?.rows as PgSearchRawType[]; - // concat same data_id - const filterRows: PgSearchRawType[] = []; - let set = new Set(); - for (const row of rows) { - if (!set.has(row.data_id)) { - filterRows.push(row); - set.add(row.data_id); - } - } - return { - results: filterRows.map((item) => ({ + results: rows.map((item) => ({ id: item.id, collectionId: item.collection_id, - dataId: item.data_id, score: item.score })) }; @@ -184,7 +151,11 @@ export const embeddingRecall = async ( } }; -// bill +export const checkDataExist = async (id: string) => { + const { rows } = await PgClient.query(`SELECT id FROM ${PgDatasetTableName} WHERE id=${id};`); + + return rows.length > 0; +}; export const getVectorCountByTeamId = async (teamId: string) => { const total = await PgClient.count(PgDatasetTableName, { where: [['team_id', String(teamId)]] @@ -193,15 +164,20 @@ export const getVectorCountByTeamId = async (teamId: string) => { return total; }; export const getVectorDataByTime = async (start: Date, end: Date) => { - const { rows } = await PgClient.query<{ id: string; data_id: string }>(`SELECT id, data_id + const { rows } = await PgClient.query<{ + id: string; + team_id: string; + dataset_id: string; + }>(`SELECT id, team_id, dataset_id FROM ${PgDatasetTableName} - WHERE createTime BETWEEN '${dayjs(start).format('YYYY-MM-DD')}' AND '${dayjs(end).format( - 'YYYY-MM-DD 23:59:59' + WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs(end).format( + 'YYYY-MM-DD HH:mm:ss' )}'; `); return rows.map((item) => ({ id: item.id, - dataId: item.data_id + datasetId: item.dataset_id, + teamId: item.team_id })); }; diff --git a/packages/service/common/vectorStore/type.d.ts b/packages/service/common/vectorStore/type.d.ts index 99807dcee..0f7624662 100644 --- a/packages/service/common/vectorStore/type.d.ts +++ b/packages/service/common/vectorStore/type.d.ts @@ -7,6 +7,5 @@ declare global { export type EmbeddingRecallItemType = { id: string; collectionId: string; - dataId: string; score: number; }; diff --git a/packages/service/core/ai/embedding/index.ts b/packages/service/core/ai/embedding/index.ts index 2b785ab0a..7c8e86b48 100644 --- a/packages/service/core/ai/embedding/index.ts +++ b/packages/service/core/ai/embedding/index.ts @@ -18,10 +18,9 @@ export async function getVectorsByText({ } try { - // 获取 chatAPI const ai = getAIApi(); - // 把输入的内容转成向量 + // input text to vector const result = await ai.embeddings .create({ model, @@ -38,7 +37,7 @@ export async function getVectorsByText({ } return { - tokens: res.usage.total_tokens || 0, + charsLength: input.length, vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding))) }; }); @@ -53,7 +52,9 @@ export async function getVectorsByText({ function unityDimensional(vector: number[]) { if (vector.length > 1536) { - console.log(`当前向量维度为: ${vector.length}, 向量维度不能超过 1536, 已自动截取前 1536 维度`); + console.log( + `The current vector dimension is ${vector.length}, and the vector dimension cannot exceed 1536. The first 1536 dimensions are automatically captured` + ); return vector.slice(0, 1536); } let resultVector = vector; diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index aefbd7d01..bf71c8d65 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -11,6 +11,8 @@ import { appCollectionName } from '../app/schema'; import { userCollectionName } from '../../support/user/schema'; import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; +export const ChatItemCollectionName = 'chatitems'; + const ChatItemSchema = new Schema({ teamId: { type: Schema.Types.ObjectId, @@ -79,20 +81,23 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ teamId: 1 }); - ChatItemSchema.index({ time: -1 }); - ChatItemSchema.index({ appId: 1 }); - ChatItemSchema.index({ chatId: 1 }); - ChatItemSchema.index({ obj: 1 }); - ChatItemSchema.index({ userGoodFeedback: 1 }); - ChatItemSchema.index({ userBadFeedback: 1 }); - ChatItemSchema.index({ customFeedbacks: 1 }); - ChatItemSchema.index({ adminFeedback: 1 }); + ChatItemSchema.index({ dataId: 1 }, { background: true }); + /* delete by app; + delete by chat id; + get chat list; + get chat logs; + close custom feedback; + */ + ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true }); + ChatItemSchema.index({ userGoodFeedback: 1 }, { background: true }); + ChatItemSchema.index({ userBadFeedback: 1 }, { background: true }); + ChatItemSchema.index({ customFeedbacks: 1 }, { background: true }); + ChatItemSchema.index({ adminFeedback: 1 }, { background: true }); } catch (error) { console.log(error); } export const MongoChatItem: Model = - models['chatItem'] || model('chatItem', ChatItemSchema); + models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema); MongoChatItem.syncIndexes(); diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 637e69550..52cc92526 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -1,13 +1,12 @@ import { connectionMongo, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d'; -import { ChatRoleMap, ChatSourceMap } from '@fastgpt/global/core/chat/constants'; +import { ChatSourceMap } from '@fastgpt/global/core/chat/constants'; import { TeamCollectionName, TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; import { appCollectionName } from '../app/schema'; -import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; export const chatCollectionName = 'chat'; @@ -48,7 +47,8 @@ const ChatSchema = new Schema({ default: '' }, top: { - type: Boolean + type: Boolean, + default: false }, source: { type: String, @@ -73,10 +73,16 @@ const ChatSchema = new Schema({ }); try { - ChatSchema.index({ appId: 1 }); - ChatSchema.index({ tmbId: 1 }); - ChatSchema.index({ shareId: 1 }); - ChatSchema.index({ updateTime: -1 }); + ChatSchema.index({ chatId: 1 }, { background: true }); + // get user history + ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }, { background: true }); + // delete by appid; clear history; init chat; update chat; auth chat; + ChatSchema.index({ appId: 1, chatId: 1 }, { background: true }); + + // get chat logs; + ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true }); + // get share chat history + ChatSchema.index({ shareId: 1, outLinkUid: 1 }, { background: true }); } catch (error) { console.log(error); } diff --git a/packages/service/core/chat/controller.ts b/packages/service/core/chat/controller.ts index 59d465f8d..094fc11bf 100644 --- a/packages/service/core/chat/controller.ts +++ b/packages/service/core/chat/controller.ts @@ -3,10 +3,12 @@ import { MongoChatItem } from './chatItemSchema'; import { addLog } from '../../common/system/log'; export async function getChatItems({ + appId, chatId, limit = 30, field }: { + appId: string; chatId?: string; limit?: number; field: string; @@ -15,7 +17,10 @@ export async function getChatItems({ return { history: [] }; } - const history = await MongoChatItem.find({ chatId }, field).sort({ _id: -1 }).limit(limit).lean(); + const history = await MongoChatItem.find({ appId, chatId }, field) + .sort({ _id: -1 }) + .limit(limit) + .lean(); history.reverse(); @@ -23,10 +28,12 @@ export async function getChatItems({ } export const addCustomFeedbacks = async ({ + appId, chatId, chatItemId, feedbacks }: { + appId: string; chatId?: string; chatItemId?: string; feedbacks: string[]; diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index d0e7c14d2..4a609088f 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,6 +1,20 @@ -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; +import { + CollectionWithDatasetType, + DatasetCollectionSchemaType +} from '@fastgpt/global/core/dataset/type'; +import { MongoDatasetTraining } from '../training/schema'; +import { delay } from '@fastgpt/global/common/system/utils'; +import { MongoDatasetData } from '../data/schema'; +import { delImgByRelatedId } from '../../../common/file/image/controller'; +import { deleteDatasetDataVector } from '../../../common/vectorStore/controller'; +import { delFileByFileIdList } from '../../../common/file/gridfs/controller'; +import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; export async function createOneCollection({ teamId, @@ -85,20 +99,50 @@ export function createDefaultCollection({ }); } -// check same collection -export const getSameRawTextCollection = async ({ - datasetId, - hashRawText +/** + * delete collection and it related data + */ +export async function delCollectionAndRelatedSources({ + collections }: { - datasetId: string; - hashRawText?: string; -}) => { - if (!hashRawText) return undefined; + collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; +}) { + if (collections.length === 0) return; - const collection = await MongoDatasetCollection.findOne({ - datasetId, - hashRawText + const teamId = collections[0].teamId; + + if (!teamId) return Promise.reject('teamId is not exist'); + + const collectionIds = collections.map((item) => String(item._id)); + const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean); + const relatedImageIds = collections + .map((item) => item?.metadata?.relatedImgId || '') + .filter(Boolean); + + // delete training data + await MongoDatasetTraining.deleteMany({ + teamId, + collectionId: { $in: collectionIds } }); - return collection; -}; + await delay(2000); + + // delete dataset.datas + await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }); + // delete pg data + await deleteDatasetDataVector({ teamId, collectionIds }); + + // delete file and imgs + await Promise.all([ + delImgByRelatedId(relatedImageIds), + delFileByFileIdList({ + bucketName: BucketNameEnum.dataset, + fileIdList + }) + ]); + + // delete collections + await MongoDatasetCollection.deleteMany({ + _id: { $in: collectionIds } + }); +} diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index f2b5f8def..4b3d4ca8d 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,7 +1,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; -import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant'; +import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; import { DatasetCollectionName } from '../schema'; import { TeamCollectionName, @@ -91,11 +91,19 @@ const DatasetCollectionSchema = new Schema({ }); try { - DatasetCollectionSchema.index({ teamId: 1 }); - DatasetCollectionSchema.index({ datasetId: 1 }); - DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, parentId: 1 }); - DatasetCollectionSchema.index({ updateTime: -1 }); - DatasetCollectionSchema.index({ hashRawText: -1 }); + // auth file + DatasetCollectionSchema.index({ teamId: 1, fileId: 1 }, { background: true }); + + // list collection; deep find collections + DatasetCollectionSchema.index( + { + teamId: 1, + datasetId: 1, + parentId: 1, + updateTime: -1 + }, + { background: true } + ); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 225e01b22..9fc3cdf21 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -4,16 +4,32 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { MongoDatasetTraining } from '../training/schema'; import { urlsFetch } from '../../../common/string/cheerio'; -import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + DatasetCollectionTypeEnum, + TrainingModeEnum +} from '@fastgpt/global/core/dataset/constants'; import { hashStr } from '@fastgpt/global/common/string/tools'; /** * get all collection by top collectionId */ -export async function findCollectionAndChild(id: string, fields = '_id parentId name metadata') { +export async function findCollectionAndChild({ + teamId, + datasetId, + collectionId, + fields = '_id parentId name metadata' +}: { + teamId: string; + datasetId: string; + collectionId: string; + fields?: string; +}) { async function find(id: string) { // find children - const children = await MongoDatasetCollection.find({ parentId: id }, fields); + const children = await MongoDatasetCollection.find( + { teamId, datasetId, parentId: id }, + fields + ).lean(); let collections = children; @@ -25,8 +41,8 @@ export async function findCollectionAndChild(id: string, fields = '_id parentId return collections; } const [collection, childCollections] = await Promise.all([ - MongoDatasetCollection.findById(id, fields), - find(id) + MongoDatasetCollection.findById(collectionId, fields), + find(collectionId) ]); if (!collection) { @@ -107,8 +123,8 @@ export const getCollectionAndRawText = async ({ }); return { - title: result[0].title, - rawText: result[0].content + title: result[0]?.title, + rawText: result[0]?.content }; } @@ -121,7 +137,7 @@ export const getCollectionAndRawText = async ({ })(); const hashRawText = hashStr(rawText); - const isSameRawText = col.hashRawText === hashRawText; + const isSameRawText = rawText && col.hashRawText === hashRawText; return { collection: col, @@ -161,8 +177,7 @@ export const reloadCollectionChunks = async ({ // split data const { chunks } = splitText2Chunks({ text: newRawText, - chunkLen: col.chunkSize || 512, - countTokens: false + chunkLen: col.chunkSize || 512 }); // insert to training queue diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index 20faa55ae..65f9ed876 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -1,24 +1,47 @@ -import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type'; +import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from './collection/schema'; import { MongoDataset } from './schema'; +import { delCollectionAndRelatedSources } from './collection/controller'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ -export async function findDatasetIdTreeByTopDatasetId( - id: string, - result: string[] = [] -): Promise { - let allChildrenIds = [...result]; +export async function findDatasetAndAllChildren({ + teamId, + datasetId, + fields +}: { + teamId: string; + datasetId: string; + fields?: string; +}): Promise { + const find = async (id: string) => { + const children = await MongoDataset.find( + { + teamId, + parentId: id + }, + fields + ).lean(); - // find children - const children = await MongoDataset.find({ parentId: id }); + let datasets = children; - for (const child of children) { - const grandChildrenIds = await findDatasetIdTreeByTopDatasetId(child._id, result); - allChildrenIds = allChildrenIds.concat(grandChildrenIds); + for (const child of children) { + const grandChildrenIds = await find(child._id); + datasets = datasets.concat(grandChildrenIds); + } + + return datasets; + }; + const [dataset, childDatasets] = await Promise.all([ + MongoDataset.findById(datasetId), + find(datasetId) + ]); + + if (!dataset) { + return Promise.reject('Dataset not found'); } - return [String(id), ...allChildrenIds]; + return [dataset, ...childDatasets]; } export async function getCollectionWithDataset(collectionId: string) { @@ -30,3 +53,22 @@ export async function getCollectionWithDataset(collectionId: string) { } return data; } + +/* delete all data by datasetIds */ +export async function delDatasetRelevantData({ datasets }: { datasets: DatasetSchemaType[] }) { + if (!datasets.length) return; + + const teamId = datasets[0].teamId; + const datasetIds = datasets.map((item) => String(item._id)); + + // Get _id, teamId, fileId, metadata.relatedImgId for all collections + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds } + }, + '_id teamId fileId metadata' + ).lean(); + + await delCollectionAndRelatedSources({ collections }); +} diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index e1af0ad0b..ebb463c1e 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -1,87 +1,2 @@ import { MongoDatasetData } from './schema'; -import { MongoDatasetTraining } from '../training/schema'; -import { delFileByFileIdList, delFileByMetadata } from '../../../common/file/gridfs/controller'; -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { MongoDatasetCollection } from '../collection/schema'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { delImgByFileIdList } from '../../../common/file/image/controller'; import { deleteDatasetDataVector } from '../../../common/vectorStore/controller'; - -/* delete all data by datasetIds */ -export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) { - datasetIds = datasetIds.map((item) => String(item)); - - // delete training data(There could be a training mission) - await MongoDatasetTraining.deleteMany({ - datasetId: { $in: datasetIds } - }); - - await delay(2000); - - // delete dataset.datas - await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } }); - // delete pg data - await deleteDatasetDataVector({ datasetIds }); - - // delete collections - await MongoDatasetCollection.deleteMany({ - datasetId: { $in: datasetIds } - }); - - // delete related files - await Promise.all( - datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id })) - ); -} -/** - * delete all data by collectionIds - */ -export async function delCollectionRelevantData({ - collectionIds, - fileIds -}: { - collectionIds: string[]; - fileIds: string[]; -}) { - collectionIds = collectionIds.filter(Boolean).map((item) => String(item)); - const filterFileIds = fileIds.filter(Boolean).map((item) => String(item)); - - // delete training data - await MongoDatasetTraining.deleteMany({ - collectionId: { $in: collectionIds } - }); - - await delay(2000); - - // delete dataset.datas - await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } }); - // delete pg data - await deleteDatasetDataVector({ collectionIds }); - - // delete collections - await MongoDatasetCollection.deleteMany({ - _id: { $in: collectionIds } - }); - - // delete file and imgs - await Promise.all([ - delImgByFileIdList(filterFileIds), - delFileByFileIdList({ - bucketName: BucketNameEnum.dataset, - fileIdList: filterFileIds - }) - ]); -} -/** - * delete one data by mongoDataId - */ -export async function delDatasetDataByDataId({ - collectionId, - mongoDataId -}: { - collectionId: string; - mongoDataId: string; -}) { - await deleteDatasetDataVector({ collectionId, dataIds: [mongoDataId] }); - await MongoDatasetData.findByIdAndDelete(mongoDataId); -} diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 8681cc9fc..8209ae786 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -10,7 +10,7 @@ import { DatasetColCollectionName } from '../collection/schema'; import { DatasetDataIndexTypeEnum, DatasetDataIndexTypeMap -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; export const DatasetDataCollectionName = 'dataset.datas'; @@ -71,6 +71,7 @@ const DatasetDataSchema = new Schema({ ], default: [] }, + updateTime: { type: Date, default: () => new Date() @@ -85,13 +86,18 @@ const DatasetDataSchema = new Schema({ }); try { - DatasetDataSchema.index({ teamId: 1 }); - DatasetDataSchema.index({ datasetId: 1 }); - DatasetDataSchema.index({ collectionId: 1 }); - DatasetDataSchema.index({ updateTime: -1 }); - DatasetDataSchema.index({ collectionId: 1, q: 1, a: 1 }); + // same data check + DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true }); + // list collection and count data; list data + DatasetDataSchema.index( + { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 }, + { background: true } + ); // full text index - DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' }); + DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true }); + // Recall vectors after data matching + DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true }); + DatasetDataSchema.index({ updateTime: 1 }, { background: true }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 827487a23..fc7135037 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -5,7 +5,7 @@ import { DatasetStatusEnum, DatasetStatusMap, DatasetTypeMap -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { TeamCollectionName, TeamMemberCollectionName diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 8d7742763..410103571 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; -import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; +import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { DatasetColCollectionName } from '../collection/schema'; import { DatasetCollectionName } from '../schema'; import { @@ -102,11 +102,11 @@ const TrainingDataSchema = new Schema({ }); try { - TrainingDataSchema.index({ teamId: 1 }); + // lock training data; delete training data + TrainingDataSchema.index({ teamId: 1, collectionId: 1 }); + // get training data and sort TrainingDataSchema.index({ weight: -1 }); TrainingDataSchema.index({ lockTime: 1 }); - TrainingDataSchema.index({ datasetId: 1 }); - TrainingDataSchema.index({ collectionId: 1 }); TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 }); } catch (error) { console.log(error); diff --git a/packages/service/package.json b/packages/service/package.json index baa926fb5..997c343cf 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -9,13 +9,11 @@ "dayjs": "^1.11.7", "encoding": "^0.1.13", "jsonwebtoken": "^9.0.2", - "mammoth": "^1.6.0", "mongoose": "^7.0.2", "multer": "1.4.5-lts.1", "next": "13.5.2", "nextjs-cors": "^2.1.2", "node-cron": "^3.0.3", - "pdfjs-dist": "^4.0.269", "pg": "^8.10.0", "tunnel": "^0.0.6" }, diff --git a/packages/service/support/permission/auth/app.ts b/packages/service/support/permission/auth/app.ts index 5970a0a56..14c72e233 100644 --- a/packages/service/support/permission/auth/app.ts +++ b/packages/service/support/permission/auth/app.ts @@ -6,7 +6,7 @@ import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { parseHeaderCert } from '../controller'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; // 模型使用权校验 export async function authApp({ @@ -24,7 +24,7 @@ export async function authApp({ > { const result = await parseHeaderCert(props); const { teamId, tmbId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { app, isOwner, canWrite } = await (async () => { // get app diff --git a/packages/service/support/permission/auth/dataset.ts b/packages/service/support/permission/auth/dataset.ts index 3b92ec8cd..c793714eb 100644 --- a/packages/service/support/permission/auth/dataset.ts +++ b/packages/service/support/permission/auth/dataset.ts @@ -13,8 +13,9 @@ import { } from '@fastgpt/global/core/dataset/type'; import { getFileById } from '../../../common/file/gridfs/controller'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { MongoDatasetCollection } from '../../../core/dataset/collection/schema'; export async function authDatasetByTmbId({ teamId, @@ -27,7 +28,7 @@ export async function authDatasetByTmbId({ datasetId: string; per: AuthModeType['per']; }) { - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { dataset, isOwner, canWrite } = await (async () => { const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean(); @@ -107,7 +108,7 @@ export async function authDatasetCollection({ } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { collection, isOwner, canWrite } = await (async () => { const collection = await getCollectionWithDataset(collectionId); @@ -163,47 +164,40 @@ export async function authDatasetFile({ } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const { role } = await getTeamInfoByTmbId({ tmbId }); - const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId }); + const [file, collection] = await Promise.all([ + getFileById({ bucketName: BucketNameEnum.dataset, fileId }), + MongoDatasetCollection.findOne({ + teamId, + fileId + }) + ]); if (!file) { return Promise.reject(CommonErrEnum.fileNotFound); } - if (file.metadata.teamId !== teamId) { + if (!collection) { return Promise.reject(DatasetErrEnum.unAuthDatasetFile); } - const { dataset } = await authDataset({ - ...props, - datasetId: file.metadata.datasetId, - per - }); - const isOwner = - role !== TeamMemberRoleEnum.visitor && - (String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner); + // file role = collection role + try { + const { isOwner, canWrite } = await authDatasetCollection({ + ...props, + collectionId: collection._id, + per + }); - const canWrite = - isOwner || - (role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public); - - if (per === 'r' && !isOwner && dataset.permission !== PermissionTypeEnum.public) { + return { + userId, + teamId, + tmbId, + file, + isOwner, + canWrite + }; + } catch (error) { return Promise.reject(DatasetErrEnum.unAuthDatasetFile); } - if (per === 'w' && !canWrite) { - return Promise.reject(DatasetErrEnum.unAuthDatasetFile); - } - if (per === 'owner' && !isOwner) { - return Promise.reject(DatasetErrEnum.unAuthDatasetFile); - } - - return { - userId, - teamId, - tmbId, - file, - isOwner, - canWrite - }; } diff --git a/packages/service/support/permission/auth/openapi.ts b/packages/service/support/permission/auth/openapi.ts index 4fd444ef9..736cecc9c 100644 --- a/packages/service/support/permission/auth/openapi.ts +++ b/packages/service/support/permission/auth/openapi.ts @@ -2,7 +2,7 @@ import { AuthResponseType } from '@fastgpt/global/support/permission/type'; import { AuthModeType } from '../type'; import { OpenApiSchema } from '@fastgpt/global/support/openapi/type'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { MongoOpenApi } from '../../openapi/schema'; import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; @@ -21,7 +21,7 @@ export async function authOpenApiKeyCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { openapi, isOwner, canWrite } = await (async () => { const openapi = await MongoOpenApi.findOne({ _id: id, teamId }); diff --git a/packages/service/support/permission/auth/outLink.ts b/packages/service/support/permission/auth/outLink.ts index 030f5d90f..f13c921f0 100644 --- a/packages/service/support/permission/auth/outLink.ts +++ b/packages/service/support/permission/auth/outLink.ts @@ -9,7 +9,7 @@ import { MongoApp } from '../../../core/app/schema'; import { OutLinkErrEnum } from '@fastgpt/global/common/error/code/outLink'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; /* crud outlink permission */ export async function authOutLinkCrud({ @@ -27,7 +27,7 @@ export async function authOutLinkCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { app, outLink, isOwner, canWrite } = await (async () => { const outLink = await MongoOutLink.findOne({ _id: outLinkId, teamId }); diff --git a/packages/service/support/permission/auth/plugin.ts b/packages/service/support/permission/auth/plugin.ts index 6bbedd008..f6a53785c 100644 --- a/packages/service/support/permission/auth/plugin.ts +++ b/packages/service/support/permission/auth/plugin.ts @@ -1,7 +1,7 @@ import { AuthResponseType } from '@fastgpt/global/support/permission/type'; import { AuthModeType } from '../type'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { MongoPlugin } from '../../../core/plugin/schema'; import { PluginErrEnum } from '@fastgpt/global/common/error/code/plugin'; @@ -23,7 +23,7 @@ export async function authPluginCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { plugin, isOwner, canWrite } = await (async () => { const plugin = await MongoPlugin.findOne({ _id: id, teamId }); @@ -73,7 +73,7 @@ export async function authPluginCanUse({ } if (source === PluginSourceEnum.personal) { - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const plugin = await MongoPlugin.findOne({ _id: pluginId, teamId }); if (!plugin) { return Promise.reject(PluginErrEnum.unExist); diff --git a/packages/service/support/permission/auth/user.ts b/packages/service/support/permission/auth/user.ts index bbcb2a1d1..e8da32072 100644 --- a/packages/service/support/permission/auth/user.ts +++ b/packages/service/support/permission/auth/user.ts @@ -3,7 +3,7 @@ import { AuthModeType } from '../type'; import { TeamItemType } from '@fastgpt/global/support/user/team/type'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { UserErrEnum } from '../../../../global/common/error/code/user'; export async function authUserNotVisitor(props: AuthModeType): Promise< @@ -13,7 +13,7 @@ export async function authUserNotVisitor(props: AuthModeType): Promise< } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const team = await getTeamInfoByTmbId({ tmbId }); + const team = await getTmbInfoByTmbId({ tmbId }); if (team.role === TeamMemberRoleEnum.visitor) { return Promise.reject(UserErrEnum.binVisitor); @@ -38,7 +38,7 @@ export async function authUserRole(props: AuthModeType): Promise< } > { const result = await parseHeaderCert(props); - const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId: result.tmbId }); + const { role: userRole, canWrite } = await getTmbInfoByTmbId({ tmbId: result.tmbId }); return { ...result, diff --git a/packages/service/support/permission/limit/dataset.ts b/packages/service/support/permission/limit/dataset.ts index a214465e2..796587e23 100644 --- a/packages/service/support/permission/limit/dataset.ts +++ b/packages/service/support/permission/limit/dataset.ts @@ -14,7 +14,7 @@ export const checkDatasetLimit = async ({ const usedSize = await getVectorCountByTeamId(teamId); if (usedSize + insertLen >= maxSize) { - return Promise.reject(`数据库容量已满，无法继续添加。可以在账号页面进行扩容。`); + return Promise.reject(`数据库容量不足，无法继续添加。可以在账号页面进行扩容。`); } return; }; diff --git a/packages/service/support/user/controller.ts b/packages/service/support/user/controller.ts index b2020d193..361a9029b 100644 --- a/packages/service/support/user/controller.ts +++ b/packages/service/support/user/controller.ts @@ -1,6 +1,6 @@ import { UserType } from '@fastgpt/global/support/user/type'; import { MongoUser } from './schema'; -import { getTeamInfoByTmbId, getUserDefaultTeam } from './team/controller'; +import { getTmbInfoByTmbId, getUserDefaultTeam } from './team/controller'; import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; @@ -21,16 +21,16 @@ export async function getUserDetail({ tmbId?: string; userId?: string; }): Promise { - const team = await (async () => { + const tmb = await (async () => { if (tmbId) { - return getTeamInfoByTmbId({ tmbId }); + return getTmbInfoByTmbId({ tmbId }); } if (userId) { return getUserDefaultTeam({ userId }); } return Promise.reject(ERROR_ENUM.unAuthorization); })(); - const user = await MongoUser.findById(team.userId); + const user = await MongoUser.findById(tmb.userId); if (!user) { return Promise.reject(ERROR_ENUM.unAuthorization); @@ -44,7 +44,7 @@ export async function getUserDetail({ timezone: user.timezone, promotionRate: user.promotionRate, openaiAccount: user.openaiAccount, - team + team: tmb }; } diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index a3e4d6515..15e75c491 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -56,9 +56,18 @@ const UserSchema = new Schema({ timezone: { type: String, default: 'Asia/Shanghai' + }, + lastLoginTmbId: { + type: Schema.Types.ObjectId } }); +try { + UserSchema.index({ createTime: -1 }); +} catch (error) { + console.log(error); +} + export const MongoUser: Model = models[userCollectionName] || model(userCollectionName, UserSchema); MongoUser.syncIndexes(); diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts index 9efd97009..212f8db63 100644 --- a/packages/service/support/user/team/controller.ts +++ b/packages/service/support/user/team/controller.ts @@ -8,7 +8,7 @@ import { import { MongoTeamMember } from './teamMemberSchema'; import { MongoTeam } from './teamSchema'; -async function getTeam(match: Record): Promise { +async function getTeamMember(match: Record): Promise { const tmb = (await MongoTeamMember.findOne(match).populate('teamId')) as TeamMemberWithTeamSchema; if (!tmb) { @@ -31,11 +31,11 @@ async function getTeam(match: Record): Promise { }; } -export async function getTeamInfoByTmbId({ tmbId }: { tmbId: string }) { +export async function getTmbInfoByTmbId({ tmbId }: { tmbId: string }) { if (!tmbId) { return Promise.reject('tmbId or userId is required'); } - return getTeam({ + return getTeamMember({ _id: new Types.ObjectId(tmbId), status: notLeaveStatus }); @@ -45,7 +45,7 @@ export async function getUserDefaultTeam({ userId }: { userId: string }) { if (!userId) { return Promise.reject('tmbId or userId is required'); } - return getTeam({ + return getTeamMember({ userId: new Types.ObjectId(userId), defaultTeam: true }); diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index eaeb9fba9..7a58783b8 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -24,11 +24,11 @@ const TeamSchema = new Schema({ }, balance: { type: Number, - default: 2 * PRICE_SCALE + default: 0 }, maxSize: { type: Number, - default: 5 + default: 3 }, limit: { lastExportDatasetTime: { @@ -41,7 +41,7 @@ const TeamSchema = new Schema({ }); try { - TeamSchema.index({ lastDatasetBillTime: -1 }); + // TeamSchema.index({ createTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/support/wallet/bill/controller.ts b/packages/service/support/wallet/bill/controller.ts index 9f9f10488..2e9c9f705 100644 --- a/packages/service/support/wallet/bill/controller.ts +++ b/packages/service/support/wallet/bill/controller.ts @@ -25,15 +25,13 @@ export const createTrainingBill = async ({ { moduleName: 'wallet.moduleName.index', model: vectorModel, - inputTokens: 0, - outputTokens: 0, + charsLength: 0, amount: 0 }, { moduleName: 'wallet.moduleName.qa', model: agentModel, - inputTokens: 0, - outputTokens: 0, + charsLength: 0, amount: 0 } ], diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index 25c8b00c1..b6795d0bb 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -52,10 +52,8 @@ const BillSchema = new Schema({ }); try { - BillSchema.index({ teamId: 1 }); - BillSchema.index({ tmbId: 1 }); - BillSchema.index({ tmbId: 1, time: 1 }); - BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 }); + BillSchema.index({ teamId: 1, tmbId: 1, time: -1 }); + BillSchema.index({ time: 1 }, { expireAfterSeconds: 180 * 24 * 60 * 60 }); } catch (error) { console.log(error); } diff --git a/packages/web/common/file/read/html.ts b/packages/web/common/file/read/html.ts index 04eb911ab..403d4a5d9 100644 --- a/packages/web/common/file/read/html.ts +++ b/packages/web/common/file/read/html.ts @@ -17,5 +17,5 @@ export const readHtmlFile = async ({ uploadImgController }); - return { rawText: rawText }; + return { rawText: simpleMd }; }; diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 15a935dd6..d464f5e67 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -15,15 +15,19 @@ export const iconPaths = { 'common/courseLight': () => import('./icons/common/courseLight.svg'), 'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), + 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), + 'common/folderFill': () => import('./icons/common/folderFill.svg'), 'common/fullScreenLight': () => import('./icons/common/fullScreenLight.svg'), 'common/gitFill': () => import('./icons/common/gitFill.svg'), + 'common/gitInlight': () => import('./icons/common/gitInlight.svg'), 'common/gitLight': () => import('./icons/common/gitLight.svg'), 'common/googleFill': () => import('./icons/common/googleFill.svg'), 'common/importLight': () => import('./icons/common/importLight.svg'), 'common/inviteLight': () => import('./icons/common/inviteLight.svg'), 'common/language/en': () => import('./icons/common/language/en.svg'), 'common/language/zh': () => import('./icons/common/language/zh.svg'), + 'common/linkBlue': () => import('./icons/common/linkBlue.svg'), 'common/loading': () => import('./icons/common/loading.svg'), 'common/navbar/pluginFill': () => import('./icons/common/navbar/pluginFill.svg'), 'common/navbar/pluginLight': () => import('./icons/common/navbar/pluginLight.svg'), @@ -37,10 +41,13 @@ export const iconPaths = { 'common/retryLight': () => import('./icons/common/retryLight.svg'), 'common/rightArrowLight': () => import('./icons/common/rightArrowLight.svg'), 'common/routePushLight': () => import('./icons/common/routePushLight.svg'), + 'common/saveFill': () => import('./icons/common/saveFill.svg'), 'common/searchLight': () => import('./icons/common/searchLight.svg'), + 'common/selectLight': () => import('./icons/common/selectLight.svg'), 'common/settingLight': () => import('./icons/common/settingLight.svg'), 'common/text/t': () => import('./icons/common/text/t.svg'), 'common/tickFill': () => import('./icons/common/tickFill.svg'), + 'common/uploadFileFill': () => import('./icons/common/uploadFileFill.svg'), 'common/viewLight': () => import('./icons/common/viewLight.svg'), 'common/voiceLight': () => import('./icons/common/voiceLight.svg'), copy: () => import('./icons/copy.svg'), @@ -52,7 +59,12 @@ export const iconPaths = { 'core/app/logsLight': () => import('./icons/core/app/logsLight.svg'), 'core/app/markLight': () => import('./icons/core/app/markLight.svg'), 'core/app/questionGuide': () => import('./icons/core/app/questionGuide.svg'), - 'core/app/tts': () => import('./icons/core/app/tts.svg'), + 'core/app/simpleMode/ai': () => import('./icons/core/app/simpleMode/ai.svg'), + 'core/app/simpleMode/chat': () => import('./icons/core/app/simpleMode/chat.svg'), + 'core/app/simpleMode/dataset': () => import('./icons/core/app/simpleMode/dataset.svg'), + 'core/app/simpleMode/template': () => import('./icons/core/app/simpleMode/template.svg'), + 'core/app/simpleMode/tts': () => import('./icons/core/app/simpleMode/tts.svg'), + 'core/app/simpleMode/variable': () => import('./icons/core/app/simpleMode/variable.svg'), 'core/app/ttsFill': () => import('./icons/core/app/ttsFill.svg'), 'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'), 'core/app/variable/select': () => import('./icons/core/app/variable/select.svg'), @@ -76,12 +88,15 @@ export const iconPaths = { 'core/dataset/commonDataset': () => import('./icons/core/dataset/commonDataset.svg'), 'core/dataset/datasetFill': () => import('./icons/core/dataset/datasetFill.svg'), 'core/dataset/datasetLight': () => import('./icons/core/dataset/datasetLight.svg'), - 'core/dataset/folderDataset': () => import('./icons/core/dataset/folderDataset.svg'), + 'core/dataset/fileCollection': () => import('./icons/core/dataset/fileCollection.svg'), 'core/dataset/fullTextRecall': () => import('./icons/core/dataset/fullTextRecall.svg'), + 'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'), 'core/dataset/mixedRecall': () => import('./icons/core/dataset/mixedRecall.svg'), 'core/dataset/modeEmbedding': () => import('./icons/core/dataset/modeEmbedding.svg'), 'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'), + 'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'), 'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'), + 'core/modules/flowLight': () => import('./icons/core/modules/flowLight.svg'), 'core/modules/previewLight': () => import('./icons/core/modules/previewLight.svg'), 'core/modules/variable': () => import('./icons/core/modules/variable.svg'), 'core/modules/welcomeText': () => import('./icons/core/modules/welcomeText.svg'), @@ -91,6 +106,15 @@ export const iconPaths = { empty: () => import('./icons/empty.svg'), export: () => import('./icons/export.svg'), 'file/csv': () => import('./icons/file/csv.svg'), + 'file/fill/csv': () => import('./icons/file/fill/csv.svg'), + 'file/fill/doc': () => import('./icons/file/fill/doc.svg'), + 'file/fill/file': () => import('./icons/file/fill/file.svg'), + 'file/fill/folder': () => import('./icons/file/fill/folder.svg'), + 'file/fill/html': () => import('./icons/file/fill/html.svg'), + 'file/fill/manual': () => import('./icons/file/fill/manual.svg'), + 'file/fill/markdown': () => import('./icons/file/fill/markdown.svg'), + 'file/fill/pdf': () => import('./icons/file/fill/pdf.svg'), + 'file/fill/txt': () => import('./icons/file/fill/txt.svg'), 'file/html': () => import('./icons/file/html.svg'), 'file/indexImport': () => import('./icons/file/indexImport.svg'), 'file/manualImport': () => import('./icons/file/manualImport.svg'), @@ -103,10 +127,14 @@ export const iconPaths = { kbTest: () => import('./icons/kbTest.svg'), menu: () => import('./icons/menu.svg'), minus: () => import('./icons/minus.svg'), + 'modal/edit': () => import('./icons/modal/edit.svg'), + 'modal/manualDataset': () => import('./icons/modal/manualDataset.svg'), + 'modal/selectSource': () => import('./icons/modal/selectSource.svg'), more: () => import('./icons/more.svg'), out: () => import('./icons/out.svg'), 'phoneTabbar/me': () => import('./icons/phoneTabbar/me.svg'), - 'phoneTabbar/more': () => import('./icons/phoneTabbar/more.svg'), + 'phoneTabbar/tool': () => import('./icons/phoneTabbar/tool.svg'), + 'phoneTabbar/toolFill': () => import('./icons/phoneTabbar/toolFill.svg'), save: () => import('./icons/save.svg'), stop: () => import('./icons/stop.svg'), 'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'), diff --git a/packages/web/components/common/Icon/icons/common/closeLight.svg b/packages/web/components/common/Icon/icons/common/closeLight.svg index 9fc9b8537..2f09ccdee 100644 --- a/packages/web/components/common/Icon/icons/common/closeLight.svg +++ b/packages/web/components/common/Icon/icons/common/closeLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/errorFill.svg b/packages/web/components/common/Icon/icons/common/errorFill.svg new file mode 100644 index 000000000..c73b125d9 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/errorFill.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/folderFill.svg b/packages/web/components/common/Icon/icons/common/folderFill.svg new file mode 100644 index 000000000..7e8e49578 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/folderFill.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/gitInlight.svg b/packages/web/components/common/Icon/icons/common/gitInlight.svg new file mode 100644 index 000000000..354a96496 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/gitInlight.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/importLight.svg b/packages/web/components/common/Icon/icons/common/importLight.svg index e62871c3b..8e3f5b4d6 100644 --- a/packages/web/components/common/Icon/icons/common/importLight.svg +++ b/packages/web/components/common/Icon/icons/common/importLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/linkBlue.svg b/packages/web/components/common/Icon/icons/common/linkBlue.svg new file mode 100644 index 000000000..c28f59906 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/linkBlue.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/overviewLight.svg b/packages/web/components/common/Icon/icons/common/overviewLight.svg index 3d107058e..a56fb7fd3 100644 --- a/packages/web/components/common/Icon/icons/common/overviewLight.svg +++ b/packages/web/components/common/Icon/icons/common/overviewLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/questionLight.svg b/packages/web/components/common/Icon/icons/common/questionLight.svg index f6fc6095e..f09e09270 100644 --- a/packages/web/components/common/Icon/icons/common/questionLight.svg +++ b/packages/web/components/common/Icon/icons/common/questionLight.svg @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/saveFill.svg b/packages/web/components/common/Icon/icons/common/saveFill.svg new file mode 100644 index 000000000..12e305f89 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/saveFill.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/selectLight.svg b/packages/web/components/common/Icon/icons/common/selectLight.svg new file mode 100644 index 000000000..f8ed98e79 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/selectLight.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/settingLight.svg b/packages/web/components/common/Icon/icons/common/settingLight.svg index a4de19777..c74dd2022 100644 --- a/packages/web/components/common/Icon/icons/common/settingLight.svg +++ b/packages/web/components/common/Icon/icons/common/settingLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/uploadFileFill.svg b/packages/web/components/common/Icon/icons/common/uploadFileFill.svg new file mode 100644 index 000000000..f10644c29 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/uploadFileFill.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/viewLight.svg b/packages/web/components/common/Icon/icons/common/viewLight.svg index 29f97bbc0..2ed165b24 100644 --- a/packages/web/components/common/Icon/icons/common/viewLight.svg +++ b/packages/web/components/common/Icon/icons/common/viewLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/logsLight.svg b/packages/web/components/common/Icon/icons/core/app/logsLight.svg index bcb7caaa1..00496ba98 100644 --- a/packages/web/components/common/Icon/icons/core/app/logsLight.svg +++ b/packages/web/components/common/Icon/icons/core/app/logsLight.svg @@ -1,14 +1,4 @@ - - - - - + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/questionGuide.svg b/packages/web/components/common/Icon/icons/core/app/questionGuide.svg index 4903f93dc..fba55289b 100644 --- a/packages/web/components/common/Icon/icons/core/app/questionGuide.svg +++ b/packages/web/components/common/Icon/icons/core/app/questionGuide.svg @@ -1,5 +1,5 @@ - + + d="M1.66675 7.97028C1.66675 6.0661 1.66675 5.11401 2.03733 4.38671C2.3633 3.74695 2.88343 3.22682 3.52318 2.90085C4.25048 2.53027 5.20257 2.53027 7.10675 2.53027H12.8934C14.7976 2.53027 15.7497 2.53027 16.477 2.90085C17.1167 3.22682 17.6369 3.74695 17.9628 4.38671C18.3334 5.11401 18.3334 6.06609 18.3334 7.97027V10.9413C18.3334 12.8455 18.3334 13.7976 17.9628 14.5249C17.6369 15.1647 17.1167 15.6848 16.477 16.0108C15.7497 16.3814 14.7976 16.3814 12.8934 16.3814H10.0744L7.42003 18.5835C6.99384 18.937 6.34598 18.7611 6.15717 18.2405L5.4777 16.3672C4.58619 16.3392 4.0053 16.2564 3.52318 16.0108C2.88343 15.6848 2.3633 15.1647 2.03733 14.5249C1.66675 13.7976 1.66675 12.8455 1.66675 10.9414V7.97028ZM6.38847 9.48176C6.98678 9.48176 7.4718 8.99673 7.4718 8.39843C7.4718 7.80012 6.98678 7.31509 6.38847 7.31509C5.79016 7.31509 5.30514 7.80012 5.30514 8.39843C5.30514 8.99673 5.79016 9.48176 6.38847 9.48176ZM14.695 8.39843C14.695 8.99673 14.21 9.48176 13.6117 9.48176C13.0134 9.48176 12.5284 8.99673 12.5284 8.39843C12.5284 7.80012 13.0134 7.31509 13.6117 7.31509C14.21 7.31509 14.695 7.80012 14.695 8.39843ZM12.7802 11.6057C13.0281 11.3091 12.9887 10.8677 12.692 10.6197C12.3954 10.3718 11.954 10.4112 11.706 10.7078C11.3434 11.1416 10.7759 11.4343 10.0383 11.4343C9.29561 11.4343 8.59499 11.0648 8.29232 10.7057C8.04317 10.4101 7.60156 10.3724 7.30595 10.6216C7.01034 10.8707 6.97268 11.3123 7.22183 11.6079C7.80834 12.3038 8.91134 12.8343 10.0383 12.8343C11.1704 12.8343 12.1389 12.3729 12.7802 11.6057Z" + fill="#00A9A6" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg new file mode 100644 index 000000000..f06e1a3cc --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg new file mode 100644 index 000000000..0a14859f0 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg new file mode 100644 index 000000000..353761b43 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg new file mode 100644 index 000000000..d47abbdbd --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg new file mode 100644 index 000000000..4b9be223c --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg new file mode 100644 index 000000000..a4b5a181d --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/tts.svg b/packages/web/components/common/Icon/icons/core/app/tts.svg deleted file mode 100644 index 2679946e8..000000000 --- a/packages/web/components/common/Icon/icons/core/app/tts.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg index dc84253cb..0e81dfad8 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg @@ -1,4 +1,4 @@ - + + d="M18.0527 6.60938C17.6133 5.56055 16.9805 4.61914 16.1758 3.81055C15.3711 3.00391 14.4316 2.36914 13.3828 1.92969C12.3105 1.47852 11.1738 1.25 9.99999 1.25H9.96093C8.77928 1.25586 7.63671 1.49023 6.56053 1.95117C5.52147 2.39648 4.58983 3.0293 3.79296 3.83594C2.99608 4.64258 2.37108 5.58008 1.93749 6.625C1.49022 7.70703 1.26366 8.85742 1.26952 10.041C1.27538 11.3965 1.5996 12.7422 2.20507 13.9453V16.9141C2.20507 17.4102 2.60741 17.8125 3.10155 17.8125H6.06639C7.26757 18.4199 8.60936 18.7441 9.96288 18.75H10.0039C11.1719 18.75 12.3027 18.5234 13.3691 18.0801C14.4121 17.6445 15.3496 17.0195 16.1523 16.2207C16.957 15.4219 17.5898 14.4883 18.0332 13.4473C18.4922 12.3691 18.7265 11.2227 18.7324 10.0391C18.7363 8.84961 18.5058 7.69531 18.0527 6.60938ZM6.10155 10.9375C5.58593 10.9375 5.166 10.5176 5.166 10C5.166 9.48242 5.58593 9.0625 6.10155 9.0625C6.61718 9.0625 7.0371 9.48242 7.0371 10C7.0371 10.5176 6.61913 10.9375 6.10155 10.9375ZM9.99999 10.9375C9.48436 10.9375 9.06444 10.5176 9.06444 10C9.06444 9.48242 9.48436 9.0625 9.99999 9.0625C10.5156 9.0625 10.9355 9.48242 10.9355 10C10.9355 10.5176 10.5156 10.9375 9.99999 10.9375ZM13.8984 10.9375C13.3828 10.9375 12.9629 10.5176 12.9629 10C12.9629 9.48242 13.3828 9.0625 13.8984 9.0625C14.414 9.0625 14.834 9.48242 14.834 10C14.834 10.5176 14.414 10.9375 13.8984 10.9375Z" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg new file mode 100644 index 000000000..8d6950d62 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg deleted file mode 100644 index 602393396..000000000 --- a/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg new file mode 100644 index 000000000..98f100724 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg new file mode 100644 index 000000000..7fa919420 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg @@ -0,0 +1,11 @@ + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/modules/flowLight.svg b/packages/web/components/common/Icon/icons/core/modules/flowLight.svg new file mode 100644 index 000000000..3394ba30d --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/modules/flowLight.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/edit.svg b/packages/web/components/common/Icon/icons/edit.svg index f2a5c97ea..1fff0787e 100644 --- a/packages/web/components/common/Icon/icons/edit.svg +++ b/packages/web/components/common/Icon/icons/edit.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/export.svg b/packages/web/components/common/Icon/icons/export.svg index cf4cb222e..5570c2de7 100644 --- a/packages/web/components/common/Icon/icons/export.svg +++ b/packages/web/components/common/Icon/icons/export.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/csv.svg b/packages/web/components/common/Icon/icons/file/fill/csv.svg similarity index 100% rename from projects/app/public/imgs/files/csv.svg rename to packages/web/components/common/Icon/icons/file/fill/csv.svg diff --git a/projects/app/public/imgs/files/doc.svg b/packages/web/components/common/Icon/icons/file/fill/doc.svg similarity index 100% rename from projects/app/public/imgs/files/doc.svg rename to packages/web/components/common/Icon/icons/file/fill/doc.svg diff --git a/projects/app/public/imgs/files/file.svg b/packages/web/components/common/Icon/icons/file/fill/file.svg similarity index 100% rename from projects/app/public/imgs/files/file.svg rename to packages/web/components/common/Icon/icons/file/fill/file.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/folder.svg b/packages/web/components/common/Icon/icons/file/fill/folder.svg new file mode 100644 index 000000000..7e8e49578 --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/folder.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/html.svg b/packages/web/components/common/Icon/icons/file/fill/html.svg similarity index 100% rename from projects/app/public/imgs/files/html.svg rename to packages/web/components/common/Icon/icons/file/fill/html.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/manual.svg b/packages/web/components/common/Icon/icons/file/fill/manual.svg new file mode 100644 index 000000000..e5a941b52 --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/manual.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/markdown.svg b/packages/web/components/common/Icon/icons/file/fill/markdown.svg similarity index 100% rename from projects/app/public/imgs/files/markdown.svg rename to packages/web/components/common/Icon/icons/file/fill/markdown.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/pdf.svg b/packages/web/components/common/Icon/icons/file/fill/pdf.svg new file mode 100644 index 000000000..80bb8ac0f --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/pdf.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/txt.svg b/packages/web/components/common/Icon/icons/file/fill/txt.svg similarity index 100% rename from projects/app/public/imgs/files/txt.svg rename to packages/web/components/common/Icon/icons/file/fill/txt.svg diff --git a/packages/web/components/common/Icon/icons/file/markdown.svg b/packages/web/components/common/Icon/icons/file/markdown.svg index bc4b63601..bc6c862f1 100644 --- a/packages/web/components/common/Icon/icons/file/markdown.svg +++ b/packages/web/components/common/Icon/icons/file/markdown.svg @@ -1 +1,11 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/file/pdf.svg b/packages/web/components/common/Icon/icons/file/pdf.svg index 3767414c6..c89b1326f 100644 --- a/packages/web/components/common/Icon/icons/file/pdf.svg +++ b/packages/web/components/common/Icon/icons/file/pdf.svg @@ -1 +1,12 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/kbTest.svg b/packages/web/components/common/Icon/icons/kbTest.svg index e4e4fa901..f57202895 100644 --- a/packages/web/components/common/Icon/icons/kbTest.svg +++ b/packages/web/components/common/Icon/icons/kbTest.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/edit.svg b/packages/web/components/common/Icon/icons/modal/edit.svg new file mode 100644 index 000000000..284100b4f --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/edit.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/manualDataset.svg b/packages/web/components/common/Icon/icons/modal/manualDataset.svg new file mode 100644 index 000000000..5f72aafd5 --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/manualDataset.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/selectSource.svg b/packages/web/components/common/Icon/icons/modal/selectSource.svg new file mode 100644 index 000000000..43d81ac75 --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/selectSource.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/more.svg b/packages/web/components/common/Icon/icons/phoneTabbar/more.svg deleted file mode 100644 index 19608a0f9..000000000 --- a/packages/web/components/common/Icon/icons/phoneTabbar/more.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg new file mode 100644 index 000000000..b77170d86 --- /dev/null +++ b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg b/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg new file mode 100644 index 000000000..e7cfcad60 --- /dev/null +++ b/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg b/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg index 4ab652201..f9c852323 100644 --- a/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg +++ b/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg @@ -1,6 +1,4 @@ - - + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/permission/privateLight.svg b/packages/web/components/common/Icon/icons/support/permission/privateLight.svg index a72b6f44a..12be6e9de 100644 --- a/packages/web/components/common/Icon/icons/support/permission/privateLight.svg +++ b/packages/web/components/common/Icon/icons/support/permission/privateLight.svg @@ -1,8 +1,4 @@ - - - + + \ No newline at end of file diff --git a/packages/web/components/common/MyModal/index.tsx b/packages/web/components/common/MyModal/index.tsx new file mode 100644 index 000000000..1549965f3 --- /dev/null +++ b/packages/web/components/common/MyModal/index.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { + Modal, + ModalOverlay, + ModalContent, + ModalHeader, + ModalCloseButton, + ModalContentProps, + Box, + Image +} from '@chakra-ui/react'; +import MyIcon from '../Icon'; + +export interface MyModalProps extends ModalContentProps { + iconSrc?: string; + title?: any; + isCentered?: boolean; + isOpen: boolean; + onClose?: () => void; + isPc?: boolean; +} + +const CustomModal = ({ + isOpen, + onClose, + iconSrc, + title, + children, + isCentered, + w = 'auto', + maxW = ['90vw', '600px'], + ...props +}: MyModalProps) => { + return ( + onClose && onClose()} + autoFocus={false} + isCentered={isCentered} + > + + + {!title && onClose && } + {!!title && ( + + {iconSrc && ( + <> + {iconSrc.startsWith('/') ? ( + + ) : ( + + )} + + )} + {title} + + {onClose && ( + + )} + + )} + + + {children} + + + + ); +}; + +export default CustomModal; diff --git a/packages/web/components/common/MyTooltip/index.tsx b/packages/web/components/common/MyTooltip/index.tsx new file mode 100644 index 000000000..90f0ee128 --- /dev/null +++ b/packages/web/components/common/MyTooltip/index.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Tooltip, TooltipProps } from '@chakra-ui/react'; + +interface Props extends TooltipProps { + forceShow?: boolean; +} + +const MyTooltip = ({ children, shouldWrapChildren = true, ...props }: Props) => { + return ( + + {children} + + ); +}; + +export default MyTooltip; diff --git a/packages/web/components/common/Radio/LeftRadio.tsx b/packages/web/components/common/Radio/LeftRadio.tsx new file mode 100644 index 000000000..6d97ba84c --- /dev/null +++ b/packages/web/components/common/Radio/LeftRadio.tsx @@ -0,0 +1,112 @@ +import React from 'react'; +import { Box, Flex, useTheme, Grid, type GridProps } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyTooltip from '../MyTooltip'; + +// @ts-ignore +interface Props extends GridProps { + list: { + title: string; + desc?: string; + value: any; + children?: React.ReactNode; + tooltip?: string; + }[]; + align?: 'flex-top' | 'center'; + value: any; + defaultBg?: string; + activeBg?: string; + onChange: (e: any) => void; +} + +const LeftRadio = ({ + list, + value, + align = 'flex-top', + px = 3, + py = 4, + defaultBg = 'myGray.50', + activeBg = 'primary.50', + onChange, + ...props +}: Props) => { + const { t } = useTranslation(); + const theme = useTheme(); + + return ( + + {list.map((item) => ( + + onChange(item.value)} + > + + + + + + + + {typeof item.title === 'string' ? t(item.title) : item.title} + + {!!item.desc && ( + + {t(item.desc)} + + )} + {item?.children} + + + + ))} + + ); +}; + +export default LeftRadio; diff --git a/packages/web/components/common/Tabs/RowTabs.tsx b/packages/web/components/common/Tabs/RowTabs.tsx new file mode 100644 index 000000000..339e534d2 --- /dev/null +++ b/packages/web/components/common/Tabs/RowTabs.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import { Flex, Box } from '@chakra-ui/react'; +import MyIcon from '../Icon'; + +type Props = { + list: { + icon?: string; + label: string | React.ReactNode; + value: string; + }[]; + value: string; + onChange: (e: string) => void; +}; + +const RowTabs = ({ list, value, onChange }: Props) => { + return ( + + {list.map((item) => ( + onChange(item.value) + })} + > + {item.icon && } + {item.label} + + ))} + + ); +}; + +export default RowTabs; diff --git a/packages/web/components/common/Textarea/PromptEditor/Editor.tsx b/packages/web/components/common/Textarea/PromptEditor/Editor.tsx new file mode 100644 index 000000000..3b5ee9063 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/Editor.tsx @@ -0,0 +1,135 @@ +import { useState, useRef } from 'react'; +import { LexicalComposer } from '@lexical/react/LexicalComposer'; +import { PlainTextPlugin } from '@lexical/react/LexicalPlainTextPlugin'; +import { ContentEditable } from '@lexical/react/LexicalContentEditable'; +import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin'; +import { OnChangePlugin } from '@lexical/react/LexicalOnChangePlugin'; +import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary'; +import VariablePickerPlugin from './plugins/VariablePickerPlugin'; +import { Box } from '@chakra-ui/react'; +import styles from './index.module.scss'; +import VariablePlugin from './plugins/VariablePlugin'; +import { VariableNode } from './plugins/VariablePlugin/node'; +import { EditorState, LexicalEditor } from 'lexical'; +import { textToEditorState } from './utils'; +import OnBlurPlugin from './plugins/OnBlurPlugin'; +import MyIcon from '../../Icon'; +import { PickerMenuItemType } from './type.d'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; + +export default function Editor({ + h = 200, + showResize = true, + showOpenModal = true, + onOpenModal, + variables, + onChange, + onBlur, + defaultValue, + placeholder = '' +}: { + h?: number; + showResize?: boolean; + showOpenModal?: boolean; + onOpenModal?: () => void; + variables: PickerMenuItemType[]; + onChange?: (editorState: EditorState) => void; + onBlur?: (editor: LexicalEditor) => void; + defaultValue?: string; + placeholder?: string; +}) { + const key = useRef(getNanoid(6)); + const [height, setHeight] = useState(h); + const [initialConfig, setInitialConfig] = useState({ + namespace: 'promptEditor', + nodes: [VariableNode], + editorState: textToEditorState(defaultValue), + onError: (error: Error) => { + throw error; + } + }); + const initialY = useRef(0); + + const handleMouseDown = (e: React.MouseEvent) => { + initialY.current = e.clientY; + + const handleMouseMove = (e: MouseEvent) => { + const deltaY = e.clientY - initialY.current; + setHeight((prevHeight) => (prevHeight + deltaY < h * 0.5 ? h * 0.5 : prevHeight + deltaY)); + initialY.current = e.clientY; + }; + + const handleMouseUp = () => { + document.removeEventListener('mousemove', handleMouseMove); + document.removeEventListener('mouseup', handleMouseUp); + }; + + document.addEventListener('mousemove', handleMouseMove); + document.addEventListener('mouseup', handleMouseUp); + }; + + return ( + + + } + placeholder={ + + + {placeholder} + + + } + ErrorBoundary={LexicalErrorBoundary} + /> + + onChange?.(e)} /> + + + + + {showResize && ( + + + + )} + {showOpenModal && ( + + + + )} + + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/index.module.scss b/packages/web/components/common/Textarea/PromptEditor/index.module.scss new file mode 100644 index 000000000..7cba8bdc5 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/index.module.scss @@ -0,0 +1,24 @@ +.contentEditable { + position: relative; + height: 100%; + width: 100%; + border: 1px solid var(--chakra-colors-borderColor-base); + border-radius: var(--chakra-radii-md); + padding: 8px 12px; + background: var(--chakra-colors-gray-50); + font-size: 13px; + overflow-y: auto; +} + +.contentEditable:focus { + outline: none; + border: 1px solid; + border-color: var(--chakra-colors-primary-600); + background: #fff; + box-shadow: 0px 0px 0px 2.4px rgba(51, 112, 255, 0.15); +} + +.variable { + color: var(--chakra-colors-primary-600); + padding: 0 2px; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/index.tsx b/packages/web/components/common/Textarea/PromptEditor/index.tsx new file mode 100644 index 000000000..a13d297bd --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/index.tsx @@ -0,0 +1,83 @@ +import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'; +import React from 'react'; +import { editorStateToText } from './utils'; +import Editor from './Editor'; +import MyModal from '../../MyModal'; +import { useTranslation } from 'next-i18next'; +import { $getRoot, EditorState, type LexicalEditor } from 'lexical'; +import { PickerMenuItemType } from './type.d'; +import { useCallback, useTransition } from 'react'; + +const PromptEditor = ({ + showOpenModal = true, + variables = [], + defaultValue, + onChange, + onBlur, + h, + placeholder, + title +}: { + showOpenModal?: boolean; + variables?: PickerMenuItemType[]; + defaultValue?: string; + onChange?: (text: string) => void; + onBlur?: (text: string) => void; + h?: number; + placeholder?: string; + title?: string; +}) => { + const { isOpen, onOpen, onClose } = useDisclosure(); + + const [, startSts] = useTransition(); + + const { t } = useTranslation(); + + const onChangeInput = useCallback((editorState: EditorState) => { + const text = editorState.read(() => $getRoot().getTextContent()); + const formatValue = text.replaceAll('\n\n', '\n').replaceAll('}}{{', '}} {{'); + onChange?.(formatValue); + }, []); + const onBlurInput = useCallback((editor: LexicalEditor) => { + startSts(() => { + const text = editorStateToText(editor).replaceAll('\n\n', '\n').replaceAll('}}{{', '}} {{'); + onBlur?.(text); + }); + }, []); + + return ( + <> + + + + + + + + + + + ); +}; +export default React.memo(PromptEditor); diff --git a/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx b/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx new file mode 100644 index 000000000..9b85f33e0 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx @@ -0,0 +1,96 @@ +import { Box, Button, Image } from '@chakra-ui/react'; + +export default function ComfirmVar({ + newVariables, + onCancel, + onConfirm +}: { + newVariables: string[]; + onCancel: () => void; + onConfirm: () => void; +}) { + return ( + <> + + + + + + + 引用了未定义的变量，是否自动添加？ + + + {newVariables.map((item, index) => ( + + + {`{{`} + {item} + {`}}`} + + + ))} + + + + + + + + + + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx new file mode 100644 index 000000000..e5bcc2594 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx @@ -0,0 +1,24 @@ +import { useEffect } from 'react'; +import { BLUR_COMMAND, COMMAND_PRIORITY_EDITOR, LexicalEditor } from 'lexical'; +import { mergeRegister } from '@lexical/utils'; +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; + +export default function OnBlurPlugin({ onBlur }: { onBlur?: (editor: LexicalEditor) => void }) { + const [editor] = useLexicalComposerContext(); + + useEffect(() => { + return mergeRegister( + editor.registerCommand( + BLUR_COMMAND, + () => { + if (onBlur) onBlur(editor); + + return true; + }, + COMMAND_PRIORITY_EDITOR + ) + ); + }, [editor, onBlur]); + + return null; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx new file mode 100644 index 000000000..8ab9ce975 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx @@ -0,0 +1,143 @@ +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; +import { LexicalTypeaheadMenuPlugin } from '@lexical/react/LexicalTypeaheadMenuPlugin'; +import { $createTextNode, $getSelection, $isRangeSelection, TextNode } from 'lexical'; +import * as React from 'react'; +import { useCallback, useMemo, useState } from 'react'; +import * as ReactDOM from 'react-dom'; +import { VariableInputEnum } from '@fastgpt/global/core/module/constants'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '../../../../Icon'; +import { Box, Flex } from '@chakra-ui/react'; +import { useBasicTypeaheadTriggerMatch } from '../../utils'; +import { PickerMenuItemType } from '../../type.d'; + +export default function VariablePickerPlugin({ variables }: { variables: PickerMenuItemType[] }) { + const [editor] = useLexicalComposerContext(); + const [queryString, setQueryString] = useState(null); + const { t } = useTranslation(); + + const checkForTriggerMatch = useBasicTypeaheadTriggerMatch('{', { + minLength: 0 + }); + + const VariableTypeList = useMemo( + () => [ + { + title: t('core.module.variable.input type'), + icon: 'core/app/variable/input', + value: VariableInputEnum.input + }, + { + title: t('core.module.variable.textarea type'), + icon: 'core/app/variable/textarea', + value: VariableInputEnum.textarea + }, + { + title: t('core.module.variable.select type'), + icon: 'core/app/variable/select', + value: VariableInputEnum.select + } + ], + [t] + ); + + // const options: Array = useMemo(() => { + // // const newVariableOption = { + // // label: t('common.Add New') + "变量", + // // key: 'new_variable', + // // icon: 'core/modules/variable' + // // }; + // return [ + // ...variables.map((item) => ({ + // ...item, + // icon: VariableTypeList.find((type) => type.value === item.type)?.icon + // })) + // // newVariableOption + // ]; + // }, [VariableTypeList, t, variables]); + + const onSelectOption = useCallback( + (selectedOption: any, nodeToRemove: TextNode | null, closeMenu: () => void) => { + editor.update(() => { + const selection = $getSelection(); + if (!$isRangeSelection(selection) || selectedOption == null) { + return; + } + if (nodeToRemove) { + nodeToRemove.remove(); + } + selection.insertNodes([$createTextNode(`{{${selectedOption.key}}}`)]); + closeMenu(); + }); + }, + [editor] + ); + + return ( + { + if (anchorElementRef.current == null) { + return null; + } + return anchorElementRef.current && variables.length + ? ReactDOM.createPortal( + + {variables.map((item, index) => ( + { + setHighlightedIndex(index); + selectOptionAndCleanUp(item); + }} + onMouseEnter={() => { + setHighlightedIndex(index); + }} + > + + {`${item.key}(${item.label})`} + + ))} + , + anchorElementRef.current + ) + : null; + }} + /> + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx new file mode 100644 index 000000000..3927c7bc7 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx @@ -0,0 +1,49 @@ +import { TextNode } from 'lexical'; +import { mergeRegister } from '@lexical/utils'; +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; +import { useCallback, useEffect, useMemo } from 'react'; + +import { getHashtagRegexString } from './utils'; +import { registerLexicalTextEntity } from '../../utils'; +import { $createVariableNode, VariableNode } from './node'; +import { PickerMenuItemType } from '../../type'; + +const REGEX = new RegExp(getHashtagRegexString(), 'i'); + +export default function VariablePlugin({ variables }: { variables: PickerMenuItemType[] }) { + const [editor] = useLexicalComposerContext(); + useEffect(() => { + if (!editor.hasNodes([VariableNode])) + throw new Error('VariablePlugin: VariableNode not registered on editor'); + }, [editor]); + + const variableKeys: Array = useMemo(() => { + return variables.map((item) => item.key); + }, [variables]); + + const createVariableNode = useCallback((textNode: TextNode): VariableNode => { + return $createVariableNode(textNode.getTextContent()); + }, []); + + const getVariableMatch = useCallback((text: string) => { + const matches = REGEX.exec(text); + if (!matches) return null; + + if (variableKeys.indexOf(matches[3]) === -1) return null; + const hashtagLength = matches[3].length + 4; + const startOffset = matches.index; + const endOffset = startOffset + hashtagLength; + return { + end: endOffset, + start: startOffset + }; + }, []); + + useEffect(() => { + mergeRegister( + ...registerLexicalTextEntity(editor, getVariableMatch, VariableNode, createVariableNode) + ); + }, [createVariableNode, editor, getVariableMatch]); + + return null; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts new file mode 100644 index 000000000..536f25b90 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts @@ -0,0 +1,49 @@ +import type { NodeKey, EditorConfig, LexicalNode, SerializedTextNode } from 'lexical'; + +import { TextNode, $applyNodeReplacement } from 'lexical'; +import { addClassNamesToElement } from '@lexical/utils'; +import styles from '../../index.module.scss'; + +export class VariableNode extends TextNode { + static getType(): string { + return 'variable'; + } + + static clone(node: VariableNode): VariableNode { + return new VariableNode(node.__text, node.__key); + } + + constructor(text: string, key?: NodeKey) { + super(text, key); + } + + createDOM(config: EditorConfig): HTMLElement { + const element = super.createDOM(config); + addClassNamesToElement(element, styles.variable); + return element; + } + + static importJSON(serializedNode: SerializedTextNode): TextNode { + const node = $createVariableNode(serializedNode.text); + node.setFormat(serializedNode.format); + node.setDetail(serializedNode.detail); + node.setMode(serializedNode.mode); + node.setStyle(serializedNode.style); + return node; + } + + exportJSON(): SerializedTextNode { + return { + ...super.exportJSON(), + type: 'variable' + }; + } +} + +export function $createVariableNode(text: string): VariableNode { + return $applyNodeReplacement(new VariableNode(text)); +} + +export function $isVariableNode(node: LexicalNode | null | undefined): node is VariableNode { + return node instanceof VariableNode; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts new file mode 100644 index 000000000..5ec8230ad --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts @@ -0,0 +1,223 @@ +function getHashtagRegexStringChars(): Readonly<{ + alpha: string; + alphanumeric: string; + leftChars: string; + rightChars: string; +}> { + // Latin accented characters + // Excludes 0xd7 from the range + // (the multiplication sign, confusable with "x"). + // Also excludes 0xf7, the division sign + const latinAccents = + '\xC0-\xD6' + + '\xD8-\xF6' + + '\xF8-\xFF' + + '\u0100-\u024F' + + '\u0253-\u0254' + + '\u0256-\u0257' + + '\u0259' + + '\u025B' + + '\u0263' + + '\u0268' + + '\u026F' + + '\u0272' + + '\u0289' + + '\u028B' + + '\u02BB' + + '\u0300-\u036F' + + '\u1E00-\u1EFF'; + + // Cyrillic (Russian, Ukrainian, etc.) + const nonLatinChars = + '\u0400-\u04FF' + // Cyrillic + '\u0500-\u0527' + // Cyrillic Supplement + '\u2DE0-\u2DFF' + // Cyrillic Extended A + '\uA640-\uA69F' + // Cyrillic Extended B + '\u0591-\u05BF' + // Hebrew + '\u05C1-\u05C2' + + '\u05C4-\u05C5' + + '\u05C7' + + '\u05D0-\u05EA' + + '\u05F0-\u05F4' + + '\uFB12-\uFB28' + // Hebrew Presentation Forms + '\uFB2A-\uFB36' + + '\uFB38-\uFB3C' + + '\uFB3E' + + '\uFB40-\uFB41' + + '\uFB43-\uFB44' + + '\uFB46-\uFB4F' + + '\u0610-\u061A' + // Arabic + '\u0620-\u065F' + + '\u066E-\u06D3' + + '\u06D5-\u06DC' + + '\u06DE-\u06E8' + + '\u06EA-\u06EF' + + '\u06FA-\u06FC' + + '\u06FF' + + '\u0750-\u077F' + // Arabic Supplement + '\u08A0' + // Arabic Extended A + '\u08A2-\u08AC' + + '\u08E4-\u08FE' + + '\uFB50-\uFBB1' + // Arabic Pres. Forms A + '\uFBD3-\uFD3D' + + '\uFD50-\uFD8F' + + '\uFD92-\uFDC7' + + '\uFDF0-\uFDFB' + + '\uFE70-\uFE74' + // Arabic Pres. Forms B + '\uFE76-\uFEFC' + + '\u200C-\u200C' + // Zero-Width Non-Joiner + '\u0E01-\u0E3A' + // Thai + '\u0E40-\u0E4E' + // Hangul (Korean) + '\u1100-\u11FF' + // Hangul Jamo + '\u3130-\u3185' + // Hangul Compatibility Jamo + '\uA960-\uA97F' + // Hangul Jamo Extended-A + '\uAC00-\uD7AF' + // Hangul Syllables + '\uD7B0-\uD7FF' + // Hangul Jamo Extended-B + '\uFFA1-\uFFDC'; // Half-width Hangul + + const charCode = String.fromCharCode; + + const cjkChars = + '\u30A1-\u30FA\u30FC-\u30FE' + // Katakana (full-width) + '\uFF66-\uFF9F' + // Katakana (half-width) + '\uFF10-\uFF19\uFF21-\uFF3A' + + '\uFF41-\uFF5A' + // Latin (full-width) + '\u3041-\u3096\u3099-\u309E' + // Hiragana + '\u3400-\u4DBF' + // Kanji (CJK Extension A) + `\u4E00-\u9FFF${ + // Kanji (Unified) + // Disabled as it breaks the Regex. + // charCode(0x20000) + '-' + charCode(0x2A6DF) + // Kanji (CJK Extension B) + charCode(0x2a700) + }-${ + charCode(0x2b73f) // Kanji (CJK Extension C) + }${charCode(0x2b740)}-${ + charCode(0x2b81f) // Kanji (CJK Extension D) + }${charCode(0x2f800)}-${charCode(0x2fa1f)}\u3003\u3005\u303B`; // Kanji (CJK supplement) + + const otherChars = latinAccents + nonLatinChars + cjkChars; + // equivalent of \p{L} + + const unicodeLetters = + '\u0041-\u005A\u0061-\u007A\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6' + + '\u00F8-\u0241\u0250-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u037A\u0386' + + '\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481' + + '\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587' + + '\u05D0-\u05EA\u05F0-\u05F2\u0621-\u063A\u0640-\u064A\u066E-\u066F' + + '\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710' + + '\u0712-\u072F\u074D-\u076D\u0780-\u07A5\u07B1\u0904-\u0939\u093D\u0950' + + '\u0958-\u0961\u097D\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0' + + '\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1' + + '\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33' + + '\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D' + + '\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD' + + '\u0AD0\u0AE0-\u0AE1\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30' + + '\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83' + + '\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F' + + '\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0C05-\u0C0C\u0C0E-\u0C10' + + '\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C60-\u0C61\u0C85-\u0C8C' + + '\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE' + + '\u0CE0-\u0CE1\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39' + + '\u0D60-\u0D61\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6' + + '\u0E01-\u0E30\u0E32-\u0E33\u0E40-\u0E46\u0E81-\u0E82\u0E84\u0E87-\u0E88' + + '\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7' + + '\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB2-\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6' + + '\u0EDC-\u0EDD\u0F00\u0F40-\u0F47\u0F49-\u0F6A\u0F88-\u0F8B\u1000-\u1021' + + '\u1023-\u1027\u1029-\u102A\u1050-\u1055\u10A0-\u10C5\u10D0-\u10FA\u10FC' + + '\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D' + + '\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0' + + '\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310' + + '\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C' + + '\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u1700-\u170C\u170E-\u1711' + + '\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7' + + '\u17DC\u1820-\u1877\u1880-\u18A8\u1900-\u191C\u1950-\u196D\u1970-\u1974' + + '\u1980-\u19A9\u19C1-\u19C7\u1A00-\u1A16\u1D00-\u1DBF\u1E00-\u1E9B' + + '\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D' + + '\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC' + + '\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC' + + '\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u2094\u2102\u2107' + + '\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D' + + '\u212F-\u2131\u2133-\u2139\u213C-\u213F\u2145-\u2149\u2C00-\u2C2E' + + '\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96' + + '\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6' + + '\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005-\u3006\u3031-\u3035' + + '\u303B-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF' + + '\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5' + + '\u4E00-\u9FBB\uA000-\uA48C\uA800-\uA801\uA803-\uA805\uA807-\uA80A' + + '\uA80C-\uA822\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9' + + '\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C' + + '\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F' + + '\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A' + + '\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7' + + '\uFFDA-\uFFDC'; + + // equivalent of \p{Mn}\p{Mc} + const unicodeAccents = + '\u0300-\u036F\u0483-\u0486\u0591-\u05B9\u05BB-\u05BD\u05BF' + + '\u05C1-\u05C2\u05C4-\u05C5\u05C7\u0610-\u0615\u064B-\u065E\u0670' + + '\u06D6-\u06DC\u06DF-\u06E4\u06E7-\u06E8\u06EA-\u06ED\u0711\u0730-\u074A' + + '\u07A6-\u07B0\u0901-\u0903\u093C\u093E-\u094D\u0951-\u0954\u0962-\u0963' + + '\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7-\u09C8\u09CB-\u09CD\u09D7' + + '\u09E2-\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D' + + '\u0A70-\u0A71\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD' + + '\u0AE2-\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B43\u0B47-\u0B48\u0B4B-\u0B4D' + + '\u0B56-\u0B57\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7' + + '\u0C01-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56' + + '\u0C82-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5-\u0CD6' + + '\u0D02-\u0D03\u0D3E-\u0D43\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D82-\u0D83' + + '\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E31\u0E34-\u0E3A' + + '\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB-\u0EBC\u0EC8-\u0ECD\u0F18-\u0F19' + + '\u0F35\u0F37\u0F39\u0F3E-\u0F3F\u0F71-\u0F84\u0F86-\u0F87\u0F90-\u0F97' + + '\u0F99-\u0FBC\u0FC6\u102C-\u1032\u1036-\u1039\u1056-\u1059\u135F' + + '\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17B6-\u17D3\u17DD' + + '\u180B-\u180D\u18A9\u1920-\u192B\u1930-\u193B\u19B0-\u19C0\u19C8-\u19C9' + + '\u1A17-\u1A1B\u1DC0-\u1DC3\u20D0-\u20DC\u20E1\u20E5-\u20EB\u302A-\u302F' + + '\u3099-\u309A\uA802\uA806\uA80B\uA823-\uA827\uFB1E\uFE00-\uFE0F' + + '\uFE20-\uFE23'; + + // equivalent of \p{Dn} + const unicodeDigits = + '\u0030-\u0039\u0660-\u0669\u06F0-\u06F9\u0966-\u096F\u09E6-\u09EF' + + '\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0BE6-\u0BEF\u0C66-\u0C6F' + + '\u0CE6-\u0CEF\u0D66-\u0D6F\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F29' + + '\u1040-\u1049\u17E0-\u17E9\u1810-\u1819\u1946-\u194F\u19D0-\u19D9' + + '\uFF10-\uFF19'; + + // An alpha char is a unicode chars including unicode marks or + // letter or char in otherChars range + const alpha = unicodeLetters; + + // A numeric character is any with the number digit property, or + // underscore. These characters can be included in hashtags, but a hashtag + // cannot have only these characters. + const numeric = `${unicodeDigits}_`; + + // Alphanumeric char is any alpha char or a unicode char with decimal + // number property \p{Nd} + const alphanumeric = alpha + numeric; + const leftChars = '{'; + const rightChars = '}'; + + return { + alpha, + alphanumeric, + leftChars, + rightChars + }; +} + +export function getHashtagRegexString(): string { + const { leftChars, rightChars } = getHashtagRegexStringChars(); + + const hashLeftCharList = `[${leftChars}]`; + const hashRightCharList = `[${rightChars}]`; + + // A hashtag contains characters, numbers and underscores, + // but not all numbers. + const hashtag = + `(${hashLeftCharList})` + + `(${hashLeftCharList})([a-zA-Z0-9_]{0,29}` + + `)(${hashRightCharList})(${hashRightCharList})`; + + return hashtag; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/type.d.ts b/packages/web/components/common/Textarea/PromptEditor/type.d.ts new file mode 100644 index 000000000..b96de6e54 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/type.d.ts @@ -0,0 +1,5 @@ +export type PickerMenuItemType = { + key: string; + label: string; + icon?: string; +}; diff --git a/packages/web/components/common/Textarea/PromptEditor/utils.ts b/packages/web/components/common/Textarea/PromptEditor/utils.ts new file mode 100644 index 000000000..a401ff039 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/utils.ts @@ -0,0 +1,275 @@ +import type { Klass, LexicalEditor, LexicalNode } from 'lexical'; +import type { EntityMatch } from '@lexical/text'; +import { $createTextNode, $getRoot, $isTextNode, TextNode } from 'lexical'; +import { useCallback } from 'react'; + +export function registerLexicalTextEntity( + editor: LexicalEditor, + getMatch: (text: string) => null | EntityMatch, + targetNode: Klass, + createNode: (textNode: TextNode) => T +): Array<() => void> { + const isTargetNode = (node: LexicalNode | null | undefined): node is T => { + return node instanceof targetNode; + }; + + const replaceWithSimpleText = (node: TextNode): void => { + const textNode = $createTextNode(node.getTextContent()); + textNode.setFormat(node.getFormat()); + node.replace(textNode); + }; + + const getMode = (node: TextNode): number => { + return node.getLatest().__mode; + }; + + const textNodeTransform = (node: TextNode) => { + if (!node.isSimpleText()) { + return; + } + + const prevSibling = node.getPreviousSibling(); + let text = node.getTextContent(); + let currentNode = node; + let match; + + if ($isTextNode(prevSibling)) { + const previousText = prevSibling.getTextContent(); + const combinedText = previousText + text; + const prevMatch = getMatch(combinedText); + + if (isTargetNode(prevSibling)) { + if (prevMatch === null || getMode(prevSibling) !== 0) { + replaceWithSimpleText(prevSibling); + + return; + } else { + const diff = prevMatch.end - previousText.length; + + if (diff > 0) { + const concatText = text.slice(0, diff); + const newTextContent = previousText + concatText; + prevSibling.select(); + prevSibling.setTextContent(newTextContent); + + if (diff === text.length) { + node.remove(); + } else { + const remainingText = text.slice(diff); + node.setTextContent(remainingText); + } + + return; + } + } + } else if (prevMatch === null || prevMatch.start < previousText.length) { + return; + } + } + + // eslint-disable-next-line no-constant-condition + while (true) { + match = getMatch(text); + let nextText = match === null ? '' : text.slice(match.end); + text = nextText; + + if (nextText === '') { + const nextSibling = currentNode.getNextSibling(); + + if ($isTextNode(nextSibling)) { + nextText = currentNode.getTextContent() + nextSibling.getTextContent(); + const nextMatch = getMatch(nextText); + + if (nextMatch === null) { + if (isTargetNode(nextSibling)) { + replaceWithSimpleText(nextSibling); + } else { + nextSibling.markDirty(); + } + + return; + } else if (nextMatch.start !== 0) { + return; + } + } + } else { + const nextMatch = getMatch(nextText); + + if (nextMatch !== null && nextMatch.start === 0) { + return; + } + } + + if (match === null) { + return; + } + + if (match.start === 0 && $isTextNode(prevSibling) && prevSibling.isTextEntity()) { + continue; + } + + let nodeToReplace; + + if (match.start === 0) { + [nodeToReplace, currentNode] = currentNode.splitText(match.end); + } else { + [, nodeToReplace, currentNode] = currentNode.splitText(match.start, match.end); + } + + const replacementNode = createNode(nodeToReplace); + replacementNode.setFormat(nodeToReplace.getFormat()); + nodeToReplace.replace(replacementNode); + + if (currentNode == null) { + return; + } + } + }; + + const reverseNodeTransform = (node: T) => { + const text = node.getTextContent(); + const match = getMatch(text); + + if (match === null || match.start !== 0) { + replaceWithSimpleText(node); + + return; + } + + if (text.length > match.end) { + // This will split out the rest of the text as simple text + node.splitText(match.end); + + return; + } + + const prevSibling = node.getPreviousSibling(); + + if ($isTextNode(prevSibling) && prevSibling.isTextEntity()) { + replaceWithSimpleText(prevSibling); + replaceWithSimpleText(node); + } + + const nextSibling = node.getNextSibling(); + + if ($isTextNode(nextSibling) && nextSibling.isTextEntity()) { + replaceWithSimpleText(nextSibling); + + // This may have already been converted in the previous block + if (isTargetNode(node)) { + replaceWithSimpleText(node); + } + } + }; + + const removePlainTextTransform = editor.registerNodeTransform(TextNode, textNodeTransform); + const removeReverseNodeTransform = editor.registerNodeTransform( + targetNode, + reverseNodeTransform + ); + + return [removePlainTextTransform, removeReverseNodeTransform]; +} + +export function textToEditorState(text: string = '') { + const paragraph = text?.split('\n'); + + return JSON.stringify({ + root: { + children: paragraph.map((p) => { + return { + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: p, + type: 'text', + version: 1 + } + ], + direction: 'ltr', + format: '', + indent: 0, + type: 'paragraph', + version: 1 + }; + }), + direction: 'ltr', + format: '', + indent: 0, + type: 'root', + version: 1 + } + }); +} + +export function editorStateToText(editor: LexicalEditor) { + const stringifiedEditorState = JSON.stringify(editor.getEditorState().toJSON()); + const parsedEditorState = editor.parseEditorState(stringifiedEditorState); + const editorStateTextString = parsedEditorState.read(() => $getRoot().getTextContent()); + const compressedText = editorStateTextString.replace(/\n+/g, '\n\n'); + + return compressedText; +} + +const varRegex = /\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}/g; +export const getVars = (value: string) => { + if (!value) return []; + // .filter((item) => { + // return ![CONTEXT_PLACEHOLDER_TEXT, HISTORY_PLACEHOLDER_TEXT, QUERY_PLACEHOLDER_TEXT, PRE_PROMPT_PLACEHOLDER_TEXT].includes(item) + // }) + const keys = + value + .match(varRegex) + ?.map((item) => { + return item.replace('{{', '').replace('}}', ''); + }) + .filter((key) => key.length <= 10) || []; + const keyObj: Record = {}; + // remove duplicate keys + const res: string[] = []; + keys.forEach((key) => { + if (keyObj[key]) return; + + keyObj[key] = true; + res.push(key); + }); + return res; +}; + +export type MenuTextMatch = { + leadOffset: number; + matchingString: string; + replaceableString: string; +}; +export type TriggerFn = (text: string, editor: LexicalEditor) => MenuTextMatch | null; +export const PUNCTUATION = '\\.,\\+\\*\\?\\$\\@\\|#{}\\(\\)\\^\\-\\[\\]\\\\/!%\'"~=<>_:;'; +export function useBasicTypeaheadTriggerMatch( + trigger: string, + { minLength = 1, maxLength = 75 }: { minLength?: number; maxLength?: number } +): TriggerFn { + return useCallback( + (text: string) => { + const validChars = `[^${trigger}${PUNCTUATION}\\s]`; + const TypeaheadTriggerRegex = new RegExp( + `([^${trigger}]|^)(` + `[${trigger}]` + `((?:${validChars}){0,${maxLength}})` + ')$' + ); + const match = TypeaheadTriggerRegex.exec(text); + if (match !== null) { + const maybeLeadingWhitespace = match[1]; + const matchingString = match[3]; + if (matchingString.length >= minLength) { + return { + leadOffset: match.index + maybeLeadingWhitespace.length, + matchingString, + replaceableString: match[2] + }; + } + } + return null; + }, + [maxLength, minLength, trigger] + ); +} diff --git a/packages/web/hooks/useStep.tsx b/packages/web/hooks/useStep.tsx new file mode 100644 index 000000000..2d33c8c24 --- /dev/null +++ b/packages/web/hooks/useStep.tsx @@ -0,0 +1,98 @@ +import { + Box, + Flex, + IconButton, + Step, + StepDescription, + StepIcon, + StepIndicator, + StepNumber, + StepSeparator, + StepStatus, + StepTitle, + Stepper, + css, + useSteps +} from '@chakra-ui/react'; +import React, { useCallback, useState } from 'react'; + +export const useMyStep = ({ + defaultStep = 0, + steps = [] +}: { + defaultStep?: number; + steps: { title?: string; description?: string }[]; +}) => { + const { activeStep, goToNext, goToPrevious } = useSteps({ + index: defaultStep, + count: steps.length + }); + + const MyStep = useCallback( + () => ( + + {steps.map((step, index) => ( + + + } + incomplete={ + + {index + 1} + + } + active={ + + {index + 1} + + } + /> + + + + {step.title} + + + + + ))} + + ), + [steps, activeStep] + ); + + return { + activeStep, + goToNext, + goToPrevious, + MyStep + }; +}; diff --git a/packages/web/package.json b/packages/web/package.json index 119b245f6..3972fdef1 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -21,7 +21,11 @@ "react": "18.2.0", "react-dom": "18.2.0", "react-i18next": "^12.3.1", - "turndown": "^7.1.2" + "turndown": "^7.1.2", + "lexical":"0.12.6", + "@lexical/react": "0.12.6", + "@lexical/utils": "0.12.6", + "@lexical/text": "0.12.6" }, "devDependencies": { "@types/react": "18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eaa3df1c3..c23056652 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,9 +96,6 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2 - mammoth: - specifier: ^1.6.0 - version: registry.npmmirror.com/mammoth@1.6.0 mongoose: specifier: ^7.0.2 version: registry.npmmirror.com/mongoose@7.0.2 @@ -114,9 +111,6 @@ importers: node-cron: specifier: ^3.0.3 version: registry.npmmirror.com/node-cron@3.0.3 - pdfjs-dist: - specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) pg: specifier: ^8.10.0 version: registry.npmmirror.com/pg@8.10.0 @@ -175,6 +169,15 @@ importers: '@fingerprintjs/fingerprintjs': specifier: ^4.2.1 version: registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1 + '@lexical/react': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/react@0.12.6(lexical@0.12.6)(react-dom@18.2.0)(react@18.2.0)(yjs@13.6.10) + '@lexical/text': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) '@monaco-editor/react': specifier: ^4.6.0 version: registry.npmmirror.com/@monaco-editor/react@4.6.0(monaco-editor@0.45.0)(react-dom@18.2.0)(react@18.2.0) @@ -184,6 +187,9 @@ importers: joplin-turndown-plugin-gfm: specifier: ^1.0.12 version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12 + lexical: + specifier: 0.12.6 + version: registry.npmmirror.com/lexical@0.12.6 mammoth: specifier: ^1.6.0 version: registry.npmmirror.com/mammoth@1.6.0 @@ -192,7 +198,7 @@ importers: version: registry.npmmirror.com/next-i18next@13.3.0(i18next@22.5.1)(next@13.5.2)(react-i18next@12.3.1)(react@18.2.0) pdfjs-dist: specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) + version: registry.npmmirror.com/pdfjs-dist@4.0.269 react: specifier: 18.2.0 version: registry.npmmirror.com/react@18.2.0 @@ -3464,6 +3470,7 @@ packages: resolution: {integrity: sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@emotion/memoize/-/memoize-0.7.4.tgz} name: '@emotion/memoize' version: 0.7.4 + requiresBuild: true dev: false optional: true @@ -3920,12 +3927,297 @@ packages: '@jridgewell/resolve-uri': registry.npmmirror.com/@jridgewell/resolve-uri@3.1.1 '@jridgewell/sourcemap-codec': registry.npmmirror.com/@jridgewell/sourcemap-codec@1.4.15 - registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13): + registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-rJFp7tXzawCrMWWRsjCR80dZoIkLJ/EPgPmTk3xqpc+9ntlwbkm3LUOdFmgN+pshnhiZTQBwbFqg/QbsA1Pw9g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/clipboard/-/clipboard-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/clipboard/0.12.6 + name: '@lexical/clipboard' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/html': registry.npmmirror.com/@lexical/html@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-D0IBKLzDFfVqk+3KPlJd2gWIq+h5QOsVn5Atz/Eh2eLRpOakSsZiRjmddsijoLsZbvgo1HObRPQAoeATRPWIzg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/code/-/code-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/code/0.12.6 + name: '@lexical/code' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + prismjs: registry.npmmirror.com/prismjs@1.29.0 + dev: false + + registry.npmmirror.com/@lexical/dragon@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-VKbXzdtF6qizwESx7Zag/VGiYKMAc+xpJF7tcwv5SH8I4bnseoozafzxRG6AE7J9nzGwO74ypKqPmmpP9e20BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/dragon/-/dragon-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/dragon/0.12.6 + name: '@lexical/dragon' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/hashtag@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-SiEId/IBIqUKJJKGg8HSumalfKGxtZQJRkF7Q50FqFSU906V1lcM1jkU7aVw0hiuEHg3H+vFBmNTRdXKyoibsw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/hashtag/-/hashtag-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/hashtag/0.12.6 + name: '@lexical/hashtag' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/history@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-3vvbUF6XHuk/9985IQIXP15g+nr7SlwsPrd2IteOg6aNF+HeE2ttJS5dOiSJLnVZm+AX0OMgejMC1uU2uiZOtA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/history/-/history-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/history/0.12.6 + name: '@lexical/history' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/html@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-HVlJLCkazLbLpxdw0mwMkteQuv6OMkJTlAi6qGJimtuqJLm45BpaQ16PTpUmFWpWeIHL2XpvcDX/lj5fm68XPA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/html/-/html-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/html/0.12.6 + name: '@lexical/html' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-mrFFWR0EZ9liRUzHZqb2ijUDZqkCM+bNsyYqLh4I1CrJpzQtakyIEJt/GzYz4IHmmsRqwcc2zXUP/4kENjjPlQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/link/-/link-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/link/0.12.6 + name: '@lexical/link' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-9DFe8vpSxZ8NQZ/67ZFNiRptB3XPa7mUl0Rmd5WpbJHJHmiORyngYkYgKOW56T/TCtYcLfkTbctMhsIt8OeIqQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/list/-/list-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/list/0.12.6 + name: '@lexical/list' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/mark@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-utk6kgTSTuzmM0+B4imGTGwC4gQRCQ4hHEZTVbkIDbONvjbo9g6xfbTO9g6Qxs2h7Zt0Q2eDA7RG4nwC3vN1KQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/mark/-/mark-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/mark/0.12.6 + name: '@lexical/mark' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/markdown@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-q1cQ4w6KYxUF1N6nGwJTZwn8szLo0kbr8DzI62samZTxeztA0ByMSZLzvO5LSGhgeDremuWx5oa97s2qJMQZFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/markdown/-/markdown-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/markdown/0.12.6 + name: '@lexical/markdown' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/code': registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6) + '@lexical/link': registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/rich-text': registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/text': registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + transitivePeerDependencies: + - '@lexical/clipboard' + - '@lexical/selection' + dev: false + + registry.npmmirror.com/@lexical/offset@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-5NgIaWCvMuOQNf3SZSNn459QfsN7SmLl+Tu4ISqxyZRoMV5Sfojzion9MjCVmt1YSsIS4B29NYQvGQ/li1saOw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/offset/-/offset-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/offset/0.12.6 + name: '@lexical/offset' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/overflow@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-4TZJhTGkn7xvR+rumSYW9U/OIsbith0kVGOvZZf+DM/t9fb0IVQWWSWmMlXJ5XNt/qXLFof3HFyJhK84dsN3NA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/overflow/-/overflow-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/overflow/0.12.6 + name: '@lexical/overflow' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/plain-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-YF+EaWGQIxR1SHgeSuPrrqqSK8RYDxGv9RYyuIPvWXpt3M9NWw7hyAn7zxmXGgv2BhIicyHGPy5CyQgt3Mkb/w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/plain-text/-/plain-text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/plain-text/0.12.6 + name: '@lexical/plain-text' + version: 0.12.6 + peerDependencies: + '@lexical/clipboard': 0.12.6 + '@lexical/selection': 0.12.6 + '@lexical/utils': 0.12.6 + lexical: 0.12.6 + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/react@0.12.6(lexical@0.12.6)(react-dom@18.2.0)(react@18.2.0)(yjs@13.6.10): + resolution: {integrity: sha512-Pto4wsVwrnY94tzcCXP2kWukQejSRPDfwOPd+EFh8dUzj+L7fa9Pze2wVgCRZpEohwfbcgAdEsvmSbhz+yGkog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/react/-/react-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/react/0.12.6 + name: '@lexical/react' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + react: '>=17.x' + react-dom: '>=17.x' + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/code': registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6) + '@lexical/dragon': registry.npmmirror.com/@lexical/dragon@0.12.6(lexical@0.12.6) + '@lexical/hashtag': registry.npmmirror.com/@lexical/hashtag@0.12.6(lexical@0.12.6) + '@lexical/history': registry.npmmirror.com/@lexical/history@0.12.6(lexical@0.12.6) + '@lexical/link': registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/mark': registry.npmmirror.com/@lexical/mark@0.12.6(lexical@0.12.6) + '@lexical/markdown': registry.npmmirror.com/@lexical/markdown@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(lexical@0.12.6) + '@lexical/overflow': registry.npmmirror.com/@lexical/overflow@0.12.6(lexical@0.12.6) + '@lexical/plain-text': registry.npmmirror.com/@lexical/plain-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/rich-text': registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/table': registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6) + '@lexical/text': registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + '@lexical/yjs': registry.npmmirror.com/@lexical/yjs@0.12.6(lexical@0.12.6)(yjs@13.6.10) + lexical: registry.npmmirror.com/lexical@0.12.6 + react: registry.npmmirror.com/react@18.2.0 + react-dom: registry.npmmirror.com/react-dom@18.2.0(react@18.2.0) + react-error-boundary: registry.npmmirror.com/react-error-boundary@3.1.4(react@18.2.0) + transitivePeerDependencies: + - yjs + dev: false + + registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-fRZHy2ug6Pq+pJK7trr9phTGaD2ba3If8o36dphOsl27MtUllpz68lcXL6mUonzJhAu4um1e9u7GFR3dLp+cVA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/rich-text/-/rich-text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/rich-text/0.12.6 + name: '@lexical/rich-text' + version: 0.12.6 + peerDependencies: + '@lexical/clipboard': 0.12.6 + '@lexical/selection': 0.12.6 + '@lexical/utils': 0.12.6 + lexical: 0.12.6 + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-HJBEazVwOe6duyHV6+vB/MS4kUBlCV05Cfcigdx8HlLLFQRWPqHrTpaxKz4jfb9ar0SlI2W1BUNbySAxMkC/HQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/selection/-/selection-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/selection/0.12.6 + name: '@lexical/selection' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-rUh9/fN831T6UpNiPuzx0x6HNi/eQ7W5AQrVBwwzEwkbwAqnE0n28DP924AUbX72UsQNHtywgmDApMoEV7W2iQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/table/-/table-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/table/0.12.6 + name: '@lexical/table' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-WfqfH9gvPAx9Hi9wrJDWECdvt6turPQXImCRI657LVfsP2hHh4eHpcSnd3YYH313pv98HPWmeMstBbEieYwTpQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/text/-/text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/text/0.12.6 + name: '@lexical/text' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-hK5r/TD2nH5TfWSiCxy7/lh0s11qJcI1wo++PBQOR9o937pQ+/Zr/1tMOc8MnrTpl89mtmYtPfWW3f++HH1Yog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/utils/-/utils-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/utils/0.12.6 + name: '@lexical/utils' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/table': registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/yjs@0.12.6(lexical@0.12.6)(yjs@13.6.10): + resolution: {integrity: sha512-I/Yf/Qm8/ydU983kWpFBlDFNFQXLYur5uaAimTSBcJuqHmy3cv1xM7Xrq4BtM+0orKgWJt8vR8cLVIU9sAmzfw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/yjs/-/yjs-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/yjs/0.12.6 + name: '@lexical/yjs' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + yjs: '>=13.5.22' + dependencies: + '@lexical/offset': registry.npmmirror.com/@lexical/offset@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + yjs: registry.npmmirror.com/yjs@13.6.10 + dev: false + + registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11: resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz} - id: registry.npmmirror.com/@mapbox/node-pre-gyp/1.0.11 name: '@mapbox/node-pre-gyp' version: 1.0.11 hasBin: true + requiresBuild: true dependencies: detect-libc: registry.npmmirror.com/detect-libc@2.0.2 https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1 @@ -5391,6 +5683,7 @@ packages: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abbrev/-/abbrev-1.1.1.tgz} name: abbrev version: 1.1.1 + requiresBuild: true dev: false optional: true @@ -5437,6 +5730,7 @@ packages: name: agent-base version: 6.0.2 engines: {node: '>= 6.0.0'} + requiresBuild: true dependencies: debug: registry.npmmirror.com/debug@4.3.4 transitivePeerDependencies: @@ -5539,6 +5833,7 @@ packages: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/aproba/-/aproba-2.0.0.tgz} name: aproba version: 2.0.0 + requiresBuild: true dev: false optional: true @@ -5547,6 +5842,7 @@ packages: name: are-we-there-yet version: 2.0.0 engines: {node: '>=10'} + requiresBuild: true dependencies: delegates: registry.npmmirror.com/delegates@1.0.0 readable-stream: registry.npmmirror.com/readable-stream@3.6.2 @@ -6113,15 +6409,14 @@ packages: name: caniuse-lite version: 1.0.30001574 - registry.npmmirror.com/canvas@2.11.2(encoding@0.1.13): + registry.npmmirror.com/canvas@2.11.2: resolution: {integrity: sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/canvas/-/canvas-2.11.2.tgz} - id: registry.npmmirror.com/canvas/2.11.2 name: canvas version: 2.11.2 engines: {node: '>=6'} requiresBuild: true dependencies: - '@mapbox/node-pre-gyp': registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13) + '@mapbox/node-pre-gyp': registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11 nan: registry.npmmirror.com/nan@2.18.0 simple-get: registry.npmmirror.com/simple-get@3.1.1 transitivePeerDependencies: @@ -6272,6 +6567,7 @@ packages: name: chownr version: 2.0.0 engines: {node: '>=10'} + requiresBuild: true dev: false optional: true @@ -6417,6 +6713,7 @@ packages: name: color-support version: 1.1.3 hasBin: true + requiresBuild: true dev: false optional: true @@ -6513,6 +6810,7 @@ packages: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/console-control-strings/-/console-control-strings-1.1.0.tgz} name: console-control-strings version: 1.1.0 + requiresBuild: true dev: false optional: true @@ -7192,6 +7490,7 @@ packages: name: decompress-response version: 4.2.1 engines: {node: '>=8'} + requiresBuild: true dependencies: mimic-response: registry.npmmirror.com/mimic-response@2.1.0 dev: false @@ -7259,6 +7558,7 @@ packages: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz} name: delegates version: 1.0.0 + requiresBuild: true dev: false optional: true @@ -7296,6 +7596,7 @@ packages: name: detect-libc version: 2.0.2 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -7534,6 +7835,7 @@ packages: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz} name: emoji-regex version: 8.0.0 + requiresBuild: true registry.npmmirror.com/emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/emoji-regex/-/emoji-regex-9.2.2.tgz} @@ -8657,6 +8959,7 @@ packages: name: fs-minipass version: 2.1.0 engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: registry.npmmirror.com/minipass@3.3.6 dev: false @@ -8704,6 +9007,7 @@ packages: name: gauge version: 3.0.2 engines: {node: '>=10'} + requiresBuild: true dependencies: aproba: registry.npmmirror.com/aproba@2.0.0 color-support: registry.npmmirror.com/color-support@1.1.3 @@ -8925,6 +9229,7 @@ packages: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/has-unicode/-/has-unicode-2.0.1.tgz} name: has-unicode version: 2.0.1 + requiresBuild: true dev: false optional: true @@ -9118,6 +9423,7 @@ packages: name: https-proxy-agent version: 5.0.1 engines: {node: '>= 6'} + requiresBuild: true dependencies: agent-base: registry.npmmirror.com/agent-base@6.0.2 debug: registry.npmmirror.com/debug@4.3.4 @@ -9635,6 +9941,12 @@ packages: engines: {node: '>=10'} dev: true + registry.npmmirror.com/isomorphic.js@0.2.5: + resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isomorphic.js/-/isomorphic.js-0.2.5.tgz} + name: isomorphic.js + version: 0.2.5 + dev: false + registry.npmmirror.com/iterator.prototype@1.1.2: resolution: {integrity: sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/iterator.prototype/-/iterator.prototype-1.1.2.tgz} name: iterator.prototype @@ -9884,6 +10196,22 @@ packages: type-check: registry.npmmirror.com/type-check@0.4.0 dev: true + registry.npmmirror.com/lexical@0.12.6: + resolution: {integrity: sha512-Nlfjc+k9cIWpOMv7XufF0Mv09TAXSemNAuAqFLaOwTcN+RvhvYTDtVLSp9D9r+5I097fYs1Vf/UYwH2xEpkFfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lexical/-/lexical-0.12.6.tgz} + name: lexical + version: 0.12.6 + dev: false + + registry.npmmirror.com/lib0@0.2.88: + resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lib0/-/lib0-0.2.88.tgz} + name: lib0 + version: 0.2.88 + engines: {node: '>=16'} + hasBin: true + dependencies: + isomorphic.js: registry.npmmirror.com/isomorphic.js@0.2.5 + dev: false + registry.npmmirror.com/lie@3.3.0: resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lie/-/lie-3.3.0.tgz} name: lie @@ -10120,6 +10448,7 @@ packages: name: make-dir version: 3.1.0 engines: {node: '>=8'} + requiresBuild: true dependencies: semver: registry.npmmirror.com/semver@6.3.1 dev: false @@ -10369,6 +10698,7 @@ packages: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/memory-pager/-/memory-pager-1.5.0.tgz} name: memory-pager version: 1.5.0 + requiresBuild: true dev: false optional: true @@ -10804,6 +11134,7 @@ packages: name: mimic-response version: 2.1.0 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -10836,6 +11167,7 @@ packages: name: minipass version: 3.3.6 engines: {node: '>=8'} + requiresBuild: true dependencies: yallist: registry.npmmirror.com/yallist@4.0.0 dev: false @@ -10846,6 +11178,7 @@ packages: name: minipass version: 5.0.0 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -10854,6 +11187,7 @@ packages: name: minizlib version: 2.1.2 engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: registry.npmmirror.com/minipass@3.3.6 yallist: registry.npmmirror.com/yallist@4.0.0 @@ -10875,6 +11209,7 @@ packages: version: 1.0.4 engines: {node: '>=10'} hasBin: true + requiresBuild: true dev: false optional: true @@ -10997,6 +11332,7 @@ packages: resolution: {integrity: sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nan/-/nan-2.18.0.tgz} name: nan version: 2.18.0 + requiresBuild: true dev: false optional: true @@ -11189,6 +11525,7 @@ packages: version: 5.0.0 engines: {node: '>=6'} hasBin: true + requiresBuild: true dependencies: abbrev: registry.npmmirror.com/abbrev@1.1.1 dev: false @@ -11213,6 +11550,7 @@ packages: resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/npmlog/-/npmlog-5.0.1.tgz} name: npmlog version: 5.0.1 + requiresBuild: true dependencies: are-we-there-yet: registry.npmmirror.com/are-we-there-yet@2.0.0 console-control-strings: registry.npmmirror.com/console-control-strings@1.1.0 @@ -11616,14 +11954,13 @@ packages: sha.js: registry.npmmirror.com/sha.js@2.4.11 dev: true - registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13): + registry.npmmirror.com/pdfjs-dist@4.0.269: resolution: {integrity: sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pdfjs-dist/-/pdfjs-dist-4.0.269.tgz} - id: registry.npmmirror.com/pdfjs-dist/4.0.269 name: pdfjs-dist version: 4.0.269 engines: {node: '>=18'} optionalDependencies: - canvas: registry.npmmirror.com/canvas@2.11.2(encoding@0.1.13) + canvas: registry.npmmirror.com/canvas@2.11.2 path2d-polyfill: registry.npmmirror.com/path2d-polyfill@2.0.1 transitivePeerDependencies: - encoding @@ -11990,6 +12327,19 @@ packages: react: registry.npmmirror.com/react@18.2.0 scheduler: registry.npmmirror.com/scheduler@0.23.0 + registry.npmmirror.com/react-error-boundary@3.1.4(react@18.2.0): + resolution: {integrity: sha512-uM9uPzZJTF6wRQORmSrvOIgt4lJ9MC1sNgEOj2XGsDTRE4kmpWxg7ENK9EWNKJRMAOY9z0MuF4yIfl6gp4sotA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/react-error-boundary/-/react-error-boundary-3.1.4.tgz} + id: registry.npmmirror.com/react-error-boundary/3.1.4 + name: react-error-boundary + version: 3.1.4 + engines: {node: '>=10', npm: '>=6'} + peerDependencies: + react: '>=16.13.1' + dependencies: + '@babel/runtime': registry.npmmirror.com/@babel/runtime@7.23.7 + react: registry.npmmirror.com/react@18.2.0 + dev: false + registry.npmmirror.com/react-fast-compare@3.2.2: resolution: {integrity: sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/react-fast-compare/-/react-fast-compare-3.2.2.tgz} name: react-fast-compare @@ -12694,6 +13044,7 @@ packages: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/set-blocking/-/set-blocking-2.0.0.tgz} name: set-blocking version: 2.0.0 + requiresBuild: true dev: false optional: true @@ -12780,6 +13131,7 @@ packages: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/simple-concat/-/simple-concat-1.0.1.tgz} name: simple-concat version: 1.0.1 + requiresBuild: true dev: false optional: true @@ -12787,6 +13139,7 @@ packages: resolution: {integrity: sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/simple-get/-/simple-get-3.1.1.tgz} name: simple-get version: 3.1.1 + requiresBuild: true dependencies: decompress-response: registry.npmmirror.com/decompress-response@4.2.1 once: registry.npmmirror.com/once@1.4.0 @@ -12886,6 +13239,7 @@ packages: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz} name: sparse-bitfield version: 3.0.3 + requiresBuild: true dependencies: memory-pager: registry.npmmirror.com/memory-pager@1.5.0 dev: false @@ -13181,6 +13535,7 @@ packages: name: tar version: 6.2.0 engines: {node: '>=10'} + requiresBuild: true dependencies: chownr: registry.npmmirror.com/chownr@2.0.0 fs-minipass: registry.npmmirror.com/fs-minipass@2.1.0 @@ -14115,6 +14470,7 @@ packages: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/wide-align/-/wide-align-1.1.5.tgz} name: wide-align version: 1.1.5 + requiresBuild: true dependencies: string-width: registry.npmmirror.com/string-width@4.2.3 dev: false @@ -14183,6 +14539,15 @@ packages: engines: {node: '>= 14'} dev: true + registry.npmmirror.com/yjs@13.6.10: + resolution: {integrity: sha512-1JcyQek1vaMyrDm7Fqfa+pvHg/DURSbVo4VmeN7wjnTKB/lZrfIPhdCj7d8sboK6zLfRBJXegTjc9JlaDd8/Zw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yjs/-/yjs-13.6.10.tgz} + name: yjs + version: 13.6.10 + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + dependencies: + lib0: registry.npmmirror.com/lib0@0.2.88 + dev: false + registry.npmmirror.com/yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yocto-queue/-/yocto-queue-0.1.0.tgz} name: yocto-queue diff --git a/projects/app/data/simpleTemplates/fastgpt-simple.json b/projects/app/data/simpleTemplates/fastgpt-simple.json index 819691ff6..baf397405 100644 --- a/projects/app/data/simpleTemplates/fastgpt-simple.json +++ b/projects/app/data/simpleTemplates/fastgpt-simple.json @@ -1,6 +1,6 @@ { - "name": "极简模板", - "desc": "极简模板\n已内置参数细节", + "name": "core.app.template.Simple template", + "desc": "core.app.template.Simple template tip", "systemForm": { "aiSettings": { "model": true, diff --git a/projects/app/next.config.js b/projects/app/next.config.js index ef9310e53..aa2da3e4b 100644 --- a/projects/app/next.config.js +++ b/projects/app/next.config.js @@ -45,7 +45,7 @@ const nextConfig = { }, transpilePackages: ['@fastgpt/*'], experimental: { - serverComponentsExternalPackages: ['mongoose', 'pg'], + serverComponentsExternalPackages: ['mongoose', 'pg', '@chakra-ui/react', '@lexical/react'], outputFileTracingRoot: path.join(__dirname, '../../') } }; diff --git a/projects/app/package.json b/projects/app/package.json index baaaa052c..a90754dbc 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -1,6 +1,6 @@ { "name": "app", - "version": "4.6.6", + "version": "4.6.7", "private": false, "scripts": { "dev": "next dev", diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index 95e1767fa..15150a81f 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,12 +1,11 @@ -### Fast GPT V4.6.6 +### Fast GPT V4.6.7 -1. 新增 - Http 模块请求头支持 Json 编辑器。 -2. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。 -3. 新增 - [问题补全模块](https://doc.fastgpt.in/docs/workflow/modules/coreferenceresolution/) -5. 新增 - [文本编辑模块](https://doc.fastgpt.in/docs/workflow/modules/text_editor/) -6. 新增 - [判断器模块](https://doc.fastgpt.in/docs/workflow/modules/tfswitch/) -7. 新增 - [自定义反馈模块](https://doc.fastgpt.in/docs/workflow/modules/custom_feedback/) -8. 新增 - 【内容提取】模块支持选择模型，以及字段枚举 -9. [使用文档](https://doc.fastgpt.in/docs/intro/) -10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) -11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) +1. 修改了知识库UI及新的导入交互方式。 +2. 优化知识库和对话的数据索引，加快数据操作。 +3. 知识库 openAPI，支持通过 API 操作知识库。 +4. 新增 - 输入框变量提示。输入 { 号后将会获得可用变量提示。根据社区针对高级编排的反馈，我们计划于 2 月份的版本中，优化变量内容，支持模块的局部变量以及更多全局变量写入。 +5. 修复 - API 对话时，chatId 冲突问题。 +6. 修复 - Iframe 嵌入网页可能导致的 window.onLoad 冲突。 +7. [使用文档](https://doc.fastgpt.in/docs/intro/) +8. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) +9. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) diff --git a/projects/app/public/imgs/files/collection.svg b/projects/app/public/imgs/files/collection.svg deleted file mode 100644 index dbdb227ad..000000000 --- a/projects/app/public/imgs/files/collection.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/folder.svg b/projects/app/public/imgs/files/folder.svg index 602393396..7e8e49578 100644 --- a/projects/app/public/imgs/files/folder.svg +++ b/projects/app/public/imgs/files/folder.svg @@ -1 +1,5 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/link.svg b/projects/app/public/imgs/files/link.svg deleted file mode 100644 index 68534ba48..000000000 --- a/projects/app/public/imgs/files/link.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/manual.svg b/projects/app/public/imgs/files/manual.svg deleted file mode 100644 index fa8b3106a..000000000 --- a/projects/app/public/imgs/files/manual.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/mark.svg b/projects/app/public/imgs/files/mark.svg deleted file mode 100644 index 8c86e0099..000000000 --- a/projects/app/public/imgs/files/mark.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/pdf.svg b/projects/app/public/imgs/files/pdf.svg deleted file mode 100644 index 90940bb31..000000000 --- a/projects/app/public/imgs/files/pdf.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/modal/folder.svg b/projects/app/public/imgs/modal/folder.svg deleted file mode 100644 index 78d213c83..000000000 --- a/projects/app/public/imgs/modal/folder.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/module/ai.svg b/projects/app/public/imgs/module/ai.svg index 2fd7b1177..252540059 100644 --- a/projects/app/public/imgs/module/ai.svg +++ b/projects/app/public/imgs/module/ai.svg @@ -1,11 +1,5 @@ - - + - + d="M15.128 4.05015H16.25C18.3211 4.05015 20 5.72908 20 7.80011V12.8001C20 14.8711 18.3211 16.55 16.25 16.55H3.75C1.67895 16.55 0 14.8711 0 12.8001V7.80011C0 5.72908 1.67895 4.05015 3.75 4.05015H4.872L4.4153 1.31012C4.31315 0.697277 4.7272 0.117632 5.34005 0.015483C5.9529 -0.0866661 6.53255 0.32738 6.6347 0.940225L7.1347 3.9402C7.14085 3.977 7.1451 4.0137 7.1476 4.05015H12.8524C12.8549 4.01365 12.8591 3.977 12.8653 3.9402L13.3653 0.940225C13.4675 0.32738 14.0471 -0.0866661 14.66 0.015483C15.2728 0.117632 15.6869 0.697277 15.5847 1.31012L15.128 4.05015ZM4.9 9.3001V10.3001C4.9 10.9214 5.4037 11.4251 6.025 11.4251C6.6463 11.4251 7.15 10.9214 7.15 10.3001V9.3001C7.15 8.6788 6.6463 8.17511 6.025 8.17511C5.4037 8.17511 4.9 8.6788 4.9 9.3001ZM12.85 9.3001V10.3001C12.85 10.9214 13.3537 11.4251 13.975 11.4251C14.5963 11.4251 15.1 10.9214 15.1 10.3001V9.3001C15.1 8.6788 14.5963 8.17511 13.975 8.17511C13.3537 8.17511 12.85 8.6788 12.85 9.3001ZM5 20C4.3787 20 3.875 19.4963 3.875 18.875C3.875 18.2537 4.3787 17.75 5 17.75H15C15.6213 17.75 16.125 18.2537 16.125 18.875C16.125 19.4963 15.6213 20 15 20H5Z" + fill="#3370ff" /> \ No newline at end of file diff --git a/projects/app/public/imgs/module/templates.png b/projects/app/public/imgs/module/templates.png deleted file mode 100644 index b6f7e49b8..000000000 Binary files a/projects/app/public/imgs/module/templates.png and /dev/null differ diff --git a/projects/app/public/js/iframe.js b/projects/app/public/js/iframe.js index 55603d636..5624597e9 100644 --- a/projects/app/public/js/iframe.js +++ b/projects/app/public/js/iframe.js @@ -98,4 +98,4 @@ function embedChatbot() { ChatBtn.appendChild(ChatBtnDiv); document.body.appendChild(ChatBtn); } -document.body.onload = embedChatbot; +window.addEventListener('load', embedChatbot); diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 3b3ba828e..535a84216 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -1,8 +1,6 @@ { "App": "App", - "Cancel": "No", - "Confirm": "Yes", - "Create New": "Create", + "Create New": "", "Export": "Export", "Folder": "Folder", "Move": "Move", @@ -39,7 +37,6 @@ "Logs Title": "Title", "Mark Count": "Mark Count", "My Apps": "My Apps", - "Open AI Advanced Settings": "Advanced Settings", "Output Field Settings": "Output Field Settings", "Paste Config": "Paste Config", "To Chat": "To Chat Page", @@ -76,7 +73,9 @@ "Copy Successful": "Copy Successful", "Course": "", "Create Failed": "Create Failed", + "Create New": "Create", "Create Success": "Create Success", + "Create Time": "Create time", "Custom Title": "Custom Title", "Delete": "Delete", "Delete Failed": "Delete Failed", @@ -85,49 +84,63 @@ "Delete Warning": "Warning", "Done": "Done", "Edit": "Edit", + "Exit": "Exit", "Expired Time": "Expired", "File": "File", "Filed is repeat": "Filed is repeated", "Filed is repeated": "", + "Finish": "Finish", "Input": "Input", "Intro": "Intro", "Last Step": "Last", + "Last use time": "Last use time", "Load Failed": "Load Failed", "Loading": "Loading", "Max credit": "Credit", "Max credit tips": "What is the maximum amount of money that can be consumed by the link? If the link is exceeded, it will be banned. -1 indicates no limit.", + "More settings": "More settings", "Name": "Name", "Name Can": "Name Can't Be Empty", "Name is empty": "Name is empty", "New Create": "Create", "Next Step": "Next", "Not open": "Close", + "Number of words": "{{amount}} words", "OK": "OK", "Opened": "Opened", "Output": "Output", "Params": "Params", "Password inconsistency": "Password inconsistency", "Please Input Name": "Please Input Name", + "Price used": "Usage", + "Read document": "Read document", + "Read intro": "Read intro", "Rename": "Rename", "Rename Failed": "Rename Failed", "Rename Success": "Rename Success", "Request Error": "Request Error", "Require Input": "Required", + "Root folder": "Root folder", "Save": "Save", "Save Failed": "Save Failed", "Save Success": "Save Success", "Search": "Search", "Select File Failed": "Select File Failed", "Select One Folder": "Select a folder", + "Select template": "Select template", "Set Avatar": "Set Avatar", "Set Name": "Make a nice name", + "Setting": "Setting", "Status": "Status", + "Submit failed": "Submit failed", "Submit success": "Update Success", "Team": "Team", "Test": "Test", "Time": "Time", + "Un used": "Unused", "UnKnow": "UnKnow", "UnKnow Source": "UnKnow Source", + "Unlimited": "Unlimited", "Update Failed": "Update Failed", "Update Success": "Update Success", "Update Successful": "Update Successful", @@ -135,6 +148,7 @@ "Update success": "Update success", "Upload File Failed": "Upload File Failed", "Username": "UserName", + "Waiting": "Waiting", "Website": "Website", "avatar": { "Select Avatar": "Select Avatar", @@ -151,17 +165,27 @@ "Common Tip": "No data" }, "error": { + "Select avatar failed": "Select avatar failed", "Update error": "Update error", "unKnow": "There was an accident" }, "export": "", "file": { + "Empty file tip": "The file content is empty. The file may be unreadable or pure image file content.", "File Content": "File Content", "File Name": "File Name", "File content can not be empty": "File content can not be empty", "Filename Can not Be Empty": "Filename Can not Be Empty", "Read File Error": "Read file error", - "Select file amount limit 100": "You can select a maximum of 100 files at a time" + "Select and drag file tip": "Click or drag the file here to upload", + "Select failed": "Select file failed", + "Select file amount limit": "A maximum of {{max}} files can be selected", + "Select file amount limit 100": "You can select a maximum of 100 files at a time", + "Some file size exceeds limit": "Some files exceed: {{maxSize}}, have been filtered", + "Support file type": "Support {{fileType}} files", + "Support max count": "A maximum of {{maxCount}} files are supported.", + "Support max size": "Maximum for a single file {{maxSize}}.", + "Upload failed": "Upload failed" }, "folder": { "Drag Tip": "Click and move", @@ -186,6 +210,11 @@ "Help Chatbot": "Chatbot Helper", "Use Helper": "UsingHelp" }, + "time": { + "Just now": "Just now", + "The day before yesterday": "2 days", + "Yesterday": "Yesterday" + }, "ui": { "textarea": { "Magnifying": "Magnifying" @@ -193,7 +222,12 @@ } }, "core": { + "Chat": "Chat", + "Chat test": "Chat test", "Max Token": "MaxTokens", + "Start chat": "Start chat", + "Total chars": "Total chars: {{total}}", + "Total tokens": "Tokens: {{total}}", "ai": { "Model": "Model", "Prompt": "Prompt", @@ -203,18 +237,37 @@ } }, "app": { + "Ai response": "Ai response", + "Api request": "Api request", + "Api request desc": "Access to the existing system through API, or enterprise micro, flying book, etc", + "App intro": "App intro", "App params config": "App Config", "Chat Variable": "", + "External using": "External use", + "Make a brief introduction of your app": "Make a brief introduction of your app", + "Max tokens": "Max tokens", + "Name and avatar": "Avatar & Name", "Next Step Guide": "Next step guide", "Question Guide": "", "Question Guide Tip": "At the end of the conversation, three leading questions will be asked.", + "Quote prompt": "Quote prompt", + "Quote templates": "Quote templates", + "Random": "Random", "Save and preview": "Save", "Select TTS": "Select TTS", + "Select app from template": "Select from the template", + "Select quote template": "Select quote template", + "Set a name for your app": "App name", + "Share link": "Share", + "Share link desc": "Share links with other users and use them directly without logging in", + "Share link desc detail": "You can share the model directly with other users to have a conversation, and the other user can have a conversation directly without logging in. Note that this function will consume the balance of your account, please keep the link!", "Simple Config Tip": "Only basic functions are included. For complex agent functions, use advanced orchestration.", "TTS": "Audio Speech", "TTS Tip": "After this function is enabled, the voice playback function can be used after each conversation. Use of this feature may incur additional charges.", + "Temperature": "Temperature", "Welcome Text": "Welcome Text", "create app": "Create App", + "deterministic": "Deterministic", "edit": { "Confirm Save App Tip": "The application may be in advanced orchestration mode, and the advanced orchestration configuration will be overwritten after saving, please confirm!", "Open cfr": "Open Cfr", @@ -225,6 +278,10 @@ "cfr background prompt": "Chat background description", "cfr background tip": "Describing the scope of the current conversation makes it easier for AI to complete first or vague questions, thereby enhancing the knowledge base's ability to continue conversations.\nIf the value is empty, the problem completion function is not used for the \"first problem\".\nIf the value is none, the problem completion function is not used." }, + "error": { + "App name can not be empty": "App name can not be empty", + "Get app failed": "Get app failed" + }, "feedback": { "Custom feedback": "Custom feedback", "close custom feedback": "Close Feedback" @@ -232,6 +289,11 @@ "logs": { "Source And Time": "Source & Time" }, + "navbar": { + "External": "External", + "Flow mode": "Flow mode", + "Simple mode": "Simple mode" + }, "outLink": { "Can Drag": "Icon Drag", "Default open": "Default Open", @@ -247,9 +309,39 @@ "Web Link": "Web Link" }, "setting": "App Setting", + "share": { + "Amount limit tip": "A maximum of 10 groups can be created", + "Create link": "Create share", + "Create link tip": "The creation is successful. The share address has been copied and can be shared directly", + "Ip limit title": "IP limiting (person/minute)", + "Is response quote": "Response quote", + "Not share link": "No share link created", + "Role check": "Custom role check" + }, "simple": { "mode template select": "Template" }, + "template": { + "Classify and dataset": "Classification + Dataset", + "Classify and dataset desc": "Classify the user's problems first, then perform different actions according to the different types of problems.", + "Common template": "Common", + "Common template tip": "Common template\nCan completely self-configure AI properties and knowledge base", + "Dataset and guide": "Dataset + dialogue guide", + "Dataset and guide desc": "Conduct a knowledge base search each time a question is asked, inject the search results into the LLM model for reference answers ", + "Guide and variables": "Dialogue guide + Variables ", + "Guide and variables desc": "You can send a prompt at the beginning of the conversation, or ask the user to fill in something as a variable for the conversation ", + "Simple chat": "Simple chat", + "Simple chat desc": "An extremely simple AI conversation application ", + "Simple template": "Simple", + "Simple template tip": "Simple template\nHas built-in parameter details" + }, + "tip": { + "Add a intro to app": "Add a intro to app", + "chatNodeSystemPromptTip": "Indicates the fixed guide word of the model. If this content is adjusted, the model chat direction can be guided. The content is fixed at the beginning of the context. You can use variables such as {{language}}", + "userGuideTip": "You can set the guide language before the session, set global variables, set next guidelines ", + "variableTip": "You can ask the user to fill in something as a specific variable for this round of conversation before the conversation starts. This module is located after the opening boot.\nvariables can be injected into other modules with string input in the form of {{variable key}}, such as: prompts, qualifiers, etc.", + "welcomeTextTip": "Before each conversation begins, send an initial content. Support standard Markdown syntax, additional tags can be used :\n[shortcut button]: The user can send the question directly after clicking " + }, "tts": { "Close": "NoUse", "Model alloy": "Female - Alloy", @@ -293,8 +385,8 @@ "Restart": "Restart", "Select File": "Select file", "Select Image": "Select Image", - "Select Mark Kb": "Select Dataset", - "Select Mark Kb Desc": "Select a dataset to store the expected answers", + "Select dataset": "Select Dataset", + "Select dataset Desc": "Select a dataset to store the expected answers", "Send Message": "Send Message", "Speaking": "I'm listening...", "Start Chat": "Start Chat", @@ -306,7 +398,7 @@ "Chat error": "Chat error", "Messages empty": "Interface content is empty, maybe the text is too long ~", "Select dataset empty": "You didn't choose any dataset.", - "user input empty": "User question is empty" + "User input empty": "User question is empty" }, "feedback": { "Close User Good Feedback": "", @@ -369,10 +461,17 @@ "Stop Speech": "Stop" } }, + "common": { + "tip": { + "leave page": "Content has been modified, are you sure to leave the page?" + } + }, "dataset": { "All Dataset": "All Dataset", "Avatar": "Avatar", "Choose Dataset": "Choose Dataset", + "Chunk amount": "Chunks", + "Collection": "Collection", "Common Dataset": "Common Dataset", "Common Dataset Desc": "Knowledge bases can be built by importing files, web links, or manual entry", "Create dataset": "Create Dataset", @@ -385,6 +484,7 @@ "Empty Dataset Tips": "There is no knowledge base yet, go create one!", "File collection": "File collection", "Folder Dataset": "Folder", + "Folder placeholder": "This is a folder", "Go Dataset": "To Dataset", "Intro Placeholder": "This dataset has not yet been introduced~", "Manual collection": "Manual collection", @@ -393,14 +493,20 @@ "Quote Length": "Quote Length", "Read Dataset": "Read Dataset", "Search score tip": "{{scoreText}}Here are the rankings and scores:\n----\n{{detailScore}}", + "Select dataset": "Select dataset", "Set Empty Result Tip": ",Response empty text", "Set Website Config": "Configuring Website", "Similarity": "Similarity", "Sync Time": "Update Time", + "Table collection": "Table collection", + "Text collection": "Text collection", + "Total chunks": "Chunks: {{total}}", "Website Dataset": "Website Sync", "Website Dataset Desc": "Web site synchronization allows you to build a knowledge base directly from a web link", "collection": { "Click top config website": "Config", + "Collection name": "Collection name", + "Collection raw text": "Collection raw text", "Empty Tip": "The collection is empty", "QA Prompt": "QA Prompt", "Start Sync Tip": "Are you sure to start synchronizing data? The old data will be deleted and then re-acquired, please confirm!", @@ -434,11 +540,6 @@ "sameRaw": "The content has not changed and no update is required.", "success": "Start synchronization" } - }, - "training": { - "type chunk": "Chunk", - "type manual": "Manual", - "type qa": "QA" } }, "data": { @@ -451,6 +552,7 @@ "Default Index Tip": "Cannot be edited, the default index will use the text of [related data content] and [auxiliary data] to generate an index directly, if the default index is not needed, you can delete it. Each piece of data must have more than one index. After all indexes are deleted, a default index is automatically generated.", "Edit": "Edit Data", "Empty Tip": "This collection has no data yet", + "Main Content": "Main content", "Search data placeholder": "Search relevant data", "Too Long": "Content is too long", "Total Amount": "{{total}} Chunks", @@ -460,46 +562,82 @@ "unit": "pieces" }, "error": { + "Data not found": "The data does not exist or has been deleted", "Start Sync Failed": "Start Sync Failed", + "Template does not exist": "Template does not exist", "unAuthDataset": "No access to this knowledge base ", "unAuthDatasetCollection": "Not authorized to manipulate this data set ", "unAuthDatasetData": "Not authorized to manipulate this data ", "unAuthDatasetFile": "No permission to manipulate this file ", "unCreateCollection": "No permission to manipulate this data ", - "unLinkCollection": "not a network link collection " + "unLinkCollection": "not a network link collection" }, "file": "File", "folder": "Folder", "import": { + "Auto process": "Auto", + "Auto process desc": "Automatically set segmentation and preprocessing rules", "CSV Import": "CSV QA Import", "CSV Import Tip": "Import q and a from csv, data is required to be sorted out in advance", - "Chunk Range": "Range: 100~{{max}}", + "Chunk Range": "Range: {{min}}~{{max}}", "Chunk Split": "Chunk Split", - "Chunk Split Tip": "Select the files and split the by sentences", + "Chunk Split Tip": "After the text is segmented according to certain rules, it is converted into a format that can conduct semantic search, which is suitable for most scenarios.", "Chunk length": "Chunk length", "Csv format error": "The csv file format is incorrect, please ensure that the index and content columns are two", + "Custom file": "", + "Custom process": "Custom rule", + "Custom process desc": "Customize scoring and preprocessing rules", + "Custom prompt": "Custom prompt", "Custom split char": "Custom split char", "Custom split char Tips": "Allows you to block according to custom delimiters. It is usually used for processed data, using specific delimiters to precisely block it.", - "Embedding Estimated Price Tips": "Index billing: {{price}}/1k tokens", - "Estimated Price": "Estimated Price", + "Custom text": "Custom text", + "Custom text desc": "Manually enter a piece of text as the collection", + "Data Preprocessing": "Data Preprocessing", + "Data file progress": "Data upload progress", + "Data process params": "Data process params", + "Down load csv template": "Down load csv template", + "Embedding Estimated Price Tips": "Index billing: {{price}}/1k chars", + "Estimated Price": "Estimated Price: : {{amount}}{{unit}}", "Estimated Price Tips": "QA charges\nInput: {{inputPrice}}/1k tokens\nOutput: {{outputPrice}}/1k tokens", "Fetch Error": "Get link failed", "Fetch Url": "Url", "Fetch url placeholder": "Up to 10 links, one per line.", "Fetch url tip": "Only static links can be read, please check the results", - "Ideal chunk length": "Ideal chunk length", + "File chunk amount": "Chunks: {{amount}}", + "File list": "Files", + "Ideal chunk length": "Chunk length", "Ideal chunk length Tips": "Segment by end symbol. We recommend that your document should be properly punctuated to ensure that each complete sentence length does not exceed this value \n Chinese document recommended 400~1000\n English document recommended 600~1200", "Import Failed": "Import Failed", "Import Success Tip": "The {{num}} group data is imported successfully. Please wait for training.", "Import Tip": "This task cannot be terminated and takes some time to generate indexes. Please confirm the import. If the balance is insufficient, the unfinished task will be suspended and can continue after topping up.", + "Link name": "Link name", + "Link name placeholder": "Only static links are supported\nOne per line, up to 10 links at a time", + "Local file": "Local file", + "Local file desc": "Upload files in PDF, TXT, DOCX and other formats", "Only Show First 50 Chunk": "Show only part", - "QA Estimated Price Tips": "QA charges\nInput: {{inputPrice}}/1k tokens\nOutput: {{outputPrice}}/1k tokens", + "Preview chunks": "Chunks", + "Preview raw text": "Preview file text (max show 10000 words)", + "Process way": "Process way", + "QA Estimated Price Tips": "QA billing: {{price}}/1k characters (including input and output)", "QA Import": "QA Split", - "QA Import Tip": "Select the files and let the LLM automatically generate QA", + "QA Import Tip": "According to certain rules, the text is broken into a larger paragraph, and the AI is invoked to generate a question and answer pair for the paragraph.", "Re Preview": "RePreview", + "Select file": "Select file", + "Select source": "Select source", "Set Chunk Error": "Split chunks error", + "Source name": "Source name", + "Sources list": "Sources", + "Start upload": "Start", "Total Chunk Preview": "Chunk Preview: {{totalChunks}} ", - "Total tokens": "Tokens" + "Total files": "Total {{total}} files", + "Total tokens": "Tokens", + "Training mode": "Training mode", + "Upload data": "Upload data", + "Upload file progress": "File upload progress", + "Upload status": "Upload status", + "Upload success": "Upload success", + "Web link": "Web link", + "Web link desc": "Fetch static web content as a collection" }, "link": "Link", "search": { @@ -512,6 +650,7 @@ "Min Similarity": "Min Similarity", "Min Similarity Tips": "The similarity of different index models is different, please use the search test to select the appropriate value", "Params Setting": "Params Setting", + "Quote index": "Quote index", "Rank": "Rank", "Rank Tip": "Ranking in all data", "ReRank": "ReRank", @@ -520,24 +659,30 @@ "Rerank score": "ReRank score", "Score": "Score", "Search type": "Type", + "Source id": "Source ID", + "Source name": "Source", "Top K": "Top K", "mode": { - "embedding": "Vector search", + "embedding": "Vector recall", "embedding desc": "Use vectors for text correlation queries", - "fullTextRecall": "Full text search ", + "fullTextRecall": "Full text recall ", "fullTextRecall desc": "Using traditional full-text search, suitable for finding data with specific keywords and main predicates", - "mixedRecall": "Mixedrecall", + "mixedRecall": "Mixed recall", "mixedRecall desc": "Returns the combined results of vector and full-text searches, sorted using the RRF algorithm." }, "score": { "embedding": "Embedding", - "embedding desc": "", - "fullText": "Full text", - "fullText desc": "", + "embedding desc": "Text correlation query using vectors", + "fullText": "FullText", + "fullText desc": "Calculate the score of the same keyword, ranging from 0 to infinity.", + "fullTextRecall": "Full text recall", + "fullTextRecall desc": "Using traditional full-text search, suitable for finding specific data with keywords and main predicates ", + "mixedRecall": "Mixed recall", + "mixedRecall desc": "Returns the combined results of vector and full-text searches, sorted using the RRF algorithm.", "reRank": "ReRank", - "reRank desc": "", - "rrf": "RRF Merge", - "rrf desc": "" + "reRank desc": "The correlation degree between sentences was calculated by ReRank model, ranging from 0 to 1.", + "rrf": "RRF", + "rrf desc": "Multiple search results are combined by inverting calculation." }, "search mode": "Search Mode" }, @@ -562,13 +707,15 @@ }, "training": { "Agent queue": "QA wait list", + "Chunk mode": "Chunk split", "Full": "Expect more than 5 minutes", "Leisure": "Leisure", + "Manual": "Manual import", + "Manual mode": "", + "QA mode": "QA learning", "Vector queue": "Vector wait list", "Waiting": "Waiting", - "Website Sync": "Website Sync", - "type chunk": "Chunk", - "type qa": "QA" + "Website Sync": "Website Sync" }, "website": { "Base Url": "BaseUrl", @@ -577,12 +724,13 @@ "Confirm Create Tips": "Confirm to synchronize the site, the synchronization task will start later, please confirm!", "Confirm Update Tips": "Are you sure to update the site configuration? The synchronization starts immediately with the new configuration. Please confirm", "Selector": "Selector", - "Selector Course": "Selector using tutorial", + "Selector Course": "Instructions", "Start Sync": "Start Sync", "UnValid Website Tip": "Your site may not be static and cannot be synchronized" } }, "module": { + "Add question type": "Add type", "Data Type": "Data Type", "Field Description": "Description", "Field Name": "Name", @@ -590,7 +738,8 @@ "Field key": "Key", "Input Type": "Input Type", "Plugin output must connect": "Custom outputs must all be connected", - "Variable": "Variable", + "Unlink tip": "[{{name}}] An unfilled or unconnected parameter exists", + "Variable": "Variables", "Variable Setting": "Variable Setting", "edit": { "Field Already Exist": "Key already exist", @@ -606,28 +755,38 @@ "Add Input": "Add Input", "Input Number": "Input: {{length}}", "description": { - "Http Request Header": "", - "Http Request Url": "", - "TFSwitch textarea": "", - "anyInput": "", - "cfr background": "The background knowledge of the current conversation makes it easy to complete the first question and the fuzzy question, and only needs to briefly describe the scope of the current conversation.", - "dynamic input": "", - "textEditor textarea": "The passed variable can be referenced by {{key}}." + "Background": "", + "Http Request Header": "User-defined request header, please strictly fill in the JSON string.\n1. Make sure the last attribute has no commas\n2. Make sure key contains double quotes\nFor example: {\"Authorization\":\"Bearer xxx\"}", + "Http Request Url": "New HTTP request address. If two 'request addresses' appear, the module can be deleted and rejoined, and the latest module configuration will be pulled.", + "Quote": "Object array format, structure:\n[{q:' question ',a:' answer '}]", + "Response content": "You can use \\n to achieve continuous line wrapping.\nReplies can be achieved by external module input, which overwrites the content currently filled in.\nIf passed non-string type data will be automatically converted to a string", + "TFSwitch textarea": "Allows you to define a number of strings to achieve false matching, one per line, and supports regular expressions.", + "anyInput": "Can pass anything ", + "cfr background": "Describes the scope of the current conversation, making it easier for the AI to complete first or vague questions, thereby enhancing the knowledge base's ability to continue conversations. If\nis empty, the question completion function is not used in the first conversation. ", + "dynamic input": "Receives parameters dynamically added by the user and will be tiled in at run time ", + "textEditor textarea": "The passed variable can be referenced by {{key}}. Variables support only strings or numbers." }, "label": { - "Http Request Header": "", - "Http Request Method": "", - "Http Request Url": "", - "TFSwitch textarea": "", - "aiModel": "", - "anyInput": "", + "Background": "Background", + "Classify model": "Classify model", + "Http Request Header": "Request header ", + "Http Request Method": "Request Method", + "Http Request Url": "Request address ", + "LLM": "", + "Quote": "Quote", + "Response content": "Response content", + "Select dataset": "Select dataset", + "TFSwitch textarea": "Custom False matching rule ", + "aiModel": "AI model ", + "anyInput": "Any input ", "cfr background": "Background", "chat history": "chat history", - "switch": "Switch", - "textEditor textarea": "Text Edit", + "switch": "Trigger", + "textEditor textarea": "Text edit", "user question": "User question" }, "placeholder": { + "Classify background": "For example:\n1.AIGC (Artificial Intelligence Generates content) refers to the automatic or semi-automatic generation of digital content, such as text, images, music, videos, and so on, using artificial intelligence technologies. AIGC technologies include, but are not limited to, natural language processing, computer vision, machine learning, and deep learning. These technologies can create new content or modify existing content to meet specific creative, educational, entertainment or information needs.", "cfr background": "Questions about the introduction and use of python. \nThe current dialogue is related to the game GTA5." } }, @@ -646,20 +805,52 @@ "Add Output": "Add Output", "Output Number": "Output: {{length}}", "description": { - "running done": "Triggered when the module call ends" + "Ai response content": "Will be triggered after the stream reply is complete", + "New context": "Concatenate the reply content with history and return it as a new context", + "Quote": "Always return an array, if you want the search results to be empty to perform additional operations, you need to use the above two inputs and the trigger of the target module", + "running done": "Triggered when the module call finish" }, "label": { - "cfr result": "", - "result false": "", - "result true": "", - "running done": "End of module call ", + "Ai response content": "Response Content", + "New context": "New context", + "Quote": "Quote", + "Search result empty": "Search result empty", + "Search result not empty": "Search result not empty", + "cfr result": "Response text", + "result false": "False", + "result true": "True", + "running done": "done", "text": "Text output" } }, "template": { - "TFSwitch": "", - "TFSwitch intro": "", + "Ai chat": "LLM Chat", + "Ai chat intro": "Request LLM chat", + "Assigned reply": "Assigned reply", + "Assigned reply intro": "The module can respond directly to a specified piece of content. Often used to guide and prompt", + "Chat entrance": "Chat entrance", + "Chat entrance intro": "When the user sends a content, the flow will start from this module.", + "Classify question": "Classify question", + "Classify question intro": "Determine the type of question based on the user's history and current issue. Multiple sets of question types can be added, here is a template example: \n type 1: Hello\ntype 2: Questions about 'use'\ntype 3: Questions about 'purchase'\ntype 4: Other questions", + "Dataset search": "Dataset search", + "Dataset search intro": "Invoke the Dataset search capability to find content that may be relevant to the problem", + "External module": "External call", + "Extract field": "Text content extraction ", + "Extract field intro": "Can extract specified data from the text, such as: sql statements, search keywords, code, etc.", + "Function module": " Function call", + "Guide module": "Guides", + "Http request": "Http request", + "Http request intro": " Can issue an HTTP request to implement more complex operations (Internet search, database query, etc.)", + "My plugin module": "Personal plugins", + "Response module": "Text output", + "Running app": "Running app", + "Running app intro": "You can select a different app to run", + "System input module": "System input", + "TFSwitch": "TF Switch", + "TFSwitch intro": "Output True False based on what is passed in. By default, false is printed when the content passed in is false, undefined, null, 0, none. You can also add some custom strings to supplement the output of false.", + "Tool module": "Tools", "UnKnow Module": "UnKnow Module", + "User guide": "User guide", "cfr": "Coreference resolution", "cfr intro": "Refine the current issue based on history, making it more conducive to knowledge base search, while improving continuous conversation capabilities.", "textEditor": "Text Editor", @@ -717,10 +908,10 @@ "Files": "{{total}} Files", "Folder Name": "Input folder name", "Insert Data": "Insert", - "Manual collection Tip": "Manual Collections allow you to create a custom container to hold data", "Manual Data": "Manual Data", "Manual Input": "Manual Input", "Manual Mark": "Manual Mark", + "Manual collection Tip": "Manual Collections allow you to create a custom container to hold data", "Mark Data": "Mark Data", "Move Failed": "Move Failed", "Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while", @@ -859,9 +1050,9 @@ }, "navbar": { "Account": "Account", - "Apps": "Apps", + "Apps": "App", "Chat": "Chat", - "Datasets": "DataSets", + "Datasets": "DataSet", "Module": "Module", "Plugin": "Plugin", "Store": "Store", @@ -914,6 +1105,15 @@ "Update Your Plugin": "Update Plugin" }, "support": { + "openapi": { + "Api baseurl": "Baseurl", + "Api manager": "API key management", + "Copy success": "The API address has been copied", + "Max usage": "Max usage", + "New api key": "New API key", + "New api key tip": "Please keep your secret key, the secret key will not be displayed again~", + "Usage": "Usage" + }, "outlink": { "share": { "Response Quote": "Show Quote", @@ -1067,6 +1267,7 @@ "App name": "App name", "Audio Speech": "Audio Speech", "Bill Module": "Bill Detail", + "Chars length": "Chars length", "Data Length": "Data length", "Dataset store": "", "Duration": "Duration(s)", diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 9b0512844..d50730b7b 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -1,8 +1,6 @@ { "App": "应用", - "Cancel": "取消", - "Confirm": "确认", - "Create New": "新建", + "Create New": "", "Export": "导出", "Folder": "文件夹", "Move": "移动", @@ -39,7 +37,6 @@ "Logs Title": "标题", "Mark Count": "标注答案数量", "My Apps": "我的应用", - "Open AI Advanced Settings": "高级配置", "Output Field Settings": "输出字段编辑", "Paste Config": "粘贴配置", "To Chat": "前去对话", @@ -76,7 +73,9 @@ "Copy Successful": "复制成功", "Course": "", "Create Failed": "创建异常", + "Create New": "新建", "Create Success": "创建成功", + "Create Time": "创建时间", "Custom Title": "自定义标题", "Delete": "删除", "Delete Failed": "删除失败", @@ -85,49 +84,63 @@ "Delete Warning": "删除警告", "Done": "完成", "Edit": "编辑", + "Exit": "退出", "Expired Time": "过期时间", "File": "文件", "Filed is repeat": "", "Filed is repeated": "字段重复了", + "Finish": "完成", "Input": "输入", "Intro": "介绍", "Last Step": "上一步", + "Last use time": "最后使用时间", "Load Failed": "加载失败", "Loading": "加载中", "Max credit": "最大金额", "Max credit tips": "该链接最大可消耗多少金额，超出后链接将被禁止使用。-1 代表无限制。", + "More settings": "更多设置", "Name": "名称", "Name Can": "名称不能为空", "Name is empty": "名称不能为空", "New Create": "新建", "Next Step": "下一步", "Not open": "未开启", + "Number of words": "{{amount}}字", "OK": "好的", "Opened": "已开启", "Output": "输出", "Params": "参数", "Password inconsistency": "两次密码不一致", "Please Input Name": "请输入名称", + "Price used": "金额消耗", + "Read document": "查看文档", + "Read intro": "查看说明", "Rename": "重命名", "Rename Failed": "重命名失败", "Rename Success": "重命名成功", "Request Error": "请求异常", "Require Input": "必填", + "Root folder": "根目录", "Save": "保存", "Save Failed": "保存失败", "Save Success": "保存成功", "Search": "搜索", "Select File Failed": "选择文件异常", "Select One Folder": "选择一个目录", + "Select template": "选择模板", "Set Avatar": "点击设置头像", "Set Name": "取个名字", + "Setting": "设置", "Status": "状态", + "Submit failed": "提交失败", "Submit success": "提交成功", "Team": "团队", "Test": "测试", "Time": "时间", + "Un used": "未使用", "UnKnow": "未知", "UnKnow Source": "未知来源", + "Unlimited": "无限制", "Update Failed": "更新异常", "Update Success": "更新成功", "Update Successful": "更新成功", @@ -135,6 +148,7 @@ "Update success": "更新成功", "Upload File Failed": "上传文件失败", "Username": "用户名", + "Waiting": "等待中", "Website": "网站", "avatar": { "Select Avatar": "点击选择头像", @@ -151,17 +165,27 @@ "Common Tip": "没有什么数据噢~" }, "error": { + "Select avatar failed": "头像选择异常", "Update error": "更新失败", "unKnow": "出现了点意外~" }, "export": "", "file": { + "Empty file tip": "文件内容为空，可能该文件无法读取或为纯图片文件内容。", "File Content": "文件内容", "File Name": "文件名", "File content can not be empty": "文件内容不能为空", "Filename Can not Be Empty": "文件名不能为空", "Read File Error": "解析文件失败", - "Select file amount limit 100": "每次最多选择100个文件" + "Select and drag file tip": "点击或拖动文件到此处上传", + "Select failed": "选择文件异常", + "Select file amount limit": "最多选择 {{max}} 个文件", + "Select file amount limit 100": "每次最多选择100个文件", + "Some file size exceeds limit": "部分文件超出: {{maxSize}}，已被过滤", + "Support file type": "支持 {{fileType}} 类型文件", + "Support max count": "最多支持 {{maxCount}} 个文件。", + "Support max size": "单个文件最大 {{maxSize}}。", + "Upload failed": "上传异常" }, "folder": { "Drag Tip": "点我可拖动", @@ -186,6 +210,11 @@ "Help Chatbot": "机器人助手", "Use Helper": "使用帮助" }, + "time": { + "Just now": "刚刚", + "The day before yesterday": "前天", + "Yesterday": "昨天" + }, "ui": { "textarea": { "Magnifying": "放大" @@ -193,7 +222,12 @@ } }, "core": { + "Chat": "对话", + "Chat test": "测试对话", "Max Token": "单条数据上限", + "Start chat": "立即对话", + "Total chars": "总字符数: {{total}}", + "Total tokens": "总 Tokens: {{total}}", "ai": { "Model": "AI 模型", "Prompt": "提示词", @@ -203,18 +237,37 @@ } }, "app": { + "Ai response": "返回AI内容", + "Api request": "API 访问", + "Api request desc": "通过 API 接入到已有系统中，或企微、飞书等", + "App intro": "应用介绍", "App params config": "应用配置", "Chat Variable": "对话框变量", + "External using": "外部使用途径", + "Make a brief introduction of your app": "给你的 AI 应用一个介绍", + "Max tokens": "回复上限", + "Name and avatar": "头像 & 名称", "Next Step Guide": "下一步指引", "Question Guide": "问题引导", "Question Guide Tip": "对话结束后，会为生成 3 个引导性问题。", + "Quote prompt": "引用模板提示词", + "Quote templates": "引用内容模板", + "Random": "发散", "Save and preview": "保存并预览", "Select TTS": "选择语音播放模式", + "Select app from template": "从模板中选择", + "Select quote template": "选择引用提示模板", + "Set a name for your app": "给应用设置一个名称", + "Share link": "免登录窗口", + "Share link desc": "分享链接给其他用户，无需登录即可直接进行使用", + "Share link desc detail": "可以直接分享该模型给其他用户去进行对话，对方无需登录即可直接进行对话。注意，这个功能会消耗你账号的余额，请保管好链接！", "Simple Config Tip": "仅包含基础功能，复杂 agent 功能请使用高级编排。", "TTS": "语音播报", "TTS Tip": "开启后，每次对话后可使用语音播放功能。使用该功能可能产生额外费用。", + "Temperature": "温度", "Welcome Text": "对话开场白", "create app": "创建属于你的 AI 应用", + "deterministic": "严谨", "edit": { "Confirm Save App Tip": "该应用可能为高级编排模式，保存后将会覆盖高级编排配置，请确认！", "Open cfr": "开启自动补全", @@ -225,6 +278,10 @@ "cfr background prompt": "对话背景描述", "cfr background tip": "描述当前对话的范围，便于AI补全首次问题或模糊的问题，从而增强知识库连续对话的能力。\n值为空时，表示【首个问题】不使用问题补全功能。\n值为 none 时，表示不使用问题补全功能。" }, + "error": { + "App name can not be empty": "应用名不能为空", + "Get app failed": "获取应用异常" + }, "feedback": { "Custom feedback": "自定义反馈", "close custom feedback": "关闭反馈" @@ -232,6 +289,11 @@ "logs": { "Source And Time": "来源 & 时间" }, + "navbar": { + "External": "外部使用", + "Flow mode": "高级编排", + "Simple mode": "简易配置" + }, "outLink": { "Can Drag": "图标可拖拽", "Default open": "默认打开", @@ -247,9 +309,39 @@ "Web Link": "网络链接" }, "setting": "应用信息设置", + "share": { + "Amount limit tip": "最多创建10组", + "Create link": "创建新链接", + "Create link tip": "创建成功。已复制分享地址，可直接分享使用", + "Ip limit title": "IP限流（人/分钟）", + "Is response quote": "返回引用", + "Not share link": "没有创建分享链接", + "Role check": "身份校验" + }, "simple": { "mode template select": "简易模板" }, + "template": { + "Classify and dataset": "问题分类 + 知识库", + "Classify and dataset desc": "先对用户的问题进行分类，再根据不同类型问题，执行不同的操作", + "Common template": "通用模板", + "Common template tip": "通用模板\n可完全自行配置AI属性和知识库", + "Dataset and guide": "知识库 + 对话引导", + "Dataset and guide desc": "每次提问时进行一次知识库搜索，将搜索结果注入 LLM 模型进行参考回答", + "Guide and variables": "对话引导 + 变量", + "Guide and variables desc": "可以在对话开始发送一段提示，或者让用户填写一些内容，作为本次对话的变量", + "Simple chat": "简单的对话", + "Simple chat desc": "一个极其简单的 AI 对话应用", + "Simple template": "简易模板", + "Simple template tip": "极简模板\n已内置参数细节" + }, + "tip": { + "Add a intro to app": "快来给应用一个介绍~", + "chatNodeSystemPromptTip": "模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}", + "userGuideTip": "可以在对话前设置引导语，设置全局变量，设置下一步指引", + "variableTip": "可以在对话开始前，要求用户填写一些内容作为本轮对话的特定变量。该模块位于开场引导之后。\n变量可以通过 {{变量key}} 的形式注入到其他模块 string 类型的输入中，例如：提示词、限定词等", + "welcomeTextTip": "每次对话开始前，发送一个初始内容。支持标准 Markdown 语法，可使用的额外标记:\n[快捷按键]: 用户点击后可以直接发送该问题" + }, "tts": { "Close": "不使用", "Model alloy": "女声 - Alloy", @@ -293,8 +385,8 @@ "Restart": "重开对话", "Select File": "选择文件", "Select Image": "选择图片", - "Select Mark Kb": "选择知识库", - "Select Mark Kb Desc": "选择一个知识库存储预期答案", + "Select dataset": "选择知识库", + "Select dataset Desc": "选择一个知识库存储预期答案", "Send Message": "发送", "Speaking": "我在听，请说...", "Start Chat": "开始对话", @@ -306,7 +398,7 @@ "Chat error": "对话出现异常", "Messages empty": "接口内容为空，可能文本超长了~", "Select dataset empty": "你没有选择知识库", - "user input empty": "传入的用户问题为空" + "User input empty": "传入的用户问题为空" }, "feedback": { "Close User Good Feedback": "", @@ -369,10 +461,17 @@ "Stop Speech": "停止" } }, + "common": { + "tip": { + "leave page": "内容已修改，确认离开页面吗？" + } + }, "dataset": { "All Dataset": "全部知识库", "Avatar": "知识库头像", "Choose Dataset": "关联知识库", + "Chunk amount": "分段数", + "Collection": "数据集", "Common Dataset": "通用知识库", "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "Create dataset": "创建一个知识库", @@ -385,6 +484,7 @@ "Empty Dataset Tips": "还没有知识库，快去创建一个吧！", "File collection": "文件数据集", "Folder Dataset": "文件夹", + "Folder placeholder": "这是一个目录", "Go Dataset": "前往知识库", "Intro Placeholder": "这个知识库还没有介绍~", "Manual collection": "手动数据集", @@ -393,14 +493,20 @@ "Quote Length": "引用内容长度", "Read Dataset": "查看知识库详情", "Search score tip": "{{scoreText}}下面是详细排名和得分情况:\n----\n{{detailScore}}", + "Select dataset": "选择知识库", "Set Empty Result Tip": ",未搜索到内容时回复指定内容", "Set Website Config": "开始配置网站信息", "Similarity": "相关度", "Sync Time": "最后更新时间", + "Table collection": "表格数据集", + "Text collection": "文本数据集", + "Total chunks": "总分段: {{total}}", "Website Dataset": "Web 站点同步", "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", "collection": { "Click top config website": "点击配置网站", + "Collection name": "数据集名称", + "Collection raw text": "数据集内容", "Empty Tip": "数据集空空如也", "QA Prompt": "QA 拆分引导词", "Start Sync Tip": "确认开始同步数据？将会删除旧数据后重新获取，请确认！", @@ -436,9 +542,6 @@ } }, "training": { - "type chunk": "直接分段", - "type manual": "手动", - "type qa": "问答拆分" } }, "data": { @@ -451,6 +554,7 @@ "Default Index Tip": "无法编辑，默认索引会使用【相关数据内容】与【辅助数据】的文本直接生成索引，如不需要默认索引，可删除。 每条数据必须保证有一个以上索引，所有索引被删除后，会自动生成默认索引。", "Edit": "编辑数据", "Empty Tip": "这个集合还没有数据~", + "Main Content": "主要内容", "Search data placeholder": "搜索相关数据", "Too Long": "总长度超长了", "Total Amount": "{{total}} 组", @@ -460,7 +564,9 @@ "unit": "条" }, "error": { + "Data not found": "数据不存在或已被删除", "Start Sync Failed": "开始同步失败", + "Template does not exist": "模板不存在", "unAuthDataset": "无权操作该知识库", "unAuthDatasetCollection": "无权操作该数据集", "unAuthDatasetData": "无权操作该数据", @@ -471,35 +577,69 @@ "file": "文件", "folder": "目录", "import": { + "Auto process": "自动", + "Auto process desc": "自动设置分割和预处理规则", "CSV Import": "CSV 导入", "CSV Import Tip": "通过批量导入问答对，要求提前整理好数据", - "Chunk Range": "范围: 100~{{max}}", + "Chunk Range": "范围: {{min}}~{{max}}", "Chunk Split": "直接分段", - "Chunk Split Tip": "选择文本文件，直接将其按分段进行处理", + "Chunk Split Tip": "将文本按一定的规则进行分段处理后，转成可进行语义搜索的格式，适合绝大多数场景。", "Chunk length": "分块总量", "Csv format error": "csv 文件格式有误,请确保 index 和 content 两列", + "Custom file": "自定义文本", + "Custom process": "自定义规则", + "Custom process desc": "自定义设置分制和预处理规则", + "Custom prompt": "自定义提示词", "Custom split char": "自定义分隔符", "Custom split char Tips": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据，使用特定的分隔符来精确分块。", - "Embedding Estimated Price Tips": "索引计费: {{price}}/1k tokens", - "Estimated Price": "预估价格", + "Custom text": "自定义文本", + "Custom text desc": "手动输入一段文本作为数据集", + "Data Preprocessing": "数据处理", + "Data file progress": "数据上传进度", + "Data process params": "数据处理参数", + "Down load csv template": "点击下载 CSV 模板", + "Embedding Estimated Price Tips": "索引计费: {{price}}/1k字符", + "Estimated Price": "预估价格: {{amount}}{{unit}}", "Estimated Price Tips": "QA计费为\n输入: {{inputPrice}}/1k tokens\n输出: {{outputPrice}}/1k tokens", "Fetch Error": "获取链接失败", "Fetch Url": "网络链接", "Fetch url placeholder": "最多10个链接，每行一个。", "Fetch url tip": "仅支持读取静态链接，请注意检查结果", + "File chunk amount": "分段: {{amount}}", + "File list": "文件列表", "Ideal chunk length": "理想分块长度", "Ideal chunk length Tips": "按结束符号进行分段。我们建议您的文档应合理的使用标点符号，以确保每个完整的句子长度不要超过该值\n中文文档建议400~1000\n英文文档建议600~1200", "Import Failed": "导入文件失败", "Import Success Tip": "共成功导入 {{num}} 组数据，请耐心等待训练.", "Import Tip": "该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。", + "Link name": "网络链接", + "Link name placeholder": "仅支持静态链接，如果上传后数据为空，可能该链接无法被读取\n每行一个，每次最多 10 个链接", + "Local file": "本地文件", + "Local file desc": "上传 PDF, TXT, DOCX 等格式的文件", "Only Show First 50 Chunk": "仅展示部分", - "QA Estimated Price Tips": "QA计费为\n输入: {{inputPrice}}/1k tokens\n输出: {{outputPrice}}/1k tokens", + "Preview chunks": "分段预览", + "Preview raw text": "预览源文本（最多展示10000字）", + "Process way": "处理方式", + "QA Estimated Price Tips": "QA计费为: {{price}}元/1k 字符(包含输入和输出)", "QA Import": "QA拆分", - "QA Import Tip": "选择文本文件，让大模型自动生成问答对", + "QA Import Tip": "根据一定规则，将文本拆成一段较大的段落，调用 AI 为该段落生成问答对。", "Re Preview": "重新生成预览", + "Select file": "选择文件", + "Select source": "选择来源", "Set Chunk Error": "文本分段异常", + "Source name": "来源名", + "Sources list": "来源列表", + "Start upload": "开始上传", "Total Chunk Preview": "分段预览({{totalChunks}}组)", - "Total tokens": "总Tokens" + "Total files": "共 {{total}} 个文件", + "Total tokens": "总Tokens", + "Training mode": "训练模式", + "Upload data": "上传数据", + "Upload file progress": "文件上传进度", + "Upload status": "上传状态", + "Upload success": "上传成功", + "Web link": "网页链接", + "Web link desc": "读取静态网页内容作为数据集" }, "link": "链接", "search": { @@ -512,6 +652,7 @@ "Min Similarity": "最低相关度", "Min Similarity Tips": "不同索引模型的相关度有区别，请通过搜索测试来选择合适的数值，使用 ReRank 时，相关度可能会很低。", "Params Setting": "搜索参数设置", + "Quote index": "第几个引用", "Rank": "排名", "Rank Tip": "在所有数据中的排名", "ReRank": "结果重排", @@ -520,6 +661,8 @@ "Rerank score": "结果重排得分", "Score": "得分", "Search type": "类型", + "Source id": "来源ID", + "Source name": "引用来源名", "Top K": "单次搜索上限", "mode": { "embedding": "语义检索", @@ -534,6 +677,10 @@ "embedding desc": "通过计算向量之间的距离获取得分，范围为 0~1。", "fullText": "全文检索", "fullText desc": "计算相同关键词的得分，范围为 0~无穷。", + "fullTextRecall": "", + "fullTextRecall desc": "", + "mixedRecall": "", + "mixedRecall desc": "", "reRank": "结果重排", "reRank desc": "通过 ReRank 模型计算句子之间的关联度，范围为 0~1。", "rrf": "综合排名", @@ -562,13 +709,15 @@ }, "training": { "Agent queue": "QA训练排队", + "Chunk mode": "直接分段", "Full": "预计5分钟以上", "Leisure": "空闲", + "Manual": "", + "Manual mode": "手动导入", + "QA mode": "问答拆分", "Vector queue": "索引排队", "Waiting": "预计5分钟", - "Website Sync": "Web 站点同步", - "type chunk": "直接分段", - "type qa": "问答拆分" + "Website Sync": "Web 站点同步" }, "website": { "Base Url": "根地址", @@ -577,12 +726,13 @@ "Confirm Create Tips": "确认同步该站点，同步任务将随后开启，请确认！", "Confirm Update Tips": "确认更新站点配置？会立即按新的配置开始同步，请确认！", "Selector": "选择器", - "Selector Course": "选择器使用教程", + "Selector Course": "使用教程", "Start Sync": "开始同步", "UnValid Website Tip": "您的站点可能非静态站点，无法同步" } }, "module": { + "Add question type": "添加问题类型", "Data Type": "数据类型", "Field Description": "字段描述", "Field Name": "字段名", @@ -590,6 +740,7 @@ "Field key": "字段 Key", "Input Type": "输入类型", "Plugin output must connect": "自定义输出必须全部连接", + "Unlink tip": "【{{name}}】存在未填或未连接参数", "Variable": "参数变量", "Variable Setting": "变量设置", "edit": { @@ -606,8 +757,11 @@ "Add Input": "添加入参", "Input Number": "入参: {{length}}", "description": { + "Background": "你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。", "Http Request Header": "自定义请求头，请严格填入JSON字符串。\n1. 确保最后一个属性没有逗号\n2. 确保 key 包含双引号\n例如: {\"Authorization\":\"Bearer xxx\"}", "Http Request Url": "新的HTTP请求地址。如果出现两个“请求地址”，可以删除该模块重新加入，会拉取最新的模块配置。", + "Quote": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]", + "Response content": "可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串", "TFSwitch textarea": "允许定义一些字符串来实现 false 匹配，每行一个，支持正则表达式。", "anyInput": "可传入任意内容", "cfr background": "描述当前对话的范围，便于AI补全首次问题或模糊的问题，从而增强知识库连续对话的能力。\n为空时，表示【首次对话】不使用问题补全功能。", @@ -615,9 +769,15 @@ "textEditor textarea": "可以通过 {{key}} 的方式引用传入的变量。变量仅支持字符串或数字。" }, "label": { + "Background": "背景知识", + "Classify model": "分类模型", "Http Request Header": "请求头", "Http Request Method": "请求方式", "Http Request Url": "请求地址", + "LLM": "AI 模型", + "Quote": "引用内容", + "Response content": "回复的内容", + "Select dataset": "选择知识库", "TFSwitch textarea": "自定义 False 匹配规则", "aiModel": "AI 模型", "anyInput": "任意内容输入", @@ -628,6 +788,7 @@ "user question": "用户问题" }, "placeholder": { + "Classify background": "例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。", "cfr background": "关于 python 的介绍和使用等问题。\n当前对话与游戏《GTA5》有关。" } }, @@ -646,9 +807,17 @@ "Add Output": "添加出参", "Output Number": "出参: {{length}}", "description": { + "Ai response content": "将在 stream 回复完毕后触发", + "New context": "将本次回复内容拼接上历史记录，作为新的上下文返回", + "Quote": "始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器", "running done": "模块调用结束时触发" }, "label": { + "Ai response content": "AI回复内容", + "New context": "新的上下文", + "Quote": "引用内容", + "Search result empty": "搜索结果为空", + "Search result not empty": "搜索结果不为空", "cfr result": "补全结果", "result false": "False", "result true": "True", @@ -657,9 +826,33 @@ } }, "template": { + "Ai chat": "AI 对话", + "Ai chat intro": "AI 大模型对话", + "Assigned reply": "指定回复", + "Assigned reply intro": "该模块可以直接回复一段指定的内容。常用于引导、提示", + "Chat entrance": "对话入口", + "Chat entrance intro": "当用户发送一个内容后，流程将会从这个模块开始执行。", + "Classify question": "问题分类", + "Classify question intro": "根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于商品“使用”问题\n类型3: 关于商品“购买”问题\n类型4: 其他问题", + "Dataset search": "知识库搜索", + "Dataset search intro": "调用知识库搜索能力，查找有可能与问题相关的内容", + "External module": "外部调用", + "Extract field": "文本内容提取", + "Extract field intro": "可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等", + "Function module": "功能调用", + "Guide module": "引导模块", + "Http request": "Http 请求", + "Http request intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）", + "My plugin module": "个人插件", + "Response module": "文本输出", + "Running app": "应用调用", + "Running app intro": "可以选择一个其他应用进行调用", + "System input module": "系统输入", "TFSwitch": "判断器", "TFSwitch intro": "根据传入的内容进行 True False 输出。默认情况下，当传入的内容为 false, undefined, null, 0, none 时，会输出 false。你也可以增加一些自定义的字符串来补充输出 false 的内容。", + "Tool module": "工具", "UnKnow Module": "未知模块", + "User guide": "用户引导", "cfr": "问题补全", "cfr intro": "根据历史记录，完善当前问题，使其更利于知识库搜索，同时提高连续对话能力。", "textEditor": "文本加工", @@ -717,10 +910,10 @@ "Files": "文件: {{total}}个", "Folder Name": "输入文件夹名称", "Insert Data": "插入", - "Manual collection Tip": "手动数据集允许创建一个空的容器装入数据", "Manual Data": "手动录入", "Manual Input": "手动录入", "Manual Mark": "手动标注", + "Manual collection Tip": "手动数据集允许创建一个空的容器装入数据", "Mark Data": "标注数据", "Move Failed": "移动出现错误~", "Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间", @@ -914,6 +1107,15 @@ "Update Your Plugin": "更新插件" }, "support": { + "openapi": { + "Api baseurl": "API根地址", + "Api manager": "API 秘钥管理", + "Copy success": "已复制 API 地址", + "Max usage": "最大额度(￥)", + "New api key": "新的 API 秘钥", + "New api key tip": "请保管好你的秘钥，秘钥不会再次展示~", + "Usage": "已用额度(￥)" + }, "outlink": { "share": { "Response Quote": "返回引用", @@ -1067,6 +1269,7 @@ "App name": "应用名", "Audio Speech": "语音播报", "Bill Module": "扣费模块", + "Chars length": "文本长度", "Data Length": "数据长度", "Dataset store": "知识库存储", "Duration": "时长（秒）", diff --git a/projects/app/src/components/ChatBox/FeedbackModal.tsx b/projects/app/src/components/ChatBox/FeedbackModal.tsx index a29cfba5a..b05f11581 100644 --- a/projects/app/src/components/ChatBox/FeedbackModal.tsx +++ b/projects/app/src/components/ChatBox/FeedbackModal.tsx @@ -56,7 +56,7 @@ const FeedbackModal = ({ ); diff --git a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx index 75b3125fd..57abc080e 100644 --- a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx +++ b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx @@ -1,23 +1,10 @@ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useTransition } from 'react'; import { NodeProps } from 'reactflow'; -import { - Box, - Flex, - Textarea, - useTheme, - Table, - Thead, - Tbody, - Tr, - Th, - Td, - TableContainer, - Switch -} from '@chakra-ui/react'; +import { Box, Flex, Textarea, useTheme } from '@chakra-ui/react'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { FlowModuleItemType, ModuleItemType } from '@fastgpt/global/core/module/type.d'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; -import { welcomeTextTip, variableTip } from '@fastgpt/global/core/module/template/tip'; +import { welcomeTextTip } from '@fastgpt/global/core/module/template/tip'; import { onChangeNode } from '../../FlowProvider'; import VariableEdit from '../modules/VariableEdit'; @@ -29,6 +16,7 @@ import type { VariableItemType } from '@fastgpt/global/core/module/type.d'; import QGSwitch from '@/components/core/module/Flow/components/modules/QGSwitch'; import TTSSelect from '@/components/core/module/Flow/components/modules/TTSSelect'; import { splitGuideModule } from '@fastgpt/global/core/module/utils'; +import { useTranslation } from 'next-i18next'; const NodeUserGuide = React.memo(function NodeUserGuide({ data }: { data: FlowModuleItemType }) { const theme = useTheme(); @@ -56,19 +44,18 @@ export default function Node({ data }: NodeProps) { return ; } export function WelcomeText({ data }: { data: FlowModuleItemType }) { + const { t } = useTranslation(); const { inputs, moduleId } = data; + const [, startTst] = useTransition(); - const welcomeText = useMemo( - () => inputs.find((item) => item.key === ModuleInputKeyEnum.welcomeText), - [inputs] - ); + const welcomeText = inputs.find((item) => item.key === ModuleInputKeyEnum.welcomeText); return ( <> - 开场白 - + {t('core.app.Welcome Text')} + @@ -79,16 +66,18 @@ export function WelcomeText({ data }: { data: FlowModuleItemType }) { resize={'both'} defaultValue={welcomeText.value} bg={'myWhite.500'} - placeholder={welcomeTextTip} + placeholder={t(welcomeTextTip)} onChange={(e) => { - onChangeNode({ - moduleId, - key: ModuleInputKeyEnum.welcomeText, - type: 'updateInput', - value: { - ...welcomeText, - value: e.target.value - } + startTst(() => { + onChangeNode({ + moduleId, + key: ModuleInputKeyEnum.welcomeText, + type: 'updateInput', + value: { + ...welcomeText, + value: e.target.value + } + }); }); }} /> diff --git a/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx b/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx deleted file mode 100644 index d16d1fff8..000000000 --- a/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* Abandon */ -import React, { useCallback, useMemo, useState } from 'react'; -import { NodeProps } from 'reactflow'; -import { Box, Button, Table, Thead, Tbody, Tr, Th, Td, TableContainer } from '@chakra-ui/react'; -import { AddIcon } from '@chakra-ui/icons'; -import NodeCard from '../../render/NodeCard'; -import { FlowModuleItemType } from '@fastgpt/global/core/module/type.d'; -import Container from '../../modules/Container'; -import { VariableInputEnum, ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; -import type { VariableItemType } from '@fastgpt/global/core/module/type.d'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import { customAlphabet } from 'nanoid'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); -import VariableEditModal, { addVariable } from '../../modules/VariableEdit'; -import { onChangeNode } from '../../../FlowProvider'; - -export const defaultVariable: VariableItemType = { - id: nanoid(), - key: 'key', - label: 'label', - type: VariableInputEnum.input, - required: true, - maxLen: 50, - enums: [{ value: '' }] -}; - -const NodeUserGuide = ({ data }: NodeProps) => { - const { inputs, moduleId } = data; - - const variables = useMemo( - () => - (inputs.find((item) => item.key === ModuleInputKeyEnum.variables) - ?.value as VariableItemType[]) || [], - [inputs] - ); - - return ( - <> - - - - onChangeNode({ - moduleId, - key: ModuleInputKeyEnum.variables, - type: 'updateInput', - value: { - ...inputs.find((item) => item.key === ModuleInputKeyEnum.variables), - value: e - } - }) - } - /> - - - - ); -}; -export default React.memo(NodeUserGuide); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx index b843b5e73..fb9728102 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useMemo } from 'react'; import type { FlowNodeInputItemType } from '@fastgpt/global/core/module/node/type'; import { Box } from '@chakra-ui/react'; import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant'; @@ -6,7 +6,7 @@ import dynamic from 'next/dynamic'; import InputLabel from './Label'; import type { RenderInputProps } from './type.d'; -import { getFlowStore, type useFlowProviderStoreType } from '../../../FlowProvider'; +import { useFlowProviderStore } from '../../../FlowProvider'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; const RenderList: { @@ -77,8 +77,8 @@ type Props = { moduleId: string; CustomComponent?: Record React.ReactNode>; }; -const RenderInput = ({ flowInputList, moduleId, CustomComponent = {} }: Props) => { - const [mode, setMode] = useState('app'); +const RenderInput = ({ flowInputList, moduleId, CustomComponent }: Props) => { + const { mode } = useFlowProviderStore(); const sortInputs = useMemo( () => @@ -109,48 +109,43 @@ const RenderInput = ({ flowInputList, moduleId, CustomComponent = {} }: Props) = [mode, sortInputs] ); - useEffect(() => { - async () => { - const { mode } = await getFlowStore(); - setMode(mode); - }; - }, []); + const memoCustomComponent = useMemo(() => CustomComponent || {}, [CustomComponent]); - return ( - <> - {filterInputs.map((input) => { - const RenderComponent = (() => { - if (input.type === FlowNodeInputTypeEnum.custom && CustomComponent[input.key]) { - return <>{CustomComponent[input.key]({ ...input })}; - } - const Component = RenderList.find((item) => item.types.includes(input.type))?.Component; + const Render = useMemo(() => { + return filterInputs.map((input) => { + const RenderComponent = (() => { + if (input.type === FlowNodeInputTypeEnum.custom && memoCustomComponent[input.key]) { + return <>{memoCustomComponent[input.key]({ ...input })}; + } + const Component = RenderList.find((item) => item.types.includes(input.type))?.Component; - if (!Component) return null; - return ; - })(); + if (!Component) return null; + return ; + })(); - return ( - - {input.key === ModuleInputKeyEnum.userChatInput && ( - - )} - {input.type !== FlowNodeInputTypeEnum.hidden && ( - <> - {!!input.label && ( - - )} - {!!RenderComponent && ( - - {RenderComponent} - - )} - - )} - - ); - })} - - ); + return ( + + {input.key === ModuleInputKeyEnum.userChatInput && ( + + )} + {input.type !== FlowNodeInputTypeEnum.hidden && ( + <> + {!!input.label && ( + + )} + {!!RenderComponent && ( + + {RenderComponent} + + )} + + )} + + ); + }); + }, [memoCustomComponent, filterInputs, mode, moduleId]); + + return <>{Render}; }; export default React.memo(RenderInput); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx index fff3d7ef6..56ec626e0 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx @@ -1,20 +1,39 @@ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import type { RenderInputProps } from '../type'; -import { onChangeNode } from '../../../../FlowProvider'; +import { getFlowStore, onChangeNode, useFlowProviderStoreType } from '../../../../FlowProvider'; import { Box, Button, Flex, Grid, useDisclosure, useTheme } from '@chakra-ui/react'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { SelectedDatasetType } from '@fastgpt/global/core/module/api'; import Avatar from '@/components/Avatar'; -import DatasetSelectModal from '@/components/core/module/DatasetSelectModal'; import { useQuery } from '@tanstack/react-query'; +import { useTranslation } from 'next-i18next'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; +import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; +import { chatModelList } from '@/web/common/system/staticData'; -const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { +import dynamic from 'next/dynamic'; +import MyIcon from '@fastgpt/web/components/common/Icon'; + +const DatasetSelectModal = dynamic(() => import('@/components/core/module/DatasetSelectModal')); +const DatasetParamsModal = dynamic(() => import('@/components/core/module/DatasetParamsModal')); + +const SelectDatasetRender = ({ inputs = [], item, moduleId }: RenderInputProps) => { + const { t } = useTranslation(); const theme = useTheme(); + const [nodes, setNodes] = useState([]); + const [data, setData] = useState({ + searchMode: DatasetSearchModeEnum.embedding, + limit: 5, + similarity: 0.5, + usingReRank: false + }); + const { allDatasets, loadAllDatasets } = useDatasetStore(); const { - isOpen: isOpenKbSelect, - onOpen: onOpenKbSelect, - onClose: onCloseKbSelect + isOpen: isOpenDatasetSelect, + onOpen: onOpenDatasetSelect, + onClose: onCloseDatasetSelect } = useDisclosure(); const selectedDatasets = useMemo(() => { @@ -22,14 +41,68 @@ const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { return allDatasets.filter((dataset) => value?.find((item) => item.datasetId === dataset._id)); }, [allDatasets, item.value]); + const tokenLimit = useMemo(() => { + let maxTokens = 3000; + + nodes.forEach((item) => { + if (item.type === FlowNodeTypeEnum.chatNode) { + const model = + item.data.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || ''; + const quoteMaxToken = + chatModelList.find((item) => item.model === model)?.quoteMaxToken || 3000; + + maxTokens = Math.max(maxTokens, quoteMaxToken); + } + }); + + return maxTokens; + }, [nodes]); + + const { + isOpen: isOpenDatasetPrams, + onOpen: onOpenDatasetParams, + onClose: onCloseDatasetParams + } = useDisclosure(); + useQuery(['loadAllDatasets'], loadAllDatasets); + useEffect(() => { + inputs.forEach((input) => { + // @ts-ignore + if (data[input.key] !== undefined) { + setData((state) => ({ + ...state, + [input.key]: input.value + })); + } + }); + }, [inputs]); + + useEffect(() => { + async () => { + const { nodes } = await getFlowStore(); + setNodes(nodes); + }; + }, []); + return ( <> - + {/* } + onClick={onOpenDatasetParams} + > + {t('core.dataset.search.Params Setting')} + */} {selectedDatasets.map((item) => ( { ))} - {isOpenKbSelect && ( + {isOpenDatasetSelect && ( { onChangeNode({ @@ -68,9 +141,32 @@ const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { } }); }} - onClose={onCloseKbSelect} + onClose={onCloseDatasetSelect} /> )} + {/* {isOpenDatasetPrams && ( + { + for (let key in e) { + const item = inputs.find((input) => input.key === key); + if (!item) continue; + onChangeNode({ + moduleId, + type: 'updateInput', + key, + value: { + ...item, + //@ts-ignore + value: e[key] + } + }); + } + }} + /> + )} */} ); }; diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx index 38fc9c138..e31f2ed1a 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx @@ -1,9 +1,9 @@ import React, { useEffect, useMemo, useState } from 'react'; import type { RenderInputProps } from '../type'; -import { getFlowStore, onChangeNode, useFlowProviderStoreType } from '../../../../FlowProvider'; +import { onChangeNode, useFlowProviderStore } from '../../../../FlowProvider'; import { Button, useDisclosure } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; import { chatModelList } from '@/web/common/system/staticData'; @@ -11,7 +11,7 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import DatasetParamsModal from '@/components/core/module/DatasetParamsModal'; const SelectDatasetParam = ({ inputs = [], moduleId }: RenderInputProps) => { - const [nodes, setNodes] = useState([]); + const { nodes } = useFlowProviderStore(); const { t } = useTranslation(); const [data, setData] = useState({ @@ -52,47 +52,44 @@ const SelectDatasetParam = ({ inputs = [], moduleId }: RenderInputProps) => { }); }, [inputs]); - useEffect(() => { - async () => { - const { nodes } = await getFlowStore(); - setNodes(nodes); - }; - }, []); + const Render = useMemo(() => { + return ( + <> + } + onClick={onOpen} + > + {t('core.dataset.search.Params Setting')} + + {isOpen && ( + { + for (let key in e) { + const item = inputs.find((input) => input.key === key); + if (!item) continue; + onChangeNode({ + moduleId, + type: 'updateInput', + key, + value: { + ...item, + //@ts-ignore + value: e[key] + } + }); + } + }} + /> + )} + + ); + }, [data, inputs, isOpen, moduleId, onClose, onOpen, t, tokenLimit]); - return ( - <> - } - onClick={onOpen} - > - {t('core.dataset.search.Params Setting')} - - {isOpen && ( - { - for (let key in e) { - const item = inputs.find((input) => input.key === key); - if (!item) continue; - onChangeNode({ - moduleId, - type: 'updateInput', - key, - value: { - ...item, - //@ts-ignore - value: e[key] - } - }); - } - }} - /> - )} - - ); + return Render; }; export default React.memo(SelectDatasetParam); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx index 9bffa9ece..af5a0e040 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx @@ -1,38 +1,53 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useMemo, useTransition } from 'react'; import type { RenderInputProps } from '../type'; -import { onChangeNode } from '../../../../FlowProvider'; +import { useFlowProviderStore, onChangeNode } from '../../../../FlowProvider'; import { useTranslation } from 'next-i18next'; -import PromptTextarea from '@/components/common/Textarea/PromptTextarea'; +import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor'; +import { + formatVariablesIcon, + getGuideModule, + splitGuideModule +} from '@fastgpt/global/core/module/utils'; const TextareaRender = ({ item, moduleId }: RenderInputProps) => { const { t } = useTranslation(); + const [, startTst] = useTransition(); + const { nodes } = useFlowProviderStore(); - const update = useCallback( - (value: string) => { - onChangeNode({ - moduleId, - type: 'updateInput', - key: item.key, - value: { - ...item, - value - } + // get variable + const variables = useMemo( + () => + formatVariablesIcon( + splitGuideModule(getGuideModule(nodes.map((node) => node.data)))?.variableModules || [] + ), + [nodes] + ); + + const onChange = useCallback( + (e: string) => { + startTst(() => { + onChangeNode({ + moduleId, + type: 'updateInput', + key: item.key, + value: { + ...item, + value: e + } + }); }); }, [item, moduleId] ); return ( - { - update(e.target.value); - }} + onChange={onChange} /> ); }; diff --git a/projects/app/src/components/core/module/Flow/index.tsx b/projects/app/src/components/core/module/Flow/index.tsx index fb66b0ebe..a8a04f4a7 100644 --- a/projects/app/src/components/core/module/Flow/index.tsx +++ b/projects/app/src/components/core/module/Flow/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import React, { useMemo } from 'react'; import ReactFlow, { Background, Controls, ReactFlowProvider } from 'reactflow'; import { Box, Flex, IconButton, useDisclosure } from '@chakra-ui/react'; import { SmallCloseIcon } from '@chakra-ui/icons'; @@ -15,7 +15,6 @@ import 'reactflow/dist/style.css'; const NodeSimple = dynamic(() => import('./components/nodes/NodeSimple')); const nodeTypes: Record<`${FlowNodeTypeEnum}`, any> = { [FlowNodeTypeEnum.userGuide]: dynamic(() => import('./components/nodes/NodeUserGuide')), - [FlowNodeTypeEnum.variable]: dynamic(() => import('./components/nodes/abandon/NodeVariable')), [FlowNodeTypeEnum.questionInput]: dynamic(() => import('./components/nodes/NodeQuestionInput')), [FlowNodeTypeEnum.historyNode]: NodeSimple, [FlowNodeTypeEnum.chatNode]: NodeSimple, @@ -38,6 +37,16 @@ const Container = React.memo(function Container() { const { reactFlowWrapper, nodes, onNodesChange, edges, onEdgesChange, onConnect } = useFlowProviderStore(); + const memoRenderTools = useMemo( + () => ( + <> + + + + ), + [] + ); + return ( - - + {memoRenderTools} ); }); @@ -81,48 +89,54 @@ const Flow = ({ onClose: onCloseTemplate } = useDisclosure(); + const memoRenderContainer = useMemo(() => { + return ( + { + e.preventDefault(); + return false; + }} + > + {/* open module template */} + } + transform={isOpenTemplate ? '' : 'rotate(135deg)'} + transition={'0.2s ease'} + aria-label={''} + zIndex={1} + boxShadow={'2px 2px 6px #85b1ff'} + onClick={() => { + isOpenTemplate ? onCloseTemplate() : onOpenTemplate(); + }} + /> + + + + + + ); + }, [data, isOpenTemplate, onCloseTemplate, onOpenTemplate, templates]); + return ( {Header} - { - e.preventDefault(); - return false; - }} - > - {/* open module template */} - } - transform={isOpenTemplate ? '' : 'rotate(135deg)'} - transition={'0.2s ease'} - aria-label={''} - zIndex={1} - boxShadow={'2px 2px 6px #85b1ff'} - onClick={() => { - isOpenTemplate ? onCloseTemplate() : onOpenTemplate(); - }} - /> - - - - - + {memoRenderContainer} diff --git a/projects/app/src/components/support/apikey/Table.tsx b/projects/app/src/components/support/apikey/Table.tsx index 6f751e176..16cb5edaa 100644 --- a/projects/app/src/components/support/apikey/Table.tsx +++ b/projects/app/src/components/support/apikey/Table.tsx @@ -81,7 +81,7 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { - API 秘钥管理 + {t('support.openapi.Api manager')} {feConfigs?.docUrl && ( { ml={1} color={'primary.500'} > - 查看文档 + {t('common.Read document')} )} @@ -106,10 +106,10 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { borderRadius={'md'} cursor={'pointer'} userSelect={'none'} - onClick={() => copyData(baseUrl, '已复制 API 地址')} + onClick={() => copyData(baseUrl, t('support.openapi.Copy success'))} > - API根地址 + {t('support.openapi.Api baseurl')} {baseUrl} @@ -127,7 +127,7 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { }) } > - 新建 + {t('common.New Create')} @@ -137,16 +137,16 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => {
{t('Name')} Api Key - 已用额度(￥) + {t('support.openapi.Usage')} {feConfigs?.isPlus && ( <> - 最大额度(￥) - 过期时间 + {t('support.openapi.Max usage')} + {t('common.Expired Time')} )} - 创建时间 - 最后一次使用时间 + {t('common.Create Time')} + {t('common.Last use time')}
@@ -158,7 +158,11 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { {usage} {feConfigs?.isPlus && ( <> - {limit?.credit && limit?.credit > -1 ? `${limit?.credit}` : '无限制'} + + {limit?.credit && limit?.credit > -1 + ? `${limit?.credit}` + : t('common.Unlimited')} + {limit?.expiredTime ? dayjs(limit?.expiredTime).format('YYYY/MM/DD\nHH:mm') @@ -168,7 +172,9 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { )} {dayjs(createTime).format('YYYY/MM/DD\nHH:mm:ss')} - {lastUsedTime ? dayjs(lastUsedTime).format('YYYY/MM/DD\nHH:mm:ss') : '没有使用过'} + {lastUsedTime + ? dayjs(lastUsedTime).format('YYYY/MM/DD\nHH:mm:ss') + : t('common.Un used')} @@ -229,10 +235,10 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { title={ - 新的 API 秘钥 + {t('support.openapi.New api key')} - 请保管好你的秘钥，秘钥不会再次展示~ + {t('support.openapi.New api key tip')} } @@ -359,14 +365,14 @@ function EditKeyModal({ diff --git a/projects/app/src/components/support/wallet/Price.tsx b/projects/app/src/components/support/wallet/Price.tsx index 6182da4b1..7a3104ed3 100644 --- a/projects/app/src/components/support/wallet/Price.tsx +++ b/projects/app/src/components/support/wallet/Price.tsx @@ -33,20 +33,16 @@ ${chatModelList md: ` | 模型 | 价格(￥) | | --- | --- | -${vectorModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k tokens |`).join('\n')} +${vectorModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')} ` }, { title: '文件预处理模型(QA 拆分)', describe: '', md: ` -| 模型 | 输入价格(￥) | 输出价格(￥) | -| --- | --- | --- | -${qaModelList - ?.map( - (item) => `| ${item.name} | ${item.inputPrice}/1k tokens | ${item.outputPrice}/1k tokens |` - ) - .join('\n')} +| 模型 | 价格(￥) | +| --- | --- | +${qaModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')} ` }, { @@ -84,14 +80,6 @@ ${qgModelList (item) => `| ${item.name} | ${item.inputPrice}/1k tokens | ${item.outputPrice}/1k tokens |` ) .join('\n')}` - }, - { - title: '重排模型(增强检索 & 混合检索)', - describe: '', - md: ` -| 模型 | 价格(￥) | -| --- | --- | -${reRankModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')}` }, { title: '语音播放', diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index 6cad6a2f8..960f56feb 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -2,9 +2,9 @@ import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import type { RequestPaging } from '@/types'; -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import type { SearchTestItemType } from '@/types/core/dataset'; import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; diff --git a/projects/app/src/global/core/app/constants.ts b/projects/app/src/global/core/app/constants.ts index 0eb6f7bdf..b5392e6b2 100644 --- a/projects/app/src/global/core/app/constants.ts +++ b/projects/app/src/global/core/app/constants.ts @@ -1,10 +1,10 @@ import { AppSimpleEditConfigTemplateType } from '@fastgpt/global/core/app/type.d'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; export const SimpleModeTemplate_FastGPT_Universal: AppSimpleEditConfigTemplateType = { id: 'fastgpt-universal', - name: '通用模板', - desc: '通用模板\n可完全自行配置AI属性和知识库', + name: 'core.app.template.Common template', + desc: 'core.app.template.Common template tip', systemForm: { aiSettings: { model: true, diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 5ec3c0180..e942fe637 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -3,7 +3,7 @@ import { DatasetSearchModeEnum, DatasetTypeEnum, TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { DatasetDataIndexItemType, SearchDataResponseItemType diff --git a/projects/app/src/global/core/prompt/AIChat.ts b/projects/app/src/global/core/prompt/AIChat.ts index 2965df48d..b8c7a2690 100644 --- a/projects/app/src/global/core/prompt/AIChat.ts +++ b/projects/app/src/global/core/prompt/AIChat.ts @@ -53,8 +53,8 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [ 回答要求： - 如果你不清楚答案，你需要澄清。 -- 避免提及你是从 data 获取的知识。 -- 保持答案与 data 中描述的一致。 +- 避免提及你是从 获取的知识。 +- 保持答案与 中描述的一致。 - 使用 Markdown 语法优化回答格式。 - 使用与问题相同的语言回答。 @@ -88,8 +88,8 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [ 3. 如果无关，你直接拒绝回答本次问题。 回答要求： -- 避免提及你是从 data 获取的知识。 -- 保持答案与 data 中描述的一致。 +- 避免提及你是从 获取的知识。 +- 保持答案与 中描述的一致。 - 使用 Markdown 语法优化回答格式。 - 使用与问题相同的语言回答。 diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index 428397ad7..b329486c1 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useState } from 'react'; import type { AppProps } from 'next/app'; import Script from 'next/script'; import Head from 'next/head'; diff --git a/projects/app/src/pages/account/components/BillDetail.tsx b/projects/app/src/pages/account/components/BillDetail.tsx index 6ade1f896..c3b7b6cee 100644 --- a/projects/app/src/pages/account/components/BillDetail.tsx +++ b/projects/app/src/pages/account/components/BillDetail.tsx @@ -30,7 +30,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void hasTokens, hasInputTokens, hasOutputTokens, - hasTextLen, + hasCharsLen, hasDuration, hasDataLen } = useMemo(() => { @@ -38,7 +38,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void let hasTokens = false; let hasInputTokens = false; let hasOutputTokens = false; - let hasTextLen = false; + let hasCharsLen = false; let hasDuration = false; let hasDataLen = false; @@ -46,24 +46,21 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void if (item.model !== undefined) { hasModel = true; } - if (item.tokenLen !== undefined) { + if (typeof item.tokenLen === 'number') { hasTokens = true; } - if (item.inputTokens !== undefined) { + if (typeof item.inputTokens === 'number') { hasInputTokens = true; } - if (item.outputTokens !== undefined) { + if (typeof item.outputTokens === 'number') { hasOutputTokens = true; } - if (item.textLen !== undefined) { - hasTextLen = true; + if (typeof item.charsLength === 'number') { + hasCharsLen = true; } - if (item.duration !== undefined) { + if (typeof item.duration === 'number') { hasDuration = true; } - if (item.dataLen !== undefined) { - hasDataLen = true; - } }); return { @@ -71,7 +68,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void hasTokens, hasInputTokens, hasOutputTokens, - hasTextLen, + hasCharsLen, hasDuration, hasDataLen }; @@ -123,9 +120,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void {hasTokens && {t('wallet.bill.Token Length')}} {hasInputTokens && {t('wallet.bill.Input Token Length')}} {hasOutputTokens && {t('wallet.bill.Output Token Length')}} - {hasTextLen && {t('wallet.bill.Text Length')}} + {hasCharsLen && {t('wallet.bill.Text Length')}} {hasDuration && {t('wallet.bill.Duration')}} - {hasDataLen && {t('wallet.bill.Data Length')}} 费用(￥) @@ -137,10 +133,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void {hasTokens && {item.tokenLen ?? '-'}} {hasInputTokens && {item.inputTokens ?? '-'}} {hasOutputTokens && {item.outputTokens ?? '-'}} - {hasTextLen && {item.textLen ?? '-'}} + {hasCharsLen && {item.charsLength ?? '-'}} {hasDuration && {item.duration ?? '-'}} - {hasDataLen && {item.dataLen ?? '-'}} - {formatStorePrice2Read(item.amount)} ))} diff --git a/projects/app/src/pages/account/components/Info.tsx b/projects/app/src/pages/account/components/Info.tsx index b2f1743ce..5ee0f63bd 100644 --- a/projects/app/src/pages/account/components/Info.tsx +++ b/projects/app/src/pages/account/components/Info.tsx @@ -108,12 +108,12 @@ const UserInfo = () => { }); } catch (err: any) { toast({ - title: typeof err === 'string' ? err : '头像选择异常', + title: typeof err === 'string' ? err : t('common.error.Select avatar failed'), status: 'warning' }); } }, - [onclickSave, toast, userInfo] + [onclickSave, t, toast, userInfo] ); useQuery(['init'], initUserInfo, { diff --git a/projects/app/src/pages/api/admin/initv46-2.ts b/projects/app/src/pages/api/admin/initv46-2.ts deleted file mode 100644 index e99de6a22..000000000 --- a/projects/app/src/pages/api/admin/initv46-2.ts +++ /dev/null @@ -1,184 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; - -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { defaultQAModels } from '@fastgpt/global/core/ai/model'; - -let success = 0; -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50 } = req.body as { limit: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - success = 0; - - try { - await Promise.allSettled([ - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null - PgClient.query( - `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);` - ), // team_id varchar - PgClient.query( - `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);` - ), // tmb_id varchar - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null - ]); - } catch (error) {} - - try { - await initPgData(); - } catch (error) {} - - await MongoDataset.updateMany( - {}, - { - agentModel: defaultQAModels[0].model - } - ); - - jsonRes(res, { - data: await init(limit), - message: - '初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。' - }); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -type PgItemType = { - id: string; - q: string; - a: string; - dataset_id: string; - collection_id: string; - team_id: string; - tmb_id: string; -}; - -async function initPgData() { - const limit = 10; - const { rows } = await PgClient.query<{ user_id: string }>(` - SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null'; -`); - console.log('init pg', rows.length); - let success = 0; - for (let i = 0; i < limit; i++) { - init(i); - } - - async function init(index: number): Promise { - const userId = rows[index]?.user_id; - if (!userId) return; - try { - const tmb = await getUserDefaultTeam({ userId }); - console.log(tmb); - - // update pg - await PgClient.query( - `Update ${PgDatasetTableName} set team_id = '${String(tmb.teamId)}', tmb_id = '${String( - tmb.tmbId - )}' where user_id = '${userId}' AND team_id='null';` - ); - console.log(++success); - init(index + limit); - } catch (error) { - if (error === 'default team not exist') { - return; - } - console.log(error); - await delay(1000); - return init(index); - } - } -} - -async function init(limit: number): Promise { - const { rows: idList } = await PgClient.query<{ id: string }>( - `SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL` - ); - - console.log('totalCount', idList.length); - if (idList.length === 0) return; - - for (let i = 0; i < limit; i++) { - initData(i); - } - - async function initData(index: number): Promise { - const dataId = idList[index]?.id; - if (!dataId) { - console.log('done'); - return; - } - // get limit data where data_id is null - const { rows } = await PgClient.query( - `SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};` - ); - const data = rows[0]; - if (!data) { - console.log('done'); - return; - } - - let id = ''; - try { - // create mongo data and update data_id - const { _id } = await MongoDatasetData.create({ - teamId: data.team_id.trim(), - tmbId: data.tmb_id.trim(), - datasetId: data.dataset_id, - collectionId: data.collection_id, - q: data.q, - a: data.a, - fullTextToken: '', - indexes: [ - { - defaultIndex: !data.a, - type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk, - dataId: data.id, - text: data.q - } - ] - }); - id = _id; - // update pg data_id - await PgClient.query( - `UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};` - ); - - console.log(++success); - return initData(index + limit); - } catch (error) { - console.log(error); - console.log(data); - - try { - if (id) { - await MongoDatasetData.findByIdAndDelete(id); - } - } catch (error) {} - await delay(500); - return initData(index); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv46-fix.ts b/projects/app/src/pages/api/admin/initv46-fix.ts deleted file mode 100644 index 8ddd8e070..000000000 --- a/projects/app/src/pages/api/admin/initv46-fix.ts +++ /dev/null @@ -1,173 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; - -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { Types, connectionMongo } from '@fastgpt/service/common/mongo'; -import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; -import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; - -let success = 0; -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50 } = req.body as { limit: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - success = 0; - - await init(limit); - await initCollectionFileTeam(limit); - - jsonRes(res, {}); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -type PgItemType = { - id: string; - q: string; - a: string; - dataset_id: string; - collection_id: string; - data_id: string; -}; - -async function init(limit: number): Promise { - const { rows } = await PgClient.query<{ id: string; data_id: string }>( - `SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id` - ); - - console.log('totalCount', rows.length); - - await delay(2000); - - if (rows.length === 0) return; - - for (let i = 0; i < limit; i++) { - initData(i); - } - - async function initData(index: number): Promise { - const item = rows[index]; - if (!item) { - console.log('done'); - return; - } - // get mongo - const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId'); - if (!mongoData) { - return initData(index + limit); - } - - try { - // find team owner - const db = connectionMongo?.connection?.db; - const TeamMember = db.collection(TeamMemberCollectionName); - - const tmb = await TeamMember.findOne({ - teamId: new Types.ObjectId(mongoData.teamId), - role: 'owner' - }); - - if (!tmb) { - return initData(index + limit); - } - - // update mongo and pg tmb_id - await MongoDatasetData.findByIdAndUpdate(item.data_id, { - tmbId: tmb._id - }); - await PgClient.query( - `UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'` - ); - - console.log(++success); - - return initData(index + limit); - } catch (error) { - console.log(error); - await delay(500); - return initData(index); - } - } -} - -async function initCollectionFileTeam(limit: number) { - /* init user default Team */ - const DatasetFile = getGFSCollection('dataset'); - const matchWhere = { - $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }] - }; - const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([ - { - $match: matchWhere - }, - { - $group: { - _id: '$metadata.userId', // 按 metadata.userId 分组以去重 - userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]).toArray(); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const item of userArr) { - await Promise.all(item.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ - userId - }); - - await DatasetFile.updateMany( - { - 'metadata.userId': String(userId), - ...matchWhere - }, - { - $set: { - 'metadata.teamId': String(tmb.teamId), - 'metadata.tmbId': String(tmb.tmbId) - } - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv46.ts b/projects/app/src/pages/api/admin/initv46.ts deleted file mode 100644 index 4af44b89a..000000000 --- a/projects/app/src/pages/api/admin/initv46.ts +++ /dev/null @@ -1,330 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema'; -import { - createDefaultTeam, - getUserDefaultTeam -} from '@fastgpt/service/support/user/team/controller'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { UserModelSchema } from '@fastgpt/global/support/user/type'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; -import { MongoOutLink } from '@fastgpt/service/support/outLink/schema'; -import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema'; -import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; -import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; -import { MongoPlugin } from '@fastgpt/service/core/plugin/schema'; -import { POST } from '@fastgpt/service/common/api/plusRequest'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; -import { FastGPTProUrl } from '@fastgpt/service/common/system/constants'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - - await initDefaultTeam(limit, maxSize); - await initMongoTeamId(limit); - await initDatasetAndApp(); - await initCollectionFileTeam(limit); - - if (FastGPTProUrl) { - POST('/admin/init46'); - } - - await initPgData(); - - jsonRes(res, { - data: {} - }); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -async function initDefaultTeam(limit: number, maxSize: number) { - /* init user default Team */ - const users = await MongoUser.find({}, '_id balance'); - console.log('init user default team', users.length); - // 100 组一次 - const userArr: UserModelSchema[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - let success = 0; - for await (const users of userArr) { - await Promise.all(users.map(init)); - success += limit; - console.log(success); - } - - async function init(user: UserModelSchema): Promise { - try { - await createDefaultTeam({ - userId: user._id, - balance: user.balance, - maxSize - }); - } catch (error) { - console.log(error); - - await delay(1000); - return init(user); - } - } -} -async function initMongoTeamId(limit: number) { - const mongoSchema = [ - { - label: 'MongoPlugin', - schema: MongoPlugin - }, - { - label: 'MongoChat', - schema: MongoChat - }, - { - label: 'MongoChatItem', - schema: MongoChatItem - }, - { - label: 'MongoApp', - schema: MongoApp - }, - { - label: 'MongoDataset', - schema: MongoDataset - }, - { - label: 'MongoDatasetCollection', - schema: MongoDatasetCollection - }, - { - label: 'MongoDatasetTraining', - schema: MongoDatasetTraining - }, - { - label: 'MongoBill', - schema: MongoBill - }, - { - label: 'MongoOutLink', - schema: MongoOutLink - }, - { - label: 'MongoOpenApi', - schema: MongoOpenApi - } - ]; - /* init user default Team */ - - for await (const item of mongoSchema) { - console.log('start init', item.label); - await initTeamTmbId(item.schema); - console.log('finish init', item.label); - } - - async function initTeamTmbId(schema: any) { - const emptyWhere = { - $or: [{ teamId: { $exists: false } }, { teamId: null }] - }; - const uniqueUsersWithNoTeamId = await schema.aggregate([ - { - $match: emptyWhere - }, - { - $group: { - _id: '$userId', // 按 userId 分组以去重 - userId: { $first: '$userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const users of userArr) { - await Promise.all(users.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ userId }); - - await schema.updateMany( - { - userId, - ...emptyWhere - }, - { - teamId: tmb.teamId, - tmbId: tmb.tmbId - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } - } -} -async function initDatasetAndApp() { - await MongoDataset.updateMany( - {}, - { - $set: { - permission: PermissionTypeEnum.private - } - } - ); - await MongoApp.updateMany( - {}, - { - $set: { - permission: PermissionTypeEnum.private - } - } - ); -} -async function initCollectionFileTeam(limit: number) { - /* init user default Team */ - const DatasetFile = getGFSCollection('dataset'); - const matchWhere = { - $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }] - }; - const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([ - { - $match: matchWhere - }, - { - $group: { - _id: '$metadata.userId', // 按 metadata.userId 分组以去重 - userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]).toArray(); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const item of userArr) { - await Promise.all(item.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ - userId - }); - - await DatasetFile.updateMany( - { - 'metadata.userId': String(userId), - ...matchWhere - }, - { - $set: { - 'metadata.teamId': String(tmb.teamId), - 'metadata.tmbId': String(tmb.tmbId) - } - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } -} -async function initPgData() { - const limit = 10; - // add column - try { - await Promise.allSettled([ - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`) - ]); - } catch (error) { - console.log(error); - console.log('column exists'); - } - - const { rows } = await PgClient.query<{ user_id: string }>(` - SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL; -`); - console.log('init pg', rows.length); - let success = 0; - for (let i = 0; i < limit; i++) { - init(i); - } - async function init(index: number): Promise { - const userId = rows[index]?.user_id; - if (!userId) return; - try { - const tmb = await getUserDefaultTeam({ userId }); - // update pg - await PgClient.query( - `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;` - ); - console.log(++success); - init(index + limit); - } catch (error) { - if (error === 'default team not exist') { - return; - } - console.log(error); - await delay(1000); - return init(index); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv463.ts b/projects/app/src/pages/api/admin/initv463.ts index 446d4f69f..3142b98d0 100644 --- a/projects/app/src/pages/api/admin/initv463.ts +++ b/projects/app/src/pages/api/admin/initv463.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; let success = 0; diff --git a/projects/app/src/pages/api/admin/initv467.ts b/projects/app/src/pages/api/admin/initv467.ts new file mode 100644 index 000000000..e8cc89270 --- /dev/null +++ b/projects/app/src/pages/api/admin/initv467.ts @@ -0,0 +1,106 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; +import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; +import { MongoImage } from '@fastgpt/service/common/file/image/schema'; +import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type'; +import { delay } from '@fastgpt/global/common/system/utils'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; + +let success = 0; +let deleteImg = 0; +/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { test = false } = req.body as { test: boolean }; + await authCert({ req, authRoot: true }); + await connectToDatabase(); + success = 0; + deleteImg = 0; + + // 取消 pg tmb_id 和 data_id 的null + await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id DROP NOT NULL;`); + await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN data_id DROP NOT NULL;`); + + // 重新绑定 images 和 collections + const images = await MongoImage.find( + { 'metadata.fileId': { $exists: true } }, + '_id metadata' + ).lean(); + + // 去除 fileId 相同的数据 + const fileIdMap = new Map(); + images.forEach((image) => { + // @ts-ignore + const fileId = image.metadata?.fileId; + if (!fileIdMap.has(fileId) && fileId) { + fileIdMap.set(fileId, image); + } + }); + const images2 = Array.from(fileIdMap.values()); + + console.log('total image list', images2.length); + + for await (const image of images2) { + await initImages(image, test); + } + + jsonRes(res, { + data: success, + message: 'success' + }); + } catch (error) { + console.log(error); + + jsonRes(res, { + code: 500, + error + }); + } +} +export const initImages = async (image: MongoImageSchemaType, test: boolean): Promise => { + try { + //@ts-ignore + const fileId = image.metadata.fileId as string; + if (!fileId) return; + + // 找到集合 + const collection = await MongoDatasetCollection.findOne({ fileId }, '_id metadata').lean(); + + if (!collection) { + deleteImg++; + console.log('deleteImg', deleteImg); + + if (test) return; + return MongoImage.deleteOne({ _id: image._id }); + } + + const relatedImageId = getNanoid(24); + + // update image + if (!test) { + await Promise.all([ + MongoImage.updateMany( + { 'metadata.fileId': fileId }, + { $set: { 'metadata.relatedId': relatedImageId } } + ), + MongoDatasetCollection.findByIdAndUpdate(collection._id, { + $set: { + 'metadata.relatedImgId': relatedImageId + } + }) + ]); + } + + success++; + console.log('success', success); + } catch (error) { + console.log(error); + + await delay(1000); + return initImages(image, test); + } +}; diff --git a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts deleted file mode 100644 index 69e3d6bd8..000000000 --- a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { - delFileByFileIdList, - getGFSCollection -} from '@fastgpt/service/common/file/gridfs/controller'; -import { addLog } from '@fastgpt/service/common/system/log'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { delay } from '@fastgpt/global/common/system/utils'; - -/* - check dataset.files data. If there is no match in dataset.collections, delete it -*/ -let deleteFileAmount = 0; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { - startDay = 10, - endDay = 3, - limit = 30 - } = req.body as { startDay?: number; endDay?: number; limit?: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - - // start: now - maxDay, end: now - 3 day - const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000); - const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000); - deleteFileAmount = 0; - - checkFiles(start, end, limit); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - addLog.error(`check valid dataset files error`, error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -export async function checkFiles(start: Date, end: Date, limit: number) { - const collection = getGFSCollection('dataset'); - const where = { - uploadDate: { $gte: start, $lte: end } - }; - - // 1. get all _id - const ids = await collection - .find(where, { - projection: { - _id: 1 - } - }) - .toArray(); - console.log('total files', ids.length); - - for (let i = 0; i < limit; i++) { - check(i); - } - - async function check(index: number): Promise { - const id = ids[index]; - if (!id) { - console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`); - - return; - } - try { - const { _id } = id; - - // 2. find fileId in dataset.collections - const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id }); - - // 3. if not found, delete file - if (hasCollection === 0) { - await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] }); - console.log('delete file', _id); - deleteFileAmount++; - } - index % 100 === 0 && console.log(index); - return check(index + limit); - } catch (error) { - console.log(error); - await delay(2000); - return check(index); - } - } -} diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index 76101589d..3f3d135e5 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -18,32 +18,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< try { const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); - const { files, bucketName, metadata } = await upload.doUpload(req, res); - - filePaths = files.map((file) => file.path); + const { file, bucketName, metadata } = await upload.doUpload(req, res); + filePaths = [file.path]; await connectToDatabase(); if (!bucketName) { throw new Error('bucketName is empty'); } - const upLoadResults = await Promise.all( - files.map((file) => - uploadFile({ - teamId, - tmbId, - bucketName, - path: file.path, - filename: file.originalname, - metadata: { - ...metadata, - contentType: file.mimetype, - userId - } - }) - ) - ); + const upLoadResults = await uploadFile({ + teamId, + tmbId, + bucketName, + path: file.path, + filename: file.originalname, + contentType: file.mimetype, + metadata: metadata + }); jsonRes(res, { data: upLoadResults diff --git a/projects/app/src/pages/api/common/system/getInitData.ts b/projects/app/src/pages/api/common/system/getInitData.ts index 8eb660f35..7bfe9c072 100644 --- a/projects/app/src/pages/api/common/system/getInitData.ts +++ b/projects/app/src/pages/api/common/system/getInitData.ts @@ -55,7 +55,8 @@ const defaultFeConfigs: FastGPTFeConfigsType = { websiteSyncLimitMinuted: 0 }, scripts: [], - favicon: '/favicon.ico' + favicon: '/favicon.ico', + uploadFileMaxSize: 500 }; export async function getInitConfig() { diff --git a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts index ff93cac97..6f2a469db 100644 --- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts +++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts @@ -7,7 +7,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d'; import type { ModuleItemType } from '@fastgpt/global/core/module/type'; import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; import { getExtractModel } from '@/service/core/ai/model'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -37,7 +37,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { return [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -49,7 +49,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -58,7 +58,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ @@ -93,7 +93,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -189,7 +189,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: @@ -268,7 +268,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { const modules: ModuleItemType[] = [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -280,7 +280,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -289,17 +289,13 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ { moduleId: 'vuc92c', key: 'userChatInput' - }, - { - moduleId: 'chatModule', - key: 'userChatInput' } ] } @@ -307,7 +303,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { }, { moduleId: 'datasetSearch', - name: '知识库搜索', + name: 'core.module.template.Dataset search', avatar: '/imgs/module/db.png', flowType: 'datasetSearchNode', showStatus: true, @@ -447,6 +443,18 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { valueType: 'boolean', type: 'source', targets: [] + }, + { + key: 'userChatInput', + label: 'core.module.input.label.user question', + type: 'hidden', + valueType: 'string', + targets: [ + { + moduleId: 'chatModule', + key: 'userChatInput' + } + ] } ] }, @@ -473,7 +481,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -569,7 +577,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: diff --git a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts index fa75ca327..140df7cb3 100644 --- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts +++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts @@ -33,7 +33,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { return [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -45,7 +45,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -54,7 +54,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ @@ -89,7 +89,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -185,7 +185,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: @@ -264,7 +264,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { const modules: ModuleItemType[] = [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -276,7 +276,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -285,17 +285,13 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ { moduleId: 'vuc92c', key: 'userChatInput' - }, - { - moduleId: 'chatModule', - key: 'userChatInput' } ] } @@ -303,7 +299,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { }, { moduleId: 'datasetSearch', - name: '知识库搜索', + name: 'core.module.template.Dataset search', avatar: '/imgs/module/db.png', flowType: 'datasetSearchNode', showStatus: true, @@ -457,6 +453,18 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { valueType: 'boolean', type: 'source', targets: [] + }, + { + key: 'userChatInput', + label: 'core.module.input.label.user question', + type: 'hidden', + valueType: 'string', + targets: [ + { + moduleId: 'chatModule', + key: 'userChatInput' + } + ] } ] }, @@ -483,7 +491,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -579,7 +587,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: diff --git a/projects/app/src/pages/api/core/app/getChatLogs.ts b/projects/app/src/pages/api/core/app/getChatLogs.ts index 66f198509..8e74264ee 100644 --- a/projects/app/src/pages/api/core/app/getChatLogs.ts +++ b/projects/app/src/pages/api/core/app/getChatLogs.ts @@ -8,6 +8,7 @@ import { Types } from '@fastgpt/service/common/mongo'; import { addDays } from 'date-fns'; import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d'; import { authApp } from '@fastgpt/service/support/permission/auth/app'; +import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -28,8 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' }); const where = { - appId: new Types.ObjectId(appId), teamId: new Types.ObjectId(teamId), + appId: new Types.ObjectId(appId), updateTime: { $gte: new Date(dateStart), $lte: new Date(dateEnd) @@ -41,18 +42,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) { $match: where }, { $lookup: { - from: 'chatitems', - let: { chat_id: '$chatId' }, + from: ChatItemCollectionName, + let: { chatId: '$chatId' }, pipeline: [ { $match: { $expr: { $and: [ - { $eq: ['$chatId', '$$chat_id'] }, - { $eq: ['$appId', new Types.ObjectId(appId)] } + { $eq: ['$appId', new Types.ObjectId(appId)] }, + { $eq: ['$chatId', '$$chatId'] } ] } } + }, + { + $project: { + userGoodFeedback: 1, + userBadFeedback: 1, + customFeedbacks: 1, + adminFeedback: 1 + } } ], as: 'chatitems' diff --git a/projects/app/src/pages/api/core/chat/clearHistories.ts b/projects/app/src/pages/api/core/chat/clearHistories.ts index 05cbb1ca1..2c1ffc662 100644 --- a/projects/app/src/pages/api/core/chat/clearHistories.ts +++ b/projects/app/src/pages/api/core/chat/clearHistories.ts @@ -35,16 +35,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) return Promise.reject('Param are error'); })(); - console.log(match); // find chatIds const list = await MongoChat.find(match, 'chatId').lean(); const idList = list.map((item) => item.chatId); await MongoChatItem.deleteMany({ + appId, chatId: { $in: idList } }); await MongoChat.deleteMany({ + appId, chatId: { $in: idList } }); diff --git a/projects/app/src/pages/api/core/chat/delHistory.ts b/projects/app/src/pages/api/core/chat/delHistory.ts index 39baca624..d4623b49f 100644 --- a/projects/app/src/pages/api/core/chat/delHistory.ts +++ b/projects/app/src/pages/api/core/chat/delHistory.ts @@ -23,9 +23,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChatItem.deleteMany({ + appId, chatId }); await MongoChat.findOneAndRemove({ + appId, chatId }); diff --git a/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts b/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts index dcc8cac4a..36ae9ad5b 100644 --- a/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts +++ b/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts @@ -27,6 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + appId, + chatId, dataId: chatItemId }, { diff --git a/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts b/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts index 9c3678d32..619a66b21 100644 --- a/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts +++ b/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts @@ -2,14 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import type { - AdminUpdateFeedbackParams, - CloseCustomFeedbackParams -} from '@/global/core/chat/api.d'; +import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d'; import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; import { autChatCrud } from '@/service/support/permission/auth/chat'; -/* 初始化我的聊天框，需要身份验证 */ +/* remove custom feedback */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); @@ -29,13 +26,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await authCert({ req, authToken: true }); await MongoChatItem.findOneAndUpdate( - { dataId: chatItemId }, + { appId, chatId, dataId: chatItemId }, { $unset: { [`customFeedbacks.${index}`]: 1 } } ); - await MongoChatItem.findOneAndUpdate( - { dataId: chatItemId }, - { $pull: { customFeedbacks: null } } - ); jsonRes(res); } catch (err) { diff --git a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts index d1655e738..6cdd98675 100644 --- a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts +++ b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts @@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + appId, chatId, dataId: chatItemId }, diff --git a/projects/app/src/pages/api/core/chat/getHistories.ts b/projects/app/src/pages/api/core/chat/getHistories.ts index fe68beea8..c81f8cae6 100644 --- a/projects/app/src/pages/api/core/chat/getHistories.ts +++ b/projects/app/src/pages/api/core/chat/getHistories.ts @@ -31,8 +31,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) if (appId) { const { tmbId } = await authCert({ req, authToken: true }); return { - appId, tmbId, + appId, source: ChatSourceEnum.online }; } diff --git a/projects/app/src/pages/api/core/chat/init.ts b/projects/app/src/pages/api/core/chat/init.ts index 9f759f0ee..4bbdace8f 100644 --- a/projects/app/src/pages/api/core/chat/init.ts +++ b/projects/app/src/pages/api/core/chat/init.ts @@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) appId, per: 'r' }), - chatId ? MongoChat.findOne({ chatId }) : undefined + chatId ? MongoChat.findOne({ appId, chatId }) : undefined ]); // auth chat permission @@ -41,6 +41,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) // get app and history const { history } = await getChatItems({ + appId, chatId, limit: 30, field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${ diff --git a/projects/app/src/pages/api/core/chat/item/delete.ts b/projects/app/src/pages/api/core/chat/item/delete.ts index 7a1b8c043..6d7318707 100644 --- a/projects/app/src/pages/api/core/chat/item/delete.ts +++ b/projects/app/src/pages/api/core/chat/item/delete.ts @@ -25,8 +25,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChatItem.deleteOne({ - dataId: contentId, - chatId + appId, + chatId, + dataId: contentId }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/chat/item/getSpeech.ts b/projects/app/src/pages/api/core/chat/item/getSpeech.ts index 4c8629a19..17d9a647b 100644 --- a/projects/app/src/pages/api/core/chat/item/getSpeech.ts +++ b/projects/app/src/pages/api/core/chat/item/getSpeech.ts @@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) try { pushAudioSpeechBill({ model: model, - textLen: input.length, + charsLength: input.length, tmbId, teamId, source: authType2BillSource({ authType }) diff --git a/projects/app/src/pages/api/core/chat/outLink/init.ts b/projects/app/src/pages/api/core/chat/outLink/init.ts index 134203527..27bb33138 100644 --- a/projects/app/src/pages/api/core/chat/outLink/init.ts +++ b/projects/app/src/pages/api/core/chat/outLink/init.ts @@ -26,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) // auth app permission const [tmb, chat, app] = await Promise.all([ MongoTeamMember.findById(shareChat.tmbId, '_id userId').populate('userId', 'avatar').lean(), - MongoChat.findOne({ chatId, shareId }).lean(), + MongoChat.findOne({ appId, chatId, shareId }).lean(), MongoApp.findById(appId).lean() ]); @@ -40,6 +40,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } const { history } = await getChatItems({ + appId: app._id, chatId, limit: 30, field: `dataId obj value userGoodFeedback userBadFeedback ${ diff --git a/projects/app/src/pages/api/core/chat/updateHistory.ts b/projects/app/src/pages/api/core/chat/updateHistory.ts index 8e983fd7d..634336c9a 100644 --- a/projects/app/src/pages/api/core/chat/updateHistory.ts +++ b/projects/app/src/pages/api/core/chat/updateHistory.ts @@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChat.findOneAndUpdate( - { chatId }, + { appId, chatId }, { ...(customTitle !== undefined && { customTitle }), ...(top !== undefined && { top }) diff --git a/projects/app/src/pages/api/core/dataset/allDataset.ts b/projects/app/src/pages/api/core/dataset/allDataset.ts index e6dc77c32..e1d854018 100644 --- a/projects/app/src/pages/api/core/dataset/allDataset.ts +++ b/projects/app/src/pages/api/core/dataset/allDataset.ts @@ -6,7 +6,7 @@ import { getVectorModel } from '@/service/core/ai/model'; import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; /* get all dataset by teamId or tmbId */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { diff --git a/projects/app/src/pages/api/core/dataset/collection/create/file.ts b/projects/app/src/pages/api/core/dataset/collection/create/file.ts new file mode 100644 index 000000000..e208ead8c --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/create/file.ts @@ -0,0 +1,85 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import { removeFilesByPaths } from '@fastgpt/service/common/file/utils'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +/** + * Creates the multer uploader + */ +const upload = getUploadModel({ + maxSize: 500 * 1024 * 1024 +}); + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + let filePaths: string[] = []; + + const { datasetId } = req.query as { datasetId: string }; + + try { + await connectToDatabase(); + + const { teamId, tmbId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + per: 'w', + datasetId + }); + + const { file, bucketName, data } = await upload.doUpload( + req, + res + ); + filePaths = [file.path]; + + if (!file || !bucketName) { + throw new Error('file is empty'); + } + + const { fileMetadata, collectionMetadata, ...collectionData } = data; + + // upload file and create collection + const fileId = await uploadFile({ + teamId, + tmbId, + bucketName, + path: file.path, + filename: file.originalname, + contentType: file.mimetype, + metadata: fileMetadata + }); + + // create collection + const collectionId = await createOneCollection({ + ...collectionData, + metadata: collectionMetadata, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.file, + fileId + }); + + jsonRes(res, { + data: collectionId + }); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } + + removeFilesByPaths(filePaths); +} + +export const config = { + api: { + bodyParser: false + } +}; diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts b/projects/app/src/pages/api/core/dataset/collection/create/link.ts similarity index 96% rename from projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts rename to projects/app/src/pages/api/core/dataset/collection/create/link.ts index 7a4f74807..7ccd0bcc6 100644 --- a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/link.ts @@ -7,7 +7,10 @@ import { connectToDatabase } from '@/service/mongo'; import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts b/projects/app/src/pages/api/core/dataset/collection/create/text.ts similarity index 87% rename from projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts rename to projects/app/src/pages/api/core/dataset/collection/create/text.ts index 6e131c4dc..59f8267fd 100644 --- a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/text.ts @@ -7,11 +7,14 @@ import { connectToDatabase } from '@/service/mongo'; import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; -import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller'; import { hashStr } from '@fastgpt/global/common/string/tools'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -39,8 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< text, chunkLen: chunkSize, overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0, - customReg: chunkSplitter ? [chunkSplitter] : [], - countTokens: false + customReg: chunkSplitter ? [chunkSplitter] : [] }); // 2. check dataset limit @@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // 4. push chunks to training queue - const insertResults = await pushDataToDatasetCollection({ + const insertResults = await pushDataToTrainingQueue({ teamId, tmbId, collectionId, diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 9544badec..d4c7835c4 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils'; -import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; +import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('CollectionIdId is required'); } - await authDatasetCollection({ + const { teamId, collection } = await authDatasetCollection({ req, authToken: true, authApiKey: true, @@ -24,13 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // find all delete id - const collections = await findCollectionAndChild(collectionId, '_id fileId'); - const delIdList = collections.map((item) => item._id); + const collections = await findCollectionAndChild({ + teamId, + datasetId: collection.datasetId._id, + collectionId, + fields: '_id teamId fileId metadata' + }); // delete - await delCollectionRelevantData({ - collectionIds: delIdList, - fileIds: collections.map((item) => item?.fileId || '').filter(Boolean) + await delCollectionAndRelatedSources({ + collections }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 4ffb1e94c..844b5ce02 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -7,7 +7,7 @@ import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type. import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import { PagingData } from '@/types'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { startQueue } from '@/service/utils/tools'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; @@ -87,7 +87,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< { $match: { $expr: { - $eq: ['$collectionId', '$$id'] + $and: [{ $eq: ['$teamId', match.teamId] }, { $eq: ['$collectionId', '$$id'] }] } } }, @@ -105,7 +105,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< { $match: { $expr: { - $eq: ['$collectionId', '$$id'] + $and: [ + { $eq: ['$teamId', match.teamId] }, + { $eq: ['$datasetId', match.datasetId] }, + { $eq: ['$collectionId', '$$id'] } + ] } } }, diff --git a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts index b7a1b9042..4aba219f6 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts @@ -6,11 +6,11 @@ import { getCollectionAndRawText, reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils'; -import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; +import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionSyncResultEnum, DatasetCollectionTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; @@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('CollectionIdId is required'); } - const { collection, tmbId } = await authDatasetCollection({ + const { collection, teamId, tmbId } = await authDatasetCollection({ req, authToken: true, collectionId, @@ -87,9 +87,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // delete old collection - await delCollectionRelevantData({ - collectionIds: [collection._id], - fileIds: collection.fileId ? [collection.fileId] : [] + await delCollectionAndRelatedSources({ + collections: [collection] }); jsonRes(res, { diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index d7533443d..e2e200c67 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -5,7 +5,7 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import type { CreateDatasetParams } from '@/global/core/dataset/api.d'; import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller'; import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { diff --git a/projects/app/src/pages/api/core/dataset/data/delete.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts index 5bf4df034..b945d3e56 100644 --- a/projects/app/src/pages/api/core/dataset/data/delete.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -3,7 +3,8 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { connectToDatabase } from '@/service/mongo'; import { authDatasetData } from '@/service/support/permission/auth/dataset'; -import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } // 凭证校验 - const { datasetData } = await authDatasetData({ + const { teamId, datasetData } = await authDatasetData({ req, authToken: true, authApiKey: true, @@ -25,11 +26,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); - await delDatasetDataByDataId({ - collectionId: datasetData.collectionId, - mongoDataId: dataId + // update mongo data update time + await MongoDatasetData.findByIdAndUpdate(dataId, { + updateTime: new Date() }); + // delete vector data + await deleteDatasetDataVector({ + teamId, + idList: datasetData.indexes.map((item) => item.dataId) + }); + + // delete mongo data + await MongoDatasetData.findByIdAndDelete(dataId); + jsonRes(res, { data: 'success' }); diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index 0a794bfe3..7196383e6 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -71,12 +71,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // Duplicate data check await hasSameValue({ + teamId, collectionId, q: formatQ, a: formatA }); - const { insertId, tokens } = await insertData2Dataset({ + const { insertId, charsLength } = await insertData2Dataset({ teamId, tmbId, datasetId, @@ -91,7 +92,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: vectorModelData.model }); diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index e014ec7a4..a22eed4d5 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -20,11 +20,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< pageSize = Math.min(pageSize, 30); // 凭证校验 - await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' }); + const { teamId, collection } = await authDatasetCollection({ + req, + authToken: true, + authApiKey: true, + collectionId, + per: 'r' + }); searchText = searchText.replace(/'/g, ''); const match = { + teamId, + datasetId: collection.datasetId._id, collectionId, ...(searchText ? { diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index 5e05a7fde..146a21e52 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -3,13 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import type { PushDataResponse } from '@/global/core/api/datasetRes.d'; import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; -import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -41,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex }); jsonRes(res, { - data: await pushDataToDatasetCollection({ + data: await pushDataToTrainingQueue({ ...req.body, teamId, tmbId diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index 41279637a..7f7102d4b 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -31,7 +31,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // auth team balance await authTeamBalance(teamId); - const { tokens } = await updateData2Dataset({ + const { charsLength } = await updateData2Dataset({ dataId: id, q, a, @@ -42,7 +42,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: vectorModel }); diff --git a/projects/app/src/pages/api/core/dataset/delete.ts b/projects/app/src/pages/api/core/dataset/delete.ts index 6b8f796c6..af030a7fb 100644 --- a/projects/app/src/pages/api/core/dataset/delete.ts +++ b/projects/app/src/pages/api/core/dataset/delete.ts @@ -2,32 +2,35 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; -import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller'; -import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; +import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller'; +import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id } = req.query as { + const { id: datasetId } = req.query as { id: string; }; - if (!id) { + if (!datasetId) { throw new Error('缺少参数'); } // auth owner - await authDataset({ req, authToken: true, datasetId: id, per: 'owner' }); + const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' }); - const deletedIds = await findDatasetIdTreeByTopDatasetId(id); + const datasets = await findDatasetAndAllChildren({ + teamId, + datasetId + }); // delete all dataset.data and pg data - await delDatasetRelevantData({ datasetIds: deletedIds }); + await delDatasetRelevantData({ datasets }); // delete dataset data await MongoDataset.deleteMany({ - _id: { $in: deletedIds } + _id: { $in: datasets.map((d) => d._id) } }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/dataset/exportAll.ts b/projects/app/src/pages/api/core/dataset/exportAll.ts index 44ef68650..b0d9453e0 100644 --- a/projects/app/src/pages/api/core/dataset/exportAll.ts +++ b/projects/app/src/pages/api/core/dataset/exportAll.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { addLog } from '@fastgpt/service/common/system/log'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; +import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { checkExportDatasetLimit, @@ -30,7 +30,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes }); - const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId); + const datasets = await findDatasetAndAllChildren({ + teamId, + datasetId, + fields: '_id' + }); res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; '); @@ -42,7 +46,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex a: string; }>( { - datasetId: { $in: exportIds } + teamId, + datasetId: { $in: datasets.map((d) => d._id) } }, 'q a' ) diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index 9ebea8e33..07f54546c 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts index b0fb6246d..b0cbb3a43 100644 --- a/projects/app/src/pages/api/core/dataset/searchTest.ts +++ b/projects/app/src/pages/api/core/dataset/searchTest.ts @@ -47,7 +47,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // model: global.chatModels[0].model // }); - const { searchRes, tokens, ...result } = await searchDatasetData({ + const { searchRes, charsLength, ...result } = await searchDatasetData({ + teamId, rawQuery: text, queries: [text], model: dataset.vectorModel, @@ -62,7 +63,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { total } = pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: dataset.vectorModel, source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt }); diff --git a/projects/app/src/pages/api/plugins/customFeedback/index.ts b/projects/app/src/pages/api/plugins/customFeedback/index.ts index f6e401ca6..cae9ede8a 100644 --- a/projects/app/src/pages/api/plugins/customFeedback/index.ts +++ b/projects/app/src/pages/api/plugins/customFeedback/index.ts @@ -12,6 +12,7 @@ type Props = HttpBodyType<{ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { const { + appId, chatId, responseChatItemId: chatItemId, data: { defaultFeedback, customFeedback } @@ -30,6 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // wait the chat finish setTimeout(() => { addCustomFeedbacks({ + appId, chatId, chatItemId, feedbacks: [feedback] diff --git a/projects/app/src/pages/api/support/user/account/loginByPassword.ts b/projects/app/src/pages/api/support/user/account/loginByPassword.ts index 197ed8700..44917e3e1 100644 --- a/projects/app/src/pages/api/support/user/account/loginByPassword.ts +++ b/projects/app/src/pages/api/support/user/account/loginByPassword.ts @@ -10,7 +10,7 @@ import { UserStatusEnum } from '@fastgpt/global/support/user/constant'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { username, password, tmbId = '' } = req.body as PostLoginProps; + const { username, password } = req.body as PostLoginProps; if (!username || !password) { throw new Error('缺少参数'); @@ -40,7 +40,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) throw new Error('密码错误'); } - const userDetail = await getUserDetail({ tmbId, userId: user._id }); + const userDetail = await getUserDetail({ + tmbId: user?.lastLoginTmbId, + userId: user._id + }); + + MongoUser.findByIdAndUpdate(user._id, { + lastLoginTmbId: userDetail.team.tmbId + }); const token = createJWT(userDetail); setCookie(res, token); diff --git a/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts b/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts new file mode 100644 index 000000000..4ead99cd3 --- /dev/null +++ b/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts @@ -0,0 +1,37 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { size } = req.query as { + size: string; + }; + + // 凭证校验 + const { teamId } = await authCert({ req, authToken: true }); + + if (!size) { + return jsonRes(res); + } + + const numberSize = Number(size); + + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: numberSize + }); + + jsonRes(res); + } catch (err) { + res.status(500); + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts b/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts index b7db92ed4..7a8ba7c13 100644 --- a/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts +++ b/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts @@ -1,29 +1,32 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d'; import { getQAModel, getVectorModel } from '@/service/core/ai/model'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { name, vectorModel, agentModel } = req.body as CreateTrainingBillProps; + const { name, datasetId } = req.body as CreateTrainingBillProps; - const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true }); - - const vectorModelData = getVectorModel(vectorModel); - const agentModelData = getQAModel(agentModel); + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'w' + }); const { billId } = await createTrainingBill({ teamId, tmbId, appName: name, billSource: BillSourceEnum.training, - vectorModel: vectorModelData.name, - agentModel: agentModelData.name + vectorModel: getVectorModel(dataset.vectorModel).name, + agentModel: getQAModel(dataset.agentModel).name }); jsonRes(res, { diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts index 22fd91dce..189ab13c4 100644 --- a/projects/app/src/pages/api/v1/audio/transcriptions.ts +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -17,11 +17,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex try { const { - files, - metadata: { duration, shareId } + file, + data: { duration } } = await upload.doUpload<{ duration: number; shareId?: string }>(req, res); - filePaths = files.map((file) => file.path); + filePaths = [file.path]; const { teamId, tmbId } = await authCert({ req, authToken: true }); @@ -29,8 +29,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex throw new Error('whisper model not found'); } - const file = files[0]; - if (!file) { throw new Error('file not found'); } diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 2ad353878..f703072ab 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -195,7 +195,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex }); // get and concat history - const { history } = await getChatItems({ chatId, limit: 30, field: `dataId obj value` }); + const { history } = await getChatItems({ + appId: app._id, + chatId, + limit: 30, + field: `dataId obj value` + }); const concatHistories = history.concat(chatMessages); const responseChatItemId: string | undefined = messages[messages.length - 1].dataId; diff --git a/projects/app/src/pages/api/v1/embeddings.ts b/projects/app/src/pages/api/v1/embeddings.ts index 4371c2609..e1e1b283d 100644 --- a/projects/app/src/pages/api/v1/embeddings.ts +++ b/projects/app/src/pages/api/v1/embeddings.ts @@ -33,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex await authTeamBalance(teamId); - const { tokens, vectors } = await getVectorsByText({ input: query, model }); + const { charsLength, vectors } = await getVectorsByText({ input: query, model }); res.json({ object: 'list', @@ -44,15 +44,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex })), model, usage: { - prompt_tokens: tokens, - total_tokens: tokens + prompt_tokens: charsLength, + total_tokens: charsLength } }); const { total } = pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model, billId, source: getBillSourceByAuthType({ authType }) diff --git a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx index 3b797d97a..f1932ec3f 100644 --- a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx +++ b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx @@ -62,7 +62,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ }); if (unconnected) { - const msg = `【${t(item.name)}】存在未填或未连接参数`; + const msg = t('core.module.Unlink tip', { name: t(item.name) }); toast({ status: 'warning', @@ -82,8 +82,8 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ permission: undefined }); }, - successToast: '保存配置成功', - errorToast: '保存配置异常', + successToast: t('common.Save Success'), + errorToast: t('common.Save Failed'), onSuccess() { ChatTestRef.current?.resetChatTest(); } @@ -98,22 +98,23 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ alignItems={'center'} userSelect={'none'} > - - } - borderColor={'myGray.300'} - variant={'whiteBase'} - aria-label={''} - onClick={openConfirmOut(async () => { - const modules = await flow2ModulesAndCheck(); - if (modules) { - await onclickSave(modules); - } - onClose(); - }, onClose)} - /> - + } + borderRadius={'50%'} + w={'26px'} + h={'26px'} + borderColor={'myGray.300'} + variant={'whiteBase'} + aria-label={''} + onClick={openConfirmOut(async () => { + const modules = await flow2ModulesAndCheck(); + if (modules) { + await onclickSave(modules); + } + onClose(); + }, onClose)} + /> {app.name} @@ -154,7 +155,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ onClick={() => setTestModules(undefined)} /> ) : ( - + } @@ -171,9 +172,9 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ )} - + } + icon={} size={'smSquare'} isLoading={isLoading} aria-label={'save'} diff --git a/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx b/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx index 11dbd0761..b1298c9cd 100644 --- a/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx +++ b/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx @@ -44,12 +44,18 @@ const Render = ({ app, onClose }: Props) => { initData(JSON.parse(JSON.stringify(app.modules))); }, [app.modules]); - return } />; + const memoRender = useMemo(() => { + return } />; + }, [app, moduleTemplates.length, onClose]); + + return memoRender; }; export default React.memo(function FlowEdit(props: Props) { + const filterAppIds = useMemo(() => [props.app._id], [props.app._id]); + return ( - + ); diff --git a/projects/app/src/pages/app/detail/components/InfoModal.tsx b/projects/app/src/pages/app/detail/components/InfoModal.tsx index 6786ea4b8..3185c741b 100644 --- a/projects/app/src/pages/app/detail/components/InfoModal.tsx +++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx @@ -52,7 +52,7 @@ const InfoModal = ({ }); const [refresh, setRefresh] = useState(false); - // 提交保存模型修改 + // submit config const { mutate: saveSubmitSuccess, isLoading: btnLoading } = useRequest({ mutationFn: async (data: AppSchema) => { await updateAppDetail(data._id, { @@ -66,18 +66,17 @@ const InfoModal = ({ onSuccess && onSuccess(); onClose(); toast({ - title: '更新成功', + title: t('common.Update Success'), status: 'success' }); }, - errorToast: '更新失败' + errorToast: t('common.Update Failed') }); - // 提交保存表单失败 const saveSubmitError = useCallback(() => { // deep search message const deepSearch = (obj: any): string => { - if (!obj) return '提交表单错误'; + if (!obj) return t('common.Submit failed'); if (!!obj.message) { return obj.message; } @@ -89,7 +88,7 @@ const InfoModal = ({ duration: 4000, isClosable: true }); - }, [errors, toast]); + }, [errors, t, toast]); const saveUpdateModel = useCallback( () => handleSubmit((data) => saveSubmitSuccess(data), saveSubmitError)(), @@ -111,12 +110,12 @@ const InfoModal = ({ setRefresh((state) => !state); } catch (err: any) { toast({ - title: getErrText(err, '头像选择异常'), + title: getErrText(err, t('common.error.Select avatar failed')), status: 'warning' }); } }, - [setValue, toast] + [setValue, t, toast] ); return ( @@ -127,7 +126,7 @@ const InfoModal = ({ title={t('core.app.setting')} > - 头像 & 名称 + {t('core.app.Name and avatar')} onOpenSelectFile()} /> - 应用介绍 + {t('core.app.App intro')} {/* 该介绍主要用于记忆和在应用市场展示 @@ -158,7 +157,7 @@ const InfoModal = ({