v4.5.1 (#417)

.github/workflows/preview-image.yml

@@ -15,7 +15,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: recursive # Fetch submodules
+          fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # Install dependencies only when needed
-FROM node:current-alpine AS deps
+FROM node:18.15-alpine AS deps
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk add --no-cache libc6-compat && npm install -g pnpm
 WORKDIR /app
@@ -11,12 +11,12 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
-RUN \
+RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
-  [ -f pnpm-lock.yaml ] && pnpm install || \
+
-  (echo "Lockfile not found." && exit 1)
+RUN pnpm install
 
 # Rebuild the source code only when needed
-FROM node:current-alpine AS builder
+FROM node:18.15-alpine AS builder
 WORKDIR /app
 
 ARG name
@@ -33,7 +33,7 @@ ENV NEXT_TELEMETRY_DISABLED 1
 RUN npm install -g pnpm
 RUN pnpm --filter=$name run build
 
-FROM node:current-alpine AS runner
+FROM node:18.15-alpine AS runner
 WORKDIR /app
 
 ARG name

docSite/content/docs/development/configuration.md

@@ -54,7 +54,7 @@ weight: 520
       "defaultSystemChatPrompt": ""
     }
   ],
-  "QAModel": [ // QA 拆分模型
+  "QAModels": [ // QA 拆分模型
     { 
       "model": "gpt-3.5-turbo-16k",
       "name": "GPT35-16k",

docSite/content/docs/development/openApi.md

@@ -120,7 +120,7 @@ event: answer
 data: [DONE]
 
 event: appStreamResponse
-data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"kb_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"kb_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"kb_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
+data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"dataset_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"dataset_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"dataset_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
 
 ```
 {{< /markdownify >}}
@@ -150,21 +150,21 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",
@@ -487,21 +487,21 @@ curl --location --request POST '{{host}}/shareAuth/finish' \
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",

docSite/content/docs/installation/upgrading/45.md

@@ -62,10 +62,16 @@ ALTER EXTENSION vector UPDATE;
 alter system set maintenance_work_mem = '2400MB'; 
 select pg_reload_conf();
 
+-- 重构数据库索引和排序
+REINDEX DATABASE postgres;
+ALTER DATABASE postgres REFRESH COLLATION VERSION;
+
 -- 开始构建索引，该索引构建时间非常久，直接关掉终端即可，不要使用 ctrl+c 关闭
 CREATE INDEX CONCURRENTLY vector_index ON modeldata USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
 -- 可以再次连接数据库，输入下方命令。如果看到 "vector_index" hnsw (vector vector_ip_ops) WITH (m='16', ef_construction='64') 则代表构建完成（注意，后面没有 INVALID）
 \d modeldata
+
+
 ```
 
 ## 版本新功能介绍

docSite/content/docs/installation/upgrading/451.md

@@ -0,0 +1,34 @@
+---
+title: 'V4.5.1(需进行初始化)'
+description: 'FastGPT V4.5.1 更新'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 839
+---
+
+## 执行初始化 API
+
+发起 1 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名）
+
+1. https://xxxxx/api/admin/initv451
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv451' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化内容：
+1. rename 数据库字段
+2. 初始化 Mongo APP 表中知识库的相关字段
+3. 初始化 PG 和 Mongo 的内容，为每个文件创建一个集合（存储 Mongo 中），并反馈赋值给 PG。
+
+**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可**
+
+## 功能介绍
+
+### Fast GPT V4.5.1
+
+1. 新增知识库文件夹管理
+

docSite/content/docs/workflow/modules/ai_chat.md

@@ -42,17 +42,17 @@ weight: 123
 
 ```ts
 type DataType = {
-  kb_id?: string;
+  dataset_id?: string;
   id?: string;
   q: string;
   a: string;
   source?: string;
 };
-// 如果是外部引入的内容，尽量不要携带 kb_id 和 id
+// 如果是外部引入的内容，尽量不要携带 dataset_id 和 id
 const quoteList: DataType[] = [
-  { kb_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
+  { dataset_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
 ];
 ```
 

packages/common/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "@fastgpt/common",
-  "version": "1.0.0",
-  "dependencies": {
-    "mongoose": "^7.0.2",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1",
-    "axios": "^1.5.1",
-    "nextjs-cors": "^2.1.2",
-    "next": "13.5.2"
-  },
-  "devDependencies": {
-    "@types/node": "^20.8.5"
-  }
-}

packages/core/dataset/type.d.ts

@@ -1,13 +0,0 @@
-import { DatasetTypeEnum } from './constant';
-
-export type DatasetSchemaType = {
-  _id: string;
-  userId: string;
-  parentId: string;
-  updateTime: Date;
-  avatar: string;
-  name: string;
-  vectorModel: string;
-  tags: string[];
-  type: `${DatasetTypeEnum}`;
-};

packages/core/dataset/utils.ts

@@ -1,8 +0,0 @@
-import { datasetSpecialIds } from './constant';
-import { strIsLink } from '@fastgpt/common/tools/str';
-
-export function isSpecialFileId(id: string) {
-  if (datasetSpecialIds.includes(id)) return true;
-  if (strIsLink(id)) return true;
-  return false;
-}

packages/core/package.json

@@ -1,14 +0,0 @@
-{
-  "name": "@fastgpt/core",
-  "version": "1.0.0",
-  "dependencies": {
-    "@fastgpt/common": "workspace:*",
-    "@fastgpt/support": "workspace:*",
-    "encoding": "^0.1.13",
-    "openai": "^4.12.1",
-    "tunnel": "^0.0.6"
-  },
-  "devDependencies": {
-    "@types/tunnel": "^0.0.4"
-  }
-}

packages/global/common/error/errorCode.ts

@@ -27,7 +27,8 @@ export enum ERROR_ENUM {
   insufficientQuota = 'insufficientQuota',
   unAuthModel = 'unAuthModel',
   unAuthApiKey = 'unAuthApiKey',
-  unAuthKb = 'unAuthKb',
+  unAuthDataset = 'unAuthDataset',
+  unAuthDatasetCollection = 'unAuthDatasetCollection',
   unAuthFile = 'unAuthFile'
 }
 export const ERROR_RESPONSE: Record<
@@ -57,9 +58,9 @@ export const ERROR_RESPONSE: Record<
     message: '无权使用该模型',
     data: null
   },
-  [ERROR_ENUM.unAuthKb]: {
+  [ERROR_ENUM.unAuthDataset]: {
     code: 512,
-    statusText: ERROR_ENUM.unAuthKb,
+    statusText: ERROR_ENUM.unAuthDataset,
     message: '无权使用该知识库',
     data: null
   },
@@ -74,5 +75,11 @@ export const ERROR_RESPONSE: Record<
     statusText: ERROR_ENUM.unAuthApiKey,
     message: 'Api Key 不合法',
     data: null
+  },
+  [ERROR_ENUM.unAuthDatasetCollection]: {
+    code: 515,
+    statusText: ERROR_ENUM.unAuthDatasetCollection,
+    message: '无权使用该知识库文件',
+    data: null
   }
 };

packages/global/common/error/utils.ts

@@ -0,0 +1,5 @@
+export const getErrText = (err: any, def = '') => {
+  const msg: string = typeof err === 'string' ? err : err?.message || def || '';
+  msg && console.log('error =>', msg);
+  return msg;
+};

packages/global/common/file/icon.ts

@@ -1,4 +1,4 @@
-import { strIsLink } from './str';
+import { strIsLink } from '../string/tools';
 
 export const fileImgs = [
   { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
@@ -10,14 +10,7 @@ export const fileImgs = [
 ];
 
 export function getFileIcon(name = '') {
-  return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src;
+  return (
-}
+    fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || '/imgs/files/file.svg'
-export function getSpecialFileIcon(name = '') {
+  );
-  if (name === 'manual') {
-    return '/imgs/files/manual.svg';
-  } else if (name === 'mark') {
-    return '/imgs/files/mark.svg';
-  } else if (strIsLink(name)) {
-    return '/imgs/files/link.svg';
-  }
 }

packages/global/common/file/tools.ts

@@ -0,0 +1,9 @@
+export const formatFileSize = (bytes: number): string => {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+};

packages/global/common/parentFolder/type.d.ts

@@ -0,0 +1,4 @@
+export type ParentTreePathItemType = {
+  parentId: string;
+  parentName: string;
+};

packages/global/common/string/tools.ts

@@ -13,10 +13,10 @@ export const hashStr = (psw: string) => {
 /* simple text, remove chinese space and extra \n */
 export const simpleText = (text: string) => {
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');
   text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
   text = text.replace(/[\x00-\x08]/g, ' ');
-  text = text.replace(/\r\n|\r/g, '\n');
 
   return text;
 };

packages/global/common/system/types/index.d.ts

@@ -1,6 +1,3 @@
-import type { Mongoose } from '../mongo';
-import type { Logger } from 'winston';
-
 export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
@@ -36,8 +33,6 @@ export type SystemEnvType = {
 };
 
 declare global {
-  var mongodb: Mongoose | undefined;
-  var logger: Logger;
   var feConfigs: FeConfigsType;
   var systemEnv: SystemEnvType;
 }

packages/global/common/time/timezone.ts

@@ -1,4 +1,3 @@
-import { loginOut } from '@/web/support/api/user';
 import timezones from 'timezones-list';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
@@ -7,23 +6,6 @@ import timezone from 'dayjs/plugin/timezone';
 dayjs.extend(utc);
 dayjs.extend(timezone);
 
-const tokenKey = 'token';
-export const clearToken = () => {
-  try {
-    loginOut();
-    localStorage.removeItem(tokenKey);
-  } catch (error) {
-    error;
-  }
-};
-
-export const setToken = (token: string) => {
-  localStorage.setItem(tokenKey, token);
-};
-export const getToken = () => {
-  return localStorage.getItem(tokenKey) || '';
-};
-
 /**
  * Returns the offset from UTC in hours for the current locale.
  * @param {string} timeZone Timezone to get offset for

packages/global/core/dataset/constant.ts

@@ -12,11 +12,37 @@ export const DatasetTypeMap = {
   }
 };
 
-export enum FileStatusEnum {
+export enum DatasetCollectionTypeEnum {
-  embedding = 'embedding',
+  file = 'file',
-  ready = 'ready'
+  folder = 'folder',
+  link = 'link',
+  virtual = 'virtual'
 }
 
+export const DatasetCollectionTypeMap = {
+  [DatasetCollectionTypeEnum.file]: {
+    name: 'dataset.file'
+  },
+  [DatasetCollectionTypeEnum.folder]: {
+    name: 'dataset.folder'
+  },
+  [DatasetCollectionTypeEnum.link]: {
+    name: 'dataset.link'
+  },
+  [DatasetCollectionTypeEnum.virtual]: {
+    name: 'dataset.Virtual File'
+  }
+};
+
+export enum TrainingModeEnum {
+  'qa' = 'qa',
+  'index' = 'index'
+}
+export const TrainingTypeMap = {
+  [TrainingModeEnum.qa]: 'qa',
+  [TrainingModeEnum.index]: 'index'
+};
+
 export enum DatasetSpecialIdEnum {
   manual = 'manual',
   mark = 'mark'

packages/global/core/dataset/type.d.ts

@@ -0,0 +1,75 @@
+import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant';
+
+export type DatasetSchemaType = {
+  _id: string;
+  userId: string;
+  parentId: string;
+  updateTime: Date;
+  avatar: string;
+  name: string;
+  vectorModel: string;
+  tags: string[];
+  type: `${DatasetTypeEnum}`;
+};
+
+export type DatasetCollectionSchemaType = {
+  _id: string;
+  userId: string;
+  datasetId: string;
+  parentId?: string;
+  name: string;
+  type: `${DatasetCollectionTypeEnum}`;
+  updateTime: Date;
+  metadata: {
+    fileId?: string;
+    rawLink?: string;
+    pgCollectionId?: string;
+  };
+};
+
+export type DatasetTrainingSchemaType = {
+  _id: string;
+  userId: string;
+  datasetId: string;
+  datasetCollectionId: string;
+  billId: string;
+  expireAt: Date;
+  lockTime: Date;
+  mode: `${TrainingModeEnum}`;
+  model: string;
+  prompt: string;
+  q: string;
+  a: string;
+};
+
+/* ================= dataset ===================== */
+
+/* ================= collection ===================== */
+
+/* ================= data ===================== */
+export type PgDataItemType = {
+  id: string;
+  q: string;
+  a: string;
+  dataset_id: string;
+  collection_id: string;
+};
+export type DatasetChunkItemType = {
+  q: string;
+  a: string;
+};
+export type DatasetDataItemType = DatasetChunkItemType & {
+  id: string;
+  datasetId: string;
+  collectionId: string;
+  sourceName: string;
+  sourceId?: string;
+};
+
+/* ============= search =============== */
+export type SearchDataResultItemType = PgDataItemType & {
+  score: number;
+};
+export type SearchDataResponseItemType = DatasetDataItemType & {
+  score: number;
+};

packages/global/core/dataset/utils.ts

@@ -0,0 +1,21 @@
+import { DatasetCollectionTypeEnum } from './constant';
+import { getFileIcon } from '../../common/file/icon';
+
+export function getCollectionIcon(
+  type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
+  name = ''
+) {
+  if (type === DatasetCollectionTypeEnum.folder) {
+    return '/imgs/files/folder.svg';
+  } else if (type === DatasetCollectionTypeEnum.link) {
+    return '/imgs/files/link.svg';
+  } else if (type === DatasetCollectionTypeEnum.virtual) {
+    if (name === '手动录入') {
+      return '/imgs/files/manual.svg';
+    } else if (name === '手动标注') {
+      return '/imgs/files/mark.svg';
+    }
+    return '/imgs/files/collection.svg';
+  }
+  return getFileIcon(name);
+}

packages/global/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@fastgpt/global",
+  "version": "1.0.0",
+  "dependencies": {
+    "axios": "^1.5.1",
+    "timezones-list": "^3.0.2",
+    "dayjs": "^1.11.7",
+    "encoding": "^0.1.13",
+    "openai": "^4.12.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.8.5"
+  }
+}

packages/global/support/outLink/constant.ts

@@ -1,5 +1,5 @@
 export enum OutLinkTypeEnum {
-  'share' = 'share',
+  share = 'share',
-  'iframe' = 'iframe',
+  iframe = 'iframe',
   apikey = 'apikey'
 }

packages/service/common/mongo/sessionRun.ts

@@ -0,0 +1,39 @@
+import mongoose from './index';
+
+export class MongoSession {
+  tasks: (() => Promise<any>)[] = [];
+  session: mongoose.mongo.ClientSession | null = null;
+  opts: {
+    session: mongoose.mongo.ClientSession;
+    new: boolean;
+  } | null = null;
+
+  constructor() {}
+  async init() {
+    this.session = await mongoose.startSession();
+    this.opts = { session: this.session, new: true };
+  }
+  push(
+    tasks: ((opts: {
+      session: mongoose.mongo.ClientSession;
+      new: boolean;
+    }) => () => Promise<any>)[] = []
+  ) {
+    if (!this.opts) return;
+    // this.tasks = this.tasks.concat(tasks.map((item) => item(this.opts)));
+  }
+  async run() {
+    if (!this.session || !this.opts) return;
+    try {
+      this.session.startTransaction();
+
+      const opts = { session: this.session, new: true };
+
+      await this.session.commitTransaction();
+    } catch (error) {
+      await this.session.abortTransaction();
+      console.error(error);
+    }
+    this.session.endSession();
+  }
+}

packages/service/common/mongo/type.d.ts

@@ -0,0 +1,7 @@
+import type { Mongoose } from 'mongoose';
+import type { Logger } from 'winston';
+
+declare global {
+  var mongodb: Mongoose | undefined;
+  var logger: Logger;
+}

packages/service/common/response/index.ts

@@ -14,7 +14,7 @@ export function responseWriteController({
   return (text: string | Buffer) => {
     const writeResult = res.write(text);
     if (!writeResult) {
-      readStream.pause();
+      readStream?.pause();
     }
   };
 }

packages/service/core/ai/config.ts

@@ -1,4 +1,4 @@
-import type { UserModelSchema } from '@fastgpt/support/user/type.d';
+import type { UserModelSchema } from '@fastgpt/global/support/user/type';
 import OpenAI from 'openai';
 
 export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';

packages/service/core/ai/functions/createQuestionGuide.ts

@@ -1,4 +1,4 @@
-import { ChatCompletionRequestMessage } from '../type';
+import type { ChatCompletionRequestMessage } from '@fastgpt/global/core/ai/type.d';
 import { getAIApi } from '../config';
 
 export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;

packages/service/core/dataset/auth.ts

@@ -0,0 +1,26 @@
+import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
+import { MongoDatasetCollection } from './collection/schema';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+
+export async function authCollection({
+  collectionId,
+  userId
+}: {
+  collectionId: string;
+  userId: string;
+}) {
+  const collection = await MongoDatasetCollection.findOne({
+    _id: collectionId,
+    userId
+  })
+    .populate('datasetId')
+    .lean();
+
+  if (collection) {
+    return {
+      ...collection,
+      dataset: collection.datasetId as unknown as DatasetSchemaType
+    };
+  }
+  return Promise.reject(ERROR_ENUM.unAuthDataset);
+}

packages/service/core/dataset/collection/schema.ts

@@ -0,0 +1,66 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
+import { DatasetCollectionName } from '../schema';
+
+export const DatasetColCollectionName = 'dataset.collections';
+
+const DatasetCollectionSchema = new Schema({
+  parentId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetColCollectionName,
+    default: null
+  },
+  userId: {
+    type: Schema.Types.ObjectId,
+    ref: 'user',
+    required: true
+  },
+  datasetId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetCollectionName,
+    required: true
+  },
+  name: {
+    type: String,
+    required: true
+  },
+  type: {
+    type: String,
+    enum: Object.keys(DatasetCollectionTypeMap),
+    required: true
+  },
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
+  metadata: {
+    type: {
+      fileId: {
+        type: Schema.Types.ObjectId,
+        ref: 'dataset.files'
+      },
+      rawLink: {
+        type: String,
+        default: ''
+      },
+      // 451 初始化
+      pgCollectionId: {
+        type: String
+      }
+    },
+    default: {}
+  }
+});
+
+try {
+  DatasetCollectionSchema.index({ datasetId: 1 });
+  DatasetCollectionSchema.index({ userId: 1 });
+  DatasetCollectionSchema.index({ updateTime: -1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDatasetCollection: Model<DatasetCollectionSchemaType> =
+  models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema);

packages/service/core/dataset/collection/utils.ts

@@ -0,0 +1,62 @@
+import { MongoDatasetCollection } from './schema';
+import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+
+/**
+ * get all collection by top collectionId
+ */
+export async function findCollectionAndChild(id: string, fields = '_id parentId name metadata') {
+  async function find(id: string) {
+    // find children
+    const children = await MongoDatasetCollection.find({ parentId: id }, fields);
+
+    let collections = children;
+
+    for (const child of children) {
+      const grandChildrenIds = await find(child._id);
+      collections = collections.concat(grandChildrenIds);
+    }
+
+    return collections;
+  }
+  const [collection, childCollections] = await Promise.all([
+    MongoDatasetCollection.findById(id, fields),
+    find(id)
+  ]);
+
+  if (!collection) {
+    return Promise.reject('Collection not found');
+  }
+
+  return [collection, ...childCollections];
+}
+
+export async function getDatasetCollectionPaths({
+  parentId = '',
+  userId
+}: {
+  parentId?: string;
+  userId: string;
+}): Promise<ParentTreePathItemType[]> {
+  async function find(parentId?: string): Promise<ParentTreePathItemType[]> {
+    if (!parentId) {
+      return [];
+    }
+
+    const parent = await MongoDatasetCollection.findOne({ _id: parentId, userId }, 'name parentId');
+
+    if (!parent) return [];
+
+    const paths = await find(parent.parentId);
+    paths.push({ parentId, parentName: parent.name });
+
+    return paths;
+  }
+
+  return await find(parentId);
+}
+
+export function getCollectionUpdateTime({ name, time }: { time?: Date; name: string }) {
+  if (time) return time;
+  if (name.startsWith('手动') || ['manual', 'mark'].includes(name)) return new Date('2999/9/9');
+  return new Date();
+}

packages/service/core/dataset/schema.ts

@@ -1,12 +1,14 @@
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { DatasetSchemaType } from './type';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import { DatasetTypeMap } from './constant';
+import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
+
+export const DatasetCollectionName = 'datasets';
 
 const DatasetSchema = new Schema({
   parentId: {
     type: Schema.Types.ObjectId,
-    ref: 'kb',
+    ref: DatasetCollectionName,
     default: null
   },
   userId: {
@@ -43,4 +45,11 @@ const DatasetSchema = new Schema({
   }
 });
 
-export const MongoDataset: Model<DatasetSchemaType> = models['kb'] || model('kb', DatasetSchema);
+try {
+  DatasetSchema.index({ userId: 1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDataset: Model<DatasetSchemaType> =
+  models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);

packages/service/core/dataset/training/schema.ts

@@ -1,19 +1,36 @@
 /* 模型的知识库 */
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
+import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
-import { TrainingTypeMap } from '@/constants/plugin';
+import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
+import { DatasetColCollectionName } from '../collection/schema';
+import { DatasetCollectionName } from '../schema';
+
+export const DatasetTrainingCollectionName = 'dataset.trainings';
 
-// pgList and vectorList, Only one of them will work
 const TrainingDataSchema = new Schema({
   userId: {
     type: Schema.Types.ObjectId,
     ref: 'user',
     required: true
   },
-  kbId: {
+  datasetId: {
     type: Schema.Types.ObjectId,
-    ref: 'kb',
+    ref: DatasetCollectionName,
+    required: true
+  },
+  datasetCollectionId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetColCollectionName,
+    required: true
+  },
+  billId: {
+    type: String,
+    default: ''
+  },
+  mode: {
+    type: String,
+    enum: Object.keys(TrainingTypeMap),
     required: true
   },
   expireAt: {
@@ -24,16 +41,10 @@ const TrainingDataSchema = new Schema({
     type: Date,
     default: () => new Date('2000/1/1')
   },
-  mode: {
+  model: {
     type: String,
-    enum: Object.keys(TrainingTypeMap),
     required: true
   },
-  vectorModel: {
-    type: String,
-    required: true,
-    default: 'text-embedding-ada-002'
-  },
   prompt: {
     // qa split prompt
     type: String,
@@ -41,23 +52,11 @@ const TrainingDataSchema = new Schema({
   },
   q: {
     type: String,
-    default: ''
+    required: true
   },
   a: {
     type: String,
     default: ''
-  },
-  source: {
-    type: String,
-    default: ''
-  },
-  file_id: {
-    type: String,
-    default: ''
-  },
-  billId: {
-    type: String,
-    default: ''
   }
 });
 
@@ -69,5 +68,5 @@ try {
   console.log(error);
 }
 
-export const TrainingData: Model<TrainingDateType> =
+export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
-  models['trainingData'] || model('trainingData', TrainingDataSchema);
+  models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);

packages/service/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@fastgpt/service",
+  "version": "1.0.0",
+  "dependencies": {
+    "@fastgpt/global": "workspace:*",
+    "axios": "^1.5.1",
+    "nextjs-cors": "^2.1.2",
+    "next": "13.5.2",
+    "cookie": "^0.5.0",
+    "jsonwebtoken": "^9.0.2",
+    "mongoose": "^7.0.2",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1",
+    "tunnel": "^0.0.6",
+    "encoding": "^0.1.13",
+    "openai": "^4.12.1"
+  },
+  "devDependencies": {
+    "@types/tunnel": "^0.0.4",
+    "@types/node": "^20.8.5",
+    "@types/cookie": "^0.5.2",
+    "@types/jsonwebtoken": "^9.0.3"
+  }
+}

packages/service/support/openapi/auth.ts

@@ -1,8 +1,8 @@
-import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 import { updateApiKeyUsedTime } from './tools';
 import { MongoOpenApi } from './schema';
-import { POST } from '@fastgpt/common/plusApi/request';
+import { POST } from '../../common/api/plusRequest';
-import { OpenApiSchema } from './type.d';
+import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
 
 export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
 

packages/service/support/openapi/schema.ts

@@ -1,8 +1,8 @@
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { OpenApiSchema } from './type.d';
+import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
-import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
+import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 
 const OpenApiSchema = new Schema(
   {

packages/service/support/outLink/auth.ts

@@ -1,7 +1,7 @@
 import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
 import { MongoOutLink } from './schema';
-import { POST } from '@fastgpt/common/plusApi/request';
+import { POST } from '../../common/api/plusRequest';
-import { OutLinkSchema } from './type.d';
+import { OutLinkSchema } from '@fastgpt/global/support/outLink/type';
 
 export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
 export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };

packages/service/support/outLink/schema.ts

@@ -1,7 +1,7 @@
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { OutLinkSchema as SchemaType } from './type.d';
+import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
-import { OutLinkTypeEnum } from './constant';
+import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
 
 const OutLinkSchema = new Schema({
   shareId: {

packages/service/support/user/auth.ts

@@ -4,8 +4,8 @@ import jwt from 'jsonwebtoken';
 import { authOpenApiKey } from '../openapi/auth';
 import { authOutLinkId } from '../outLink/auth';
 import { MongoUser } from './schema';
-import type { UserModelSchema } from './type.d';
+import type { UserModelSchema } from '@fastgpt/global/support/user/type';
-import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 
 export enum AuthUserTypeEnum {
   token = 'token',

packages/service/support/user/schema.ts

@@ -1,8 +1,8 @@
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { hashStr } from '@fastgpt/common/tools/str';
+import { hashStr } from '@fastgpt/global/common/string/tools';
-import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
+import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
-import type { UserModelSchema } from './type.d';
+import type { UserModelSchema } from '@fastgpt/global/support/user/type';
 
 const UserSchema = new Schema({
   username: {

packages/support/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "@fastgpt/support",
-  "version": "1.0.0",
-  "dependencies": {
-    "@fastgpt/common": "workspace:*",
-    "cookie": "^0.5.0",
-    "jsonwebtoken": "^9.0.2",
-    "axios": "^1.5.1",
-    "next": "13.5.2"
-  },
-  "devDependencies": {
-    "@types/cookie": "^0.5.2",
-    "@types/jsonwebtoken": "^9.0.3"
-  }
-}

packages/web/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@fastgpt/web",
+  "version": "1.0.0",
+  "dependencies": {
+    "axios": "^1.5.1"
+  },
+  "devDependencies": {}
+}

packages/web/tsconfig.json

@@ -14,10 +14,7 @@
     "isolatedModules": true,
     "jsx": "preserve",
     "incremental": true,
-    "baseUrl": ".",
+    "baseUrl": "."
-    "paths": {
-      "@/*": ["./*"]
-    }
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
   "exclude": ["node_modules"]

projects/app/data/config.json

@@ -2,7 +2,7 @@
   "SystemParams": {
     "vectorMaxProcess": 15,
     "qaMaxProcess": 15,
-    "pgHNSWEfSearch ": 40
+    "pgHNSWEfSearch ": 60
   },
   "ChatModels": [
     {

projects/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "4.5.0",
+  "version": "4.5.1",
   "private": false,
   "scripts": {
     "dev": "next dev",
@@ -16,9 +16,9 @@
     "@chakra-ui/system": "^2.5.8",
     "@emotion/react": "^11.10.6",
     "@emotion/styled": "^11.10.6",
-    "@fastgpt/common": "workspace:*",
+    "@fastgpt/global": "workspace:*",
-    "@fastgpt/core": "workspace:*",
+    "@fastgpt/service": "workspace:*",
-    "@fastgpt/support": "workspace:*",
+    "@fastgpt/web": "workspace:*",
     "@mozilla/readability": "^0.4.4",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
@@ -63,7 +63,6 @@
     "remark-math": "^5.1.1",
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
-    "timezones-list": "^3.0.2",
     "zustand": "^4.3.5"
   },
   "devDependencies": {

projects/app/public/docs/versionIntro.md

@@ -1,10 +1,9 @@
-### Fast GPT V4.5.0
+### Fast GPT V4.5.1
 
-1. 新增 - 升级 PgVector 插件，引入 HNSW 索引，极大加快的知识库搜索速度。
+1. 新增 - 知识库目录结构，更方便进行分类
-2. 新增 - AI对话模块，增加【返回AI内容】选项，可控制 AI 的内容不直接返回浏览器。
+2. 新增 - 升级 PgVector 插件，引入 HNSW 索引，极大加快的知识库搜索速度。
-3. 优化 - TextSplitter，采用递归拆解法。
+3. 新增 - AI对话模块，增加【返回AI内容】选项，可控制 AI 的内容不直接返回浏览器。
-4. 优化 - 高级编排 UX 性能
+4. 优化 - TextSplitter，采用递归拆解法。
-5. 优化数据集管理，区分手动录入和标注，可追数据至某个文件，保留链接读取的原始链接。
+5. [使用文档](https://doc.fastgpt.run/docs/intro/)
-6. [使用文档](https://doc.fastgpt.run/docs/intro/)
+6. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
-7. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
+7. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/)
-8. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/)

projects/app/public/imgs/files/collection.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1697612159198" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6941" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M512 928c-125.44 0-224-182.72-224-416s98.56-416 224-416 224 182.72 224 416-98.56 416-224 416z m0-768c-75.84 0-160 144.64-160 352s84.16 352 160 352 160-144.64 160-352-84.16-352-160-352z" fill="#C0C5D3" p-id="6942"></path><path d="M699.52 797.76a623.04 623.04 0 0 1-299.52-91.84C198.08 589.44 88.96 412.8 151.68 304s270.4-102.72 472.32 14.08 311.04 293.12 248.32 401.92c-30.08 51.84-93.12 77.76-172.8 77.76zM324.16 290.56c-57.6 0-100.16 16.32-117.12 45.44-37.76 65.6 45.44 210.88 224 314.56s346.88 103.04 384 37.44-45.44-210.88-224-314.56a560.32 560.32 0 0 0-266.88-82.88z" fill="#C0C5D3" p-id="6943"></path><path d="M324.48 797.76c-79.68 0-142.72-25.92-172.8-77.76-64-108.8 46.4-285.44 248.32-401.92S809.6 195.2 872.32 304s-46.4 285.44-248.32 401.92a623.04 623.04 0 0 1-299.52 91.84zM699.84 290.56a560.32 560.32 0 0 0-267.84 82.88c-179.52 103.68-262.72 248.96-224 314.56s205.44 66.24 384-37.44 262.72-248.96 224-314.56c-16-29.12-58.56-45.44-116.16-45.44z" fill="#C0C5D3" p-id="6944"></path><path d="M512 512m-64 0a64 64 0 1 0 128 0 64 64 0 1 0-128 0Z" fill="#C0C5D3" p-id="6945"></path><path d="M512 592a80 80 0 1 1 80-80 80 80 0 0 1-80 80z m0-128a48 48 0 1 0 48 48 48 48 0 0 0-48-48z" fill="#C0C5D3" p-id="6946"></path></svg>

projects/app/public/imgs/files/file.svg

@@ -1 +1 @@
-<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694327751771" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4992" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M0 0h1024v1024H0V0z" fill="#202425" opacity=".01" p-id="4993"></path><path d="M136.533333 68.266667a68.266667 68.266667 0 0 0-68.266666 68.266666v428.305067a17.066667 17.066667 0 0 0 28.842666 12.356267l237.738667-226.440534a34.133333 34.133333 0 0 1 42.496-3.6864l268.288 178.858667a17.066667 17.066667 0 0 0 22.766933-3.447467L951.978667 171.4176A17.066667 17.066667 0 0 0 955.733333 160.699733V136.533333a68.266667 68.266667 0 0 0-68.266666-68.266666H136.533333z m819.2 255.3856a17.066667 17.066667 0 0 0-30.344533-10.717867l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.652267zM614.4 955.733333a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8z m-307.2 0a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667z" fill="#FFAA44" p-id="4994"></path><path d="M73.557333 693.8624a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333zM392.533333 938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667z m307.2 0a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.6864a17.066667 17.066667 0 0 0-30.344533-10.752l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334z" fill="#11AA66" p-id="4995"></path></svg>
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1697622173220" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2398" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M722.36246914 1024.8960632h-415.54678519c-120.26260543 0-218.12969876-97.86709333-218.12969877-218.12969875v-586.42583704c0-120.26260543 97.86709333-218.12969876 218.12969877-218.12969877h415.54678519c120.26260543 0 218.12969876 97.86709333 218.12969876 218.12969877v586.42583704c0 120.26260543-97.86709333 218.12969876-218.12969876 218.12969875z m-415.54678519-898.28010667c-51.65207703 0-93.72457086 42.07249383-93.72457086 93.72457088v586.42583704c0 51.65207703 42.07249383 93.72457086 93.72457086 93.72457086h415.54678519c51.65207703 0 93.72457086-42.07249383 93.72457086-93.72457086v-586.42583704c0-51.65207703-42.07249383-93.72457086-93.72457086-93.72457088h-415.54678519z" fill="#BDD2EF" p-id="2399"></path><path d="M684.0441363 599.76969482H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885037s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885037zM465.00826075 360.66847605h-126.7352968c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885036s34.04635653-76.11885037 76.11885037-76.11885038h126.7352968c42.07249383 0 76.11885037 34.04635653 76.11885036 76.11885038s-34.04635653 76.11885037-76.11885036 76.11885036zM684.0441363 360.66847605h-25.37295013c-42.07249383 0-76.11885037-34.04635653-76.11885036-76.11885036s34.04635653-76.11885037 76.11885036-76.11885038h25.37295013c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885038s-34.17581037 76.11885037-76.11885037 76.11885036zM684.0441363 838.87091358H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885038s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885038z" fill="#2867CE" p-id="2400"></path></svg>

projects/app/public/imgs/files/url.svg

@@ -1 +0,0 @@
-<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1692418843591" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4084" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M511.5 82c-236.6 0-429 192.4-429 429 0 236.5 192.5 429 429 429 236.6 0 429-192.4 429-429 0-236.5-192.4-429-429-429z m377.6 403.8H734.3c-4-139.9-41.4-259.9-97.5-331.9C776.5 203 879 332 889.1 485.8z m-402.8-349v349h-147c5.5-175.5 68.6-322.6 147-349z m0 399.4v349c-78.4-26.4-141.4-173.5-147-349h147z m50.5 349v-349h147c-5.6 175.5-68.6 322.6-147 349z m0-399.4v-349c78.4 26.4 141.4 173.5 147 349h-147zM386.3 153.9c-56.1 72-93.5 192-97.5 331.9H133.9C144.1 332 246.5 203 386.3 153.9zM133.9 536.2h154.8c4 139.9 41.4 259.9 97.5 331.9C246.5 819 144.1 690 133.9 536.2z m502.8 331.9c56.1-72 93.5-192 97.5-331.9H889C879 690 776.5 819 636.7 868.1z" fill="#5F9BEB" p-id="4085"></path></svg>

projects/app/public/locales/en/common.json

@@ -14,6 +14,7 @@
   "UnKnow": "UnKnow",
   "Warning": "Warning",
   "app": {
+    "AI Settings": "AI Settings",
     "Advance App TestTip": "The current application is advanced editing mode \n. If you need to switch to [simple mode], please click the save button on the left",
     "App Detail": "App Detail",
     "Chat Logs Tips": "Logs record the app's online, shared, and API(chatId is existing) conversations",
@@ -39,7 +40,6 @@
     "My Apps": "My Apps",
     "Output Field Settings": "Output Field Settings",
     "Paste Config": "Paste Config",
-    "AI Settings": "AI Settings",
     "Variable Key Repeat Tip": "Variable Key Repeat",
     "module": {
       "Custom Title Tip": "The title name is displayed during the conversation"
@@ -108,9 +108,11 @@
     "Add": "Add",
     "Close": "Clow",
     "Collect": "Collect",
+    "Confirm Move": "Move here",
     "Copy": "Copy",
     "Copy Successful": "Copy Successful",
     "Course": "",
+    "Create Virtual File Failed": "Create Virtual File Failed",
     "Custom Title": "Custom Title",
     "Delete": "Delete",
     "Delete Failed": "Delete Failed",
@@ -118,29 +120,92 @@
     "Delete Warning": "Warning",
     "Edit": "Edit",
     "Expired Time": "Expired",
+    "File": "File",
     "Filed is repeat": "Filed is repeated",
     "Filed is repeated": "",
     "Input": "Input",
+    "Last Step": "Last",
     "Max credit": "Credit",
     "Max credit tips": "What is the maximum amount of money that can be consumed by the link? If the link is exceeded, it will be banned. -1 indicates no limit.",
+    "Name": "Name",
     "Name is empty": "Name is empty",
+    "Next Step": "Next",
     "Output": "Output",
     "Password inconsistency": "Password inconsistency",
     "Rename": "Rename",
     "Rename Failed": "Rename Failed",
     "Rename Success": "Rename Success",
+    "Request Error": "Request Error",
     "Search": "Search",
+    "Select One Folder": "Select a folder",
     "Status": "Status",
+    "Time": "Time",
     "Unknow": "Unknow",
+    "Unknow Source": "UnKnow Source",
     "Update Successful": "Update Successful",
-    "export": ""
+    "Update Time": "Update Time",
+    "error": {
+      "unKnow": "There was an accident"
+    },
+    "export": "",
+    "folder": {
+      "Drag Tip": "Click and move",
+      "Move Success": "Move Success",
+      "No Folder": "No Folder",
+      "Root Path": "Root Folder"
+    }
   },
   "dataset": {
+    "Chunk Length": "Chunk Length",
+    "Confirm move the folder": "Confirm Move",
     "Confirm to delete the data": "Confirm to delete the data?",
+    "Confirm to delete the file": "Are you sure to delete the file and all its data?",
+    "Create Folder": "Create Folder",
+    "Create Virtual File": "Virtual File",
+    "Delete Dataset Error": "Delete dataset failed",
+    "Edit Folder": "Edit Folder",
     "Export": "Export",
+    "File Input": "Import File",
+    "File Size": "File Size",
+    "Filename": "Filename",
+    "Files": "{{total}} Files",
+    "Folder Name": "Input folder name",
+    "Insert Data": "Insert",
+    "Manual Data": "Manual Data",
+    "Manual Input": "Manual Input",
+    "Manual Mark": "Manual Mark",
+    "Mark Data": "Mark Data",
+    "Move Failed": "Move Failed",
+    "My Dataset": "My Dataset",
     "Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while",
+    "Select Dataset": "Select Dataset",
+    "Select Folder": "Enter folder",
     "System Data Queue": "Data Queue",
-    "Training Name": "Dataset Training"
+    "Training Name": "Dataset Training",
+    "Upload Time": "Upload Time",
+    "Virtual File Tip": "Virtual files allow you to create a custom container to hold data",
+    "collections": {
+      "Click to view file": "View File Data",
+      "Click to view folder": "To Folder",
+      "Collection Embedding": "{{total}}Embedding",
+      "Confirm to delete the folder": "Are you sure to delete this folder and all its contents?",
+      "Create Training Data": "Training-{{filename}}",
+      "Create Virtual File Success": "Create Virtual File Success",
+      "Data Amount": "Data Amount",
+      "Ready": "Ready",
+      "Select Collection": "Select Collection",
+      "Select One Collection To Store": "Select the collection to store"
+    },
+    "data": {
+      "Delete Tip": "Confirm to delete the data?",
+      "File import": "File Import",
+      "Input Data": "Import Data",
+      "Input Success Tip": "Succeeded in importing data",
+      "Update Data": "Update Data",
+      "Update Success Tip": "Update data successfully"
+    },
+    "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
+    "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!"
   },
   "file": {
     "Click to download CSV template": "Click to download CSV template",
@@ -148,7 +213,6 @@
     "Create File": "Create File",
     "Create file": "Create file",
     "Drag and drop": "Drag and drop files here",
-    "Embedding": "Embedding",
     "Fetch Url": "Fetch Url",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
     "Parse": "{{name}} Parsing...",
@@ -207,31 +271,6 @@
     "desc": "AI knowledge base question and answer platform based on LLM large model",
     "slogan": "Let the AI know more about you"
   },
-  "kb": {
-    "Chunk Length": "Chunk Length",
-    "Confirm move the folder": "Confirm Move",
-    "Confirm to delete the file": "Are you sure to delete the file and all its data?",
-    "Create Folder": "Create Folder",
-    "Delete Dataset Error": "Delete dataset failed",
-    "Edit Folder": "Edit Folder",
-    "File Size": "File Size",
-    "Filename": "Filename",
-    "Files": "{{total}} Files",
-    "Folder Name": "Input folder name",
-    "Insert Data": "Insert",
-    "Manual Data": "Manual Data",
-    "Manual Input": "Manual Input",
-    "Manual Mark": "Manual Mark",
-    "Mark Data": "Mark Data",
-    "Move Failed": "Move Failed",
-    "My Dataset": "My Dataset",
-    "No Folder": "No Folder",
-    "Select Dataset": "Select Dataset",
-    "Select Folder": "Enter folder",
-    "Upload Time": "Upload Time",
-    "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
-    "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!"
-  },
   "navbar": {
     "Account": "Account",
     "Apps": "Apps",

projects/app/public/locales/zh/common.json

@@ -14,6 +14,7 @@
   "UnKnow": "未知",
   "Warning": "提示",
   "app": {
+    "AI Settings": "AI 高级配置",
     "Advance App TestTip": "当前应用为高级编排模式\n如需切换为【简易模式】请点击左侧保存按键",
     "App Detail": "应用详情",
     "Chat Logs Tips": "日志会记录该应用的在线、分享和 API(需填写 chatId) 对话记录",
@@ -39,7 +40,6 @@
     "My Apps": "我的应用",
     "Output Field Settings": "输出字段编辑",
     "Paste Config": "粘贴配置",
-    "AI Settings": "AI 高级配置",
     "Variable Key Repeat Tip": "变量 key 重复",
     "module": {
       "Custom Title Tip": "该标题名字会展示在对话过程中"
@@ -108,9 +108,11 @@
     "Add": "添加",
     "Close": "关闭",
     "Collect": "收藏",
+    "Confirm Move": "移动到这",
     "Copy": "复制",
     "Copy Successful": "复制成功",
     "Course": "",
+    "Create Virtual File Failed": "创建虚拟文件失败",
     "Custom Title": "自定义标题",
     "Delete": "删除",
     "Delete Failed": "删除失败",
@@ -118,29 +120,92 @@
     "Delete Warning": "删除警告",
     "Edit": "编辑",
     "Expired Time": "过期时间",
+    "File": "文件",
     "Filed is repeat": "",
     "Filed is repeated": "字段重复了",
     "Input": "输入",
+    "Last Step": "上一步",
     "Max credit": "最大金额",
     "Max credit tips": "该链接最大可消耗多少金额，超出后链接将被禁止使用。-1 代表无限制。",
+    "Name": "名称",
     "Name is empty": "名称不能为空",
+    "Next Step": "下一步",
     "Output": "输出",
     "Password inconsistency": "两次密码不一致",
     "Rename": "重命名",
     "Rename Failed": "重命名失败",
     "Rename Success": "重命名成功",
+    "Request Error": "请求异常",
     "Search": "搜索",
+    "Select One Folder": "选择一个目录",
     "Status": "状态",
+    "Time": "时间",
     "Unknow": "未知",
+    "Unknow Source": "未知来源",
     "Update Successful": "更新成功",
-    "export": ""
+    "Update Time": "更新时间",
+    "error": {
+      "unKnow": "出现了点意外~"
+    },
+    "export": "",
+    "folder": {
+      "Drag Tip": "点我可拖动",
+      "Move Success": "移动成功",
+      "No Folder": "这个目录空空的~",
+      "Root Path": "根目录"
+    }
   },
   "dataset": {
+    "Chunk Length": "数据总量",
+    "Confirm move the folder": "确认移动到该目录",
     "Confirm to delete the data": "确认删除该数据？",
+    "Confirm to delete the file": "确认删除该文件及其所有数据？",
+    "Create Folder": "创建文件夹",
+    "Create Virtual File": "创建虚拟文件",
+    "Delete Dataset Error": "删除知识库异常",
+    "Edit Folder": "编辑文件夹",
     "Export": "导出",
+    "File Input": "文件导入",
+    "File Size": "文件大小",
+    "Filename": "文件名",
+    "Files": "文件: {{total}}个",
+    "Folder Name": "输入文件夹名称",
+    "Insert Data": "插入",
+    "Manual Data": "手动录入",
+    "Manual Input": "手动录入",
+    "Manual Mark": "手动标注",
+    "Mark Data": "标注数据",
+    "Move Failed": "移动出现错误~",
+    "My Dataset": "我的知识库",
     "Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间",
+    "Select Dataset": "选择该知识库",
+    "Select Folder": "进入文件夹",
     "System Data Queue": "排队长度",
-    "Training Name": "数据训练"
+    "Training Name": "数据训练",
+    "Upload Time": "上传时间",
+    "Virtual File Tip": "虚拟文件允许创建一个自定义的容器装入数据",
+    "collections": {
+      "Click to view file": "点击查看文件详情",
+      "Click to view folder": "进入目录",
+      "Collection Embedding": "{{total}}组索引中",
+      "Confirm to delete the folder": "确认删除该文件夹及里面所有内容？",
+      "Create Training Data": "文件训练-{{filename}}",
+      "Create Virtual File Success": "创建虚拟文件成功",
+      "Data Amount": "数据总量",
+      "Ready": "已就绪",
+      "Select Collection": "选择文件",
+      "Select One Collection To Store": "选择一个文件进行存储"
+    },
+    "data": {
+      "Delete Tip": "确认删除该条数据？",
+      "File import": "文件导入",
+      "Input Data": "导入数据",
+      "Input Success Tip": "导入数据成功",
+      "Update Data": "更新数据",
+      "Update Success Tip": "更新数据成功"
+    },
+    "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
+    "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！"
   },
   "file": {
     "Click to download CSV template": "点击下载 CSV 模板",
@@ -148,11 +213,10 @@
     "Create File": "创建新文件",
     "Create file": "创建文件",
     "Drag and drop": "拖拽文件至此",
-    "Embedding": "索引中",
     "Fetch Url": "链接读取",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
     "Parse": "{{name}} 解析中...",
-    "Ready": "可用",
+    "Ready": "",
     "Release the mouse to upload the file": "松开鼠标上传文件",
     "Select a maximum of 10 files": "最多选择10个文件",
     "Uploading": "正在上传 {{name}}，进度: {{percent}}%",
@@ -207,31 +271,6 @@
     "desc": "基于 LLM 大模型的 AI 知识库问答平台",
     "slogan": "让 AI 更懂你的知识"
   },
-  "kb": {
-    "Chunk Length": "数据总量",
-    "Confirm move the folder": "确认移动到该目录",
-    "Confirm to delete the file": "确认删除该文件及其所有数据？",
-    "Create Folder": "创建文件夹",
-    "Delete Dataset Error": "删除知识库异常",
-    "Edit Folder": "编辑文件夹",
-    "File Size": "文件大小",
-    "Filename": "文件名",
-    "Files": "文件: {{total}}个",
-    "Folder Name": "输入文件夹名称",
-    "Insert Data": "插入",
-    "Manual Data": "手动录入",
-    "Manual Input": "手动录入",
-    "Manual Mark": "手动标注",
-    "Mark Data": "标注数据",
-    "Move Failed": "移动出现错误~",
-    "My Dataset": "我的知识库",
-    "No Folder": "没有子目录了~",
-    "Select Dataset": "选择该知识库",
-    "Select Folder": "进入文件夹",
-    "Upload Time": "上传时间",
-    "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
-    "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！"
-  },
   "navbar": {
     "Account": "账号",
     "Apps": "应用",

projects/app/src/components/ChatBox/FeedbackModal.tsx

@@ -3,7 +3,7 @@ import { ModalBody, Textarea, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '../MyModal';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { useTranslation } from 'next-i18next';
-import { userUpdateChatFeedback } from '@/web/core/api/chat';
+import { userUpdateChatFeedback } from '@/web/core/chat/api';
 
 const FeedbackModal = ({
   chatItemId,

projects/app/src/components/ChatBox/MarkModal.tsx

@@ -0,0 +1,16 @@
+import React, { useState } from 'react';
+
+const MarkModal = () => {
+  const [adminMarkData, setAdminMarkData] = useState<{
+    chatItemId: string;
+    dataId?: string;
+    datasetId?: string;
+    collectionId?: string;
+    q: string;
+    a: string;
+  }>();
+
+  return <div>MarkModal</div>;
+};
+
+export default MarkModal;

projects/app/src/components/ChatBox/QuoteModal.tsx

@@ -1,23 +1,24 @@
 import React, { useCallback, useMemo, useState } from 'react';
 import { ModalBody, Box, useTheme, Flex, Progress } from '@chakra-ui/react';
-import { getDatasetDataItemById } from '@/web/core/api/dataset';
+import { getDatasetDataItemById } from '@/web/core/dataset/api';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import { useToast } from '@/web/common/hooks/useToast';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
-import { QuoteItemType } from '@/types/chat';
 import MyIcon from '@/components/Icon';
-import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal';
+import InputDataModal, {
+  RawSourceText,
+  type InputDataType
+} from '@/pages/dataset/detail/components/InputDataModal';
 import MyModal from '../MyModal';
 import { useTranslation } from 'react-i18next';
 import { useRouter } from 'next/router';
+import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 
 const QuoteModal = ({
-  onUpdateQuote,
   rawSearch = [],
   onClose
 }: {
-  onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
+  rawSearch: SearchDataResponseItemType[];
-  rawSearch: QuoteItemType[];
   onClose: () => void;
 }) => {
   const { t } = useTranslation();
@@ -25,7 +26,7 @@ const QuoteModal = ({
   const router = useRouter();
   const { toast } = useToast();
   const { setIsLoading, Loading } = useLoading();
-  const [editDataItem, setEditDataItem] = useState<QuoteItemType>();
+  const [editInputData, setEditInputData] = useState<InputDataType>();
 
   const isShare = useMemo(() => router.pathname === '/chat/share', [router.pathname]);
 
@@ -33,18 +34,17 @@ const QuoteModal = ({
    * click edit, get new kbDataItem
    */
   const onclickEdit = useCallback(
-    async (item: QuoteItemType) => {
+    async (item: InputDataType) => {
       if (!item.id) return;
       try {
         setIsLoading(true);
         const data = await getDatasetDataItemById(item.id);
 
         if (!data) {
-          onUpdateQuote(item.id, '已删除');
           throw new Error('该数据已被删除');
         }
 
-        setEditDataItem(data);
+        setEditInputData(data);
       } catch (err) {
         toast({
           status: 'warning',
@@ -53,7 +53,7 @@ const QuoteModal = ({
       }
       setIsLoading(false);
     },
-    [setIsLoading, toast, onUpdateQuote]
+    [setIsLoading, toast]
   );
 
   return (
@@ -94,10 +94,7 @@ const QuoteModal = ({
             >
               {!isShare && (
                 <Flex alignItems={'center'} mb={1}>
-                  <RawFileText
+                  <RawSourceText sourceName={item.sourceName} sourceId={item.sourceId} />
-                    filename={item.source || t('common.Unknow') || 'Unknow'}
-                    fileId={item.file_id}
-                  />
                   <Box flex={'1'} />
                   {item.score && (
                     <>
@@ -150,16 +147,17 @@ const QuoteModal = ({
         </ModalBody>
         <Loading fixed={false} />
       </MyModal>
-      {editDataItem && (
+      {editInputData && editInputData.id && (
         <InputDataModal
-          onClose={() => setEditDataItem(undefined)}
+          onClose={() => setEditInputData(undefined)}
-          onSuccess={() => onUpdateQuote(editDataItem.id)}
+          onSuccess={() => {
-          onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
+            console.log('更新引用成功');
-          kbId={editDataItem.kb_id}
-          defaultValues={{
-            ...editDataItem,
-            dataId: editDataItem.id
           }}
+          onDelete={() => {
+            console.log('删除引用成功');
+          }}
+          datasetId={editInputData.datasetId}
+          defaultValues={editInputData}
         />
       )}
     </>

projects/app/src/components/ChatBox/ReadFeedbackModal.tsx

@@ -3,7 +3,7 @@ import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '../MyModal';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { useTranslation } from 'next-i18next';
-import { userUpdateChatFeedback } from '@/web/core/api/chat';
+import { userUpdateChatFeedback } from '@/web/core/chat/api';
 
 const ReadFeedbackModal = ({
   chatItemId,
@@ -39,14 +39,14 @@ const ReadFeedbackModal = ({
     <MyModal isOpen={true} onClose={onClose} title={t('chat.Feedback Modal')}>
       <ModalBody>{content}</ModalBody>
       <ModalFooter>
+        <Button mr={2} isLoading={isLoading} variant={'base'} onClick={mutate}>
+          {t('chat.Feedback Close')}
+        </Button>
         {!isMarked && (
-          <Button variant={'base'} mr={2} onClick={onMark}>
+          <Button mr={2} onClick={onMark}>
             {t('chat.Feedback Mark')}
           </Button>
         )}
-        <Button isLoading={isLoading} onClick={mutate}>
-          {t('chat.Feedback Close')}
-        </Button>
       </ModalFooter>
     </MyModal>
   );

projects/app/src/components/ChatBox/ResponseTags.tsx

@@ -1,8 +1,9 @@
-import React, { useCallback, useMemo, useState } from 'react';
+import React, { useMemo, useState } from 'react';
-import { ChatHistoryItemResType, ChatItemType, QuoteItemType } from '@/types/chat';
+import { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { Flex, BoxProps, useDisclosure } from '@chakra-ui/react';
 import { useTranslation } from 'react-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import dynamic from 'next/dynamic';
 import Tag from '../Tag';
 import MyTooltip from '../MyTooltip';
@@ -13,9 +14,9 @@ const ContextModal = dynamic(() => import('./ContextModal'), { ssr: false });
 const WholeResponseModal = dynamic(() => import('./WholeResponseModal'), { ssr: false });
 
 const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemResType[] }) => {
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const { t } = useTranslation();
-  const [quoteModalData, setQuoteModalData] = useState<QuoteItemType[]>();
+  const [quoteModalData, setQuoteModalData] = useState<SearchDataResponseItemType[]>();
   const [contextModalData, setContextModalData] = useState<ChatItemType[]>();
   const {
     isOpen: isOpenWholeModal,
@@ -37,14 +38,12 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
         .filter((item) => item.moduleType === FlowModuleTypeEnum.chatNode)
         .map((item) => item.quoteList)
         .flat()
-        .filter((item) => item) as QuoteItemType[],
+        .filter((item) => item) as SearchDataResponseItemType[],
       historyPreview: chatData?.historyPreview,
       runningTime: +responseData.reduce((sum, item) => sum + (item.runningTime || 0), 0).toFixed(2)
     };
   }, [responseData]);
 
-  const updateQuote = useCallback(async (quoteId: string, sourceText?: string) => {}, []);
-
   const TagStyles: BoxProps = {
     mr: 2,
     bg: 'transparent'
@@ -100,11 +99,7 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
       </MyTooltip>
 
       {!!quoteModalData && (
-        <QuoteModal
+        <QuoteModal rawSearch={quoteModalData} onClose={() => setQuoteModalData(undefined)} />
-          rawSearch={quoteModalData}
-          onUpdateQuote={updateQuote}
-          onClose={() => setQuoteModalData(undefined)}
-        />
       )}
       {!!contextModalData && (
         <ContextModal context={contextModalData} onClose={() => setContextModalData(undefined)} />

projects/app/src/components/ChatBox/SelectDataset.tsx

@@ -1,114 +0,0 @@
-import React, { useState } from 'react';
-import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react';
-import { useTranslation } from 'next-i18next';
-import { useToast } from '@/web/common/hooks/useToast';
-import Avatar from '../Avatar';
-import MyIcon from '@/components/Icon';
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
-import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
-
-const SelectDataset = ({
-  isOpen,
-  onSuccess,
-  onClose
-}: {
-  isOpen: boolean;
-  onSuccess: (kbId: string) => void;
-  onClose: () => void;
-}) => {
-  const { t } = useTranslation();
-  const theme = useTheme();
-  const { toast } = useToast();
-  const [selectedId, setSelectedId] = useState<string>();
-  const { paths, parentId, setParentId, datasets } = useDatasetSelect();
-
-  return (
-    <DatasetSelectModal
-      isOpen={isOpen}
-      paths={paths}
-      onClose={onClose}
-      parentId={parentId}
-      setParentId={setParentId}
-      tips={t('chat.Select Mark Kb Desc')}
-    >
-      <ModalBody flex={'1 0 0'} overflowY={'auto'}>
-        <Grid
-          gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
-          gridGap={3}
-          userSelect={'none'}
-        >
-          {datasets.map((item) =>
-            (() => {
-              const selected = selectedId === item._id;
-              return (
-                <Card
-                  key={item._id}
-                  p={3}
-                  border={theme.borders.base}
-                  boxShadow={'sm'}
-                  h={'80px'}
-                  cursor={'pointer'}
-                  _hover={{
-                    boxShadow: 'md'
-                  }}
-                  {...(selected
-                    ? {
-                        bg: 'myBlue.300'
-                      }
-                    : {})}
-                  onClick={() => {
-                    if (item.type === DatasetTypeEnum.folder) {
-                      setParentId(item._id);
-                    } else {
-                      setSelectedId(item._id);
-                    }
-                  }}
-                >
-                  <Flex alignItems={'center'} h={'38px'}>
-                    <Avatar src={item.avatar} w={['24px', '28px', '32px']}></Avatar>
-                    <Box ml={3} fontWeight={'bold'} fontSize={['md', 'lg', 'xl']}>
-                      {item.name}
-                    </Box>
-                  </Flex>
-                  <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
-                    <MyIcon mr={1} name="kbTest" w={'12px'} />
-                    <Box color={'myGray.500'}>{item.vectorModel.name}</Box>
-                  </Flex>
-                </Card>
-              );
-            })()
-          )}
-        </Grid>
-        {datasets.length === 0 && (
-          <Flex mt={5} flexDirection={'column'} alignItems={'center'}>
-            <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
-            <Box mt={2} color={'myGray.500'}>
-              这个目录已经没东西可选了~
-            </Box>
-          </Flex>
-        )}
-      </ModalBody>
-      <ModalFooter>
-        <Button variant={'base'} mr={2} onClick={onClose}>
-          {t('Cancel')}
-        </Button>
-        <Button
-          onClick={() => {
-            if (!selectedId) {
-              return toast({
-                status: 'warning',
-                title: t('Select value is empty')
-              });
-            }
-
-            onSuccess(selectedId);
-          }}
-        >
-          {t('Confirm')}
-        </Button>
-      </ModalFooter>
-    </DatasetSelectModal>
-  );
-};
-
-export default SelectDataset;

projects/app/src/components/ChatBox/SelectMarkCollection.tsx

@@ -0,0 +1,195 @@
+import React, { useState } from 'react';
+import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import Avatar from '../Avatar';
+import MyIcon from '@/components/Icon';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
+import dynamic from 'next/dynamic';
+import { MarkDataType } from '@/global/core/dataset/type';
+import SelectCollections from '@/web/core/dataset/components/SelectCollections';
+
+const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal'));
+
+export type AdminMarkType = {
+  dataId?: string;
+  datasetId?: string;
+  collectionId?: string;
+  q: string;
+  a?: string;
+};
+
+const SelectMarkCollection = ({
+  adminMarkData,
+  setAdminMarkData,
+  onSuccess,
+  onClose
+}: {
+  adminMarkData: AdminMarkType;
+  setAdminMarkData: (e: AdminMarkType) => void;
+  onClose: () => void;
+  onSuccess: (adminFeedback: MarkDataType) => void;
+}) => {
+  const { t } = useTranslation();
+  const theme = useTheme();
+  const [selectedDatasetId, setSelectedDatasetId] = useState<string>();
+  const [selectedDatasetCollectionIds, setSelectedDatasetCollectionIds] = useState<string[]>([]);
+  const { paths, parentId, setParentId, datasets, isLoading } = useDatasetSelect();
+
+  return (
+    <>
+      {/* select dataset */}
+      {!adminMarkData.datasetId && (
+        <DatasetSelectModal
+          isOpen
+          paths={paths}
+          onClose={onClose}
+          parentId={parentId}
+          setParentId={setParentId}
+          tips={t('chat.Select Mark Kb Desc')}
+        >
+          <ModalBody flex={'1 0 0'} overflowY={'auto'}>
+            <Grid
+              gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
+              gridGap={3}
+              userSelect={'none'}
+            >
+              {datasets.map((item) =>
+                (() => {
+                  const selected = selectedDatasetId === item._id;
+                  return (
+                    <Card
+                      key={item._id}
+                      p={3}
+                      border={theme.borders.base}
+                      boxShadow={'sm'}
+                      h={'80px'}
+                      cursor={'pointer'}
+                      _hover={{
+                        boxShadow: 'md'
+                      }}
+                      {...(selected
+                        ? {
+                            bg: 'myBlue.300'
+                          }
+                        : {})}
+                      onClick={() => {
+                        if (item.type === DatasetTypeEnum.folder) {
+                          setParentId(item._id);
+                        } else {
+                          setSelectedDatasetId(item._id);
+                        }
+                      }}
+                    >
+                      <Flex alignItems={'center'} h={'38px'}>
+                        <Avatar src={item.avatar} w={['24px', '28px', '32px']}></Avatar>
+                        <Box ml={3} fontWeight={'bold'} fontSize={['md', 'lg', 'xl']}>
+                          {item.name}
+                        </Box>
+                      </Flex>
+                      <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
+                        <MyIcon mr={1} name="kbTest" w={'12px'} />
+                        <Box color={'myGray.500'}>{item.vectorModel.name}</Box>
+                      </Flex>
+                    </Card>
+                  );
+                })()
+              )}
+            </Grid>
+            {datasets.length === 0 && (
+              <Flex mt={'10vh'} flexDirection={'column'} alignItems={'center'}>
+                <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
+                <Box mt={2} color={'myGray.500'}>
+                  这个目录已经没东西可选了~
+                </Box>
+              </Flex>
+            )}
+          </ModalBody>
+          <ModalFooter>
+            <Button
+              isLoading={isLoading}
+              isDisabled={!selectedDatasetId}
+              onClick={() => {
+                setAdminMarkData({ ...adminMarkData, datasetId: selectedDatasetId });
+              }}
+            >
+              {t('common.Next Step')}
+            </Button>
+          </ModalFooter>
+        </DatasetSelectModal>
+      )}
+
+      {/* select collection */}
+      {adminMarkData.datasetId && !adminMarkData.collectionId && (
+        <SelectCollections
+          datasetId={adminMarkData.datasetId}
+          type={'collection'}
+          title={t('dataset.collections.Select One Collection To Store')}
+          onClose={onClose}
+          onChange={({ collectionIds }) => {
+            setSelectedDatasetCollectionIds(collectionIds);
+          }}
+          CustomFooter={
+            <ModalFooter>
+              <Button
+                variant={'base'}
+                mr={2}
+                onClick={() => {
+                  setAdminMarkData({
+                    ...adminMarkData,
+                    datasetId: undefined
+                  });
+                }}
+              >
+                {t('common.Last Step')}
+              </Button>
+              <Button
+                isDisabled={selectedDatasetCollectionIds.length === 0}
+                onClick={() => {
+                  setAdminMarkData({
+                    ...adminMarkData,
+                    collectionId: selectedDatasetCollectionIds[0]
+                  });
+                }}
+              >
+                {t('common.Next Step')}
+              </Button>
+            </ModalFooter>
+          }
+        />
+      )}
+
+      {/* input data */}
+      {adminMarkData.datasetId && adminMarkData.collectionId && (
+        <InputDataModal
+          onClose={onClose}
+          datasetId={adminMarkData.datasetId}
+          defaultValues={{
+            id: adminMarkData.dataId,
+            datasetId: adminMarkData.datasetId,
+            collectionId: adminMarkData.collectionId,
+            sourceName: '手动标注',
+            q: adminMarkData.q,
+            a: adminMarkData.a
+          }}
+          onSuccess={(data) => {
+            if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) {
+              return onClose();
+            }
+
+            onSuccess({
+              dataId: data.id,
+              datasetId: adminMarkData.datasetId,
+              collectionId: adminMarkData.collectionId,
+              q: data.q,
+              a: data.a
+            });
+            onClose();
+          }}
+        />
+      )}
+    </>
+  );
+};
+
+export default React.memo(SelectMarkCollection);

projects/app/src/components/ChatBox/WholeResponseModal.tsx

@@ -8,7 +8,7 @@ import Tabs from '../Tabs';
 import MyModal from '../MyModal';
 import MyTooltip from '../MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 
 function Row({ label, value }: { label: string; value?: string | number | React.ReactNode }) {
   const theme = useTheme();

projects/app/src/components/ChatBox/index.module.scss

@@ -10,25 +10,6 @@
   }
 }
 
-.newChat {
-  .modelListContainer {
-    height: 0;
-    overflow: hidden;
-  }
-  .modelList {
-    border-radius: 6px;
-  }
-  &:hover {
-    .modelListContainer {
-      height: 60vh;
-    }
-    .modelList {
-      box-shadow: 0 0 5px rgba($color: #000000, $alpha: 0.05);
-      border: 1px solid #dee0e2;
-    }
-  }
-}
-
 .statusAnimation {
   animation: statusBox 0.8s linear infinite alternate;
 }

projects/app/src/components/ChatBox/index.tsx

@@ -8,6 +8,7 @@ import React, {
   ForwardedRef,
   useEffect
 } from 'react';
+import Script from 'next/script';
 import { throttle } from 'lodash';
 import {
   ChatHistoryItemResType,
@@ -17,7 +18,7 @@ import {
 } from '@/types/chat';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useAudioPlay } from '@/web/common/utils/voice';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
 import {
   Box,
@@ -30,7 +31,7 @@ import {
   BoxProps,
   FlexProps
 } from '@chakra-ui/react';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { eventBus } from '@/web/common/utils/eventbus';
 import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
 import { useMarkdown } from '@/web/common/hooks/useMarkdown';
@@ -38,14 +39,15 @@ import { AppModuleItemType } from '@/types/app';
 import { VariableInputEnum } from '@/constants/app';
 import { useForm } from 'react-hook-form';
 import type { MessageItemType } from '@/types/core/chat/type';
-import { fileDownload } from '@/web/common/utils/file';
+import { fileDownload } from '@/web/common/file/utils';
 import { htmlTemplate } from '@/constants/common';
 import { useRouter } from 'next/router';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { useTranslation } from 'react-i18next';
 import { customAlphabet } from 'nanoid';
-import { userUpdateChatFeedback, adminUpdateChatFeedback } from '@/web/core/api/chat';
+import { adminUpdateChatFeedback, userUpdateChatFeedback } from '@/web/core/chat/api';
+import type { AdminMarkType } from './SelectMarkCollection';
 
 import MyIcon from '@/components/Icon';
 import Avatar from '@/components/Avatar';
@@ -56,14 +58,11 @@ import dynamic from 'next/dynamic';
 const ResponseTags = dynamic(() => import('./ResponseTags'));
 const FeedbackModal = dynamic(() => import('./FeedbackModal'));
 const ReadFeedbackModal = dynamic(() => import('./ReadFeedbackModal'));
-const SelectDataset = dynamic(() => import('./SelectDataset'));
+const SelectMarkCollection = dynamic(() => import('./SelectMarkCollection'));
-const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal'));
 
 import styles from './index.module.scss';
-import Script from 'next/script';
+import { postQuestionGuide } from '@/web/core/ai/api';
-import { postQuestionGuide } from '@/web/core/api/ai';
+import { splitGuideModule } from '@/global/core/app/modules/utils';
-import { splitGuideModule } from './utils';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
 
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
 
@@ -131,7 +130,7 @@ const ChatBox = (
   const router = useRouter();
   const { t } = useTranslation();
   const { toast } = useToast();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const TextareaDom = useRef<HTMLTextAreaElement>(null);
   const chatController = useRef(new AbortController());
   const questionGuideController = useRef(new AbortController());
@@ -147,14 +146,7 @@ const ChatBox = (
     content: string;
     isMarked: boolean;
   }>();
-  const [adminMarkData, setAdminMarkData] = useState<{
+  const [adminMarkData, setAdminMarkData] = useState<AdminMarkType & { chatItemId: string }>();
-    // mark modal data
-    kbId?: string;
-    chatItemId: string;
-    dataId?: string;
-    q: string;
-    a: string;
-  }>();
   const [questionGuides, setQuestionGuide] = useState<string[]>([]);
 
   const isChatting = useMemo(
@@ -674,10 +666,11 @@ const ChatBox = (
                                 if (item.adminFeedback) {
                                   setAdminMarkData({
                                     chatItemId: item.dataId,
-                                    kbId: item.adminFeedback.kbId,
+                                    datasetId: item.adminFeedback.datasetId,
+                                    collectionId: item.adminFeedback.collectionId,
                                     dataId: item.adminFeedback.dataId,
-                                    q: chatHistory[index - 1]?.value || '',
+                                    q: item.adminFeedback.q || chatHistory[index - 1]?.value || '',
-                                    a: item.adminFeedback.content
+                                    a: item.adminFeedback.a
                                   });
                                 } else {
                                   setAdminMarkData({
@@ -798,7 +791,9 @@ const ChatBox = (
                               </Box>
                               <Box h={'1px'} bg={'myGray.300'} flex={'1'} />
                             </Flex>
-                            <Box>{item.adminFeedback.content}</Box>
+                            <Box whiteSpace={'pre'}>{`${item.adminFeedback.q || ''}${
+                              item.adminFeedback.a ? `\n${item.adminFeedback.a}` : ''
+                            }`}</Box>
                           </Box>
                         )}
                       </Card>
@@ -940,75 +935,44 @@ const ChatBox = (
         />
       )}
       {/* admin mark data */}
-      {showMarkIcon && (
+      {!!adminMarkData && (
-        <>
+        <SelectMarkCollection
-          {/* select one dataset to insert markData */}
+          adminMarkData={adminMarkData}
-          <SelectDataset
+          setAdminMarkData={(e) => setAdminMarkData({ ...e, chatItemId: adminMarkData.chatItemId })}
-            isOpen={!!adminMarkData && !adminMarkData.kbId}
+          onClose={() => setAdminMarkData(undefined)}
-            onClose={() => setAdminMarkData(undefined)}
+          onSuccess={(adminFeedback) => {
-            // @ts-ignore
+            adminUpdateChatFeedback({
-            onSuccess={(kbId) => setAdminMarkData((state) => ({ ...state, kbId }))}
+              chatItemId: adminMarkData.chatItemId,
-          />
+              ...adminFeedback
+            });
+            // update dom
+            setChatHistory((state) =>
+              state.map((chatItem) =>
+                chatItem.dataId === adminMarkData.chatItemId
+                  ? {
+                      ...chatItem,
+                      adminFeedback
+                    }
+                  : chatItem
+              )
+            );
 
-          {/* edit markData modal */}
+            if (readFeedbackData) {
-          {adminMarkData && adminMarkData.kbId && (
+              userUpdateChatFeedback({
-            <InputDataModal
+                chatItemId: readFeedbackData.chatItemId,
-              onClose={() => setAdminMarkData(undefined)}
+                userFeedback: undefined
-              onSuccess={async (data) => {
+              });
-                if (!adminMarkData.kbId || !data.dataId) {
+              setChatHistory((state) =>
-                  return setAdminMarkData(undefined);
+                state.map((chatItem) =>
-                }
+                  chatItem.dataId === readFeedbackData.chatItemId
-                const adminFeedback = {
+                    ? { ...chatItem, userFeedback: undefined }
-                  kbId: adminMarkData.kbId,
+                    : chatItem
-                  dataId: data.dataId,
+                )
-                  content: data.a
+              );
-                };
+              setReadFeedbackData(undefined);
-
+            }
-                // update dom
+          }}
-                setChatHistory((state) =>
+        />
-                  state.map((chatItem) =>
-                    chatItem.dataId === adminMarkData.chatItemId
-                      ? {
-                          ...chatItem,
-                          adminFeedback
-                        }
-                      : chatItem
-                  )
-                );
-                // request to update adminFeedback
-                try {
-                  adminUpdateChatFeedback({
-                    chatItemId: adminMarkData.chatItemId,
-                    ...adminFeedback
-                  });
-
-                  if (readFeedbackData) {
-                    userUpdateChatFeedback({
-                      chatItemId: readFeedbackData.chatItemId,
-                      userFeedback: undefined
-                    });
-                    setChatHistory((state) =>
-                      state.map((chatItem) =>
-                        chatItem.dataId === readFeedbackData.chatItemId
-                          ? { ...chatItem, userFeedback: undefined }
-                          : chatItem
-                      )
-                    );
-                    setReadFeedbackData(undefined);
-                  }
-                } catch (error) {}
-                setAdminMarkData(undefined);
-              }}
-              kbId={adminMarkData.kbId}
-              defaultValues={{
-                dataId: adminMarkData.dataId,
-                q: adminMarkData.q,
-                a: adminMarkData.a,
-                file_id: DatasetSpecialIdEnum.mark
-              }}
-            />
-          )}
-        </>
       )}
     </Flex>
   );

projects/app/src/components/DateRangePicker/index.module.scss

@@ -1,4 +0,0 @@
-.datePicker {
-  --rdp-background-color: #d6e8ff;
-  --rdp-accent-color: #0000ff;
-}

projects/app/src/components/DateRangePicker/index.tsx

@@ -4,7 +4,6 @@ import { addDays, format } from 'date-fns';
 import { type DateRange, DayPicker } from 'react-day-picker';
 import MyIcon from '../Icon';
 import 'react-day-picker/dist/style.css';
-import styles from './index.module.scss';
 import zhCN from 'date-fns/locale/zh-CN';
 
 const DateRangePicker = ({
@@ -59,6 +58,10 @@ const DateRangePicker = ({
         <Card
           position={'absolute'}
           zIndex={1}
+          css={{
+            '--rdp-background-color': '#d6e8ff',
+            ' --rdp-accent-color': '#0000ff'
+          }}
           {...(position === 'top'
             ? {
                 bottom: '40px'
@@ -69,7 +72,6 @@ const DateRangePicker = ({
             locale={zhCN}
             id="test"
             mode="range"
-            className={styles.datePicker}
             defaultMonth={defaultDate.to}
             selected={range}
             disabled={[

projects/app/src/components/EmptyTip/index.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import { Flex, Box, FlexProps } from '@chakra-ui/react';
+import MyIcon from '../Icon';
+
+type Props = FlexProps & {
+  text?: string | null;
+};
+
+const EmptyTip = ({ text, ...props }: Props) => {
+  return (
+    <Flex mt={5} flexDirection={'column'} alignItems={'center'} pt={'10vh'} {...props}>
+      <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
+      <Box mt={2} color={'myGray.500'}>
+        {text || '没有什么数据噢~'}
+      </Box>
+    </Flex>
+  );
+};
+
+export default EmptyTip;

projects/app/src/components/Layout/auth.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { useRouter } from 'next/router';
 import { useToast } from '@chakra-ui/react';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useQuery } from '@tanstack/react-query';
 
 const unAuthPage: { [key: string]: boolean } = {

projects/app/src/components/Layout/index.tsx

@@ -2,14 +2,14 @@ import React, { useEffect, useMemo } from 'react';
 import { Box, useColorMode, Flex } from '@chakra-ui/react';
 import { useRouter } from 'next/router';
 import { useLoading } from '@/web/common/hooks/useLoading';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { throttle } from 'lodash';
 import Auth from './auth';
 import Navbar from './navbar';
 import NavbarPhone from './navbarPhone';
 import { useQuery } from '@tanstack/react-query';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
-import { getUnreadCount } from '@/web/support/api/user';
+import { getUnreadCount } from '@/web/support/user/api';
 
 const pcUnShowLayoutRoute: Record<string, boolean> = {
   '/': true,
@@ -30,7 +30,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
   const router = useRouter();
   const { colorMode, setColorMode } = useColorMode();
   const { Loading } = useLoading();
-  const { loading, setScreenWidth, isPc, loadGitStar } = useGlobalStore();
+  const { loading, setScreenWidth, isPc, loadGitStar } = useSystemStore();
   const { userInfo } = useUserStore();
 
   const isChatPage = useMemo(

projects/app/src/components/Layout/navbar.tsx

@@ -1,16 +1,16 @@
 import React, { useMemo } from 'react';
 import { Box, Flex, Link } from '@chakra-ui/react';
 import { useRouter } from 'next/router';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
-import { useChatStore } from '@/web/core/store/chat';
+import { useChatStore } from '@/web/core/chat/storeChat';
 import { HUMAN_ICON } from '@/constants/chat';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import NextLink from 'next/link';
 import Badge from '../Badge';
 import Avatar from '../Avatar';
 import MyIcon from '../Icon';
 import { useTranslation } from 'next-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyTooltip from '../MyTooltip';
 
 export enum NavbarTypeEnum {
@@ -22,7 +22,7 @@ const Navbar = ({ unread }: { unread: number }) => {
   const { t } = useTranslation();
   const router = useRouter();
   const { userInfo } = useUserStore();
-  const { gitStar } = useGlobalStore();
+  const { gitStar } = useSystemStore();
   const { lastChatAppId, lastChatId } = useChatStore();
   const navbarList = useMemo(
     () => [
@@ -44,8 +44,8 @@ const Navbar = ({ unread }: { unread: number }) => {
         label: t('navbar.Datasets'),
         icon: 'dbLight',
         activeIcon: 'dbFill',
-        link: `/kb/list`,
+        link: `/dataset/list`,
-        activeLink: ['/kb/list', '/kb/detail']
+        activeLink: ['/dataset/list', '/dataset/detail']
       },
       ...(feConfigs?.show_appStore
         ? [

projects/app/src/components/Layout/navbarPhone.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo } from 'react';
 import { useRouter } from 'next/router';
 import { Flex, Box } from '@chakra-ui/react';
-import { useChatStore } from '@/web/core/store/chat';
+import { useChatStore } from '@/web/core/chat/storeChat';
 import { useTranslation } from 'react-i18next';
 import Badge from '../Badge';
 import MyIcon from '../Icon';

projects/app/src/components/Markdown/chat/Quote.tsx

@@ -1,8 +1,8 @@
 import React, { useMemo } from 'react';
 import { Box, useTheme } from '@chakra-ui/react';
-import { getFileAndOpen } from '@/web/common/utils/file';
+import { getFileAndOpen } from '@/web/common/file/utils';
 import { useToast } from '@/web/common/hooks/useToast';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 
 type QuoteItemType = {
   file_id?: string;

projects/app/src/components/MyModal/index.tsx

@@ -22,7 +22,7 @@ const MyModal = ({
   title,
   children,
   isCentered,
-  w = 'auto',
+  w = '100%',
   maxW = ['90vw', '600px'],
   ...props
 }: Props) => {
@@ -44,7 +44,12 @@ const MyModal = ({
       >
         {!!title && <ModalHeader>{title}</ModalHeader>}
         {onClose && <ModalCloseButton />}
-        <Box overflow={'overlay'} h={'100%'} display={'flex'} flexDirection={'column'}>
+        <Box
+          overflow={props.overflow || 'overlay'}
+          h={'100%'}
+          display={'flex'}
+          flexDirection={'column'}
+        >
           {children}
         </Box>
       </ModalContent>

projects/app/src/components/MyTooltip/index.tsx

@@ -1,13 +1,13 @@
 import React from 'react';
 import { Tooltip, TooltipProps } from '@chakra-ui/react';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 interface Props extends TooltipProps {
   forceShow?: boolean;
 }
 
 const MyTooltip = ({ children, forceShow = false, shouldWrapChildren = true, ...props }: Props) => {
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   return isPc || forceShow ? (
     <Tooltip
       bg={'white'}

projects/app/src/components/PromptTemplate/index.tsx

@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import MyModal from '../MyModal';
 import { Box, Button, Grid, useTheme } from '@chakra-ui/react';
-import { PromptTemplateItem } from '@fastgpt/core/ai/type';
+import { PromptTemplateItem } from '@fastgpt/global/core/ai/type.d';
 import { ModalBody, ModalFooter } from '@chakra-ui/react';
 
 const PromptTemplate = ({

projects/app/src/components/common/ParentPaths/index.tsx

@@ -0,0 +1,62 @@
+import MyIcon from '@/components/Icon';
+import { Box, Flex } from '@chakra-ui/react';
+import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+import React, { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+const ParentPaths = (props: {
+  paths?: ParentTreePathItemType[];
+  rootName?: string;
+  FirstPathDom?: React.ReactNode;
+  onClick: (parentId: string) => void;
+}) => {
+  const { t } = useTranslation();
+  const { paths = [], rootName = t('common.folder.Root Path'), FirstPathDom, onClick } = props;
+  const concatPaths = useMemo(
+    () => [
+      {
+        parentId: '',
+        parentName: rootName
+      },
+      ...paths
+    ],
+    [rootName, paths]
+  );
+
+  return paths.length === 0 && !!FirstPathDom ? (
+    <>{FirstPathDom}</>
+  ) : (
+    <Flex flex={1}>
+      {concatPaths.map((item, i) => (
+        <Flex key={item.parentId} alignItems={'center'}>
+          <Box
+            fontSize={['md', 'lg']}
+            py={1}
+            px={[0, 2]}
+            borderRadius={'md'}
+            {...(i === concatPaths.length - 1
+              ? {
+                  cursor: 'default'
+                }
+              : {
+                  cursor: 'pointer',
+                  _hover: {
+                    bg: 'myGray.100'
+                  },
+                  onClick: () => {
+                    onClick(item.parentId);
+                  }
+                })}
+          >
+            {item.parentName}
+          </Box>
+          {i !== concatPaths.length - 1 && (
+            <MyIcon name={'rightArrowLight'} color={'myGray.500'} w={['18px', '24px']} />
+          )}
+        </Flex>
+      ))}
+    </Flex>
+  );
+};
+
+export default React.memo(ParentPaths);

projects/app/src/components/core/dataset/SelectModal.tsx

@@ -1,9 +1,9 @@
-import { getDatasets, getDatasetPaths } from '@/web/core/api/dataset';
+import { getDatasets, getDatasetPaths } from '@/web/core/dataset/api';
 import MyModal from '@/components/MyModal';
 import { useQuery } from '@tanstack/react-query';
 import React, { Dispatch, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { Box, Flex, ModalHeader } from '@chakra-ui/react';
 import MyIcon from '@/components/Icon';
 
@@ -30,7 +30,7 @@ const DatasetSelectContainer = ({
   children: React.ReactNode;
 }) => {
   const { t } = useTranslation();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
 
   return (
     <MyModal isOpen={isOpen} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered>
@@ -86,11 +86,11 @@ const DatasetSelectContainer = ({
   );
 };
 
-export const useDatasetSelect = () => {
+export function useDatasetSelect() {
   const { t } = useTranslation();
   const [parentId, setParentId] = useState<string>();
 
-  const { data } = useQuery(['loadDatasetData', parentId], () =>
+  const { data, isLoading } = useQuery(['loadDatasetData', parentId], () =>
     Promise.all([getDatasets({ parentId }), getDatasetPaths(parentId)])
   );
 
@@ -98,7 +98,7 @@ export const useDatasetSelect = () => {
     () => [
       {
         parentId: '',
-        parentName: t('kb.My Dataset')
+        parentName: t('dataset.My Dataset')
       },
       ...(data?.[1] || [])
     ],
@@ -109,8 +109,9 @@ export const useDatasetSelect = () => {
     parentId,
     setParentId,
     datasets: data?.[0] || [],
-    paths
+    paths,
+    isLoading
   };
-};
+}
 
 export default DatasetSelectContainer;

projects/app/src/components/support/apikey/Table.tsx

@@ -25,14 +25,14 @@ import {
   createAOpenApiKey,
   delOpenApiById,
   putOpenApiKey
-} from '@/web/support/api/openapi';
+} from '@/web/support/openapi/api';
 import type { EditApiKeyProps } from '@/global/support/api/openapiReq';
 import { useQuery, useMutation } from '@tanstack/react-query';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import dayjs from 'dayjs';
 import { AddIcon, QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { useTranslation } from 'react-i18next';
 import MyIcon from '@/components/Icon';
 import MyModal from '@/components/MyModal';

projects/app/src/constants/flow/ModuleTemplate.ts

@@ -9,7 +9,7 @@ import {
 } from './index';
 import type { AppItemType } from '@/types/app';
 import type { FlowModuleTemplateType } from '@/types/core/app/flow';
-import { chatModelList, cqModelList } from '@/web/common/store/static';
+import { chatModelList, cqModelList } from '@/web/common/system/staticData';
 import {
   Input_Template_History,
   Input_Template_TFSwitch,
@@ -240,7 +240,7 @@ export const ChatModule: FlowModuleTemplateType = {
 };
 
 export const KBSearchModule: FlowModuleTemplateType = {
-  flowType: FlowModuleTypeEnum.kbSearchNode,
+  flowType: FlowModuleTypeEnum.datasetSearchNode,
   logo: '/imgs/module/db.png',
   name: '知识库搜索',
   intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。',
@@ -248,7 +248,7 @@ export const KBSearchModule: FlowModuleTemplateType = {
   inputs: [
     Input_Template_TFSwitch,
     {
-      key: 'kbList',
+      key: 'datasets',
       type: FlowInputItemTypeEnum.selectDataset,
       label: '关联的知识库',
       value: [],
@@ -900,7 +900,7 @@ export const appTemplates: (AppItemType & {
       {
         moduleId: 'kbSearch',
         name: '知识库搜索',
-        flowType: 'kbSearchNode',
+        flowType: 'datasetSearchNode',
         showStatus: true,
         position: {
           x: 956.0838440206068,
@@ -908,7 +908,7 @@ export const appTemplates: (AppItemType & {
         },
         inputs: [
           {
-            key: 'kbList',
+            key: 'datasets',
             type: 'custom',
             label: '关联的知识库',
             value: [],
@@ -1952,7 +1952,7 @@ export const appTemplates: (AppItemType & {
       {
         moduleId: 'fljhzy',
         name: '知识库搜索',
-        flowType: 'kbSearchNode',
+        flowType: 'datasetSearchNode',
         showStatus: true,
         position: {
           x: 1305.5374262228029,
@@ -1960,7 +1960,7 @@ export const appTemplates: (AppItemType & {
         },
         inputs: [
           {
-            key: 'kbList',
+            key: 'datasets',
             type: 'custom',
             label: '关联的知识库',
             value: [],

projects/app/src/constants/flow/index.ts

@@ -34,7 +34,7 @@ export enum FlowModuleTypeEnum {
   questionInput = 'questionInput',
   historyNode = 'historyNode',
   chatNode = 'chatNode',
-  kbSearchNode = 'kbSearchNode',
+  datasetSearchNode = 'datasetSearchNode',
   tfSwitchNode = 'tfSwitchNode',
   answerNode = 'answerNode',
   classifyQuestion = 'classifyQuestion',