v4.5.1 (#417)

projects/app/src/pages/_app.tsx

@@ -8,12 +8,12 @@ import { theme } from '@/web/styles/theme';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
-import { clientInitData, feConfigs } from '@/web/common/store/static';
+import { clientInitData, feConfigs } from '@/web/common/system/staticData';
 import { appWithTranslation, useTranslation } from 'next-i18next';
 import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
 import { useRouter } from 'next/router';
-import { useGlobalStore } from '@/web/common/store/global';
-import type { FeConfigsType } from '@fastgpt/common/type/index.d';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
 
 import 'nprogress/nprogress.css';
 import '@/web/styles/reset.scss';
@@ -38,7 +38,7 @@ function App({ Component, pageProps }: AppProps) {
   const router = useRouter();
   const { hiId } = router.query as { hiId?: string };
   const { i18n } = useTranslation();
-  const { setLastRoute } = useGlobalStore();
+  const { setLastRoute } = useSystemStore();
   const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
 
   useEffect(() => {

projects/app/src/pages/_error.tsx

@@ -1,11 +1,11 @@
 import { useEffect } from 'react';
 import { useRouter } from 'next/router';
 import { serviceSideProps } from '@/web/common/utils/i18n';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 function Error() {
   const router = useRouter();
-  const { lastRoute } = useGlobalStore();
+  const { lastRoute } = useSystemStore();
 
   useEffect(() => {
     setTimeout(() => {

projects/app/src/pages/account/components/BillDetail.tsx

@@ -14,7 +14,7 @@ import {
 import { UserBillType } from '@/types/user';
 import dayjs from 'dayjs';
 import { BillSourceMap } from '@/constants/user';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 import MyModal from '@/components/MyModal';
 import { useTranslation } from 'react-i18next';
 

projects/app/src/pages/account/components/BillTable.tsx

@@ -12,7 +12,7 @@ import {
   Button
 } from '@chakra-ui/react';
 import { BillSourceMap } from '@/constants/user';
-import { getUserBills } from '@/web/common/api/bill';
+import { getUserBills } from '@/web/common/bill/api';
 import type { UserBillType } from '@/types/user';
 import { usePagination } from '@/web/common/hooks/usePagination';
 import { useLoading } from '@/web/common/hooks/useLoading';
@@ -21,7 +21,7 @@ import MyIcon from '@/components/Icon';
 import DateRangePicker, { type DateRangeType } from '@/components/DateRangePicker';
 import { addDays } from 'date-fns';
 import dynamic from 'next/dynamic';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { useTranslation } from 'next-i18next';
 const BillDetail = dynamic(() => import('./BillDetail'));
 
@@ -32,7 +32,7 @@ const BillTable = () => {
     from: addDays(new Date(), -7),
     to: new Date()
   });
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
 
   const {
     data: bills,

projects/app/src/pages/account/components/Info.tsx

@@ -15,15 +15,15 @@ import {
 import { useForm } from 'react-hook-form';
 import { UserUpdateParams } from '@/types/user';
 import { useToast } from '@/web/common/hooks/useToast';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { UserType } from '@/types/user';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
-import { useSelectFile } from '@/web/common/hooks/useSelectFile';
-import { compressImg } from '@/web/common/utils/file';
-import { feConfigs, systemVersion } from '@/web/common/store/static';
+import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
+import { compressImg } from '@/web/common/file/utils';
+import { feConfigs, systemVersion } from '@/web/common/system/staticData';
 import { useTranslation } from 'next-i18next';
-import { timezoneList } from '@/utils/user';
+import { timezoneList } from '@fastgpt/global/common/time/timezone';
 import Loading from '@/components/Loading';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@/components/Icon';

projects/app/src/pages/account/components/InformTable.tsx

@@ -1,17 +1,17 @@
 import React from 'react';
 import { Box, Flex, useTheme } from '@chakra-ui/react';
-import { getInforms, readInform } from '@/web/support/api/user';
+import { getInforms, readInform } from '@/web/support/user/api';
 import { usePagination } from '@/web/common/hooks/usePagination';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import type { informSchema } from '@/types/mongoSchema';
 import { formatTimeToChatTime } from '@/utils/tools';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyIcon from '@/components/Icon';
 
 const BillTable = () => {
   const theme = useTheme();
   const { Loading } = useLoading();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const {
     data: informs,
     isLoading,

projects/app/src/pages/account/components/PayModal.tsx

@@ -1,14 +1,14 @@
 import React, { useState, useCallback } from 'react';
 import { ModalFooter, ModalBody, Button, Input, Box, Grid } from '@chakra-ui/react';
-import { getPayCode, checkPayResult } from '@/web/common/api/bill';
+import { getPayCode, checkPayResult } from '@/web/common/bill/api';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useTranslation } from 'react-i18next';
 import Markdown from '@/components/Markdown';
 import MyModal from '@/components/MyModal';
-import { priceMd } from '@/web/common/store/static';
+import { priceMd } from '@/web/common/system/staticData';
 
 const PayModal = ({ onClose }: { onClose: () => void }) => {
   const router = useRouter();
@@ -69,12 +69,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
       title={t('user.Pay')}
       isCentered={!payId}
     >
-      <ModalBody
-        p={0}
-        h={payId ? 'auto' : ['auto', '70vh']}
-        display={'flex'}
-        flexDirection={'column'}
-      >
+      <ModalBody p={0} minH={payId ? 'auto' : '70vh'} display={'flex'} flexDirection={'column'}>
         {!payId && (
           <>
             <Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4} px={6}>

projects/app/src/pages/account/components/PayRecordTable.tsx

@@ -11,11 +11,11 @@ import {
   Flex,
   Box
 } from '@chakra-ui/react';
-import { getPayOrders, checkPayResult } from '@/web/common/api/bill';
+import { getPayOrders, checkPayResult } from '@/web/common/bill/api';
 import { PaySchema } from '@/types/mongoSchema';
 import dayjs from 'dayjs';
 import { useQuery } from '@tanstack/react-query';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import MyIcon from '@/components/Icon';
@@ -58,7 +58,7 @@ const PayRecordTable = () => {
   );
 
   return (
-    <Box position={'relative'} h={'100%'}>
+    <Box position={'relative'} h={'100%'} overflow={'overlay'}>
       {!isInitialLoading && payOrders.length === 0 ? (
         <Flex h={'100%'} flexDirection={'column'} alignItems={'center'} justifyContent={'center'}>
           <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
@@ -67,7 +67,7 @@ const PayRecordTable = () => {
           </Box>
         </Flex>
       ) : (
-        <TableContainer py={[0, 5]} px={[3, 8]} h={'100%'} overflow={'overlay'}>
+        <TableContainer py={[0, 5]} px={[3, 8]}>
           <Table>
             <Thead>
               <Tr>

projects/app/src/pages/account/components/Promotion.tsx

@@ -16,8 +16,8 @@ import {
 } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
 import { useQuery } from '@tanstack/react-query';
-import { getPromotionInitData, getPromotionRecords } from '@/web/support/api/user';
-import { useUserStore } from '@/web/support/store/user';
+import { getPromotionInitData, getPromotionRecords } from '@/web/support/user/api';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useLoading } from '@/web/common/hooks/useLoading';
 
 import MyTooltip from '@/components/MyTooltip';

projects/app/src/pages/account/components/UpdatePswModal.tsx

@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal';
 import { useTranslation } from 'react-i18next';
 import { useForm } from 'react-hook-form';
 import { useRequest } from '@/web/common/hooks/useRequest';
-import { updatePasswordByOld } from '@/web/support/api/user';
+import { updatePasswordByOld } from '@/web/support/user/api';
 
 type FormType = {
   oldPsw: string;

projects/app/src/pages/account/index.tsx

@@ -1,17 +1,17 @@
 import React, { useCallback, useRef } from 'react';
 import { Box, Flex, useTheme } from '@chakra-ui/react';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { useRouter } from 'next/router';
 import dynamic from 'next/dynamic';
-import { clearToken } from '@/utils/user';
-import { useUserStore } from '@/web/support/store/user';
+import { clearToken } from '@/web/support/user/auth';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import PageContainer from '@/components/PageContainer';
 import SideTabs from '@/components/SideTabs';
 import Tabs from '@/components/Tabs';
 import UserInfo from './components/Info';
 import { serviceSideProps } from '@/web/common/utils/i18n';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { useTranslation } from 'react-i18next';
 import Script from 'next/script';
 
@@ -86,7 +86,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
 
   const router = useRouter();
   const theme = useTheme();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const { setUserInfo } = useUserStore();
 
   const setCurrentTab = useCallback(

projects/app/src/pages/api/admin/initv43.ts

@@ -1,35 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { PgClient } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await authUser({ req, authRoot: true });
-
-    const { rowCount } = await PgClient.query(`SELECT 1
-    FROM   information_schema.columns 
-    WHERE  table_schema = 'public'
-    AND    table_name   = '${PgDatasetTableName}'
-    AND    column_name  = 'file_id'`);
-
-    if (rowCount > 0) {
-      return jsonRes(res, {
-        data: '已经存在file_id字段'
-      });
-    }
-
-    jsonRes(res, {
-      data: await PgClient.query(
-        `ALTER TABLE ${PgDatasetTableName} ADD COLUMN file_id VARCHAR(100)`
-      )
-    });
-  } catch (error) {
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}

projects/app/src/pages/api/admin/initv44.ts

@@ -1,10 +1,10 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv441.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
-import mongoose from '@fastgpt/common/mongo';
+import mongoose from '@fastgpt/service/common/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv442.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase, Bill } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/admin/initv445.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase, App } from '@/service/mongo';
 import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow';
 import { SystemInputEnum } from '@/constants/app';

projects/app/src/pages/api/admin/initv447.ts

@@ -1,11 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
-import { Types, connectionMongo } from '@fastgpt/common/mongo';
+import { DatasetSpecialIdEnum } from '@fastgpt/global/core/dataset/constant';
+import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
 import { delay } from '@/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/admin/initv451.ts

@@ -0,0 +1,344 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { App, connectToDatabase } from '@/service/mongo';
+import { PgClient } from '@/service/pg';
+import { connectionMongo } from '@fastgpt/service/common/mongo';
+import { PgDatasetTableName } from '@/constants/plugin';
+import { FlowModuleTypeEnum } from '@/constants/flow';
+import { delay } from '@/utils/tools';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
+import { GridFSStorage } from '@/service/lib/gridfs';
+import { Types } from 'mongoose';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { limit = 50 } = req.body as { limit: number };
+    await connectToDatabase();
+
+    console.log('rename');
+    await rename();
+
+    console.log('init mongo data');
+    await initMongo(limit);
+
+    console.log('create collection');
+    await createCollection();
+
+    console.log('update pg collectionId');
+    await updatePgCollection();
+    console.log('init done');
+
+    jsonRes(res, {
+      data: {}
+    });
+  } catch (error) {
+    console.log(error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+async function rename() {
+  // rename mongo kbs -> datasets
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'kbs' })
+      .toArray();
+    if (collections.length > 0) {
+      const kbCollection = connectionMongo.connection.db.collection('kbs');
+      await kbCollection.rename('datasets', { dropTarget: true });
+      console.log('success rename kbs -> datasets');
+    }
+  } catch (error) {
+    console.log('error： rename kbs -> datasets', error);
+  }
+
+  // rename pg: kb_id -> dataset_id
+  try {
+    const { rows } = await PgClient.query(`SELECT EXISTS (
+      SELECT 1
+      FROM information_schema.columns
+      WHERE table_name = '${PgDatasetTableName}'
+      AND column_name = 'kb_id'
+  );`);
+
+    if (rows[0].exists) {
+      await PgClient.query(`ALTER TABLE ${PgDatasetTableName} RENAME COLUMN kb_id TO dataset_id`);
+      console.log('success rename kb_id -> dataset_id');
+    }
+  } catch (error) {
+    console.log('error： rename kb_id -> dataset_id', error);
+  }
+  // rename pg: file_id -> collection_id
+  try {
+    const { rows } = await PgClient.query(`SELECT EXISTS (
+      SELECT 1
+      FROM information_schema.columns
+      WHERE table_name = '${PgDatasetTableName}'
+      AND column_name = 'file_id'
+  );`);
+
+    if (rows[0].exists) {
+      await PgClient.query(
+        `ALTER TABLE ${PgDatasetTableName} RENAME COLUMN file_id TO collection_id`
+      );
+      console.log('success rename file_id -> collection_id');
+    }
+  } catch (error) {
+    console.log('error： rename file_id -> collection_id', error);
+  }
+}
+
+async function initMongo(limit: number) {
+  let success = 0;
+
+  async function initApp(limit = 100): Promise<any> {
+    // 遍历所有 app，更新 app modules 里的 FlowModuleTypeEnum.kbSearchNode
+    const apps = await App.find({ inited: false }).limit(limit);
+
+    if (apps.length === 0) return;
+
+    try {
+      await Promise.all(
+        apps.map(async (app) => {
+          const modules = app.toObject().modules;
+          // @ts-ignore
+          app.inited = true;
+
+          modules.forEach((module) => {
+            // @ts-ignore
+            if (module.flowType === 'kbSearchNode') {
+              module.flowType = FlowModuleTypeEnum.datasetSearchNode;
+              module.inputs.forEach((input) => {
+                if (input.key === 'kbList') {
+                  input.key = 'datasets';
+                  input.value?.forEach((item: any) => {
+                    item.datasetId = item.kbId;
+                  });
+                }
+              });
+            }
+          });
+
+          app.modules = JSON.parse(JSON.stringify(modules));
+          await app.save();
+        })
+      );
+      success += limit;
+      console.log('mongo init:', success);
+      return initApp(limit);
+    } catch (error) {
+      return initApp(limit);
+    }
+  }
+
+  // init app
+  await App.updateMany(
+    {},
+    {
+      $set: {
+        inited: false
+      }
+    }
+  );
+
+  const totalApp = await App.countDocuments();
+  console.log(`total app: ${totalApp}`);
+  await delay(2000);
+  console.log('start init app');
+  await initApp(limit);
+  console.log('init mongo success');
+}
+
+type RowType = { user_id: string; dataset_id: string; collection_id: string };
+async function createCollection() {
+  let success = 0;
+
+  const { rows, rowCount } = await PgClient.query(`SELECT user_id,dataset_id,collection_id
+  FROM ${PgDatasetTableName} 
+  GROUP BY user_id,collection_id, dataset_id
+  ORDER BY dataset_id`);
+
+  if (rowCount === 0) {
+    console.log('pg done');
+    return;
+  }
+  // init dataset collection
+  console.log(`total collection: ${rowCount}`);
+
+  // collectionId 的类型：manual, mark, httpLink, fileId
+  async function initCollection(row: RowType): Promise<any> {
+    try {
+      {
+        const userId = row.user_id;
+        const datasetId = row.dataset_id;
+        const collectionId = row.collection_id;
+
+        const count = await MongoDatasetCollection.countDocuments({
+          datasetId,
+          userId,
+          ['metadata.pgCollectionId']: collectionId
+        });
+        if (count > 0) {
+          console.log('collection already exist');
+          return;
+        }
+
+        if (collectionId === 'manual') {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: '手动录入',
+            type: DatasetCollectionTypeEnum.virtual,
+            updateTime: new Date('2099'),
+            metadata: {
+              pgCollectionId: collectionId
+            }
+          });
+        } else if (collectionId === 'mark') {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: '手动标注',
+            type: DatasetCollectionTypeEnum.virtual,
+            updateTime: new Date('2099'),
+            metadata: {
+              pgCollectionId: collectionId
+            }
+          });
+        } else if (strIsLink(collectionId)) {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: collectionId,
+            type: DatasetCollectionTypeEnum.link,
+            metadata: {
+              rawLink: collectionId,
+              pgCollectionId: collectionId
+            }
+          });
+        } else {
+          // find file
+          const gridFs = new GridFSStorage('dataset', userId);
+          const collection = gridFs.Collection();
+          const file = await collection.findOne({
+            _id: new Types.ObjectId(collectionId)
+          });
+
+          if (file) {
+            await MongoDatasetCollection.create({
+              parentId: null,
+              datasetId,
+              userId,
+              name: file.filename,
+              type: DatasetCollectionTypeEnum.file,
+              metadata: {
+                fileId: file._id,
+                pgCollectionId: collectionId
+              }
+            });
+          } else {
+            // no file
+            await MongoDatasetCollection.create({
+              parentId: null,
+              datasetId,
+              userId,
+              name: '未知文件',
+              type: DatasetCollectionTypeEnum.virtual,
+              metadata: {
+                pgCollectionId: collectionId
+              }
+            });
+          }
+        }
+        console.log('create collection success');
+      }
+    } catch (error) {
+      console.log(error);
+
+      await delay(2000);
+      return initCollection(row);
+    }
+  }
+
+  for await (const row of rows) {
+    await initCollection(row);
+    console.log('init collection success: ', ++success);
+  }
+}
+
+async function updatePgCollection(): Promise<any> {
+  let success = 0;
+  const limit = 10;
+  const collections = await MongoDatasetCollection.find({
+    'metadata.pgCollectionId': { $exists: true, $ne: '' }
+  }).lean();
+  console.log('total:', collections.length);
+
+  async function update(i: number): Promise<any> {
+    const item = collections[i];
+    if (!item) return;
+
+    try {
+      console.log('start', item.name, item.datasetId, item.metadata.pgCollectionId);
+      const time = Date.now();
+      if (item.metadata.pgCollectionId) {
+        const { rows } = await PgClient.select(PgDatasetTableName, {
+          fields: ['id'],
+          where: [
+            ['dataset_id', String(item.datasetId)],
+            'AND',
+            ['collection_id', String(item.metadata.pgCollectionId)]
+          ],
+          limit: 999999
+        });
+        console.log('update date total', rows.length, 'time:', Date.now() - time);
+
+        await PgClient.query(`
+    update ${PgDatasetTableName} set collection_id = '${item._id}' where dataset_id = '${String(
+      item.datasetId
+    )}' AND collection_id = '${String(item.metadata.pgCollectionId)}'
+              `);
+
+        console.log('pg update time', Date.now() - time);
+      }
+
+      // 更新 file id
+      if (item.type === 'file' && item.metadata.fileId) {
+        const collection = connectionMongo.connection.db.collection(`dataset.files`);
+        await collection.findOneAndUpdate({ _id: new Types.ObjectId(item.metadata.fileId) }, [
+          {
+            $set: {
+              'metadata.datasetId': item.datasetId,
+              'metadata.collectionId': item._id
+            }
+          }
+        ]);
+      }
+
+      await MongoDatasetCollection.findByIdAndUpdate(item._id, {
+        $unset: { 'metadata.pgCollectionId': '' }
+      });
+      console.log('success', ++success);
+
+      return update(i + limit);
+    } catch (error) {
+      console.log(error);
+
+      await delay(5000);
+      return update(i);
+    }
+  }
+
+  const arr = new Array(limit).fill(0);
+
+  return Promise.all(arr.map((_, i) => update(i)));
+}

projects/app/src/pages/api/admin/sqlTest.ts

@@ -1,45 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { connectToDatabase } from '@/service/mongo';
-import { PgClient } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
-import { Types, connectionMongo } from '@fastgpt/common/mongo';
-import { delay } from '@/utils/tools';
-import { replaceVariable } from '@/utils/common/tools/text';
-import { getVector } from '../openapi/plugin/vector';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    // await connectToDatabase();
-    // const { text, analyze, sql } = req.body as {
-    //   userId: string;
-    //   text: string;
-    //   analyze?: boolean;
-    //   sql: string;
-    // };
-    // await authUser({ req, authRoot: true });
-
-    // const vectorModel = global.vectorModels[0];
-    // const { vectors } = await getVector({
-    //   model: vectorModel.model,
-    //   input: [text]
-    // });
-
-    // const start = Date.now();
-    // const result: any = await PgClient.query(sql.replace(/\[vector\]/g, `[${vectors[0]}]`));
-
-    jsonRes(res, {
-      data: {
-        // rows: result?.[2]?.rows,
-        // time: Date.now() - start
-      }
-    });
-  } catch (error) {
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}

projects/app/src/pages/api/app/create.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { App } from '@/service/models/app';
 import type { CreateAppParams } from '@/types/app';
 import { AppTypeEnum } from '@/constants/app';

projects/app/src/pages/api/app/data/totalUsage.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from '@fastgpt/common/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { Types } from '@fastgpt/service/common/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/app/del.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, App, connectToDatabase, Collection } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */

projects/app/src/pages/api/app/detail.tsx

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */

projects/app/src/pages/api/app/getChatLogs.ts

@@ -1,10 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { PagingData } from '@/types';
 import { AppLogsListItemType } from '@/types/app';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
 import { addDays } from 'date-fns';
 import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
 

projects/app/src/pages/api/app/myApps.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { AppListItemType } from '@/types/app';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/share/collection.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Collection, App } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 模型收藏切换 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/share/getModels.ts

@@ -3,8 +3,8 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
 import type { PagingData } from '@/types';
 import type { ShareAppItem } from '@/types/app';
-import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from '@fastgpt/common/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { Types } from '@fastgpt/service/common/mongo';
 
 /* 获取模型列表 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/update.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { App } from '@/service/models/app';
 import type { AppUpdateParams } from '@/types/app';
 import { authApp } from '@/service/utils/auth';

projects/app/src/pages/api/chat/chatTest.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { sseErrRes } from '@/service/response';
 import { sseResponseEventEnum } from '@/constants/chat';
-import { responseWrite } from '@fastgpt/common/tools/stream';
+import { responseWrite } from '@fastgpt/service/common/response';
 import { AppModuleItemType } from '@/types/app';
 import { dispatchModules } from '@/pages/api/v1/chat/completions';
 import { pushChatBill } from '@/service/common/bill/push';

projects/app/src/pages/api/chat/delChatRecordByContentId.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/chat/feedback/adminUpdate.ts

@@ -2,15 +2,15 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
 import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { chatItemId, kbId, dataId, content = undefined } = req.body as AdminUpdateFeedbackParams;
+    const { chatItemId, datasetId, dataId, q, a } = req.body as AdminUpdateFeedbackParams;
 
-    if (!chatItemId || !kbId || !dataId || !content) {
+    if (!chatItemId || !datasetId || !dataId || !q) {
       throw new Error('missing parameter');
     }
 
@@ -23,9 +23,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       },
       {
         adminFeedback: {
-          kbId,
+          datasetId,
           dataId,
-          content
+          q,
+          a
         }
       }
     );

projects/app/src/pages/api/chat/history/getHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { ChatHistoryItemType } from '@/types/chat';
 import { ChatSourceEnum } from '@/constants/chat';
 

projects/app/src/pages/api/chat/history/updateChatHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export type Props = {
   chatId: string;

projects/app/src/pages/api/chat/init.ts

@@ -2,11 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
 import type { InitChatResponse } from '@/global/core/api/chatRes.d';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { ChatItemType } from '@/types/chat';
 import { authApp } from '@/service/utils/auth';
 import type { ChatSchema } from '@/types/mongoSchema';
-import { getGuideModule } from '@/components/ChatBox/utils';
+import { getGuideModule } from '@/global/core/app/modules/utils';
 import { getChatModelNameListByModules } from '@/service/core/app/module';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 

projects/app/src/pages/api/chat/removeHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { ChatSourceEnum } from '@/constants/chat';
 
 type Props = {

projects/app/src/pages/api/common/bill/createTrainingBill.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { BillSourceEnum } from '@/constants/user';
-import { CreateTrainingBillType } from '@/global/common/api/billReq.d';
+import { CreateTrainingBillType } from '@fastgpt/global/common/bill/types/billReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -1,10 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
 import { pushQuestionGuideBill } from '@/service/common/bill/push';
-import { createQuestionGuide } from '@fastgpt/core/ai/functions/createQuestionGuide';
+import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

projects/app/src/pages/api/core/dataset/allDataset.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { getVectorModel } from '@/service/core/ai/model';
 import type { DatasetsItemType } from '@/types/core/dataset';
 
@@ -12,12 +12,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const kbList = await MongoDataset.find({
+    const datasets = await MongoDataset.find({
       userId,
       type: 'dataset'
     });
 
-    const data = kbList.map((item) => ({
+    const data = datasets.map((item) => ({
       ...item.toJSON(),
       vectorModel: getVectorModel(item.vectorModel)
     }));

projects/app/src/pages/api/core/dataset/collection/create.ts

@@ -0,0 +1,87 @@
+/* 
+    Create one dataset collection
+*/
+
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { userId } = await authUser({ req, authToken: true });
+
+    const body = req.body || {};
+
+    jsonRes(res, {
+      data: await createOneCollection({
+        ...body,
+        userId
+      })
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
+
+export async function createOneCollection({
+  name,
+  parentId,
+  datasetId,
+  type,
+  metadata = {},
+  userId
+}: CreateDatasetCollectionParams & { userId: string }) {
+  const { _id } = await MongoDatasetCollection.create({
+    name,
+    userId,
+    datasetId,
+    parentId: parentId || null,
+    type,
+    metadata,
+    updateTime: getCollectionUpdateTime({ name })
+  });
+
+  // create default collection
+  if (type === DatasetCollectionTypeEnum.folder) {
+    await createDefaultCollection({
+      datasetId,
+      parentId: _id,
+      userId
+    });
+  }
+
+  return _id;
+}
+
+// create default collection
+export function createDefaultCollection({
+  name = '手动录入',
+  datasetId,
+  parentId,
+  userId
+}: {
+  name?: '手动录入' | '手动标注';
+  datasetId: string;
+  parentId?: string;
+  userId: string;
+}) {
+  return MongoDatasetCollection.create({
+    name,
+    userId,
+    datasetId,
+    parentId,
+    type: DatasetCollectionTypeEnum.virtual,
+    updateTime: new Date('2000'),
+    metadata: {}
+  });
+}

projects/app/src/pages/api/core/dataset/collection/delById.ts

@@ -0,0 +1,61 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
+import { delDataByCollectionId } from '@/service/core/dataset/data/utils';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { GridFSStorage } from '@/service/lib/gridfs';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { collectionId } = req.query as { collectionId: string };
+
+    if (!collectionId) {
+      throw new Error('CollectionIdId is required');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    // find all delete id
+    const collections = await findCollectionAndChild(collectionId, '_id metadata');
+    const delIdList = collections.map((item) => item._id);
+
+    // delete pg data
+    await delDataByCollectionId({ userId, collectionIds: delIdList });
+
+    // delete training data
+    await MongoDatasetTraining.deleteMany({
+      datasetCollectionId: { $in: delIdList },
+      userId
+    });
+
+    // delete file
+    const gridFs = new GridFSStorage('dataset', userId);
+    const fileCollection = gridFs.Collection();
+    await Promise.all(
+      collections.map(
+        (item) =>
+          //@ts-ignore
+          item.metadata?.fileId && fileCollection.findOneAndDelete({ _id: item.metadata.fileId })
+      )
+    );
+
+    // delete collection
+    await MongoDatasetCollection.deleteMany({
+      _id: { $in: delIdList },
+      userId
+    });
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -0,0 +1,37 @@
+/* 
+    Get one dataset collection detail
+*/
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { id } = req.query as { id: string };
+
+    if (!id) {
+      throw new Error('Id is required');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const collection = await MongoDatasetCollection.findOne({ _id: id, userId }).lean();
+
+    if (!collection) {
+      throw new Error('Collection not found');
+    }
+
+    jsonRes(res, {
+      data: collection
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -0,0 +1,134 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+
+import { Types } from '@fastgpt/service/common/mongo';
+import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
+import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
+import { PagingData } from '@/types';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { countCollectionData } from '@/service/core/dataset/data/utils';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { startQueue } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    let {
+      pageNum = 1,
+      pageSize = 10,
+      datasetId,
+      parentId = null,
+      searchText = '',
+      selectFolder = false,
+      simple = false
+    } = req.body as GetDatasetCollectionsProps;
+    searchText = searchText?.replace(/'/g, '');
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const match = {
+      userId: new Types.ObjectId(userId),
+      datasetId: new Types.ObjectId(datasetId),
+      parentId: parentId ? new Types.ObjectId(parentId) : null,
+      ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
+      ...(searchText
+        ? {
+            name: new RegExp(searchText, 'i')
+          }
+        : {})
+    };
+
+    if (simple) {
+      const collections = await MongoDatasetCollection.find(match, '_id name type parentId')
+        .sort({
+          updateTime: -1
+        })
+        .lean();
+      return jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
+        data: {
+          pageNum,
+          pageSize,
+          data: await Promise.all(
+            collections.map(async (item) => ({
+              ...item,
+              dataAmount: 0,
+              trainingAmount: 0
+            }))
+          ),
+          total: await MongoDatasetCollection.countDocuments(match)
+        }
+      });
+    }
+
+    const collections = await MongoDatasetCollection.aggregate([
+      {
+        $match: match
+      },
+      {
+        $lookup: {
+          from: DatasetTrainingCollectionName,
+          localField: '_id',
+          foreignField: 'datasetCollectionId',
+          as: 'trainings_amount'
+        }
+      },
+      // 统计子集合的数量和子训练的数量
+      {
+        $project: {
+          _id: 1,
+          parentId: 1,
+          fileId: 1,
+          name: 1,
+          type: 1,
+          updateTime: 1,
+          trainingAmount: { $size: '$trainings_amount' }
+        }
+      },
+      {
+        $sort: { updateTime: -1 }
+      },
+      {
+        $skip: (pageNum - 1) * pageSize
+      },
+      {
+        $limit: pageSize
+      }
+    ]);
+
+    const counts = await countCollectionData({
+      collectionIds: collections.map((item) => item._id),
+      datasetId
+    });
+
+    const data = await Promise.all(
+      collections.map(async (item, i) => ({
+        ...item,
+        dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i]
+      }))
+    );
+
+    if (data.find((item) => item.trainingAmount > 0)) {
+      startQueue(1);
+    }
+
+    // count collections
+    jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data,
+        total: await MongoDatasetCollection.countDocuments(match)
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/paths.ts

@@ -0,0 +1,29 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import type { DatasetPathItemType } from '@/types/core/dataset';
+import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils';
+import { authUser } from '@fastgpt/service/support/user/auth';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { parentId } = req.query as { parentId: string };
+
+    const { userId } = await authUser({ req, authToken: true });
+    const paths = await getDatasetCollectionPaths({
+      parentId,
+      userId
+    });
+
+    jsonRes<DatasetPathItemType[]>(res, {
+      data: paths
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/update.ts

@@ -0,0 +1,48 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { UpdateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { id, parentId, name, metadata = {} } = req.body as UpdateDatasetCollectionParams;
+
+    if (!id) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const updateFields: Record<string, any> = {
+      ...(parentId !== undefined && { parentId: parentId || null }),
+      ...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
+    };
+
+    // 将metadata的每个字段添加到updateFields中
+    for (const [key, value] of Object.entries(metadata)) {
+      updateFields[`metadata.${key}`] = value;
+    }
+
+    await MongoDatasetCollection.findOneAndUpdate(
+      {
+        _id: id,
+        userId
+      },
+      {
+        $set: updateFields
+      }
+    );
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/create.ts

@@ -1,9 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
+import { createDefaultCollection } from './collection/create';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -23,6 +24,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       type
     });
 
+    await createDefaultCollection({
+      datasetId: _id,
+      userId
+    });
+
     jsonRes(res, { data: _id });
   } catch (err) {
     jsonRes(res, {

projects/app/src/pages/api/core/dataset/data/delDataById.ts

@@ -1,20 +1,20 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { connectToDatabase } from '@/service/mongo';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    let { dataId } = req.query as {
+    const { dataId } = req.query as {
       dataId: string;
     };
 
     if (!dataId) {
-      throw new Error('缺少参数');
+      throw new Error('dataId is required');
     }
 
     // 凭证校验

projects/app/src/pages/api/core/dataset/data/exportAll.ts

@@ -1,30 +1,30 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoUser } from '@fastgpt/support/user/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { findAllChildrenIds } from '../delete';
 import QueryStream from 'pg-query-stream';
 import { PgClient } from '@/service/pg';
 import { addLog } from '@/service/utils/tools';
-import { responseWriteController } from '@fastgpt/common/tools/stream';
+import { responseWriteController } from '@fastgpt/service/common/response';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    let { kbId } = req.query as {
-      kbId: string;
+    let { datasetId } = req.query as {
+      datasetId: string;
     };
 
-    if (!kbId || !global.pgClient) {
+    if (!datasetId || !global.pgClient) {
       throw new Error('缺少参数');
     }
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const exportIds = [kbId, ...(await findAllChildrenIds(kbId))];
+    const exportIds = [datasetId, ...(await findAllChildrenIds(datasetId))];
 
     const limitMinutesAgo = new Date(
       Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
@@ -48,7 +48,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     }
 
     const { rows } = await PgClient.query(
-      `SELECT count(id) FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
+      `SELECT count(id) FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
         .map((id) => `'${id}'`)
         .join(',')})`
     );
@@ -67,10 +67,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         res.end('Error connecting to database');
         return;
       }
+      if (!client) return;
 
       // create pg select stream
       const query = new QueryStream(
-        `SELECT q, a, source FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
+        `SELECT q, a FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
           .map((id) => `'${id}'`)
           .join(',')})`
       );
@@ -84,18 +85,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         readStream: stream
       });
 
-      write('index,content,source');
+      write('index,content');
 
       // parse data every row
-      stream.on('data', ({ q, a, source }: { q: string; a: string; source?: string }) => {
+      stream.on('data', ({ q, a }: { q: string; a: string }) => {
         if (res.closed) {
           return stream.destroy();
         }
         q = q.replace(/"/g, '""');
         a = a.replace(/"/g, '""');
-        source = source?.replace(/"/g, '""');
+        // source = source?.replace(/"/g, '""');
 
-        write(`\n"${q}","${a || ''}","${source || ''}"`);
+        write(`\n"${q}","${a || ''}"`);
       });
       // finish
       stream.on('end', async () => {

projects/app/src/pages/api/core/dataset/data/getDataById.ts

@@ -1,10 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import type { PgDataItemType } from '@/types/core/dataset/data';
+import type { DatasetDataItemType, PgDataItemType } from '@fastgpt/global/core/dataset/type';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 
 export type Response = {
   id: string;
@@ -26,16 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const where: any = [['user_id', userId], 'AND', ['id', dataId]];
-
-    const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
-      fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'],
-      where,
-      limit: 1
-    });
-
     jsonRes(res, {
-      data: searchRes.rows[0]
+      data: await getDatasetDataById({ userId, id: dataId })
     });
   } catch (err) {
     jsonRes(res, {
@@ -44,3 +37,70 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
   }
 }
+
+export async function getDatasetDataById({
+  id,
+  userId
+}: {
+  id: string;
+  userId: string;
+}): Promise<DatasetDataItemType> {
+  const where: any = [['user_id', userId], 'AND', ['id', id]];
+
+  const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
+    fields: ['id', 'q', 'a', 'dataset_id', 'collection_id'],
+    where,
+    limit: 1
+  });
+
+  const data = searchRes?.rows?.[0];
+
+  if (!data) {
+    return Promise.reject('Data not found');
+  }
+
+  // find source
+  const collection = (await getDatasetDataItemInfo({ pgDataList: [data] }))[0];
+
+  if (!collection) {
+    return Promise.reject('Data Collection not found');
+  }
+
+  return {
+    id: data.id,
+    q: data.q,
+    a: data.a,
+    datasetId: data.dataset_id,
+    collectionId: data.collection_id,
+    sourceName: collection.sourceName,
+    sourceId: collection.sourceId
+  };
+}
+
+export async function getDatasetDataItemInfo({
+  pgDataList
+}: {
+  pgDataList: PgDataItemType[];
+}): Promise<DatasetDataItemType[]> {
+  const collections = await MongoDatasetCollection.find(
+    {
+      _id: { $in: pgDataList.map((item) => item.collection_id) }
+    },
+    '_id name datasetId metadata'
+  ).lean();
+
+  return pgDataList.map((item) => {
+    const collection = collections.find(
+      (collection) => String(collection._id) === item.collection_id
+    );
+    return {
+      id: item.id,
+      q: item.q,
+      a: item.a,
+      datasetId: collection?.datasetId || '',
+      collectionId: item.collection_id,
+      sourceName: collection?.name || '',
+      sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
+    };
+  });
+}

projects/app/src/pages/api/core/dataset/data/getDataList.ts

@@ -1,29 +1,23 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import type { PgDataItemType } from '@/types/core/dataset/data';
+import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
+import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
     let {
-      kbId,
       pageNum = 1,
       pageSize = 10,
       searchText = '',
-      fileId = ''
-    } = req.body as {
-      kbId: string;
-      pageNum: number;
-      pageSize: number;
-      searchText: string;
-      fileId: string;
-    };
-    if (!kbId) {
-      throw new Error('缺少参数');
+      collectionId
+    } = req.body as GetDatasetDataListProps;
+    if (!collectionId) {
+      throw new Error('collectionId is required');
     }
 
     // 凭证校验
@@ -34,20 +28,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const where: any = [
       ['user_id', userId],
       'AND',
-      ['kb_id', kbId],
-      'AND',
-      ['file_id', fileId],
-      ...(searchText
-        ? [
-            'AND',
-            `(q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%' OR source ILIKE '%${searchText}%')`
-          ]
-        : [])
+      ['collection_id', collectionId],
+      searchText ? `AND (q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%')` : ''
     ];
 
     const [searchRes, total] = await Promise.all([
-      PgClient.select<PgDataItemType>(PgDatasetTableName, {
-        fields: ['id', 'q', 'a', 'source', 'file_id'],
+      PgClient.select<DatasetDataListItemType>(PgDatasetTableName, {
+        fields: ['id', 'q', 'a'],
         where,
         order: [{ field: 'id', mode: 'DESC' }],
         limit: pageSize,

projects/app/src/pages/api/core/dataset/data/getQueueLen.ts

@@ -1,7 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { TrainingData, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -10,7 +11,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     await authUser({ req, authToken: true });
 
     // split queue data
-    const result = await TrainingData.countDocuments({
+    const result = await MongoDatasetTraining.countDocuments({
       lockTime: { $lt: new Date('2040/1/1') }
     });
 

projects/app/src/pages/api/core/dataset/data/getTrainingData.ts

@@ -1,52 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { TrainingModeEnum } from '@/constants/plugin';
-import { Types } from '@fastgpt/common/mongo';
-import { startQueue } from '@/service/utils/tools';
-
-/* 拆分数据成QA */
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await connectToDatabase();
-    const { kbId, init = false } = req.body as { kbId: string; init: boolean };
-    if (!kbId) {
-      throw new Error('参数错误');
-    }
-
-    const { userId } = await authUser({ req, authToken: true });
-
-    // split queue data
-    const result = await TrainingData.aggregate([
-      {
-        $match: {
-          userId: new Types.ObjectId(userId),
-          kbId: new Types.ObjectId(kbId)
-        }
-      },
-      {
-        $group: {
-          _id: '$mode',
-          count: { $sum: 1 }
-        }
-      }
-    ]);
-
-    jsonRes(res, {
-      data: {
-        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
-        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
-      }
-    });
-
-    if (init) {
-      startQueue();
-    }
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

projects/app/src/pages/api/core/dataset/data/insertData.ts

@@ -5,21 +5,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authDataset } from '@/service/utils/auth';
-import { authUser } from '@fastgpt/support/user/auth';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { insertData2Dataset, PgClient } from '@/service/pg';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { countPromptTokens } from '@/global/common/tiktoken';
 import { getVectorModel } from '@/service/core/ai/model';
-import { getVector } from '@/pages/api/openapi/plugin/vector';
-import { DatasetDataItemType } from '@/types/core/dataset/data';
-import { countPromptTokens } from '@/utils/common/tiktoken';
-import { authFileIdValid } from '@/service/dataset/auth';
-
-export type Props = {
-  kbId: string;
-  data: DatasetDataItemType;
-};
+import { insertData2Dataset, hasSameValue } from '@/service/core/dataset/data/utils';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -28,7 +22,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    jsonRes(res, {
+    jsonRes<string>(res, {
       data: await getVectorAndInsertDataset({
         ...req.body,
         userId
@@ -43,58 +37,59 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 });
 
 export async function getVectorAndInsertDataset(
-  props: Props & { userId: string }
+  props: SetOneDatasetDataProps & { userId: string }
 ): Promise<string> {
-  const { kbId, data, userId } = props;
-  if (!kbId || !data?.q) {
-    return Promise.reject('缺少参数');
+  let { datasetId, collectionId, q, a, userId } = props;
+
+  if (!datasetId) {
+    return Promise.reject('知识库 ID 不能为空');
   }
 
-  // auth kb
-  const kb = await authDataset({ kbId, userId });
+  if (!q) {
+    return Promise.reject('索引内容不能为空');
+  }
 
-  const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-  const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+  if (!collectionId) {
+    return Promise.reject('集合 ID 和集合类型不能同时为空');
+  }
+
+  // auth collection and get dataset
+  const collection = await MongoDatasetCollection.findOne({
+    _id: collectionId,
+    userId,
+    datasetId,
+    type: { $ne: DatasetCollectionTypeEnum.folder }
+  }).populate('datasetId', '_id vectorModel');
+
+  if (!collection) {
+    return Promise.reject('集合不存在');
+  }
+  const dataset = collection.datasetId as unknown as DatasetSchemaType;
+
+  // format data
+  const formatQ = q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+  const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
 
   // token check
-  const token = countPromptTokens(q, 'system');
+  const token = countPromptTokens(formatQ, 'system');
 
-  if (token > getVectorModel(kb.vectorModel).maxToken) {
-    return Promise.reject('Over Tokens');
+  if (token > getVectorModel(dataset.vectorModel).maxToken) {
+    return Promise.reject('Q Over Tokens');
   }
 
-  const { rows: existsRows } = await PgClient.query(`
-  SELECT COUNT(*) > 0 AS exists
-  FROM  ${PgDatasetTableName} 
-  WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND file_id='${data.file_id}' AND kb_id='${kbId}'
-`);
-  const exists = existsRows[0]?.exists || false;
-
-  if (exists) {
-    return Promise.reject('已经存在完全一致的数据');
-  }
-
-  await authFileIdValid(data.file_id);
-
-  const { vectors } = await getVector({
-    model: kb.vectorModel,
-    input: [q],
-    userId
+  // Duplicate data check
+  await hasSameValue({
+    collectionId,
+    q,
+    a
   });
 
-  const response = await insertData2Dataset({
+  return insertData2Dataset({
     userId,
-    kbId,
-    data: [
-      {
-        ...data,
-        q,
-        a,
-        vector: vectors[0]
-      }
-    ]
+    q: formatQ,
+    a: formatA,
+    collectionId,
+    datasetId,
+    model: dataset.vectorModel
   });
-
-  // @ts-ignore
-  return response?.rows?.[0]?.id || '';
 }

projects/app/src/pages/api/core/dataset/data/pushData.ts

@@ -1,18 +1,17 @@
 /* push data to training queue */
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
-import { authDataset } from '@/service/utils/auth';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
-import { TrainingModeEnum } from '@/constants/plugin';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { authCollection } from '@fastgpt/service/core/dataset/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 import { startQueue } from '@/service/utils/tools';
-import { DatasetDataItemType } from '@/types/core/dataset/data';
-import { countPromptTokens } from '@/utils/common/tiktoken';
+import { DatasetChunkItemType } from '@fastgpt/global/core/dataset/type';
+import { countPromptTokens } from '@/global/common/tiktoken';
 import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
 import type { PushDataProps } from '@/global/core/api/datasetReq.d';
-import { authFileIdValid } from '@/service/dataset/auth';
 import { getVectorModel } from '@/service/core/ai/model';
 
 const modeMap = {
@@ -23,25 +22,25 @@ const modeMap = {
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
+    const { collectionId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
 
-    if (!kbId || !Array.isArray(data)) {
-      throw new Error('KbId or data is empty');
+    if (!collectionId || !Array.isArray(data)) {
+      throw new Error('collectionId or data is empty');
     }
 
     if (modeMap[mode] === undefined) {
-      throw new Error('Mode is error');
+      throw new Error('Mode is not index or qa');
     }
 
-    if (data.length > 500) {
-      throw new Error('Data is too long, max 500');
+    if (data.length > 200) {
+      throw new Error('Data is too long, max 200');
     }
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true, authApiKey: true });
 
     jsonRes<PushDataResponse>(res, {
-      data: await pushDataToKb({
+      data: await pushDataToDatasetCollection({
         ...req.body,
         userId
       })
@@ -54,40 +53,40 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
   }
 });
 
-export async function pushDataToKb({
+export async function pushDataToDatasetCollection({
   userId,
-  kbId,
+  collectionId,
   data,
   mode,
   prompt,
   billId
 }: { userId: string } & PushDataProps): Promise<PushDataResponse> {
-  const [kb, vectorModel] = await Promise.all([
-    authDataset({
-      userId,
-      kbId
-    }),
-    (async () => {
-      if (mode === TrainingModeEnum.index) {
-        const vectorModel = (await MongoDataset.findById(kbId, 'vectorModel'))?.vectorModel;
+  // auth dataset & get training model
+  const {
+    dataset: { _id: datasetId, vectorModel }
+  } = await authCollection({
+    userId,
+    collectionId
+  });
+  const vectorModelData = getVectorModel(vectorModel);
 
-        return getVectorModel(vectorModel);
-      }
-      return global.vectorModels[0];
-    })()
-  ]);
-
-  const modeMaxToken = {
-    [TrainingModeEnum.index]: vectorModel.maxToken * 1.5,
-    [TrainingModeEnum.qa]: global.qaModels[0].maxToken * 0.8
+  const modeMap = {
+    [TrainingModeEnum.index]: {
+      maxToken: vectorModelData.maxToken * 1.5,
+      model: vectorModelData.model
+    },
+    [TrainingModeEnum.qa]: {
+      maxToken: global.qaModels[0].maxToken * 0.8,
+      model: global.qaModels[0].model
+    }
   };
 
   // filter repeat or equal content
   const set = new Set();
-  const filterResult: Record<string, DatasetDataItemType[]> = {
+  const filterResult: Record<string, DatasetChunkItemType[]> = {
     success: [],
     overToken: [],
-    fileIdInvalid: [],
+    repeat: [],
     error: []
   };
 
@@ -101,21 +100,16 @@ export async function pushDataToKb({
       const text = item.q + item.a;
 
       // count q token
-      const token = countPromptTokens(item.q, 'system');
+      const token = countPromptTokens(item.q);
 
-      if (token > modeMaxToken[mode]) {
+      if (token > modeMap[mode].maxToken) {
         filterResult.overToken.push(item);
         return;
       }
 
-      try {
-        await authFileIdValid(item.file_id);
-      } catch (error) {
-        filterResult.fileIdInvalid.push(item);
-        return;
-      }
-
-      if (!set.has(text)) {
+      if (set.has(text)) {
+        filterResult.repeat.push(item);
+      } else {
         filterResult.success.push(item);
         set.add(text);
       }
@@ -123,15 +117,17 @@ export async function pushDataToKb({
   );
 
   // 插入记录
-  const insertRes = await TrainingData.insertMany(
+  const insertRes = await MongoDatasetTraining.insertMany(
     filterResult.success.map((item) => ({
-      ...item,
       userId,
-      kbId,
+      datasetId,
+      datasetCollectionId: collectionId,
+      billId,
       mode,
       prompt,
-      billId,
-      vectorModel: vectorModel.model
+      model: modeMap[mode].model,
+      q: item.q,
+      a: item.a
     }))
   );
 

projects/app/src/pages/api/core/dataset/data/updateData.ts

@@ -1,57 +1,37 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { PgClient } from '@/service/pg';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { getVector } from '@/pages/api/openapi/plugin/vector';
-import { PgDatasetTableName } from '@/constants/plugin';
-import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq.d';
+import { updateData2Dataset } from '@/service/core/dataset/data/utils';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams;
+    const { id, datasetId, collectionId, q = '', a } = req.body as SetOneDatasetDataProps;
 
-    if (!dataId) {
+    if (!id || !collectionId) {
       throw new Error('缺少参数');
     }
 
     // auth user and get kb
-    const [{ userId }, kb] = await Promise.all([
+    const [{ userId }, dataset] = await Promise.all([
       authUser({ req, authToken: true }),
-      MongoDataset.findById(kbId, 'vectorModel')
+      MongoDataset.findById(datasetId, 'vectorModel')
     ]);
 
-    if (!kb) {
+    if (!dataset) {
       throw new Error("Can't find database");
     }
 
-    // get vector
-    const { vectors = [] } = await (async () => {
-      if (q) {
-        return getVector({
-          userId,
-          input: [q],
-          model: kb.vectorModel
-        });
-      }
-      return { vectors: [[]] };
-    })();
-
-    // 更新 pg 内容.仅修改a，不需要更新向量。
-    await PgClient.update(PgDatasetTableName, {
-      where: [['id', dataId], 'AND', ['user_id', userId]],
-      values: [
-        { key: 'a', value: a.replace(/'/g, '"') },
-        ...(q
-          ? [
-              { key: 'q', value: q.replace(/'/g, '"') },
-              { key: 'vector', value: `[${vectors[0]}]` }
-            ]
-          : [])
-      ]
+    await updateData2Dataset({
+      dataId: id,
+      userId,
+      q,
+      a,
+      model: dataset.vectorModel
     });
 
     jsonRes(res);

projects/app/src/pages/api/core/dataset/delete.ts

@@ -1,12 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -25,9 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const deletedIds = [id, ...(await findAllChildrenIds(id))];
 
     // delete training data
-    await TrainingData.deleteMany({
+    await MongoDatasetTraining.deleteMany({
       userId,
-      kbId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) }
+      datasetId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) }
     });
 
     // delete all pg data
@@ -35,15 +37,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       where: [
         ['user_id', userId],
         'AND',
-        `kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
+        `dataset_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
       ]
     });
 
     // delete related files
     const gridFs = new GridFSStorage('dataset', userId);
-    await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
+    await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByDatasetId(id)));
 
-    // delete kb data
+    // delete collections
+    await MongoDatasetCollection.deleteMany({
+      datasetId: { $in: deletedIds }
+    });
+
+    // delete dataset data
     await MongoDataset.deleteMany({
       _id: { $in: deletedIds },
       userId

projects/app/src/pages/api/core/dataset/detail.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { getVectorModel } from '@/service/core/ai/model';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

projects/app/src/pages/api/core/dataset/file/delById.ts

@@ -1,57 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { GridFSStorage } from '@/service/lib/gridfs';
-import { PgClient } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { Types } from '@fastgpt/common/mongo';
-import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
-  try {
-    await connectToDatabase();
-
-    const { fileId, kbId } = req.query as { fileId: string; kbId: string };
-
-    if (!fileId || !kbId) {
-      throw new Error('fileId and kbId is required');
-    }
-
-    // 凭证校验
-    const { userId } = await authUser({ req, authToken: true });
-
-    // other data. Delete only vector data
-    if (isSpecialFileId(fileId)) {
-      await PgClient.delete(PgDatasetTableName, {
-        where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
-      });
-    } else {
-      // auth file
-      const gridFs = new GridFSStorage('dataset', userId);
-      const bucket = gridFs.GridFSBucket();
-
-      await gridFs.findAndAuthFile(fileId);
-
-      // delete all pg data
-      await PgClient.delete(PgDatasetTableName, {
-        where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
-      });
-      // delete all training data
-      await TrainingData.deleteMany({
-        userId,
-        file_id: fileId
-      });
-
-      // delete file
-      await bucket.delete(new Types.ObjectId(fileId));
-    }
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

projects/app/src/pages/api/core/dataset/file/delEmptyFiles.ts

@@ -1,14 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
 
-    const { kbId } = req.query as { kbId: string };
+    const { datasetId } = req.query as { datasetId: string };
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
@@ -17,7 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const files = await collection.deleteMany({
       uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
-      ['metadata.kbId']: kbId,
+      ['metadata.datasetId']: datasetId,
       ['metadata.userId']: userId,
       ['metadata.datasetUsed']: { $ne: true }
     });

projects/app/src/pages/api/core/dataset/file/detail.ts

@@ -1,12 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { datasetSpecialIdMap } from '@fastgpt/core/dataset/constant';
-import { datasetSpecialIds } from '@fastgpt/core/dataset/constant';
+import { datasetSpecialIdMap } from '@fastgpt/global/core/dataset/constant';
+import { datasetSpecialIds } from '@fastgpt/global/core/dataset/constant';
 import type { GSFileInfoType } from '@/types/common/file';
-import { strIsLink } from '@fastgpt/common/tools/str';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 
@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
   try {
     await connectToDatabase();
 
-    const { fileId } = req.query as { kbId: string; fileId: string };
+    const { fileId } = req.query as { fileId: string };
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 

projects/app/src/pages/api/core/dataset/file/list.ts

@@ -1,175 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { GridFSStorage } from '@/service/lib/gridfs';
-import { PgClient, updateDataFileId } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { strIsLink } from '@fastgpt/common/tools/str';
-import {
-  FileStatusEnum,
-  DatasetSpecialIdEnum,
-  datasetSpecialIdMap,
-  datasetSpecialIds
-} from '@fastgpt/core/dataset/constant';
-import { Types } from '@fastgpt/common/mongo';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
-  try {
-    await connectToDatabase();
-
-    let {
-      pageNum = 1,
-      pageSize = 10,
-      kbId,
-      searchText = ''
-    } = req.body as { pageNum: number; pageSize: number; kbId: string; searchText: string };
-    searchText = searchText?.replace(/'/g, '');
-
-    // 凭证校验
-    const { userId } = await authUser({ req, authToken: true });
-
-    // select and count same file_id data, exclude special id
-    const pgWhere = `user_id = '${userId}' AND kb_id = '${kbId}' ${datasetSpecialIds
-      .map((item) => `AND file_id!='${item}'`)
-      .join(' ')}
-      ${searchText ? `AND source ILIKE '%${searchText}%'` : ''}`;
-
-    let [{ rows }, { rowCount: total }] = await Promise.all([
-      PgClient.query<{ file_id: string; count: number }>(`SELECT file_id, COUNT(*) AS count
-    FROM ${PgDatasetTableName}
-    where ${pgWhere}
-    GROUP BY file_id
-    ORDER BY file_id DESC
-    LIMIT ${pageSize} OFFSET ${(pageNum - 1) * pageSize};
-    `),
-      PgClient.query(`SELECT DISTINCT file_id
-    FROM ${PgDatasetTableName}
-    where ${pgWhere}
-    `)
-    ]);
-
-    // If fileId is invalid, reset it to manual
-    await Promise.all(
-      rows.map((row) => {
-        if (!strIsLink(row.file_id) && row.file_id.length !== 24) {
-          return updateDataFileId({
-            oldFileId: row.file_id,
-            userId,
-            newFileId: DatasetSpecialIdEnum.manual
-          });
-        }
-      })
-    );
-    // just filter link or fileData
-    rows = rows.filter((row) => strIsLink(row.file_id) || row.file_id.length === 24);
-
-    // find files
-    const gridFs = new GridFSStorage('dataset', userId);
-    const collection = gridFs.Collection();
-
-    async function getSpecialData() {
-      if (pageNum !== 1) return [];
-      return [
-        {
-          id: DatasetSpecialIdEnum.manual,
-          size: 0,
-          filename: datasetSpecialIdMap[DatasetSpecialIdEnum.manual].name,
-          uploadTime: new Date(),
-          status: FileStatusEnum.ready,
-          chunkLength: await PgClient.count(PgDatasetTableName, {
-            fields: ['id'],
-            where: [
-              ['user_id', userId],
-              'AND',
-              ['file_id', DatasetSpecialIdEnum.manual],
-              'AND',
-              ['kb_id', kbId]
-            ]
-          })
-        },
-        {
-          id: DatasetSpecialIdEnum.mark,
-          size: 0,
-          filename: datasetSpecialIdMap[DatasetSpecialIdEnum.mark].name,
-          uploadTime: new Date(),
-          status: FileStatusEnum.ready,
-          chunkLength: await PgClient.count(PgDatasetTableName, {
-            fields: ['id'],
-            where: [
-              ['user_id', userId],
-              'AND',
-              ['file_id', DatasetSpecialIdEnum.mark],
-              'AND',
-              ['kb_id', kbId]
-            ]
-          })
-        }
-      ];
-    }
-
-    const data = await Promise.all([
-      getSpecialData(),
-      ...rows.map(async (row) => {
-        if (!row.file_id) return null;
-        // link data
-        if (strIsLink(row.file_id)) {
-          const { rows } = await PgClient.select(PgDatasetTableName, {
-            where: [['user_id', userId], 'AND', ['file_id', row.file_id]],
-            limit: 1,
-            fields: ['source']
-          });
-          return {
-            id: row.file_id,
-            size: 0,
-            filename: rows[0]?.source || row.file_id,
-            uploadTime: new Date(),
-            status: FileStatusEnum.ready,
-            chunkLength: row.count
-          };
-        }
-        // file data
-        const file = await collection.findOne(
-          {
-            _id: new Types.ObjectId(row.file_id),
-            ['metadata.userId']: userId,
-            ['metadata.kbId']: kbId
-          },
-          {
-            projection: {
-              _id: 1,
-              filename: 1,
-              uploadDate: 1,
-              length: 1
-            }
-          }
-        );
-        if (!file) return null;
-        return {
-          id: String(file._id),
-          size: file.length,
-          filename: file.filename,
-          uploadTime: file.uploadDate,
-          status: (await TrainingData.findOne({ userId, kbId, file_id: file._id }))
-            ? FileStatusEnum.embedding
-            : FileStatusEnum.ready,
-          chunkLength: row.count
-        };
-      })
-    ]);
-
-    jsonRes(res, {
-      data: {
-        pageNum,
-        pageSize,
-        data: data.flat().filter((item) => item),
-        total
-      }
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

projects/app/src/pages/api/core/dataset/file/markUsed.ts

@@ -1,38 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { GridFSStorage } from '@/service/lib/gridfs';
-import { MarkFileUsedProps } from '@/global/core/api/datasetReq.d';
-import { Types } from '@fastgpt/common/mongo';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
-  try {
-    await connectToDatabase();
-
-    const { fileIds } = req.body as MarkFileUsedProps;
-    const { userId } = await authUser({ req, authToken: true });
-
-    const gridFs = new GridFSStorage('dataset', userId);
-    const collection = gridFs.Collection();
-
-    await collection.updateMany(
-      {
-        _id: { $in: fileIds.filter((id) => !!id).map((id) => new Types.ObjectId(id)) },
-        ['metadata.userId']: userId
-      },
-      {
-        $set: {
-          ['metadata.datasetUsed']: true
-        }
-      }
-    );
-
-    jsonRes(res, {});
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

projects/app/src/pages/api/core/dataset/file/update.ts

@@ -1,20 +1,19 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { UpdateFileProps } from '@/global/core/api/datasetReq.d';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { addLog } from '@/service/utils/tools';
-import { strIsLink } from '@fastgpt/common/tools/str';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
 
-    const { id, name, datasetUsed } = req.body as UpdateFileProps;
+    const { id, name, datasetUsed } = req.body;
     const { userId } = await authUser({ req, authToken: true });
 
     const gridFs = new GridFSStorage('dataset', userId);

projects/app/src/pages/api/core/dataset/list.ts

@@ -1,11 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { getVectorModel } from '@/service/core/ai/model';
 import type { DatasetsItemType } from '@/types/core/dataset';
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -14,14 +14,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const kbList = await MongoDataset.find({
+    const datasets = await MongoDataset.find({
       userId,
       ...(parentId !== undefined && { parentId: parentId || null }),
       ...(type && { type })
     }).sort({ updateTime: -1 });
 
     const data = await Promise.all(
-      kbList.map(async (item) => ({
+      datasets.map(async (item) => ({
         ...item.toJSON(),
         vectorModel: getVectorModel(item.vectorModel)
       }))

projects/app/src/pages/api/core/dataset/paths.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import type { DatasetPathItemType } from '@/types/core/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/core/dataset/searchTest.ts

@@ -1,53 +1,64 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { getVector } from '../../openapi/plugin/vector';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
-import type { SearchTestResponseType } from '@/global/core/api/datasetRes.d';
 import { connectToDatabase } from '@/service/mongo';
+import type {
+  SearchDataResponseItemType,
+  SearchDataResultItemType
+} from '@fastgpt/global/core/dataset/type';
+import { getDatasetDataItemInfo } from './data/getDataById';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { kbId, text } = req.body as SearchTestProps;
+    const { datasetId, text } = req.body as SearchTestProps;
 
-    if (!kbId || !text) {
+    if (!datasetId || !text) {
       throw new Error('缺少参数');
     }
 
     // 凭证校验
-    const [{ userId }, kb] = await Promise.all([
+    const [{ userId }, dataset] = await Promise.all([
       authUser({ req, authToken: true, authApiKey: true }),
-      MongoDataset.findById(kbId, 'vectorModel')
+      MongoDataset.findById(datasetId, 'vectorModel')
     ]);
 
-    if (!userId || !kb) {
+    if (!userId || !dataset) {
       throw new Error('缺少用户ID');
     }
 
     const { vectors } = await getVector({
-      model: kb.vectorModel,
+      model: dataset.vectorModel,
       userId,
       input: [text]
     });
 
-    const response: any = await PgClient.query(
+    const results: any = await PgClient.query(
       `BEGIN;
-        SET LOCAL hnsw.ef_search= ${global.systemEnv.pgHNSWEfSearch || 40};
-        select id, q, a, source, file_id, (vector <#> '[${
+        SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 60};
+        select id, q, a, dataset_id, collection_id, (vector <#> '[${
           vectors[0]
-        }]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
+        }]') * -1 AS score from ${PgDatasetTableName} where dataset_id='${datasetId}' AND user_id='${userId}' ORDER BY vector <#> '[${
           vectors[0]
         }]' limit 12;
         COMMIT;`
     );
 
-    jsonRes<SearchTestResponseType>(res, {
-      data: response?.[2]?.rows || []
+    const rows = results?.[2]?.rows as SearchDataResultItemType[];
+
+    const collectionsData = await getDatasetDataItemInfo({ pgDataList: rows });
+
+    jsonRes<SearchDataResponseItemType[]>(res, {
+      data: collectionsData.map((item, index) => ({
+        ...item,
+        score: rows[index].score
+      }))
     });
   } catch (err) {
     jsonRes(res, {

projects/app/src/pages/api/core/dataset/update.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/openapi/plugin/vector.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authBalanceByUid, authUser } from '@fastgpt/support/user/auth';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
-import { getAIApi } from '@fastgpt/core/ai/config';
+import { authBalanceByUid, authUser } from '@fastgpt/service/support/user/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import { getAIApi } from '@fastgpt/service/core/ai/config';
 import { pushGenerateVectorBill } from '@/service/common/bill/push';
 import { connectToDatabase } from '@/service/mongo';
 
@@ -44,55 +44,61 @@ export async function getVector({
   input,
   billId
 }: { userId?: string } & Props) {
-  userId && (await authBalanceByUid(userId));
+  try {
+    userId && (await authBalanceByUid(userId));
 
-  for (let i = 0; i < input.length; i++) {
-    if (!input[i]) {
-      return Promise.reject({
-        code: 500,
-        message: '向量生成模块输入内容为空'
-      });
+    for (let i = 0; i < input.length; i++) {
+      if (!input[i]) {
+        return Promise.reject({
+          code: 500,
+          message: '向量生成模块输入内容为空'
+        });
+      }
     }
-  }
 
-  // 获取 chatAPI
-  const ai = getAIApi();
+    // 获取 chatAPI
+    const ai = getAIApi();
 
-  // 把输入的内容转成向量
-  const result = await ai.embeddings
-    .create(
-      {
+    // 把输入的内容转成向量
+    const result = await ai.embeddings
+      .create(
+        {
+          model,
+          input
+        },
+        {
+          timeout: 60000
+        }
+      )
+      .then(async (res) => {
+        if (!res.data) {
+          return Promise.reject('Embedding API 404');
+        }
+        if (!res?.data?.[0]?.embedding) {
+          console.log(res?.data);
+          // @ts-ignore
+          return Promise.reject(res.data?.err?.message || 'Embedding API Error');
+        }
+        return {
+          tokenLen: res.usage.total_tokens || 0,
+          vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
+        };
+      });
+
+    userId &&
+      pushGenerateVectorBill({
+        userId,
+        tokenLen: result.tokenLen,
         model,
-        input
-      },
-      {
-        timeout: 60000
-      }
-    )
-    .then(async (res) => {
-      if (!res.data) {
-        return Promise.reject('Embedding API 404');
-      }
-      if (!res?.data?.[0]?.embedding) {
-        console.log(res?.data);
-        // @ts-ignore
-        return Promise.reject(res.data?.err?.message || 'Embedding API Error');
-      }
-      return {
-        tokenLen: res.usage.total_tokens || 0,
-        vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
-      };
-    });
+        billId
+      });
 
-  userId &&
-    pushGenerateVectorBill({
-      userId,
-      tokenLen: result.tokenLen,
-      model,
-      billId
-    });
+    return result;
+  } catch (error) {
+    console.log(`Embedding Error`, error);
 
-  return result;
+    return Promise.reject(error);
+  }
 }
 
 function unityDimensional(vector: number[]) {

projects/app/src/pages/api/openapi/v1/chat/completions.ts

@@ -1,5 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import ChatCompletion from '@/pages/api/v1/chat/completions';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/plugins/urlFetch.ts

@@ -4,9 +4,9 @@ import axios from 'axios';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import type { FetchResultItem } from '@/global/common/api/pluginRes.d';
-import { simpleText } from '@fastgpt/common/tools/str';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
+import { simpleText } from '@fastgpt/global/common/string/tools';
 import { connectToDatabase } from '@/service/mongo';
 
 export type UrlFetchResponse = FetchResultItem[];

projects/app/src/pages/api/plusApi/[...path].ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { request } from '@fastgpt/common/plusApi/request';
+import { request } from '@fastgpt/service/common/api/plusRequest';
 import type { Method } from 'axios';
 import { connectToDatabase } from '@/service/mongo';
 

projects/app/src/pages/api/support/openapi/delKey.ts

@@ -2,8 +2,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/support/openapi/getKeys.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { GetApiKeyProps } from '@/global/support/api/openapiReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/support/openapi/postKey.ts

@@ -2,8 +2,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { customAlphabet } from 'nanoid';
 import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
 

projects/app/src/pages/api/support/openapi/putKey.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/support/outLink/create.ts

@@ -1,12 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
 import { authApp } from '@/service/utils/auth';
-import { authUser } from '@fastgpt/support/user/auth';
-import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
 import { customAlphabet } from 'nanoid';
-import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
+import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
 
 /* create a shareChat */

projects/app/src/pages/api/support/outLink/delete.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* delete a shareChat by shareChatId */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/support/outLink/init.ts

@@ -1,13 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import { MongoUser } from '@fastgpt/support/user/schema';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
 import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
 import { authApp } from '@/service/utils/auth';
 import { HUMAN_ICON } from '@/constants/chat';
-import { getGuideModule } from '@/components/ChatBox/utils';
-import { authShareChatInit } from '@fastgpt/support/outLink/auth';
+import { getGuideModule } from '@/global/core/app/modules/utils';
+import { authShareChatInit } from '@fastgpt/service/support/outLink/auth';
 import { getChatModelNameListByModules } from '@/service/core/app/module';
 
 /* init share chat window */

projects/app/src/pages/api/support/outLink/list.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* get shareChat list by appId */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/support/outLink/update.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/system/file/delete.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/system/file/readUrl.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import jwt from 'jsonwebtoken';
-import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 import { GridFSStorage } from '@/service/lib/gridfs';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/system/file/upload.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { customAlphabet } from 'nanoid';
 import multer from 'multer';

projects/app/src/pages/api/system/file/uploadImage.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Image } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 type Props = { base64Img: string };
 

projects/app/src/pages/api/system/getInitData.ts

@@ -1,12 +1,11 @@
-import type { SystemEnvType } from '@/types';
-import type { FeConfigsType } from '@fastgpt/common/type/index.d';
+import type { FeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types/index.d';
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { readFileSync } from 'fs';
 import type { InitDateResponse } from '@/global/common/api/systemRes';
-import { formatPrice } from '@fastgpt/common/bill';
-import { getTikTokenEnc } from '@/utils/common/tiktoken';
-import { initHttpAgent } from '@fastgpt/core/init';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
+import { getTikTokenEnc } from '@/global/common/tiktoken';
+import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
 import {
   defaultChatModels,
   defaultQAModels,

projects/app/src/pages/api/user/account/loginByPassword.ts

@@ -1,9 +1,9 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { MongoUser } from '@fastgpt/support/user/schema';
-import { setCookie } from '@fastgpt/support/user/auth';
-import { generateToken } from '@fastgpt/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
+import { setCookie } from '@fastgpt/service/support/user/auth';
+import { generateToken } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/user/account/loginout.ts

@@ -1,7 +1,7 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { clearCookie } from '@fastgpt/support/user/auth';
+import { clearCookie } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

projects/app/src/pages/api/user/account/paySuccess.ts

@@ -1,8 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { TrainingData, connectToDatabase } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { startQueue } from '@/service/utils/tools';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -15,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
 async function unlockTask(userId: string) {
   try {
-    await TrainingData.updateMany(
+    await MongoDatasetTraining.updateMany(
       {
         userId
       },

projects/app/src/pages/api/user/account/tokenLogin.ts

@@ -1,8 +1,8 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { MongoUser } from '@fastgpt/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
 import { connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/user/account/update.ts

@@ -1,10 +1,10 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { MongoUser } from '@fastgpt/support/user/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { UserUpdateParams } from '@/types/user';
-import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config';
+import { getAIApi, openaiBaseUrl } from '@fastgpt/service/core/ai/config';
 import { connectToDatabase } from '@/service/mongo';
 
 /* update user info */

projects/app/src/pages/api/user/account/updatePasswordByOld.ts

@@ -1,8 +1,8 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { MongoUser } from '@fastgpt/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
 import { connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/user/getBill.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Bill, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { adaptBill } from '@/utils/adapt';
 import { addDays } from 'date-fns';
 

projects/app/src/pages/api/user/getPayOrders.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { Pay, connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -11,7 +11,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const records = await Pay.find({
       userId,
       status: { $ne: 'CLOSED' }
-    }).sort({ createTime: -1 });
+    })
+      .sort({ createTime: -1 })
+      .limit(100);
 
     jsonRes(res, {
       data: records

projects/app/src/pages/api/user/inform/countUnread.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Inform, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/user/inform/list.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Inform, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/user/inform/read.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Inform, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/user/inform/send.ts

@@ -2,10 +2,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Inform, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { InformTypeEnum } from '@/constants/user';
 import { startSendInform } from '@/service/events/sendInform';
-import { MongoUser } from '@fastgpt/support/user/schema';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
 
 export type Props = {
   type: `${InformTypeEnum}`;

projects/app/src/pages/api/user/promotion/getPromotionData.ts

@@ -2,9 +2,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, promotionRecord } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import mongoose from '@fastgpt/common/mongo';
-import { MongoUser } from '@fastgpt/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import mongoose from '@fastgpt/service/common/mongo';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/user/promotion/getPromotions.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, promotionRecord } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/v1/chat/completions.ts

@@ -1,10 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { authApp } from '@/service/utils/auth';
-import { authUser } from '@fastgpt/support/user/auth';
-import { AuthUserTypeEnum } from '@fastgpt/support/user/auth';
+import { authUser, AuthUserTypeEnum } from '@fastgpt/service/support/user/auth';
 import { sseErrRes, jsonRes } from '@/service/response';
 import { addLog } from '@/service/utils/tools';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat';
 import {
   dispatchHistory,
@@ -17,28 +16,28 @@ import {
   dispatchHttpRequest,
   dispatchAppRequest
 } from '@/service/moduleDispatch';
-import type { CreateChatCompletionRequest } from '@fastgpt/core/ai/type';
+import type { CreateChatCompletionRequest } from '@fastgpt/global/core/ai/type.d';
 import type { MessageItemType } from '@/types/core/chat/type';
 import { gptMessage2ChatType, textAdaptGptResponse } from '@/utils/adapt';
 import { getChatHistory } from './getHistory';
 import { saveChat } from '@/service/utils/chat/saveChat';
-import { responseWrite } from '@fastgpt/common/tools/stream';
+import { responseWrite } from '@fastgpt/service/common/response';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { FlowModuleTypeEnum, initModuleType } from '@/constants/flow';
 import { AppModuleItemType, RunningModuleItemType } from '@/types/app';
 import { pushChatBill } from '@/service/common/bill/push';
 import { BillSourceEnum } from '@/constants/user';
 import { ChatHistoryItemResType } from '@/types/chat';
-import type { UserModelSchema } from '@fastgpt/support/user/type.d';
+import type { UserModelSchema } from '@fastgpt/global/support/user/type';
 import { SystemInputEnum } from '@/constants/app';
-import { getSystemTime } from '@/utils/user';
-import { authOutLinkChat } from '@fastgpt/support/outLink/auth';
-import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/support/outLink/tools';
+import { getSystemTime } from '@fastgpt/global/common/time/timezone';
+import { authOutLinkChat } from '@fastgpt/service/support/outLink/auth';
+import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
 import requestIp from 'request-ip';
-import { replaceVariable } from '@/utils/common/tools/text';
+import { replaceVariable } from '@/global/common/string/tools';
 import type { ModuleDispatchProps } from '@/types/core/chat/type';
 import { selectShareResponse } from '@/utils/service/core/chat';
-import { updateApiKeyUsage } from '@fastgpt/support/openapi/tools';
+import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { connectToDatabase } from '@/service/mongo';
 
 type FastGptWebChatProps = {
@@ -438,7 +437,7 @@ export async function dispatchModules({
         [FlowModuleTypeEnum.questionInput]: dispatchChatInput,
         [FlowModuleTypeEnum.answerNode]: dispatchAnswer,
         [FlowModuleTypeEnum.chatNode]: dispatchChatCompletion,
-        [FlowModuleTypeEnum.kbSearchNode]: dispatchKBSearch,
+        [FlowModuleTypeEnum.datasetSearchNode]: dispatchKBSearch,
         [FlowModuleTypeEnum.classifyQuestion]: dispatchClassifyQuestion,
         [FlowModuleTypeEnum.contentExtract]: dispatchContentExtract,
         [FlowModuleTypeEnum.httpRequest]: dispatchHttpRequest,

projects/app/src/pages/api/v1/chat/getHistory.ts

@@ -1,9 +1,9 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { ChatItem, connectToDatabase } from '@/service/mongo';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
 import type { ChatItemType } from '@/types/chat';
 
 export type Props = {