Change embedding (#1463)

* rebuild embedding queue

* dataset menu

* feat: rebuild data api

* feat: ui change embedding model

* dataset ui

* feat: rebuild index ui

* rename collection

packages/global/core/dataset/type.d.ts

@@ -80,6 +80,7 @@ export type DatasetDataSchemaType = {
   a: string; // answer or custom content
   fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
+  rebuilding?: boolean;
 };
 
 export type DatasetTrainingSchemaType = {
@@ -95,6 +96,7 @@ export type DatasetTrainingSchemaType = {
   mode: `${TrainingModeEnum}`;
   model: string;
   prompt: string;
+  dataId?: string;
   q: string;
   a: string;
   chunkIndex: number;

packages/global/support/user/team/constant.ts

@@ -1,6 +1,6 @@
 export const TeamCollectionName = 'teams';
-export const TeamMemberCollectionName = 'team.members';
-export const TeamTagsCollectionName = 'team.tags';
+export const TeamMemberCollectionName = 'team_members';
+export const TeamTagsCollectionName = 'team_tags';
 
 export enum TeamMemberRoleEnum {
   owner = 'owner',

packages/service/common/buffer/rawText/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../mongo';
 const { Schema, model, models } = connectionMongo;
 import { RawTextBufferSchemaType } from './type';
 
-export const collectionName = 'buffer.rawText';
+export const collectionName = 'buffer_rawtexts';
 
 const RawTextBufferSchema = new Schema({
   sourceId: {

packages/service/common/buffer/tts/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { TTSBufferSchemaType } from './type.d';
 
-export const collectionName = 'buffer.tts';
+export const collectionName = 'buffer_tts';
 
 const TTSBufferSchema = new Schema({
   bufferId: {

packages/service/common/mongo/sessionRun.ts

@@ -12,8 +12,6 @@ export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession)
 
     return result as T;
   } catch (error) {
-    console.log(error);
-
     await session.abortTransaction();
     await session.endSession();
     return Promise.reject(error);

packages/service/common/vectorStore/pg/controller.ts

@@ -98,12 +98,15 @@ export const deleteDatasetDataVector = async (
       return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if ('idList' in props && props.idList) {
+    if ('idList' in props && Array.isArray(props.idList)) {
+      if (props.idList.length === 0) return;
       return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
 
+  if (!where) return;
+
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/app/versionSchema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
 
-export const AppVersionCollectionName = 'app.versions';
+export const AppVersionCollectionName = 'app_versions';
 
 const AppVersionSchema = new Schema({
   appId: {

packages/service/core/dataset/collection/schema.ts

@@ -8,7 +8,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetColCollectionName = 'dataset.collections';
+export const DatasetColCollectionName = 'dataset_collections';
 
 const DatasetCollectionSchema = new Schema({
   parentId: {

packages/service/core/dataset/data/schema.ts

@@ -8,7 +8,7 @@ import {
 import { DatasetCollectionName } from '../schema';
 import { DatasetColCollectionName } from '../collection/schema';
 
-export const DatasetDataCollectionName = 'dataset.datas';
+export const DatasetDataCollectionName = 'dataset_datas';
 
 const DatasetDataSchema = new Schema({
   teamId: {
@@ -73,7 +73,8 @@ const DatasetDataSchema = new Schema({
   },
   inited: {
     type: Boolean
-  }
+  },
+  rebuilding: Boolean
 });
 
 try {
@@ -90,10 +91,13 @@ try {
     { background: true }
   );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
+  // rebuild data
+  DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }, { background: true });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoDatasetData: Model<DatasetDataSchemaType> =
   models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+
 MongoDatasetData.syncIndexes();

packages/service/core/dataset/training/schema.ts

@@ -10,7 +10,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetTrainingCollectionName = 'dataset.trainings';
+export const DatasetTrainingCollectionName = 'dataset_trainings';
 
 const TrainingDataSchema = new Schema({
   teamId: {
@@ -35,8 +35,7 @@ const TrainingDataSchema = new Schema({
   },
   billId: {
     // concat bill
-    type: String,
-    default: ''
+    type: Schema.Types.ObjectId
   },
   mode: {
     type: String,
@@ -78,6 +77,9 @@ const TrainingDataSchema = new Schema({
     type: Number,
     default: 0
   },
+  dataId: {
+    type: Schema.Types.ObjectId
+  },
   indexes: {
     type: [
       {

packages/service/type/next.d.ts

@@ -0,0 +1,8 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+export type ApiRequestProps<Body = any, Query = any> = Omit<NextApiRequest, 'query' | 'body'> & {
+  query: Query;
+  body: Body;
+};
+
+export type { NextApiResponse as ApiResponseType } from 'next';

packages/web/components/common/MyDivider/index.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import { Divider, type DividerProps } from '@chakra-ui/react';
+
+const MyDivider = (props: DividerProps) => {
+  const { h } = props;
+  return <Divider my={4} borderBottomWidth={h || '1x'} {...props}></Divider>;
+};
+
+export default MyDivider;

packages/web/components/common/MyTooltip/index.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react';
+import { Box, Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react';
 
 interface Props extends TooltipProps {
   forceShow?: boolean;
@@ -9,24 +9,32 @@ const MyTooltip = ({ children, forceShow = false, shouldWrapChildren = true, ...
   const [isPc] = useMediaQuery('(min-width: 900px)');
 
   return isPc || forceShow ? (
-    <Tooltip
-      className="tooltip"
-      bg={'white'}
-      arrowShadowColor={'rgba(0,0,0,0.05)'}
-      hasArrow
-      arrowSize={12}
-      offset={[-15, 15]}
-      color={'myGray.800'}
-      px={4}
-      py={2}
-      borderRadius={'8px'}
-      whiteSpace={'pre-wrap'}
-      boxShadow={'1px 1px 10px rgba(0,0,0,0.2)'}
-      shouldWrapChildren={shouldWrapChildren}
-      {...props}
+    <Box
+      css={css({
+        '& span': {
+          display: 'block'
+        }
+      })}
     >
-      {children}
-    </Tooltip>
+      <Tooltip
+        className="tooltip"
+        bg={'white'}
+        arrowShadowColor={'rgba(0,0,0,0.05)'}
+        hasArrow
+        arrowSize={12}
+        offset={[-15, 15]}
+        color={'myGray.800'}
+        px={4}
+        py={2}
+        borderRadius={'8px'}
+        whiteSpace={'pre-wrap'}
+        boxShadow={'1px 1px 10px rgba(0,0,0,0.2)'}
+        shouldWrapChildren={shouldWrapChildren}
+        {...props}
+      >
+        {children}
+      </Tooltip>
+    </Box>
   ) : (
     <>{children}</>
   );