Fix share page whisper auth (#1161)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/string/textSplitter.ts

@@ -1,5 +1,5 @@
 import { getErrText } from '../error/utils';
-import { countPromptTokens } from './tiktoken';
+import { replaceRegChars } from './tools';
 
 /**
  * text split into chunks
@@ -31,7 +31,7 @@ export const splitText2Chunks = (props: {
   // The larger maxLen is, the next sentence is less likely to trigger splitting
   const stepReges: { reg: RegExp; maxLen: number }[] = [
     ...customReg.map((text) => ({
-      reg: new RegExp(`(${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'g'),
+      reg: new RegExp(`(${replaceRegChars(text)})`, 'g'),
       maxLen: chunkLen * 1.4
     })),
     { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },

packages/global/common/string/tools.ts

@@ -51,3 +51,5 @@ export const replaceSensitiveText = (text: string) => {
 export const getNanoid = (size = 12) => {
   return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
 };
+
+export const replaceRegChars = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

packages/global/core/module/template/system/userGuide.ts

@@ -1,6 +1,5 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
 import { FlowNodeTemplateType } from '../../type.d';
-import { userGuideTip } from '../tip';
 import {
   ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
@@ -12,8 +11,8 @@ export const UserGuideModule: FlowNodeTemplateType = {
   templateType: FlowNodeTemplateTypeEnum.userGuide,
   flowType: FlowNodeTypeEnum.userGuide,
   avatar: '/imgs/module/userGuide.png',
-  name: '全局配置',
-  intro: userGuideTip,
+  name: '系统配置',
+  intro: '可以配置应用的系统参数。',
   inputs: [
     {
       key: ModuleInputKeyEnum.welcomeText,

packages/global/core/module/template/tip.ts

@@ -1,4 +1,3 @@
 export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip';
-export const userGuideTip = 'core.app.tip.userGuideTip';
 export const welcomeTextTip = 'core.app.tip.welcomeTextTip';
 export const variableTip = 'core.app.tip.variableTip';

packages/global/support/user/team/type.d.ts

@@ -81,4 +81,5 @@ export type TeamTagItemType = {
 export type LafAccountType = {
   token: string;
   appid: string;
+  pat: string;
 };

packages/global/support/wallet/bill/type.d.ts

@@ -18,6 +18,7 @@ export type BillSchemaType = {
     month?: number;
     datasetSize?: number;
     extraPoints?: number;
+    invoice: boolean;
   };
   username: string;
 };

packages/service/common/file/image/schema.ts

@@ -35,6 +35,7 @@ try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ createTime: 1 });
+  // delete related img
   ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);

packages/service/common/vectorStore/controller.d.ts

@@ -1,10 +1,9 @@
-export type DeleteDatasetVectorProps = {
+export type DeleteDatasetVectorProps = (
+  | { id: string }
+  | { datasetIds: string[]; collectionIds?: string[] }
+  | { idList: string[] }
+) & {
   teamId: string;
-
-  id?: string;
-  datasetIds?: string[];
-  collectionIds?: string[];
-  idList?: string[];
 };
 
 export type InsertVectorProps = {

packages/service/common/vectorStore/pg/controller.ts

@@ -26,13 +26,7 @@ export async function initPg() {
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
     );
     await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id);`
-    );
-    await PgClient.query(
-      ` CREATE INDEX CONCURRENTLY IF NOT EXISTS team_collection_index ON ${PgDatasetTableName} USING btree(team_id, collection_id);`
-    );
-    await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_id_index ON ${PgDatasetTableName} USING btree(team_id, id);`
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);`
     );
     await PgClient.query(
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${PgDatasetTableName} USING btree(createtime);`
@@ -83,31 +77,33 @@ export const deleteDatasetDataVector = async (
     retry?: number;
   }
 ): Promise<any> => {
-  const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props;
+  const { teamId, retry = 2 } = props;
 
   const teamIdWhere = `team_id='${String(teamId)}' AND`;
 
   const where = await (() => {
-    if (id) return `${teamIdWhere} id=${id}`;
+    if ('id' in props && props.id) return `${teamIdWhere} id=${props.id}`;
 
-    if (datasetIds) {
-      return `${teamIdWhere} dataset_id IN (${datasetIds
+    if ('datasetIds' in props && props.datasetIds) {
+      const datasetIdWhere = `dataset_id IN (${props.datasetIds
         .map((id) => `'${String(id)}'`)
         .join(',')})`;
+
+      if ('collectionIds' in props && props.collectionIds) {
+        return `${teamIdWhere} ${datasetIdWhere} AND collection_id IN (${props.collectionIds
+          .map((id) => `'${String(id)}'`)
+          .join(',')})`;
+      }
+
+      return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if (collectionIds) {
-      return `${teamIdWhere} collection_id IN (${collectionIds
-        .map((id) => `'${String(id)}'`)
-        .join(',')})`;
-    }
-
-    if (idList) {
-      return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`;
+    if ('idList' in props && props.idList) {
+      return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
-
+  console.log(where, '===');
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/dataset/collection/controller.ts

@@ -118,6 +118,37 @@ export function createDefaultCollection({
   );
 }
 
+/* delete collection related images/files */
+export const delCollectionRelatedSource = async ({
+  collections,
+  session
+}: {
+  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  session: ClientSession;
+}) => {
+  if (collections.length === 0) return;
+
+  const teamId = collections[0].teamId;
+
+  if (!teamId) return Promise.reject('teamId is not exist');
+
+  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
+  const relatedImageIds = collections
+    .map((item) => item?.metadata?.relatedImgId || '')
+    .filter(Boolean);
+
+  // delete images
+  await delImgByRelatedId({
+    teamId,
+    relateIds: relatedImageIds,
+    session
+  });
+  // delete files
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
+  });
+};
 /**
  * delete collection and it related data
  */
@@ -134,26 +165,32 @@ export async function delCollectionAndRelatedSources({
 
   if (!teamId) return Promise.reject('teamId is not exist');
 
+  const datasetIds = Array.from(
+    new Set(
+      collections.map((item) => {
+        if (typeof item.datasetId === 'string') {
+          return String(item.datasetId);
+        }
+        return String(item.datasetId._id);
+      })
+    )
+  );
   const collectionIds = collections.map((item) => String(item._id));
-  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
-  const relatedImageIds = collections
-    .map((item) => item?.metadata?.relatedImgId || '')
-    .filter(Boolean);
+
+  await delCollectionRelatedSource({ collections, session });
 
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
+    datasetIds: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
-
   // delete dataset.datas
-  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
-  // delete imgs
-  await delImgByRelatedId({
-    teamId,
-    relateIds: relatedImageIds,
-    session
-  });
+  await MongoDatasetData.deleteMany(
+    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { session }
+  );
+
   // delete collections
   await MongoDatasetCollection.deleteMany(
     {
@@ -163,9 +200,5 @@ export async function delCollectionAndRelatedSources({
   );
 
   // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, collectionIds });
-  await delFileByFileIdList({
-    bucketName: BucketNameEnum.dataset,
-    fileIdList
-  });
+  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
 }

packages/service/core/dataset/controller.ts

@@ -1,8 +1,11 @@
 import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
 import { MongoDataset } from './schema';
-import { delCollectionAndRelatedSources } from './collection/controller';
+import { delCollectionRelatedSource } from './collection/controller';
 import { ClientSession } from '../../common/mongo';
+import { MongoDatasetTraining } from './training/schema';
+import { MongoDatasetData } from './data/schema';
+import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -82,5 +85,26 @@ export async function delDatasetRelevantData({
     '_id teamId fileId metadata'
   ).lean();
 
-  await delCollectionAndRelatedSources({ collections, session });
+  // image and file
+  await delCollectionRelatedSource({ collections, session });
+
+  // delete training data
+  await MongoDatasetTraining.deleteMany({
+    teamId,
+    datasetId: { $in: datasetIds }
+  });
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session });
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany(
+    {
+      teamId,
+      datasetId: { $in: datasetIds }
+    },
+    { session }
+  );
+
+  // no session delete: delete files, vector data
+  await deleteDatasetDataVector({ teamId, datasetIds });
 }

packages/service/core/dataset/data/controller.ts

@@ -1,2 +0,0 @@
-import { MongoDatasetData } from './schema';
-import { deleteDatasetDataVector } from '../../../common/vectorStore/controller';

packages/service/core/dataset/data/schema.ts

@@ -77,17 +77,18 @@ const DatasetDataSchema = new Schema({
 });
 
 try {
-  // list collection and count data; list data
+  // list collection and count data; list data; delete collection(relate data)
   DatasetDataSchema.index(
     { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 },
     { background: true }
   );
-  // same data check
-  DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true });
   // full text index
   DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true });
   // Recall vectors after data matching
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true });
+  DatasetDataSchema.index(
+    { teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 },
+    { background: true }
+  );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
 } catch (error) {
   console.log(error);

packages/service/core/dataset/search/controller.ts

@@ -93,6 +93,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
       {
         teamId,
         datasetId: { $in: datasetIds },
+        collectionId: { $in: results.map((item) => item.collectionId) },
         'indexes.dataId': { $in: results.map((item) => item.id?.trim()) }
       },
       'datasetId collectionId q a chunkIndex indexes'

packages/service/core/dataset/training/controller.ts

@@ -6,7 +6,6 @@ import type {
 } from '@fastgpt/global/core/dataset/api.d';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { simpleText } from '@fastgpt/global/common/string/tools';
-import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { ClientSession } from '../../../common/mongo';
 import { getLLMModel, getVectorModel } from '../../ai/model';
 import { addLog } from '../../../common/system/log';

packages/service/core/dataset/training/schema.ts

@@ -92,8 +92,8 @@ const TrainingDataSchema = new Schema({
 });
 
 try {
-  // lock training data; delete training data
-  TrainingDataSchema.index({ teamId: 1, collectionId: 1 });
+  // lock training data(teamId); delete training data
+  TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
   // get training data and sort
   TrainingDataSchema.index({ mode: 1, lockTime: 1, weight: -1 });
   TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days

packages/service/support/user/team/teamSchema.ts

@@ -42,6 +42,9 @@ const TeamSchema = new Schema({
     },
     appid: {
       type: String
+    },
+    pat: {
+      type: String
     }
   }
 });

packages/web/components/common/Icon/constants.ts

@@ -107,6 +107,7 @@ export const iconPaths = {
   'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'),
   'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'),
   'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'),
+  'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'),
   'core/modules/flowLight': () => import('./icons/core/modules/flowLight.svg'),
   'core/modules/previewLight': () => import('./icons/core/modules/previewLight.svg'),
   'core/modules/systemPlugin': () => import('./icons/core/modules/systemPlugin.svg'),

packages/web/components/common/Icon/icons/core/modules/fixview.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 30">
+    <path
+        d="M3.692 4.63c0-.53.4-.938.939-.938h5.215V0H4.708C2.13 0 0 2.054 0 4.63v5.216h3.692V4.631zM27.354 0h-5.2v3.692h5.17c.53 0 .984.4.984.939v5.215H32V4.631A4.624 4.624 0 0027.354 0zm.954 24.83c0 .532-.4.94-.939.94h-5.215v3.768h5.215c2.577 0 4.631-2.13 4.631-4.707v-5.139h-3.692v5.139zm-23.677.94c-.531 0-.939-.4-.939-.94v-5.138H0v5.139c0 2.577 2.13 4.707 4.708 4.707h5.138V25.77H4.631z">
+    </path>
+</svg>