Fix share page whisper auth (#1161)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/common/file/image/schema.ts

@@ -35,6 +35,7 @@ try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ createTime: 1 });
+  // delete related img
   ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);

packages/service/common/vectorStore/controller.d.ts

@@ -1,10 +1,9 @@
-export type DeleteDatasetVectorProps = {
+export type DeleteDatasetVectorProps = (
+  | { id: string }
+  | { datasetIds: string[]; collectionIds?: string[] }
+  | { idList: string[] }
+) & {
   teamId: string;
-
-  id?: string;
-  datasetIds?: string[];
-  collectionIds?: string[];
-  idList?: string[];
 };
 
 export type InsertVectorProps = {

packages/service/common/vectorStore/pg/controller.ts

@@ -26,13 +26,7 @@ export async function initPg() {
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
     );
     await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id);`
-    );
-    await PgClient.query(
-      ` CREATE INDEX CONCURRENTLY IF NOT EXISTS team_collection_index ON ${PgDatasetTableName} USING btree(team_id, collection_id);`
-    );
-    await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_id_index ON ${PgDatasetTableName} USING btree(team_id, id);`
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);`
     );
     await PgClient.query(
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${PgDatasetTableName} USING btree(createtime);`
@@ -83,31 +77,33 @@ export const deleteDatasetDataVector = async (
     retry?: number;
   }
 ): Promise<any> => {
-  const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props;
+  const { teamId, retry = 2 } = props;
 
   const teamIdWhere = `team_id='${String(teamId)}' AND`;
 
   const where = await (() => {
-    if (id) return `${teamIdWhere} id=${id}`;
+    if ('id' in props && props.id) return `${teamIdWhere} id=${props.id}`;
 
-    if (datasetIds) {
-      return `${teamIdWhere} dataset_id IN (${datasetIds
+    if ('datasetIds' in props && props.datasetIds) {
+      const datasetIdWhere = `dataset_id IN (${props.datasetIds
         .map((id) => `'${String(id)}'`)
         .join(',')})`;
+
+      if ('collectionIds' in props && props.collectionIds) {
+        return `${teamIdWhere} ${datasetIdWhere} AND collection_id IN (${props.collectionIds
+          .map((id) => `'${String(id)}'`)
+          .join(',')})`;
+      }
+
+      return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if (collectionIds) {
-      return `${teamIdWhere} collection_id IN (${collectionIds
-        .map((id) => `'${String(id)}'`)
-        .join(',')})`;
-    }
-
-    if (idList) {
-      return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`;
+    if ('idList' in props && props.idList) {
+      return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
-
+  console.log(where, '===');
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/dataset/collection/controller.ts

@@ -118,6 +118,37 @@ export function createDefaultCollection({
   );
 }
 
+/* delete collection related images/files */
+export const delCollectionRelatedSource = async ({
+  collections,
+  session
+}: {
+  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  session: ClientSession;
+}) => {
+  if (collections.length === 0) return;
+
+  const teamId = collections[0].teamId;
+
+  if (!teamId) return Promise.reject('teamId is not exist');
+
+  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
+  const relatedImageIds = collections
+    .map((item) => item?.metadata?.relatedImgId || '')
+    .filter(Boolean);
+
+  // delete images
+  await delImgByRelatedId({
+    teamId,
+    relateIds: relatedImageIds,
+    session
+  });
+  // delete files
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
+  });
+};
 /**
  * delete collection and it related data
  */
@@ -134,26 +165,32 @@ export async function delCollectionAndRelatedSources({
 
   if (!teamId) return Promise.reject('teamId is not exist');
 
+  const datasetIds = Array.from(
+    new Set(
+      collections.map((item) => {
+        if (typeof item.datasetId === 'string') {
+          return String(item.datasetId);
+        }
+        return String(item.datasetId._id);
+      })
+    )
+  );
   const collectionIds = collections.map((item) => String(item._id));
-  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
-  const relatedImageIds = collections
-    .map((item) => item?.metadata?.relatedImgId || '')
-    .filter(Boolean);
+
+  await delCollectionRelatedSource({ collections, session });
 
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
+    datasetIds: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
-
   // delete dataset.datas
-  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
-  // delete imgs
-  await delImgByRelatedId({
-    teamId,
-    relateIds: relatedImageIds,
-    session
-  });
+  await MongoDatasetData.deleteMany(
+    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { session }
+  );
+
   // delete collections
   await MongoDatasetCollection.deleteMany(
     {
@@ -163,9 +200,5 @@ export async function delCollectionAndRelatedSources({
   );
 
   // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, collectionIds });
-  await delFileByFileIdList({
-    bucketName: BucketNameEnum.dataset,
-    fileIdList
-  });
+  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
 }

packages/service/core/dataset/controller.ts

@@ -1,8 +1,11 @@
 import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
 import { MongoDataset } from './schema';
-import { delCollectionAndRelatedSources } from './collection/controller';
+import { delCollectionRelatedSource } from './collection/controller';
 import { ClientSession } from '../../common/mongo';
+import { MongoDatasetTraining } from './training/schema';
+import { MongoDatasetData } from './data/schema';
+import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -82,5 +85,26 @@ export async function delDatasetRelevantData({
     '_id teamId fileId metadata'
   ).lean();
 
-  await delCollectionAndRelatedSources({ collections, session });
+  // image and file
+  await delCollectionRelatedSource({ collections, session });
+
+  // delete training data
+  await MongoDatasetTraining.deleteMany({
+    teamId,
+    datasetId: { $in: datasetIds }
+  });
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session });
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany(
+    {
+      teamId,
+      datasetId: { $in: datasetIds }
+    },
+    { session }
+  );
+
+  // no session delete: delete files, vector data
+  await deleteDatasetDataVector({ teamId, datasetIds });
 }

packages/service/core/dataset/data/controller.ts

@@ -1,2 +0,0 @@
-import { MongoDatasetData } from './schema';
-import { deleteDatasetDataVector } from '../../../common/vectorStore/controller';

packages/service/core/dataset/data/schema.ts

@@ -77,17 +77,18 @@ const DatasetDataSchema = new Schema({
 });
 
 try {
-  // list collection and count data; list data
+  // list collection and count data; list data; delete collection(relate data)
   DatasetDataSchema.index(
     { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 },
     { background: true }
   );
-  // same data check
-  DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true });
   // full text index
   DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true });
   // Recall vectors after data matching
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true });
+  DatasetDataSchema.index(
+    { teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 },
+    { background: true }
+  );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
 } catch (error) {
   console.log(error);

packages/service/core/dataset/search/controller.ts

@@ -93,6 +93,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
       {
         teamId,
         datasetId: { $in: datasetIds },
+        collectionId: { $in: results.map((item) => item.collectionId) },
         'indexes.dataId': { $in: results.map((item) => item.id?.trim()) }
       },
       'datasetId collectionId q a chunkIndex indexes'

packages/service/core/dataset/training/controller.ts

@@ -6,7 +6,6 @@ import type {
 } from '@fastgpt/global/core/dataset/api.d';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { simpleText } from '@fastgpt/global/common/string/tools';
-import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { ClientSession } from '../../../common/mongo';
 import { getLLMModel, getVectorModel } from '../../ai/model';
 import { addLog } from '../../../common/system/log';

packages/service/core/dataset/training/schema.ts

@@ -92,8 +92,8 @@ const TrainingDataSchema = new Schema({
 });
 
 try {
-  // lock training data; delete training data
-  TrainingDataSchema.index({ teamId: 1, collectionId: 1 });
+  // lock training data(teamId); delete training data
+  TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
   // get training data and sort
   TrainingDataSchema.index({ mode: 1, lockTime: 1, weight: -1 });
   TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days

packages/service/support/user/team/teamSchema.ts

@@ -42,6 +42,9 @@ const TeamSchema = new Schema({
     },
     appid: {
       type: String
+    },
+    pat: {
+      type: String
     }
   }
 });