4.6.3-website dataset (#532)

projects/app/src/service/core/ai/rerank.ts

@@ -20,7 +20,8 @@ export function reRankRecall({ query, inputs }: PostReRankProps) {
         Authorization: `Bearer ${model.requestAuth}`
       }
     }
-  ).finally(() => {
+  ).then((data) => {
     console.log('rerank time:', Date.now() - start);
+    return data;
   });
 }

projects/app/src/service/core/dataset/data/controller.ts

@@ -4,7 +4,8 @@ import {
   PatchIndexesProps,
   UpdateDatasetDataProps
 } from '@fastgpt/global/core/dataset/controller';
-import { deletePgDataById, insertData2Pg, updatePgDataById } from './pg';
+import { deletePgDataById } from '@fastgpt/service/core/dataset/data/pg';
+import { insertData2Pg, updatePgDataById } from './pg';
 import { Types } from 'mongoose';
 import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
@@ -213,29 +214,3 @@ export async function updateData2Dataset({
     tokenLen
   };
 }
-
-/* delete all data by datasetIds */
-export async function delDataByDatasetId({ datasetIds }: { datasetIds: string[] }) {
-  datasetIds = datasetIds.map((item) => String(item));
-  // delete pg data
-  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
-}
-/**
- * delete all data by collectionIds
- */
-export async function delDataByCollectionId({ collectionIds }: { collectionIds: string[] }) {
-  const ids = collectionIds.map((item) => String(item));
-  // delete pg data
-  await deletePgDataById(`collection_id IN ('${ids.join("','")}')`);
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ collectionId: { $in: ids } });
-}
-/**
- * delete one data by mongoDataId
- */
-export async function deleteDataByDataId(mongoDataId: string) {
-  await deletePgDataById(['data_id', mongoDataId]);
-  await MongoDatasetData.findByIdAndDelete(mongoDataId);
-}

projects/app/src/service/core/dataset/data/pg.ts

@@ -5,7 +5,7 @@ import type {
 } from '@fastgpt/global/core/dataset/type.d';
 import { PgClient } from '@fastgpt/service/common/pg';
 import { getVectorsByText } from '@/service/core/ai/vector';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { PgSearchRawType } from '@fastgpt/global/core/dataset/api';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
@@ -103,31 +103,6 @@ export async function updatePgDataById({
   return updatePg();
 }
 
-export async function deletePgDataById(
-  where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
-) {
-  let retry = 2;
-  async function deleteData(): Promise<any> {
-    try {
-      await PgClient.delete(PgDatasetTableName, {
-        where: [where]
-      });
-    } catch (error) {
-      if (--retry < 0) {
-        return Promise.reject(error);
-      }
-      await delay(500);
-      return deleteData();
-    }
-  }
-
-  await deleteData();
-
-  return {
-    tokenLen: 0
-  };
-}
-
 // ------------------ search start ------------------
 type SearchProps = {
   text: string;

projects/app/src/service/core/dataset/utils.ts

@@ -1,5 +1,6 @@
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { cut } from '@node-rs/jieba';
+import { stopWords } from '@fastgpt/global/common/string/jieba';
 
 /**
  * Same value judgment
@@ -30,7 +31,7 @@ export function jiebaSplit({ text }: { text: string }) {
   return (
     tokens
       .map((item) => item.replace(/[^\u4e00-\u9fa5a-zA-Z0-9\s]/g, '').trim())
-      .filter(Boolean)
+      .filter((item) => item && !stopWords.has(item))
       .join(' ') || ''
   );
 }