4.6.3-website dataset (#532)

projects/app/src/service/common/api/request.ts

@@ -96,18 +96,18 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
  * @param {Object} config
  * @returns
  */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
   return request(url, params, config, 'GET');
 }
 
-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'POST');
 }
 
-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'PUT');
 }
 
-export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'DELETE');
 }

projects/app/src/service/core/ai/rerank.ts

@@ -20,7 +20,8 @@ export function reRankRecall({ query, inputs }: PostReRankProps) {
         Authorization: `Bearer ${model.requestAuth}`
       }
     }
-  ).finally(() => {
+  ).then((data) => {
     console.log('rerank time:', Date.now() - start);
+    return data;
   });
 }

projects/app/src/service/core/dataset/data/controller.ts

@@ -4,7 +4,8 @@ import {
   PatchIndexesProps,
   UpdateDatasetDataProps
 } from '@fastgpt/global/core/dataset/controller';
-import { deletePgDataById, insertData2Pg, updatePgDataById } from './pg';
+import { deletePgDataById } from '@fastgpt/service/core/dataset/data/pg';
+import { insertData2Pg, updatePgDataById } from './pg';
 import { Types } from 'mongoose';
 import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
@@ -213,29 +214,3 @@ export async function updateData2Dataset({
     tokenLen
   };
 }
-
-/* delete all data by datasetIds */
-export async function delDataByDatasetId({ datasetIds }: { datasetIds: string[] }) {
-  datasetIds = datasetIds.map((item) => String(item));
-  // delete pg data
-  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
-}
-/**
- * delete all data by collectionIds
- */
-export async function delDataByCollectionId({ collectionIds }: { collectionIds: string[] }) {
-  const ids = collectionIds.map((item) => String(item));
-  // delete pg data
-  await deletePgDataById(`collection_id IN ('${ids.join("','")}')`);
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ collectionId: { $in: ids } });
-}
-/**
- * delete one data by mongoDataId
- */
-export async function deleteDataByDataId(mongoDataId: string) {
-  await deletePgDataById(['data_id', mongoDataId]);
-  await MongoDatasetData.findByIdAndDelete(mongoDataId);
-}

projects/app/src/service/core/dataset/data/pg.ts

@@ -5,7 +5,7 @@ import type {
 } from '@fastgpt/global/core/dataset/type.d';
 import { PgClient } from '@fastgpt/service/common/pg';
 import { getVectorsByText } from '@/service/core/ai/vector';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { PgSearchRawType } from '@fastgpt/global/core/dataset/api';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
@@ -103,31 +103,6 @@ export async function updatePgDataById({
   return updatePg();
 }
 
-export async function deletePgDataById(
-  where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
-) {
-  let retry = 2;
-  async function deleteData(): Promise<any> {
-    try {
-      await PgClient.delete(PgDatasetTableName, {
-        where: [where]
-      });
-    } catch (error) {
-      if (--retry < 0) {
-        return Promise.reject(error);
-      }
-      await delay(500);
-      return deleteData();
-    }
-  }
-
-  await deleteData();
-
-  return {
-    tokenLen: 0
-  };
-}
-
 // ------------------ search start ------------------
 type SearchProps = {
   text: string;

projects/app/src/service/core/dataset/utils.ts

@@ -1,5 +1,6 @@
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { cut } from '@node-rs/jieba';
+import { stopWords } from '@fastgpt/global/common/string/jieba';
 
 /**
  * Same value judgment
@@ -30,7 +31,7 @@ export function jiebaSplit({ text }: { text: string }) {
   return (
     tokens
       .map((item) => item.replace(/[^\u4e00-\u9fa5a-zA-Z0-9\s]/g, '').trim())
-      .filter(Boolean)
+      .filter((item) => item && !stopWords.has(item))
       .join(' ') || ''
   );
 }

projects/app/src/service/moduleDispatch/chat/oneapi.ts

@@ -113,10 +113,14 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
           }
         ]
       : []),
-    ...messages.map((item) => ({
-      ...item,
-      content: modelConstantsData.vision ? formatStr2ChatContent(item.content) : item.content
-    }))
+    ...(await Promise.all(
+      messages.map(async (item) => ({
+        ...item,
+        content: modelConstantsData.vision
+          ? await formatStr2ChatContent(item.content)
+          : item.content
+      }))
+    ))
   ];
 
   const response = await ai.chat.completions.create(

projects/app/src/service/moduleDispatch/index.ts

@@ -25,28 +25,29 @@ import { dispatchAppRequest } from './tools/runApp';
 import { dispatchRunPlugin } from './plugin/run';
 import { dispatchPluginInput } from './plugin/runInput';
 import { dispatchPluginOutput } from './plugin/runOutput';
+import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 
 /* running */
 export async function dispatchModules({
   res,
-  appId,
-  chatId,
-  modules,
-  user,
   teamId,
   tmbId,
+  user,
+  appId,
+  modules,
+  chatId,
   params = {},
   variables = {},
   stream = false,
   detail = false
 }: {
   res: NextApiResponse;
-  appId: string;
-  chatId?: string;
-  modules: ModuleItemType[];
-  user: UserType;
   teamId: string;
   tmbId: string;
+  user: UserType;
+  appId: string;
+  modules: ModuleItemType[];
+  chatId?: string;
   params?: Record<string, any>;
   variables?: Record<string, any>;
   stream?: boolean;
@@ -176,15 +177,15 @@ export async function dispatchModules({
     });
     const props: ModuleDispatchProps<Record<string, any>> = {
       res,
+      teamId,
+      tmbId,
+      user,
       appId,
       chatId,
       stream,
       detail,
       variables,
       outputs: module.outputs,
-      user,
-      teamId,
-      tmbId,
       inputs: params
     };
 

projects/app/src/service/support/permission/auth/user.ts

@@ -35,13 +35,11 @@ export async function authUser({
     user: UserType;
   }
 > {
-  const { userId, teamId, tmbId } = await parseHeaderCert(props);
+  const result = await parseHeaderCert(props);
 
   return {
-    userId,
-    teamId,
-    tmbId,
-    user: await getUserAndAuthBalance({ tmbId, minBalance }),
+    ...result,
+    user: await getUserAndAuthBalance({ tmbId: result.tmbId, minBalance }),
     isOwner: true,
     canWrite: true
   };

projects/app/src/service/support/wallet/bill/push.ts

@@ -12,14 +12,18 @@ export function createBill(data: CreateBillProps) {
   if (data.total === 0) {
     addLog.info('0 Bill', data);
   }
-  POST('/support/wallet/bill/createBill', data);
+  try {
+    POST('/support/wallet/bill/createBill', data);
+  } catch (error) {}
 }
 export function concatBill(data: ConcatBillProps) {
   if (!global.systemEnv.pluginBaseUrl) return;
   if (data.total === 0) {
     addLog.info('0 Bill', data);
   }
-  POST('/support/wallet/bill/concatBill', data);
+  try {
+    POST('/support/wallet/bill/concatBill', data);
+  } catch (error) {}
 }
 
 export const pushChatBill = ({
@@ -92,7 +96,7 @@ export const pushQABill = async ({
   return { total };
 };
 
-export const pushGenerateVectorBill = async ({
+export const pushGenerateVectorBill = ({
   billId,
   teamId,
   tmbId,
@@ -250,7 +254,7 @@ export function pushReRankBill({
   source: `${BillSourceEnum}`;
 }) {
   const model = global.reRankModels[0];
-  if (!model) return;
+  if (!model) return { total: 0 };
 
   const total = model.price * PRICE_SCALE;
   const name = 'wallet.bill.ReRank';
@@ -270,4 +274,6 @@ export function pushReRankBill({
       }
     ]
   });
+
+  return { total };
 }