4.8.4 (#1746)

packages/service/common/string/tiktoken/index.ts

@@ -70,7 +70,7 @@ export const countGptMessagesTokens = (
 
       callbackMap[id] = (data) => {
         // 检测是否有内存泄漏
-        addLog.info(`Count token time: ${Date.now() - start}, token: ${data}`);
+        addLog.debug(`Count token time: ${Date.now() - start}, token: ${data}`);
         // console.log(process.memoryUsage());
 
         resolve(data);

packages/service/common/system/log.ts

@@ -1,11 +1,12 @@
 import dayjs from 'dayjs';
 import chalk from 'chalk';
+import { isProduction } from './constants';
 
 enum LogLevelEnum {
-  debug = 'debug',
-  info = 'info',
-  warn = 'warn',
-  error = 'error'
+  debug = 0,
+  info = 1,
+  warn = 2,
+  error = 3
 }
 const logMap = {
   [LogLevelEnum.debug]: {
@@ -21,20 +22,35 @@ const logMap = {
     levelLog: chalk.red('[Error]')
   }
 };
+const envLogLevelMap: Record<string, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3
+};
+
+const logLevel = (() => {
+  if (!isProduction) return LogLevelEnum.debug;
+  const envLogLevel = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
+  if (!envLogLevel || envLogLevelMap[envLogLevel] === undefined) return LogLevelEnum.info;
+  return envLogLevelMap[envLogLevel];
+})();
 
 /* add logger */
 export const addLog = {
   log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) {
+    if (level < logLevel) return;
+
     const stringifyObj = JSON.stringify(obj);
     const isEmpty = Object.keys(obj).length === 0;
 
     console.log(
       `${logMap[level].levelLog} ${dayjs().format('YYYY-MM-DD HH:mm:ss')} ${msg} ${
-        level !== 'error' && !isEmpty ? stringifyObj : ''
+        level !== LogLevelEnum.error && !isEmpty ? stringifyObj : ''
       }`
     );
 
-    level === 'error' && console.error(obj);
+    level === LogLevelEnum.error && console.error(obj);
 
     const lokiUrl = process.env.LOKI_LOG_URL as string;
     if (!lokiUrl) return;

packages/service/core/dataset/collection/controller.ts

@@ -142,17 +142,17 @@ export const delCollectionRelatedSource = async ({
     .map((item) => item?.metadata?.relatedImgId || '')
     .filter(Boolean);
 
+  // delete files
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
+  });
   // delete images
   await delImgByRelatedId({
     teamId,
     relateIds: relatedImageIds,
     session
   });
-  // delete files
-  await delFileByFileIdList({
-    bucketName: BucketNameEnum.dataset,
-    fileIdList
-  });
 };
 /**
  * delete collection and it related data
@@ -182,14 +182,16 @@ export async function delCollectionAndRelatedSources({
   );
   const collectionIds = collections.map((item) => String(item._id));
 
-  await delCollectionRelatedSource({ collections, session });
-
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
     datasetIds: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
+
+  /* file and imgs */
+  await delCollectionRelatedSource({ collections, session });
+
   // delete dataset.datas
   await MongoDatasetData.deleteMany(
     { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
@@ -199,6 +201,7 @@ export async function delCollectionAndRelatedSources({
   // delete collections
   await MongoDatasetCollection.deleteMany(
     {
+      teamId,
       _id: { $in: collectionIds }
     },
     { session }

packages/service/core/dataset/collection/utils.ts

@@ -42,7 +42,7 @@ export async function findCollectionAndChild({
     return collections;
   }
   const [collection, childCollections] = await Promise.all([
-    MongoDatasetCollection.findById(collectionId, fields),
+    MongoDatasetCollection.findById(collectionId, fields).lean(),
     find(collectionId)
   ]);
 

packages/service/core/dataset/controller.ts

@@ -82,17 +82,18 @@ export async function delDatasetRelevantData({
       teamId,
       datasetId: { $in: datasetIds }
     },
-    '_id teamId fileId metadata'
+    '_id teamId datasetId fileId metadata'
   ).lean();
 
-  // image and file
-  await delCollectionRelatedSource({ collections, session });
-
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
     datasetId: { $in: datasetIds }
   });
+
+  // image and file
+  await delCollectionRelatedSource({ collections, session });
+
   // delete dataset.datas
   await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session });
 

packages/service/core/workflow/dispatch/index.ts

@@ -407,13 +407,13 @@ export function responseStatus({
 /* get system variable */
 export function getSystemVariable({
   user,
-  appId,
+  app,
   chatId,
   responseChatItemId,
   histories = []
 }: Props) {
   return {
-    appId,
+    appId: String(app._id),
     chatId,
     responseChatItemId,
     histories,

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -43,7 +43,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
   let {
     res,
     detail,
-    appId,
+    app: { _id: appId },
     chatId,
     stream,
     responseChatItemId,

packages/service/core/workflow/dispatch/tools/runApp.ts

@@ -33,8 +33,7 @@ type Response = DispatchNodeResultType<{
 export const dispatchAppRequest = async (props: Props): Promise<Response> => {
   const {
     res,
-    teamId,
-    tmbId,
+    app: workflowApp,
     stream,
     detail,
     histories,
@@ -46,10 +45,11 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
     return Promise.reject('Input is empty');
   }
 
+  // 检查该工作流的tmb是否有调用该app的权限（不是校验对话的人，是否有权限）
   const { app: appData } = await authAppByTmbId({
     appId: app.id,
-    teamId,
-    tmbId,
+    teamId: workflowApp.teamId,
+    tmbId: workflowApp.tmbId,
     per: ReadPermissionVal
   });
 
@@ -68,7 +68,7 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
 
   const { flowResponses, flowUsages, assistantResponses } = await dispatchWorkFlow({
     ...props,
-    appId: app.id,
+    app: appData,
     runtimeNodes: storeNodes2RuntimeNodes(appData.modules, getDefaultEntryNodeIds(appData.modules)),
     runtimeEdges: initWorkflowEdgeStatus(appData.edges),
     histories: chatHistories,

packages/service/core/workflow/dispatch/tools/runLaf.ts

@@ -21,7 +21,7 @@ const UNDEFINED_SIGN = 'UNDEFINED_SIGN';
 
 export const dispatchLafRequest = async (props: LafRequestProps): Promise<LafResponse> => {
   let {
-    appId,
+    app: { _id: appId },
     chatId,
     responseChatItemId,
     variables,

packages/service/support/permission/auth/dataset.ts

@@ -198,4 +198,4 @@ export async function authDatasetFile({
   } catch (error) {
     return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
   }
-}
+}

packages/service/support/permission/controller.ts

@@ -7,8 +7,6 @@ import { AuthUserTypeEnum, PerResourceTypeEnum } from '@fastgpt/global/support/p
 import { authOpenApiKey } from '../openapi/auth';
 import { FileTokenQuery } from '@fastgpt/global/common/file/type';
 import { MongoResourcePermission } from './schema';
-import { PermissionValueType } from '@fastgpt/global/support/permission/type';
-import { mongoSessionRun } from '../../common/mongo/sessionRun';
 
 export const getResourcePermission = async ({
   resourceType,
@@ -36,41 +34,6 @@ export const getResourcePermission = async ({
 export const delResourcePermissionById = (id: string) => {
   return MongoResourcePermission.findByIdAndRemove(id);
 };
-export const updateResourcePermission = async ({
-  resourceId,
-  resourceType,
-  teamId,
-  tmbIdList,
-  permission
-}: {
-  resourceId?: string;
-  resourceType: PerResourceTypeEnum;
-  teamId: string;
-  tmbIdList: string[];
-  permission: PermissionValueType;
-}) => {
-  await mongoSessionRun((session) => {
-    return Promise.all(
-      tmbIdList.map((tmbId) =>
-        MongoResourcePermission.findOneAndUpdate(
-          {
-            resourceType,
-            teamId,
-            tmbId,
-            resourceId
-          },
-          {
-            permission
-          },
-          {
-            session,
-            upsert: true
-          }
-        )
-      )
-    );
-  });
-};
 
 /* 下面代码等迁移 */
 /* create token */

packages/service/support/permission/schema.ts

@@ -45,6 +45,11 @@ try {
       unique: true
     }
   );
+  ResourcePermissionSchema.index({
+    resourceType: 1,
+    teamId: 1,
+    resourceId: 1
+  });
 } catch (error) {
   console.log(error);
 }

packages/service/support/user/team/controller.ts

@@ -9,10 +9,7 @@ import { MongoTeamMember } from './teamMemberSchema';
 import { MongoTeam } from './teamSchema';
 import { UpdateTeamProps } from '@fastgpt/global/support/user/team/controller';
 import { getResourcePermission } from '../../permission/controller';
-import {
-  PerResourceTypeEnum,
-  ReadPermissionVal
-} from '@fastgpt/global/support/permission/constant';
+import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { TeamPermission } from '@fastgpt/global/support/permission/user/controller';
 
 async function getTeamMember(match: Record<string, any>): Promise<TeamTmbItemType> {