4.8.19-feature (#3636)

* feat: sync org from wecom, pref: member list pagination (#3549)

* feat: sync org

* chore: fe

* chore: loading

* chore: type

* pref: team member list change to pagination. Edit a sort of list apis.

* feat: member update avatar

* chore: user avatar move to tmb

* chore: init scripts move user avatar

* chore: sourceMember

* fix: list api sourceMember

* fix: member sync

* fix: pagination

* chore: adjust code

* chore: move changeOwner to pro

* chore: init v4819 script

* chore: adjust code

* chore: UserBox

* perf: scroll page code

* perf: list data

* docs:更新用户答疑 (#3576)

* docs: add custom uid docs (#3572)

* fix: pagination bug (#3577)

* 4.8.19 test (#3584)

* faet: dataset search filter

* fix: scroll page

* fix: collection list api old version (#3591)

* fix: collection list api format

* fix: type error of addSourceMemeber

* fix: scroll fetch (#3592)

* fix: yuque dataset file folder can enter (#3593)

* perf: load members；perf: yuque load；fix: workflow llm params cannot close (#3594)

* chat openapi doc

* feat: dataset openapi doc

* perf: load members

* perf: member load code

* perf: yuque load

* fix: workflow llm params cannot close

* fix: api dataset reference tag preview (#3600)

* perf: doc

* feat: chat page config

* fix: http parse (#3634)

* update doc

* fix: http parse

* fix code run node reset template (#3633)

Co-authored-by: Archer <545436317@qq.com>

* docs:faq (#3627)

* docs:faq

* docsFix

* perf: sleep plugin

* fix: selector

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>

packages/service/common/api/pagination.ts

@@ -0,0 +1,21 @@
+import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { ApiRequestProps } from '../../type/next';
+
+export function parsePaginationRequest(req: ApiRequestProps) {
+  const {
+    pageSize = 10,
+    pageNum = 1,
+    offset = 0
+  } = Object.keys(req.body).includes('pageSize')
+    ? req.body
+    : Object.keys(req.query).includes('pageSize')
+      ? req.query
+      : {};
+  if (!pageSize || (pageNum === undefined && offset === undefined)) {
+    throw new Error(CommonErrEnum.missingParams);
+  }
+  return {
+    pageSize: Number(pageSize),
+    offset: offset ? Number(offset) : (Number(pageNum) - 1) * Number(pageSize)
+  };
+}

packages/service/common/middle/cors.ts

@@ -2,7 +2,7 @@ import type { NextApiResponse, NextApiRequest } from 'next';
 import NextCors from 'nextjs-cors';
 
 export async function withNextCors(req: NextApiRequest, res: NextApiResponse) {
-  const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
+  const methods = ['GET', 'HEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
 
   const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(',');
   const origin = req.headers.origin;

packages/service/core/dataset/apiDataset/api.ts

@@ -99,7 +99,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
     if (files.some((file) => !file.id || !file.name || typeof file.type === 'undefined')) {
       return Promise.reject('Invalid file data format');
     }
-    return files;
+
+    const formattedFiles = files.map((file) => ({
+      ...file,
+      hasChild: file.type === 'folder'
+    }));
+
+    return formattedFiles;
   };
 
   const getFileContent = async ({ teamId, apiFileId }: { teamId: string; apiFileId: string }) => {

packages/service/core/dataset/search/controller.ts

@@ -284,7 +284,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
         {
           _id: { $in: collectionIdList }
         },
-        '_id name fileId rawLink externalFileId externalFileUrl',
+        '_id name fileId rawLink apiFileId externalFileId externalFileUrl',
         { ...readFromSecondary }
       ).lean()
     ]);
@@ -525,7 +525,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
         {
           _id: { $in: searchResults.map((item) => item.collectionId) }
         },
-        '_id name fileId rawLink externalFileId externalFileUrl',
+        '_id name fileId rawLink apiFileId externalFileId externalFileUrl',
         { ...readFromSecondary }
       ).lean()
     ]);

packages/service/core/dataset/utils.ts

@@ -0,0 +1,39 @@
+import { getTmbInfoByTmbId } from '../../support/user/team/controller';
+import { getResourcePermission } from '../../support/permission/controller';
+import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
+
+// TODO: 需要优化成批量获取权限
+export const filterDatasetsByTmbId = async ({
+  datasetIds,
+  tmbId
+}: {
+  datasetIds: string[];
+  tmbId: string;
+}) => {
+  const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
+
+  // First get all permissions
+  const permissions = await Promise.all(
+    datasetIds.map(async (datasetId) => {
+      const per = await getResourcePermission({
+        teamId,
+        tmbId,
+        resourceId: datasetId,
+        resourceType: PerResourceTypeEnum.dataset
+      });
+
+      if (per === undefined) return false;
+
+      const datasetPer = new DatasetPermission({
+        per,
+        isOwner: tmbPer.isOwner
+      });
+
+      return datasetPer.hasReadPer;
+    })
+  );
+
+  // Then filter datasetIds based on permissions
+  return datasetIds.filter((_, index) => permissions[index]);
+};

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
 import { MongoDataset } from '../../../dataset/schema';
 import { i18nT } from '../../../../../web/i18n/utils';
+import { filterDatasetsByTmbId } from '../../../dataset/utils';
 
 type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSearchExtensionModel]: string;
   [NodeInputKeyEnum.datasetSearchExtensionBg]: string;
   [NodeInputKeyEnum.collectionFilterMatch]: string;
+  [NodeInputKeyEnum.authTmbId]: boolean;
 }>;
 export type DatasetSearchResponse = DispatchNodeResultType<{
   [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
 ): Promise<DatasetSearchResponse> {
   const {
     runningAppInfo: { teamId },
+    runningUserInfo: { tmbId },
     histories,
     node,
     params: {
@@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
       datasetSearchUsingExtensionQuery,
       datasetSearchExtensionModel,
       datasetSearchExtensionBg,
-      collectionFilterMatch
+      collectionFilterMatch,
+      authTmbId = false
     }
   } = props as DatasetSearchProps;
 
@@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
     return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
   }
 
+  const emptyResult = {
+    quoteQA: [],
+    [DispatchNodeResponseKeyEnum.nodeResponse]: {
+      totalPoints: 0,
+      query: '',
+      limit,
+      searchMode
+    },
+    nodeDispatchUsages: [],
+    [DispatchNodeResponseKeyEnum.toolResponses]: []
+  };
+
   if (!userChatInput) {
-    return {
-      quoteQA: [],
-      [DispatchNodeResponseKeyEnum.nodeResponse]: {
-        totalPoints: 0,
-        query: '',
-        limit,
-        searchMode
-      },
-      nodeDispatchUsages: [],
-      [DispatchNodeResponseKeyEnum.toolResponses]: []
-    };
+    return emptyResult;
   }
 
   // query extension
@@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
     ? getLLMModel(datasetSearchExtensionModel)
     : undefined;
 
-  const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
-    query: userChatInput,
-    extensionModel,
-    extensionBg: datasetSearchExtensionBg,
-    histories: getHistories(6, histories)
-  });
+  const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
+    datasetSearchQueryExtension({
+      query: userChatInput,
+      extensionModel,
+      extensionBg: datasetSearchExtensionBg,
+      histories: getHistories(6, histories)
+    }),
+    authTmbId
+      ? filterDatasetsByTmbId({
+          datasetIds: datasets.map((item) => item.datasetId),
+          tmbId
+        })
+      : Promise.resolve(datasets.map((item) => item.datasetId))
+  ]);
 
+  if (datasetIds.length === 0) {
+    return emptyResult;
+  }
   // console.log(concatQueries, rewriteQuery, aiExtensionResult);
 
   // get vector
@@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
     model: vectorModel.model,
     similarity,
     limit,
-    datasetIds: datasets.map((item) => item.datasetId),
+    datasetIds,
     searchMode,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
     collectionFilterMatch

packages/service/core/workflow/dispatch/plugin/run.ts

@@ -72,6 +72,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
       showStatus: false
     };
   });
+
   const runtimeVariables = {
     ...filterSystemVariables(props.variables),
     appId: String(plugin.id)

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -127,8 +127,16 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       if (typeof val === 'object') return JSON.stringify(val);
 
       if (typeof val === 'string') {
-        const str = JSON.stringify(val);
-        return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
+        try {
+          const parsed = JSON.parse(val);
+          if (typeof parsed === 'object') {
+            return JSON.stringify(parsed);
+          }
+          return val;
+        } catch (error) {
+          const str = JSON.stringify(val);
+          return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
+        }
       }
 
       return String(val);
@@ -235,7 +243,9 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       }
       if (!httpJsonBody) return {};
       if (httpContentType === ContentTypes.json) {
-        return json5.parse(replaceJsonBodyString(httpJsonBody));
+        httpJsonBody = replaceJsonBodyString(httpJsonBody);
+        console.log(httpJsonBody);
+        return json5.parse(httpJsonBody);
       }
 
       // Raw text, xml

packages/service/support/permission/controller.ts

@@ -17,7 +17,6 @@ import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MemberGroupSchemaType } from '@fastgpt/global/support/permission/memberGroup/type';
 import { TeamMemberSchema } from '@fastgpt/global/support/user/team/type';
-import { UserModelSchema } from '@fastgpt/global/support/user/type';
 import { OrgSchemaType } from '@fastgpt/global/support/user/team/org/type';
 import { getOrgIdSetWithParentByTmbId } from './org/controllers';
 
@@ -151,13 +150,9 @@ export const getClbsAndGroupsWithInfo = async ({
         $exists: true
       }
     })
-      .populate<{ tmb: TeamMemberSchema & { user: UserModelSchema } }>({
+      .populate<{ tmb: TeamMemberSchema }>({
         path: 'tmb',
-        select: 'name userId role',
-        populate: {
-          path: 'user',
-          select: 'avatar'
-        }
+        select: 'name userId avatar'
       })
       .lean(),
     MongoResourcePermission.find({

packages/service/support/permission/org/orgSchema.ts

@@ -47,14 +47,11 @@ export const OrgSchema = new Schema(
 OrgSchema.virtual('members', {
   ref: OrgMemberCollectionName,
   localField: '_id',
-  foreignField: 'orgId'
+  foreignField: 'orgId',
+  match: function (this: OrgSchemaType) {
+    return { teamId: this.teamId };
+  }
 });
-// OrgSchema.virtual('permission', {
-//   ref: ResourcePermissionCollectionName,
-//   localField: '_id',
-//   foreignField: 'orgId',
-//   justOne: true
-// });
 
 try {
   OrgSchema.index({

packages/service/support/user/controller.ts

@@ -41,7 +41,7 @@ export async function getUserDetail({
   return {
     _id: user._id,
     username: user.username,
-    avatar: user.avatar,
+    avatar: tmb.avatar,
     timezone: user.timezone,
     promotionRate: user.promotionRate,
     team: tmb,

packages/service/support/user/schema.ts

@@ -3,7 +3,6 @@ const { Schema } = connectionMongo;
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import type { UserModelSchema } from '@fastgpt/global/support/user/type';
 import { UserStatusEnum, userStatusMap } from '@fastgpt/global/support/user/constant';
-import { getRandomUserAvatar } from '@fastgpt/global/support/user/utils';
 
 export const userCollectionName = 'users';
 
@@ -33,11 +32,6 @@ const UserSchema = new Schema({
     type: Date,
     default: () => new Date()
   },
-  avatar: {
-    type: String,
-    default: () => getRandomUserAvatar()
-  },
-
   promotionRate: {
     type: Number,
     default: 15
@@ -62,7 +56,10 @@ const UserSchema = new Schema({
     ref: userCollectionName
   },
   fastgpt_sem: Object,
-  sourceDomain: String
+  sourceDomain: String,
+
+  /** @deprecated */
+  avatar: String
 });
 
 try {

packages/service/support/user/team/controller.ts

@@ -37,7 +37,7 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamTmbItemTyp
     teamAvatar: tmb.team.avatar,
     teamName: tmb.team.name,
     memberName: tmb.name,
-    avatar: tmb.team.avatar,
+    avatar: tmb.avatar,
     balance: tmb.team.balance,
     tmbId: String(tmb._id),
     teamDomain: tmb.team?.teamDomain,

packages/service/support/user/team/teamMemberSchema.ts

@@ -7,6 +7,7 @@ import {
   TeamMemberCollectionName,
   TeamCollectionName
 } from '@fastgpt/global/support/user/team/constant';
+import { getRandomUserAvatar } from '@fastgpt/global/support/user/utils';
 
 const TeamMemberSchema = new Schema({
   teamId: {
@@ -19,6 +20,10 @@ const TeamMemberSchema = new Schema({
     ref: userCollectionName,
     required: true
   },
+  avatar: {
+    type: String,
+    default: () => getRandomUserAvatar()
+  },
   name: {
     type: String,
     default: 'Member'
@@ -39,7 +44,6 @@ const TeamMemberSchema = new Schema({
   // Abandoned
   role: {
     type: String
-    // enum: Object.keys(TeamMemberRoleMap) // disable enum validation for old data
   }
 });
 

packages/service/support/user/utils.ts

@@ -1,4 +1,7 @@
+import { SourceMemberType } from '@fastgpt/global/support/user/type';
 import { MongoTeam } from './team/teamSchema';
+import { MongoTeamMember } from './team/teamMemberSchema';
+import { ClientSession } from '../../common/mongo';
 
 /* export dataset limit */
 export const updateExportDatasetLimit = async (teamId: string) => {
@@ -67,3 +70,41 @@ export const checkWebSyncLimit = async ({
     return Promise.reject(`每个团队，每 ${limitMinutes} 分钟仅使用一次同步功能。`);
   }
 };
+
+/**
+ * This function will add a property named sourceMember to the list passed in.
+ * @param list The list to add the sourceMember property to. [TmbId] property is required.
+ * @error If member is not found, this item will be skipped.
+ * @returns The list with the sourceMember property added.
+ */
+export async function addSourceMember<T extends { tmbId: string }>({
+  list,
+  session
+}: {
+  list: T[];
+  session?: ClientSession;
+}): Promise<Array<T & { sourceMember: SourceMemberType }>> {
+  if (!Array.isArray(list)) return [];
+
+  const tmbList = await MongoTeamMember.find(
+    {
+      _id: { $in: list.map((item) => String(item.tmbId)) }
+    },
+    'tmbId name avatar status',
+    {
+      session
+    }
+  ).lean();
+
+  return list
+    .map((item) => {
+      const tmb = tmbList.find((tmb) => String(tmb._id) === String(item.tmbId));
+      if (!tmb) return;
+
+      return {
+        ...item,
+        sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status }
+      };
+    })
+    .filter(Boolean) as Array<T & { sourceMember: SourceMemberType }>;
+}