V4.8.16 dev (#3431)

* feat: add feishu & yuque dataset (#3379)

* feat: add feishu & yuque dataset

* fix ts

* fix ts

* move type position

* fix

* fix: merge interface

* fix

* feat: dingtalk sso support (#3408)

* fix: optional sso state

* feat: dingtalk bot

* feat: dingtalk sso login

* chore: move i18n to user namespace

* feat: dingtalk bot integration (#3415)

* feat: dingtalk bot integration

* docs: config dingtalk bot

* feat:sear XNG服务 (#3413)

* feat:sear XNG服务

* 补充了courseUrl

* 添加了官方文档

* 错误时返回情况修正了一下

* Tracks (#3420)

* feat: node intro

* feat: add domain track

* dingding sso login

* perf: api dataset code and add doc

* feat: tracks

* feat: searXNG plugins

* fix: ts

* feat: delete node tracks (#3423)

* fix: dingtalk bot GET verification (#3424)

* 4.8.16 test: fix: plugin inputs render；fix: ui offset (#3426)

* fix: ui offset

* perf: dingding talk

* fix: plugin inputs render

* feat: menu all folder (#3429)

* fix: recall code

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>

packages/service/common/middle/tracks/schema.ts

@@ -0,0 +1,20 @@
+import { TrackSchemaType } from '@fastgpt/global/common/middle/tracks/type';
+import { getMongoModel, Schema } from '../../mongo';
+import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants';
+
+const TrackSchema = new Schema({
+  event: { type: String, required: true, enum: Object.values(TrackEnum) },
+  uid: String,
+  teamId: String,
+  tmbId: String,
+  createTime: { type: Date, default: () => new Date() },
+  data: Object
+});
+
+try {
+  TrackSchema.index({ event: 1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const TrackModel = getMongoModel<TrackSchemaType>('track', TrackSchema);

packages/service/common/middle/tracks/utils.ts

@@ -0,0 +1,62 @@
+import { PushTrackCommonType } from '@fastgpt/global/common/middle/tracks/type';
+import { TrackModel } from './schema';
+import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants';
+import { addLog } from '../../system/log';
+import { OAuthEnum } from '@fastgpt/global/support/user/constant';
+import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { getAppLatestVersion } from '../../../core/app/version/controller';
+
+const createTrack = ({ event, data }: { event: TrackEnum; data: Record<string, any> }) => {
+  if (!global.feConfigs?.isPlus) return;
+  addLog.info('Push tracks', {
+    event,
+    ...data
+  });
+
+  const { uid, teamId, tmbId, ...props } = data;
+
+  return TrackModel.create({
+    event,
+    uid,
+    teamId,
+    tmbId,
+    data: props
+  });
+};
+export const pushTrack = {
+  login: (data: PushTrackCommonType & { type: `${OAuthEnum}` | 'password' }) => {
+    return createTrack({
+      event: TrackEnum.login,
+      data
+    });
+  },
+  createApp: (data: PushTrackCommonType & { type: AppTypeEnum }) => {
+    return createTrack({
+      event: TrackEnum.createApp,
+      data
+    });
+  },
+  createDataset: (data: PushTrackCommonType & { type: DatasetTypeEnum }) => {
+    return createTrack({
+      event: TrackEnum.createDataset,
+      data
+    });
+  },
+  countAppNodes: async (data: PushTrackCommonType & { appId: string }) => {
+    try {
+      const { nodes } = await getAppLatestVersion(data.appId);
+      const nodeTypeList = nodes.map((node) => ({
+        type: node.flowNodeType,
+        pluginId: node.pluginId
+      }));
+      return createTrack({
+        event: TrackEnum.appNodes,
+        data: {
+          ...data,
+          nodeTypeList
+        }
+      });
+    } catch (error) {}
+  }
+};

packages/service/core/chat/chatSchema.ts

@@ -72,6 +72,7 @@ const ChatSchema = new Schema({
     type: Object,
     default: {}
   },
+  pluginInputs: Array,
   metadata: {
     //For special storage
     type: Object,

packages/service/core/chat/saveChat.ts

@@ -10,6 +10,7 @@ import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/
 import { AppChatConfigType } from '@fastgpt/global/core/app/type';
 import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 import { pushChatLog } from './pushChatLog';
+import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 
 type Props = {
   chatId: string;
@@ -62,6 +63,9 @@ export async function saveChat({
       systemConfigNode: getGuideModule(nodes),
       isPublicFetch: false
     });
+    const pluginInputs = nodes?.find(
+      (node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput
+    )?.inputs;
 
     await mongoSessionRun(async (session) => {
       const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany(
@@ -89,6 +93,7 @@ export async function saveChat({
             variableList,
             welcomeText,
             variables: variables || {},
+            pluginInputs,
             title: newTitle,
             source,
             shareId,

packages/service/core/dataset/read.ts

@@ -7,8 +7,9 @@ import { TextSplitProps, splitText2Chunks } from '@fastgpt/global/common/string/
 import axios from 'axios';
 import { readRawContentByFileBuffer } from '../../common/file/read/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
-import { APIFileServer } from '@fastgpt/global/core/dataset/apiDataset';
+import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
 import { useApiDatasetRequest } from './apiDataset/api';
+import { POST } from '../../common/api/plusRequest';
 
 export const readFileRawTextByUrl = async ({
   teamId,
@@ -53,7 +54,9 @@ export const readDatasetSourceRawText = async ({
   isQAImport,
   selector,
   externalFileId,
-  apiServer
+  apiServer,
+  feishuServer,
+  yuqueServer
 }: {
   teamId: string;
   type: DatasetSourceReadTypeEnum;
@@ -63,6 +66,8 @@ export const readDatasetSourceRawText = async ({
   selector?: string; // link selector
   externalFileId?: string; // external file dataset
   apiServer?: APIFileServer; // api dataset
+  feishuServer?: FeishuServer; // feishu dataset
+  yuqueServer?: YuqueServer; // yuque dataset
 }): Promise<string> => {
   if (type === DatasetSourceReadTypeEnum.fileLocal) {
     const { rawText } = await readFileContentFromMongo({
@@ -88,28 +93,45 @@ export const readDatasetSourceRawText = async ({
     });
     return rawText;
   } else if (type === DatasetSourceReadTypeEnum.apiFile) {
-    if (!apiServer) return Promise.reject('apiServer not found');
     const rawText = await readApiServerFileContent({
       apiServer,
+      feishuServer,
+      yuqueServer,
       apiFileId: sourceId,
       teamId
     });
     return rawText;
   }
-
   return '';
 };
 
 export const readApiServerFileContent = async ({
   apiServer,
+  feishuServer,
+  yuqueServer,
   apiFileId,
   teamId
 }: {
-  apiServer: APIFileServer;
+  apiServer?: APIFileServer;
+  feishuServer?: FeishuServer;
+  yuqueServer?: YuqueServer;
   apiFileId: string;
   teamId: string;
 }) => {
-  return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, apiFileId });
+  if (apiServer) {
+    return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, apiFileId });
+  }
+
+  if (feishuServer || yuqueServer) {
+    return POST<string>(`/core/dataset/systemApiDataset`, {
+      type: 'content',
+      feishuServer,
+      yuqueServer,
+      apiFileId
+    });
+  }
+
+  return Promise.reject('No apiServer or feishuServer or yuqueServer');
 };
 
 export const rawText2Chunks = ({

packages/service/core/dataset/schema.ts

@@ -90,6 +90,12 @@ const DatasetSchema = new Schema({
   apiServer: {
     type: Object
   },
+  feishuServer: {
+    type: Object
+  },
+  yuqueServer: {
+    type: Object
+  },
 
   autoSync: Boolean,
 

packages/service/core/workflow/dispatch/index.ts

@@ -42,8 +42,7 @@ import {
   filterWorkflowEdges,
   checkNodeRunStatus,
   textAdaptGptResponse,
-  replaceEditorVariable,
-  formatVariableValByType
+  replaceEditorVariable
 } from '@fastgpt/global/core/workflow/runtime/utils';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { dispatchRunTools } from './agent/runTool/index';

packages/service/support/permission/dataset/auth.ts

@@ -172,7 +172,7 @@ export async function authDatasetCollection({
     collection: CollectionWithDatasetType;
   }
 > {
-  const { teamId, tmbId, isRoot: isRootFromHeader } = await parseHeaderCert(props);
+  const { teamId, tmbId, userId, isRoot: isRootFromHeader } = await parseHeaderCert(props);
   const collection = await getCollectionWithDataset(collectionId);
 
   if (!collection) {
@@ -187,6 +187,7 @@ export async function authDatasetCollection({
   });
 
   return {
+    userId,
     teamId,
     tmbId,
     collection,

packages/service/support/permission/type.d.ts

@@ -24,6 +24,7 @@ type authModeType = {
 export type AuthModeType = RequireAtLeastOne<authModeType, 'authApiKey' | 'authRoot' | 'authToken'>;
 
 export type AuthResponseType<T extends Permission = Permission> = {
+  userId: string;
   teamId: string;
   tmbId: string;
   authType?: `${AuthUserTypeEnum}`;

packages/service/support/user/schema.ts

@@ -49,11 +49,7 @@ const UserSchema = new Schema({
     type: String,
     default: defaultAvatars[Math.floor(Math.random() * defaultAvatars.length)]
   },
-  inviterId: {
-    // 谁邀请注册的
-    type: Schema.Types.ObjectId,
-    ref: userCollectionName
-  },
+
   promotionRate: {
     type: Number,
     default: 15
@@ -71,9 +67,14 @@ const UserSchema = new Schema({
   lastLoginTmbId: {
     type: Schema.Types.ObjectId
   },
-  fastgpt_sem: {
-    type: Object
-  }
+
+  inviterId: {
+    // 谁邀请注册的
+    type: Schema.Types.ObjectId,
+    ref: userCollectionName
+  },
+  fastgpt_sem: Object,
+  sourceDomain: String
 });
 
 try {