4.13.1 features (#5728)

* fix(api): 修复二级路由下的页面判断逻辑

在请求错误处理中，添加基础URL前缀以正确判断当前是否为外部链接页面。

* perf: use global var

* remove invalid code

* feat: response limit;perf: copy avatar image;perf: markdown parse (#5719)

* feat: response limit

* remove placeholder

* perf: copy avatar image

* perf: markdown parse

* fix: child app cannot show cite

* doc

* fix: node template bugs (#5727)

* add dataset search count track (#5721)

* add dataset search count track

* remove pro

* change to track

* remove unused

* fix

* perf: track code

---------

Co-authored-by: archer <545436317@qq.com>

* http response limit

* deploy doc

* fix: test

* doc

* remove invalid code

* remove invalid code

---------

Co-authored-by: 戴盛利 <1639499287@qq.com>
Co-authored-by: heheer <heheer@sealos.io>

packages/global/common/middle/tracks/constants.ts

@@ -4,5 +4,6 @@ export enum TrackEnum {
   useAppTemplate = 'useAppTemplate',
   createDataset = 'createDataset',
   appNodes = 'appNodes',
-  runSystemTool = 'runSystemTool'
+  runSystemTool = 'runSystemTool',
+  datasetSearch = 'datasetSearch'
 }

packages/global/common/string/swagger.ts

@@ -3,13 +3,3 @@ import SwaggerParser from '@apidevtools/swagger-parser';
 export const loadOpenAPISchemaFromUrl = async (url: string) => {
   return SwaggerParser.bundle(url);
 };
-
-export const checkOpenAPISchemaValid = async (str: string) => {
-  try {
-    const res = await SwaggerParser.validate(JSON.parse(str));
-    console.log(res);
-    return !!res;
-  } catch (error) {
-    return false;
-  }
-};

packages/global/core/chat/utils.ts

@@ -92,10 +92,13 @@ export const filterPublicNodeResponseData = ({
   responseDetail?: boolean;
 }) => {
   const publicNodeMap: Record<string, any> = {
+    [FlowNodeTypeEnum.appModule]: true,
     [FlowNodeTypeEnum.pluginModule]: true,
     [FlowNodeTypeEnum.datasetSearchNode]: true,
     [FlowNodeTypeEnum.agent]: true,
-    [FlowNodeTypeEnum.pluginOutput]: true
+    [FlowNodeTypeEnum.pluginOutput]: true,
+
+    [FlowNodeTypeEnum.runApp]: true
   };
 
   const filedMap: Record<string, boolean> = responseDetail

packages/global/core/workflow/node/constant.ts

@@ -134,9 +134,6 @@ export enum FlowNodeTypeEnum {
   classifyQuestion = 'classifyQuestion',
   contentExtract = 'contentExtract',
   httpRequest468 = 'httpRequest468',
-  runApp = 'app',
-  appModule = 'appModule',
-  pluginModule = 'pluginModule',
   pluginInput = 'pluginInput',
   pluginOutput = 'pluginOutput',
   queryExtension = 'cfr',
@@ -156,7 +153,12 @@ export enum FlowNodeTypeEnum {
   loopEnd = 'loopEnd',
   formInput = 'formInput',
   tool = 'tool',
-  toolSet = 'toolSet'
+  toolSet = 'toolSet',
+
+  // child:
+  appModule = 'appModule',
+  pluginModule = 'pluginModule',
+  runApp = 'app'
 }
 
 // node IO value type

packages/global/core/workflow/utils.ts

@@ -276,7 +276,7 @@ export const appData2FlowNodeIO = ({
           description: '',
           valueType: WorkflowIOValueTypeEnum.any,
           required: item.required,
-          list: item.enums?.map((enumItem) => ({
+          list: (item.list || item.enums)?.map((enumItem) => ({
             label: enumItem.value,
             value: enumItem.value
           }))

packages/service/common/cache/index.ts

@@ -3,6 +3,7 @@ import { getGlobalRedisConnection } from '../../common/redis';
 import type { SystemCacheKeyEnum } from './type';
 import { randomUUID } from 'node:crypto';
 import { initCache } from './init';
+import { isProduction } from '@fastgpt/global/common/system/constants';
 
 const cachePrefix = `VERSION_KEY:`;
 
@@ -48,13 +49,15 @@ export const getCachedData = async <T extends SystemCacheKeyEnum>(key: T, id?: s
   const versionKey = await getVersionKey(key, id);
   const isDisableCache = process.env.DISABLE_CACHE === 'true';
 
+  const item = global.systemCache[key];
+
   // 命中缓存
-  if (global.systemCache[key].versionKey === versionKey && !isDisableCache) {
-    return global.systemCache[key].data;
+  if ((isProduction || !item.devRefresh) && item.versionKey === versionKey && !isDisableCache) {
+    return item.data;
   }
 
-  const refreshedData = await global.systemCache[key].refreshFunc();
-  global.systemCache[key].data = refreshedData;
-  global.systemCache[key].versionKey = versionKey;
-  return global.systemCache[key].data;
+  const refreshedData = await item.refreshFunc();
+  item.data = refreshedData;
+  item.versionKey = versionKey;
+  return item.data;
 };

packages/service/common/cache/init.ts

@@ -6,7 +6,8 @@ export const initCache = () => {
     [SystemCacheKeyEnum.systemTool]: {
       versionKey: '',
       data: [],
-      refreshFunc: refreshSystemTools
+      refreshFunc: refreshSystemTools,
+      devRefresh: true
     },
     [SystemCacheKeyEnum.modelPermission]: {
       versionKey: '',

packages/service/common/cache/type.ts

@@ -15,6 +15,7 @@ type SystemCacheType = {
     versionKey: string;
     data: SystemCacheDataType[K];
     refreshFunc: () => Promise<SystemCacheDataType[K]>;
+    devRefresh?: boolean;
   };
 };
 

packages/service/common/file/image/controller.ts

@@ -55,6 +55,46 @@ export async function uploadMongoImg({
 
   return `${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(_id)}.${extension}`;
 }
+export const copyImage = async ({
+  teamId,
+  imageUrl,
+  session
+}: {
+  teamId: string;
+  imageUrl: string;
+  session?: ClientSession;
+}) => {
+  const imageId = getIdFromPath(imageUrl);
+  if (!imageId) return imageUrl;
+
+  const image = await MongoImage.findOne(
+    {
+      _id: imageId,
+      teamId
+    },
+    undefined,
+    {
+      session
+    }
+  );
+  if (!image) return imageUrl;
+
+  const [newImage] = await MongoImage.create(
+    [
+      {
+        teamId,
+        binary: image.binary,
+        metadata: image.metadata
+      }
+    ],
+    {
+      session,
+      ordered: true
+    }
+  );
+
+  return `${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(newImage._id)}.${image.metadata?.mime?.split('/')[1]}`;
+};
 
 const getIdFromPath = (path?: string) => {
   if (!path) return;

packages/service/common/middle/tracks/processor.ts

@@ -0,0 +1,79 @@
+import { delay } from '@fastgpt/global/common/system/utils';
+import { addLog } from '../../system/log';
+import { TrackModel } from './schema';
+import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants';
+
+const batchUpdateTime = Number(process.env.TRACK_BATCH_UPDATE_TIME || 10000);
+
+const getCurrentTenMinuteBoundary = () => {
+  const now = new Date();
+  const minutes = now.getMinutes();
+  const tenMinuteBoundary = Math.floor(minutes / 10) * 10;
+
+  const boundary = new Date(now);
+  boundary.setMinutes(tenMinuteBoundary, 0, 0);
+  return boundary;
+};
+
+export const trackTimerProcess = async () => {
+  while (true) {
+    await countTrackTimer();
+    await delay(batchUpdateTime);
+  }
+};
+
+export const countTrackTimer = async () => {
+  if (!global.countTrackQueue || global.countTrackQueue.size === 0) {
+    return;
+  }
+
+  const queuedItems = Array.from(global.countTrackQueue.values());
+  global.countTrackQueue = new Map();
+
+  try {
+    const currentBoundary = getCurrentTenMinuteBoundary();
+
+    const bulkOps = queuedItems
+      .map(({ event, count, data }) => {
+        if (event === TrackEnum.datasetSearch) {
+          const { teamId, datasetId } = data;
+
+          return [
+            {
+              updateOne: {
+                filter: {
+                  event,
+                  teamId,
+                  createTime: currentBoundary,
+                  'data.datasetId': datasetId
+                },
+                update: [
+                  {
+                    $set: {
+                      event,
+                      teamId,
+                      createTime: { $ifNull: ['$createTime', currentBoundary] },
+                      data: {
+                        datasetId,
+                        count: { $add: [{ $ifNull: ['$data.count', 0] }, count] }
+                      }
+                    }
+                  }
+                ],
+                upsert: true
+              }
+            }
+          ];
+        }
+        return [];
+      })
+      .flat();
+
+    if (bulkOps.length > 0) {
+      await TrackModel.bulkWrite(bulkOps);
+      addLog.info('Track timer processing success');
+    }
+  } catch (error) {
+    addLog.error('Track timer processing error', error);
+  }
+};

packages/service/common/middle/tracks/schema.ts

@@ -13,6 +13,15 @@ const TrackSchema = new Schema({
 
 try {
   TrackSchema.index({ event: 1 });
+
+  TrackSchema.index(
+    { event: 1, teamId: 1, 'data.datasetId': 1, createTime: -1 },
+    {
+      partialFilterExpression: {
+        'data.datasetId': { $exists: true }
+      }
+    }
+  );
 } catch (error) {
   console.log(error);
 }

packages/service/common/middle/tracks/type.d.ts

@@ -0,0 +1,17 @@
+export type TracksQueueType = {
+  event: TrackEnum;
+  data: Record<string, any>;
+};
+
+declare global {
+  var countTrackQueue:
+    | Map<
+        string,
+        {
+          event: TrackEnum;
+          count: number;
+          data: Record<string, any>;
+        }
+      >
+    | undefined;
+}

packages/service/common/middle/tracks/utils.ts

@@ -25,6 +25,42 @@ const createTrack = ({ event, data }: { event: TrackEnum; data: Record<string, a
     data: props
   });
 };
+
+// Run times
+const pushCountTrack = ({
+  event,
+  key,
+  data
+}: {
+  event: TrackEnum;
+  key: string;
+  data: Record<string, any>;
+}) => {
+  if (!global.feConfigs?.isPlus) return;
+  addLog.debug('Push tracks', {
+    event,
+    key
+  });
+
+  if (!global.countTrackQueue) {
+    global.countTrackQueue = new Map();
+  }
+
+  const value = global.countTrackQueue.get(key);
+  if (value) {
+    global.countTrackQueue.set(key, {
+      ...value,
+      count: value.count + 1
+    });
+  } else {
+    global.countTrackQueue.set(key, {
+      event,
+      data,
+      count: 1
+    });
+  }
+};
+
 export const pushTrack = {
   login: (data: PushTrackCommonType & { type: `${OAuthEnum}` | 'password' }) => {
     return createTrack({
@@ -73,5 +109,18 @@ export const pushTrack = {
       event: TrackEnum.runSystemTool,
       data
     });
+  },
+  datasetSearch: (data: { teamId: string; datasetIds: string[] }) => {
+    if (!data.teamId) return;
+    data.datasetIds.forEach((datasetId) => {
+      pushCountTrack({
+        event: TrackEnum.datasetSearch,
+        key: `${TrackEnum.datasetSearch}_${datasetId}`,
+        data: {
+          teamId: data.teamId,
+          datasetId
+        }
+      });
+    });
   }
 };

packages/service/common/system/constants.ts

@@ -6,3 +6,6 @@ export const isFastGPTProService = () => !!global.systemConfig;
 export const isProVersion = () => {
   return !!global.feConfigs?.isPlus;
 };
+
+export const serviceRequestMaxContentLength =
+  Number(process.env.SERVICE_REQUEST_MAX_CONTENT_LENGTH || 10) * 1024 * 1024; // 10MB

packages/service/core/app/plugin/controller.ts

@@ -549,6 +549,7 @@ const dbPluginFormat = (item: SystemPluginConfigSchemaType): SystemPluginTemplat
 /* FastsGPT-Pluign api: */
 export const refreshSystemTools = async (): Promise<SystemPluginTemplateItemType[]> => {
   const tools = await APIGetSystemToolList();
+
   // 从数据库里加载插件配置进行替换
   const systemToolsArray = await MongoSystemPlugin.find({}).lean();
   const systemTools = new Map(systemToolsArray.map((plugin) => [plugin.pluginId, plugin]));
@@ -596,7 +597,7 @@ export const refreshSystemTools = async (): Promise<SystemPluginTemplateItemType
     .map((item) => dbPluginFormat(item));
 
   const concatTools = [...formatTools, ...dbPlugins];
-  concatTools.sort((a, b) => (a.pluginOrder ?? 0) - (b.pluginOrder ?? 0));
+  concatTools.sort((a, b) => (a.pluginOrder ?? 999) - (b.pluginOrder ?? 999));
 
   global.systemToolsTypeCache = {};
   concatTools.forEach((item) => {

packages/service/core/dataset/search/controller.ts

@@ -32,10 +32,13 @@ import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
 import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { formatDatasetDataValue } from '../data/controller';
+import { pushTrack } from '../../../common/middle/tracks/utils';
 
 export type SearchDatasetDataProps = {
   histories: ChatItemType[];
   teamId: string;
+  uid?: string;
+  tmbId?: string;
   model: string;
   datasetIds: string[];
   reRankQuery: string;
@@ -900,6 +903,8 @@ export async function searchDatasetData(
   // token filter
   const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens);
 
+  pushTrack.datasetSearch({ datasetIds, teamId });
+
   return {
     searchRes: filterMaxTokensResult,
     embeddingTokens,

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -52,6 +52,7 @@ export async function dispatchDatasetSearch(
   const {
     runningAppInfo: { teamId },
     runningUserInfo: { tmbId },
+    uid,
     histories,
     node,
     params: {

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -27,6 +27,7 @@ import { addLog } from '../../../../common/system/log';
 import { SERVICE_LOCAL_HOST } from '../../../../common/system/tools';
 import { formatHttpError } from '../utils';
 import { isInternalAddress } from '../../../../common/system/utils';
+import { serviceRequestMaxContentLength } from '../../../../common/system/constants';
 
 type PropsArrType = {
   key: string;
@@ -505,6 +506,7 @@ async function fetchData({
 
   const { data: response } = await axios({
     method,
+    maxContentLength: serviceRequestMaxContentLength,
     baseURL: `http://${SERVICE_LOCAL_HOST}`,
     url,
     headers: {

packages/web/common/system/utils.ts

@@ -10,11 +10,10 @@ export const subRoute = process.env.NEXT_PUBLIC_BASE_URL;
 
 export const getWebReqUrl = (url: string = '') => {
   if (!url) return '/';
-  const baseUrl = process.env.NEXT_PUBLIC_BASE_URL;
-  if (!baseUrl) return url;
+  if (!subRoute) return url;
 
-  if (!url.startsWith('/') || url.startsWith(baseUrl)) return url;
-  return `${baseUrl}${url}`;
+  if (!url.startsWith('/') || url.startsWith(subRoute)) return url;
+  return `${subRoute}${url}`;
 };
 
 export const isMobile = () => {