V4.12.3 features (#5595)

* refactor: remove ModelProviderIdType and update related types (#5549)

* perf: model provider

* fix eval create split (#5570)

* git rebase --continuedoc

* add more variable types (#5540)

* variable types

* password

* time picker

* internal var

* file

* fix-test

* time select default value & range

* password & type render

* fix

* fix build

* fix

* move method

* split date select

* icon

* perf: variable code

* prompt editor add markdown plugin (#5556)

* editor markdown

* fix build

* pnpm lock

* add props

* update code

* fix list

* editor ui

* fix variable reset (#5586)

* perf: variables type code

* customize lexical indent (#5588)

* perf: multiple selector

* perf: tab plugin

* doc

* refactor: update workflow constants to use ToolTypeEnum (#5491)

* refactor: replace FlowNodeTemplateTypeEnum with string literals in workflow templates

* perf: tool type

---------

Co-authored-by: archer <545436317@qq.com>

* update doc

* fix: make table's row more natural while dragging it (#5596)

* feat: add APIGetTemplate function and refactor template fetching logic (#5498)

* feat: add APIGetTemplate function and refactor template fetching logic

* chore: adjust the code

* chore: update sdk

---------

Co-authored-by: FinleyGe <m13203533462@163.com>

* perf init system

* doc

* remove log

* remove i18n

* perf: variables render

---------

Co-authored-by: Ctrlz <143257420+ctrlz526@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: 伍闲犬 <whoeverimf5@gmail.com>
Co-authored-by: FinleyGe <m13203533462@163.com>

packages/service/core/app/evaluation/utils.ts

@@ -0,0 +1,147 @@
+import { evaluationFileErrors } from '@fastgpt/global/core/app/evaluation/constants';
+import { getEvaluationFileHeader } from '@fastgpt/global/core/app/evaluation/utils';
+import type { VariableItemType } from '@fastgpt/global/core/app/type';
+import { addLog } from '../../../common/system/log';
+import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
+import Papa from 'papaparse';
+
+export const parseEvaluationCSV = (rawText: string) => {
+  const parseResult = Papa.parse(rawText.trim(), {
+    skipEmptyLines: true,
+    header: false,
+    transformHeader: (header: string) => header.trim()
+  });
+
+  if (parseResult.errors.length > 0) {
+    addLog.error('CSV parsing failed', parseResult.errors);
+    throw new Error('CSV parsing failed');
+  }
+
+  return parseResult.data as string[][];
+};
+
+export const validateEvaluationFile = async (
+  rawText: string,
+  appVariables?: VariableItemType[]
+) => {
+  // Parse CSV using Papa Parse
+  const csvData = parseEvaluationCSV(rawText);
+  const dataLength = csvData.length;
+
+  // Validate file header
+  const expectedHeader = getEvaluationFileHeader(appVariables);
+  const actualHeader = csvData[0]?.join(',') || '';
+  if (actualHeader !== expectedHeader) {
+    addLog.error(`Header mismatch. Expected: ${expectedHeader}, Got: ${actualHeader}`);
+    return Promise.reject(evaluationFileErrors);
+  }
+
+  // Validate data rows count
+  if (dataLength <= 1) {
+    addLog.error('No data rows found');
+    return Promise.reject(evaluationFileErrors);
+  }
+
+  const maxRows = 1000;
+  if (dataLength - 1 > maxRows) {
+    addLog.error(`Too many rows. Max: ${maxRows}, Got: ${dataLength - 1}`);
+    return Promise.reject(evaluationFileErrors);
+  }
+
+  const headers = csvData[0];
+
+  // Get required field indices
+  const requiredFields = headers
+    .map((header, index) => ({ header: header.trim(), index }))
+    .filter(({ header }) => header.startsWith('*'));
+
+  const errors: string[] = [];
+
+  // Validate each data row
+  for (let i = 1; i < csvData.length; i++) {
+    const values = csvData[i];
+
+    // Check required fields
+    requiredFields.forEach(({ header, index }) => {
+      if (!values[index]?.trim()) {
+        errors.push(`Row ${i + 1}: required field "${header}" is empty`);
+      }
+    });
+
+    // Validate app variables
+    if (appVariables) {
+      validateRowVariables({
+        values,
+        variables: appVariables,
+        rowNum: i + 1,
+        errors
+      });
+    }
+  }
+
+  if (errors.length > 0) {
+    addLog.error(`Validation failed: ${errors.join('; ')}`);
+    return Promise.reject(evaluationFileErrors);
+  }
+
+  return { csvData, dataLength };
+};
+
+const validateRowVariables = ({
+  values,
+  variables,
+  rowNum,
+  errors
+}: {
+  values: string[];
+  variables: VariableItemType[];
+  rowNum: number;
+  errors: string[];
+}) => {
+  variables.forEach((variable, index) => {
+    const value = values[index]?.trim();
+
+    // Skip validation if value is empty and not required
+    if (!value && !variable.required) return;
+
+    switch (variable.type) {
+      case VariableInputEnum.input:
+        // Validate string length
+        if (variable.maxLength && value && value.length > variable.maxLength) {
+          errors.push(
+            `Row ${rowNum}: "${variable.label}" exceeds max length (${variable.maxLength})`
+          );
+        }
+        break;
+
+      case VariableInputEnum.numberInput:
+        // Validate number type and range
+        if (value) {
+          const numValue = Number(value);
+          if (isNaN(numValue)) {
+            errors.push(`Row ${rowNum}: "${variable.label}" must be a number`);
+          } else {
+            if (variable.min !== undefined && numValue < variable.min) {
+              errors.push(`Row ${rowNum}: "${variable.label}" below minimum (${variable.min})`);
+            }
+            if (variable.max !== undefined && numValue > variable.max) {
+              errors.push(`Row ${rowNum}: "${variable.label}" exceeds maximum (${variable.max})`);
+            }
+          }
+        }
+        break;
+
+      case VariableInputEnum.select:
+        // Validate select options
+        if (value && variable.enums?.length) {
+          const validOptions = variable.enums.map((item) => item.value);
+          if (!validOptions.includes(value)) {
+            errors.push(
+              `Row ${rowNum}: "${variable.label}" invalid option. Valid: [${validOptions.join(', ')}]`
+            );
+          }
+        }
+        break;
+    }
+  });
+};

packages/service/core/app/plugin/controller.ts

@@ -504,17 +504,17 @@ const dbPluginFormat = (item: SystemPluginConfigSchemaType): SystemPluginTemplat
 
 /* FastsGPT-Pluign api: */
 function getCachedSystemPlugins() {
-  if (!global.systemPlugins_cache) {
-    global.systemPlugins_cache = {
+  if (!global.systemToolsCache) {
+    global.systemToolsCache = {
       expires: 0,
       data: [] as SystemPluginTemplateItemType[]
     };
   }
-  return global.systemPlugins_cache;
+  return global.systemToolsCache;
 }
 
 const cleanSystemPluginCache = () => {
-  global.systemPlugins_cache = undefined;
+  global.systemToolsCache = undefined;
 };
 
 export const refetchSystemPlugins = () => {
@@ -579,15 +579,20 @@ export const getSystemTools = async (): Promise<SystemPluginTemplateItemType[]>
       .filter((item) => item.customConfig?.associatedPluginId)
       .map((item) => dbPluginFormat(item));
 
-    const plugins = [...formatTools, ...dbPlugins];
-    plugins.sort((a, b) => (a.pluginOrder ?? 0) - (b.pluginOrder ?? 0));
+    const concatTools = [...formatTools, ...dbPlugins];
+    concatTools.sort((a, b) => (a.pluginOrder ?? 0) - (b.pluginOrder ?? 0));
 
-    global.systemPlugins_cache = {
+    global.systemToolsCache = {
       expires: Date.now() + 30 * 60 * 1000, // 30 minutes
-      data: plugins
+      data: concatTools
     };
 
-    return plugins;
+    global.systemToolsTypeCache = {};
+    concatTools.forEach((item) => {
+      global.systemToolsTypeCache[item.templateType] = 1;
+    });
+
+    return concatTools;
   }
 };
 
@@ -608,10 +613,11 @@ export const getSystemToolById = async (id: string): Promise<SystemPluginTemplat
 };
 
 declare global {
-  var systemPlugins_cache:
+  var systemToolsCache:
     | {
         expires: number;
         data: SystemPluginTemplateItemType[];
       }
     | undefined;
+  var systemToolsTypeCache: Record<string, 1>;
 }

packages/service/core/app/plugin/pluginGroupSchema.ts

@@ -1,5 +1,5 @@
 import { connectionMongo, getMongoModel } from '../../../common/mongo/index';
-import { type PluginGroupSchemaType, type TGroupType } from './type';
+import { type SystemToolGroupSchemaType, type TGroupType } from './type';
 const { Schema } = connectionMongo;
 
 export const collectionName = 'app_plugin_groups';
@@ -29,7 +29,7 @@ const PluginGroupSchema = new Schema({
 
 PluginGroupSchema.index({ groupId: 1 }, { unique: true });
 
-export const MongoPluginGroups = getMongoModel<PluginGroupSchemaType>(
+export const MongoToolGroups = getMongoModel<SystemToolGroupSchemaType>(
   collectionName,
   PluginGroupSchema
 );

packages/service/core/app/plugin/type.d.ts

@@ -2,6 +2,7 @@ import { SystemPluginListItemType } from '@fastgpt/global/core/app/type';
 import { FlowNodeTemplateTypeEnum } from '@fastgpt/global/core/workflow/constants';
 import type { WorkflowTemplateBasicType } from '@fastgpt/global/core/workflow/type';
 import type { InputConfigType } from '@fastgpt/global/core/workflow/type/io';
+import type { I18nStringStrictType } from '@fastgpt/global/sdk/fastgpt-plugin';
 
 export type SystemPluginConfigSchemaType = {
   pluginId: string;
@@ -38,11 +39,11 @@ export type SystemPluginConfigSchemaType = {
 };
 
 export type TGroupType = {
-  typeName: string;
+  typeName: I18nStringStrictType | string;
   typeId: string;
 };
 
-export type PluginGroupSchemaType = {
+export type SystemToolGroupSchemaType = {
   groupId: string;
   groupAvatar: string;
   groupName: string;

packages/service/core/app/provider/controller.ts

@@ -0,0 +1,29 @@
+import { loadModelProviders } from '../../../thirdProvider/fastgptPlugin/model';
+import {
+  type langType,
+  defaultProvider,
+  formatModelProviders
+} from '@fastgpt/global/core/ai/provider';
+
+// Preload model providers
+export async function preloadModelProviders(): Promise<void> {
+  const { modelProviders, aiproxyIdMap } = await loadModelProviders();
+
+  const { ModelProviderListCache, ModelProviderMapCache } = formatModelProviders(modelProviders);
+  global.ModelProviderRawCache = modelProviders;
+  global.ModelProviderListCache = ModelProviderListCache;
+  global.ModelProviderMapCache = ModelProviderMapCache;
+
+  global.aiproxyIdMapCache = aiproxyIdMap;
+}
+
+export const getModelProviders = (language = 'en') => {
+  return global.ModelProviderListCache[language as langType] || [];
+};
+export const getModelProvider = (provider?: string, language = 'en') => {
+  if (!provider) {
+    return defaultProvider;
+  }
+
+  return ModelProviderMapCache[language as langType][provider] ?? defaultProvider;
+};

packages/service/core/app/templates/register.ts

@@ -0,0 +1,69 @@
+import { isProduction } from '@fastgpt/global/common/system/constants';
+import { PluginSourceEnum } from '@fastgpt/global/core/app/plugin/constants';
+import { type AppTemplateSchemaType } from '@fastgpt/global/core/app/type';
+import { MongoAppTemplate } from './templateSchema';
+import { pluginClient } from '../../../thirdProvider/fastgptPlugin';
+
+const getFileTemplates = async (): Promise<AppTemplateSchemaType[]> => {
+  const res = await pluginClient.workflow.getTemplateList();
+  if (res.status === 200) return res.body as AppTemplateSchemaType[];
+  else return Promise.reject(res.body);
+};
+
+const getAppTemplates = async () => {
+  const communityTemplates = await getFileTemplates();
+
+  const dbTemplates = await MongoAppTemplate.find();
+
+  // Merge db data to community templates
+  const communityTemplateConfig = communityTemplates.map((template) => {
+    const config = dbTemplates.find((t) => t.templateId === template.templateId);
+
+    if (config) {
+      return {
+        ...template,
+        isActive: config.isActive ?? template.isActive,
+        tags: config.tags ?? template.tags,
+        userGuide: config.userGuide ?? template.userGuide,
+        isQuickTemplate: config.isQuickTemplate ?? template.isQuickTemplate,
+        order: config.order ?? template.order
+      };
+    }
+
+    return template;
+  });
+
+  const res = [
+    ...communityTemplateConfig,
+    ...dbTemplates.filter((t) => !isCommunityTemplate(t.templateId))
+  ].sort((a, b) => (a.order ?? 0) - (b.order ?? 0));
+
+  return res;
+};
+
+export const getAppTemplatesAndLoadThem = async (refresh = false) => {
+  if (isProduction && global.appTemplates && global.appTemplates.length > 0 && !refresh)
+    return global.appTemplates;
+
+  if (!global.appTemplates) {
+    global.appTemplates = [];
+  }
+
+  try {
+    const appTemplates = await getAppTemplates();
+    global.appTemplates = appTemplates;
+    return appTemplates;
+  } catch (error) {
+    // @ts-ignore
+    global.appTemplates = undefined;
+    return [];
+  }
+};
+
+export const isCommunityTemplate = (templateId: string) => {
+  return templateId.startsWith(PluginSourceEnum.community);
+};
+
+declare global {
+  var appTemplates: AppTemplateSchemaType[];
+}

packages/service/core/app/tool/api.ts

@@ -1,6 +1,9 @@
+import type { I18nStringStrictType, ToolTypeEnum } from '@fastgpt/global/sdk/fastgpt-plugin';
 import { RunToolWithStream } from '@fastgpt/global/sdk/fastgpt-plugin';
 import { PluginSourceEnum } from '@fastgpt/global/core/app/plugin/constants';
 import { pluginClient, BASE_URL, TOKEN } from '../../../thirdProvider/fastgptPlugin';
+import { addLog } from '../../../common/system/log';
+import { retryFn } from '@fastgpt/global/common/system/utils';
 
 export async function APIGetSystemToolList() {
   const res = await pluginClient.tool.list();
@@ -27,3 +30,24 @@ const runToolInstance = new RunToolWithStream({
   token: TOKEN
 });
 export const APIRunSystemTool = runToolInstance.run.bind(runToolInstance);
+
+// Tool Types Cache
+type SystemToolTypeItem = {
+  type: ToolTypeEnum;
+  name: I18nStringStrictType;
+};
+
+export const getSystemToolTypes = (): Promise<SystemToolTypeItem[]> => {
+  return retryFn(async () => {
+    const res = await pluginClient.tool.getType();
+
+    if (res.status === 200) {
+      const toolTypes = res.body || [];
+
+      return toolTypes;
+    }
+
+    addLog.error('Get system tool type error', res.body);
+    return [];
+  });
+};