4.6.8-production (#822)

* Json completion (#16)

* json-completion

* fix duplicate

* fix

* fix: config json

* feat: query extension

* perf: i18n

* 468 doc

* json editor

* perf: doc

* perf: default extension model

* docker file

* doc

* perf: token count

* perf: search extension

* format

* perf: some constants data

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/string/tiktoken/index.ts

@@ -34,11 +34,6 @@ export function countPromptTokens(
   const enc = getTikTokenEnc();
   const text = `${role}\n${prompt}`;
 
-  // too large a text will block the thread
-  if (text.length > 15000) {
-    return text.length * 1.7;
-  }
-
   try {
     const encodeText = enc.encode(text);
     return encodeText.length + role.length; // 补充 role 估算值

packages/global/core/app/type.d.ts

@@ -50,7 +50,7 @@ export type AppDetailType = AppSchema & {
 // };
 // Since useform cannot infer enumeration types, all enumeration keys can only be undone manually
 export type AppSimpleEditFormType = {
-  templateId: string;
+  // templateId: string;
   aiSettings: {
     model: string;
     systemPrompt?: string | undefined;
@@ -62,14 +62,14 @@ export type AppSimpleEditFormType = {
   };
   dataset: {
     datasets: SelectedDatasetType;
-    similarity: number;
-    limit: number;
     searchMode: `${DatasetSearchModeEnum}`;
-    usingReRank: boolean;
-    searchEmptyText: string;
-  };
-  cfr: {
-    background: string;
+    similarity?: number;
+    limit?: number;
+    usingReRank?: boolean;
+    searchEmptyText?: string;
+    datasetSearchUsingExtensionQuery?: boolean;
+    datasetSearchExtensionModel?: string;
+    datasetSearchExtensionBg?: string;
   };
   userGuide: {
     welcomeText: string;
@@ -116,9 +116,6 @@ export type AppSimpleEditConfigTemplateType = {
       usingReRank: boolean;
       searchEmptyText?: boolean;
     };
-    cfr?: {
-      background?: boolean;
-    };
     userGuide?: {
       welcomeText?: boolean;
       variables?: boolean;

packages/global/core/app/utils.ts

@@ -6,9 +6,8 @@ import { getGuideModule, splitGuideModule } from '../module/utils';
 import { ModuleItemType } from '../module/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 
-export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => {
+export const getDefaultAppForm = (): AppSimpleEditFormType => {
   return {
-    templateId,
     aiSettings: {
       model: 'gpt-3.5-turbo',
       systemPrompt: '',
@@ -18,16 +17,15 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
       quoteTemplate: '',
       maxToken: 4000
     },
-    cfr: {
-      background: ''
-    },
     dataset: {
       datasets: [],
       similarity: 0.4,
       limit: 1500,
       searchEmptyText: '',
       searchMode: DatasetSearchModeEnum.embedding,
-      usingReRank: false
+      usingReRank: false,
+      datasetSearchUsingExtensionQuery: true,
+      datasetSearchExtensionBg: ''
     },
     userGuide: {
       welcomeText: '',
@@ -41,14 +39,8 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
 };
 
 /* format app modules to edit form */
-export const appModules2Form = ({
-  templateId,
-  modules
-}: {
-  modules: ModuleItemType[];
-  templateId: string;
-}) => {
-  const defaultAppForm = getDefaultAppForm(templateId);
+export const appModules2Form = ({ modules }: { modules: ModuleItemType[] }) => {
+  const defaultAppForm = getDefaultAppForm();
 
   const findInputValueByKey = (inputs: FlowNodeInputItemType[], key: string) => {
     return inputs.find((item) => item.key === key)?.value;
@@ -100,6 +92,18 @@ export const appModules2Form = ({
         module.inputs,
         ModuleInputKeyEnum.datasetSearchUsingReRank
       );
+      defaultAppForm.dataset.datasetSearchUsingExtensionQuery = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSearchUsingExtensionQuery
+      );
+      defaultAppForm.dataset.datasetSearchExtensionModel = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSearchExtensionModel
+      );
+      defaultAppForm.dataset.datasetSearchExtensionBg = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSearchExtensionBg
+      );
 
       // empty text
       const emptyOutputs =
@@ -121,11 +125,6 @@ export const appModules2Form = ({
         questionGuide: questionGuide,
         tts: ttsConfig
       };
-    } else if (module.flowType === FlowNodeTypeEnum.cfr) {
-      const value = module.inputs.find((item) => item.key === ModuleInputKeyEnum.aiSystemPrompt);
-      if (value) {
-        defaultAppForm.cfr.background = value.value;
-      }
     }
   });
 

packages/global/core/chat/type.d.ts

@@ -109,6 +109,8 @@ export type moduleDispatchResType = {
   limit?: number;
   searchMode?: `${DatasetSearchModeEnum}`;
   searchUsingReRank?: boolean;
+  extensionModel?: string;
+  extensionResult?: string;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];

packages/global/core/module/constants.ts

@@ -64,7 +64,9 @@ export enum ModuleInputKeyEnum {
   datasetMaxTokens = 'limit',
   datasetSearchMode = 'searchMode',
   datasetSearchUsingReRank = 'usingReRank',
-  datasetParamsModal = 'datasetParamsModal',
+  datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
+  datasetSearchExtensionModel = 'datasetSearchExtensionModel',
+  datasetSearchExtensionBg = 'datasetSearchExtensionBg',
 
   // context extract
   contextExtractInput = 'content',

packages/global/core/module/template/system/coreferenceResolution.ts

@@ -19,11 +19,11 @@ import { Output_Template_UserChatInput } from '../output';
 
 export const AiCFR: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.chatNode,
-  templateType: ModuleTemplateTypeEnum.tools,
+  templateType: ModuleTemplateTypeEnum.other,
   flowType: FlowNodeTypeEnum.cfr,
   avatar: '/imgs/module/cfr.svg',
-  name: 'core.module.template.cfr',
-  intro: 'core.module.template.cfr intro',
+  name: 'core.module.template.Query extension',
+  intro: '该模块已合并到知识库搜索参数中，无需单独使用。',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
@@ -39,11 +39,11 @@ export const AiCFR: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiSystemPrompt,
       type: FlowNodeInputTypeEnum.textarea,
-      label: 'core.module.input.label.cfr background',
+      label: 'core.module.input.label.Background',
       max: 300,
       valueType: ModuleIOValueTypeEnum.string,
-      description: 'core.app.edit.cfr background tip',
-      placeholder: 'core.module.input.placeholder.cfr background',
+      description: 'core.app.edit.Query extension background tip',
+      placeholder: 'core.module.QueryExtension.placeholder',
       showTargetInApp: true,
       showTargetInPlugin: true
     },

packages/global/core/module/template/system/datasetSearch.ts

@@ -37,17 +37,10 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.datasetSimilarity,
-      type: FlowNodeInputTypeEnum.hidden,
+      type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
       label: '',
       value: 0.4,
       valueType: ModuleIOValueTypeEnum.number,
-      min: 0,
-      max: 1,
-      step: 0.01,
-      markList: [
-        { label: '0', value: 0 },
-        { label: '1', value: 1 }
-      ],
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -79,13 +72,31 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       value: false
     },
     {
-      key: ModuleInputKeyEnum.datasetParamsModal,
-      type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
+      key: ModuleInputKeyEnum.datasetSearchUsingExtensionQuery,
+      type: FlowNodeInputTypeEnum.hidden,
       label: '',
-      valueType: ModuleIOValueTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.boolean,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: true
+    },
+    {
+      key: ModuleInputKeyEnum.datasetSearchExtensionModel,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '',
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
+    {
+      key: ModuleInputKeyEnum.datasetSearchExtensionBg,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '',
+      valueType: ModuleIOValueTypeEnum.string,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: ''
+    },
     Input_Template_UserChatInput
   ],
   outputs: [