4.6.8 supplement (#831)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/service/core/dataset/data/controller.ts

@@ -310,6 +310,24 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
   let usingSimilarityFilter = false;
 
   /* function */
+  const countRecallLimit = () => {
+    if (searchMode === DatasetSearchModeEnum.embedding) {
+      return {
+        embeddingLimit: 100,
+        fullTextLimit: 0
+      };
+    }
+    if (searchMode === DatasetSearchModeEnum.fullTextRecall) {
+      return {
+        embeddingLimit: 0,
+        fullTextLimit: 100
+      };
+    }
+    return {
+      embeddingLimit: 60,
+      fullTextLimit: 40
+    };
+  };
   const embeddingRecall = async ({ query, limit }: { query: string; limit: number }) => {
     const { vectors, charsLength } = await getVectorsByText({
       model: getVectorModel(model),
@@ -555,8 +573,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
 
   /* main step */
   // count limit
-  const embeddingLimit = 60;
-  const fullTextLimit = 40;
+  const { embeddingLimit, fullTextLimit } = countRecallLimit();
 
   // recall
   const { embeddingRecallResults, fullTextRecallResults, charsLength } = await multiQueryRecall({

projects/app/src/service/moduleDispatch/index.ts

@@ -21,6 +21,7 @@ import { dispatchAnswer } from './tools/answer';
 import { dispatchClassifyQuestion } from './agent/classifyQuestion';
 import { dispatchContentExtract } from './agent/extract';
 import { dispatchHttpRequest } from './tools/http';
+import { dispatchHttp468Request } from './tools/http468';
 import { dispatchAppRequest } from './tools/runApp';
 import { dispatchCFR } from './tools/cfr';
 import { dispatchRunPlugin } from './plugin/run';
@@ -38,6 +39,7 @@ const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = {
   [FlowNodeTypeEnum.classifyQuestion]: dispatchClassifyQuestion,
   [FlowNodeTypeEnum.contentExtract]: dispatchContentExtract,
   [FlowNodeTypeEnum.httpRequest]: dispatchHttpRequest,
+  [FlowNodeTypeEnum.httpRequest468]: dispatchHttp468Request,
   [FlowNodeTypeEnum.runApp]: dispatchAppRequest,
   [FlowNodeTypeEnum.pluginModule]: dispatchRunPlugin,
   [FlowNodeTypeEnum.pluginInput]: dispatchPluginInput,

projects/app/src/service/moduleDispatch/tools/answer.ts

@@ -23,7 +23,7 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
   if (stream) {
     responseWrite({
       res,
-      event: detail ? sseResponseEventEnum.answer : undefined,
+      event: detail ? sseResponseEventEnum.response : undefined,
       data: textAdaptGptResponse({
         text: `\n${formatText}`
       })
@@ -31,6 +31,6 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
   }
 
   return {
-    answerText: formatText
+    [ModuleOutputKeyEnum.answerText]: formatText
   };
 };

projects/app/src/service/moduleDispatch/tools/http.ts

@@ -1,23 +1,37 @@
 import type { moduleDispatchResType } from '@fastgpt/global/core/chat/type.d';
 import type { ModuleDispatchProps } from '@fastgpt/global/core/module/type.d';
-import { ModuleInputKeyEnum, ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
+import {
+  DYNAMIC_INPUT_KEY,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum
+} from '@fastgpt/global/core/module/constants';
 import axios from 'axios';
-import { flatDynamicParams, valueTypeFormat } from '../utils';
+import { valueTypeFormat } from '../utils';
 import { SERVICE_LOCAL_HOST } from '@fastgpt/service/common/system/tools';
 
-export type HttpRequestProps = ModuleDispatchProps<{
+type HttpRequestProps = ModuleDispatchProps<{
   [ModuleInputKeyEnum.abandon_httpUrl]: string;
   [ModuleInputKeyEnum.httpMethod]: string;
   [ModuleInputKeyEnum.httpReqUrl]: string;
-  [ModuleInputKeyEnum.httpHeader]: string;
+  [ModuleInputKeyEnum.httpHeaders]: string;
   [key: string]: any;
 }>;
-export type HttpResponse = {
+type HttpResponse = {
   [ModuleOutputKeyEnum.failed]?: boolean;
   [ModuleOutputKeyEnum.responseData]: moduleDispatchResType;
   [key: string]: any;
 };
 
+const flatDynamicParams = (params: Record<string, any>) => {
+  const dynamicParams = params[DYNAMIC_INPUT_KEY];
+  if (!dynamicParams) return params;
+  return {
+    ...params,
+    ...dynamicParams,
+    [DYNAMIC_INPUT_KEY]: undefined
+  };
+};
+
 export const dispatchHttpRequest = async (props: HttpRequestProps): Promise<HttpResponse> => {
   let {
     appId,

projects/app/src/service/moduleDispatch/tools/http468.ts

@@ -0,0 +1,280 @@
+import type { moduleDispatchResType } from '@fastgpt/global/core/chat/type.d';
+import type { ModuleDispatchProps } from '@fastgpt/global/core/module/type.d';
+import {
+  DYNAMIC_INPUT_KEY,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum
+} from '@fastgpt/global/core/module/constants';
+import axios from 'axios';
+import { valueTypeFormat } from '../utils';
+import { SERVICE_LOCAL_HOST } from '@fastgpt/service/common/system/tools';
+
+type PropsArrType = {
+  key: string;
+  type: string;
+  value: string;
+};
+type HttpRequestProps = ModuleDispatchProps<{
+  [ModuleInputKeyEnum.abandon_httpUrl]: string;
+  [ModuleInputKeyEnum.httpMethod]: string;
+  [ModuleInputKeyEnum.httpReqUrl]: string;
+  [ModuleInputKeyEnum.httpHeaders]: PropsArrType[];
+  [ModuleInputKeyEnum.httpParams]: PropsArrType[];
+  [ModuleInputKeyEnum.httpJsonBody]: string;
+  [DYNAMIC_INPUT_KEY]: Record<string, any>;
+  [key: string]: any;
+}>;
+type HttpResponse = {
+  [ModuleOutputKeyEnum.failed]?: boolean;
+  [ModuleOutputKeyEnum.responseData]: moduleDispatchResType;
+  [key: string]: any;
+};
+
+const UNDEFINED_SIGN = 'UNDEFINED_SIGN';
+
+export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<HttpResponse> => {
+  let {
+    appId,
+    chatId,
+    responseChatItemId,
+    variables,
+    outputs,
+    histories,
+    params: {
+      system_httpMethod: httpMethod = 'POST',
+      system_httpReqUrl: httpReqUrl,
+      system_httpHeader: httpHeader,
+      system_httpParams: httpParams = [],
+      system_httpJsonBody: httpJsonBody,
+      [DYNAMIC_INPUT_KEY]: dynamicInput,
+      ...body
+    }
+  } = props;
+
+  if (!httpReqUrl) {
+    return Promise.reject('Http url is empty');
+  }
+
+  const concatVariables = {
+    appId,
+    chatId,
+    responseChatItemId,
+    variables,
+    histories: histories.slice(0, 10),
+    ...body
+  };
+
+  // parse header
+  const headers = await (() => {
+    try {
+      if (!httpHeader || httpHeader.length === 0) return {};
+      // array
+      return httpHeader.reduce((acc, item) => {
+        item.key = replaceVariable(item.key, concatVariables);
+        item.value = replaceVariable(item.value, concatVariables);
+        // @ts-ignore
+        acc[item.key] = valueTypeFormat(item.value, 'string');
+        return acc;
+      }, {});
+    } catch (error) {
+      return Promise.reject('Header 为非法 JSON 格式');
+    }
+  })();
+  const params = httpParams.reduce((acc, item) => {
+    item.key = replaceVariable(item.key, concatVariables);
+    item.value = replaceVariable(item.value, concatVariables);
+    // @ts-ignore
+    acc[item.key] = valueTypeFormat(item.value, 'string');
+    return acc;
+  }, {});
+  const requestBody = await (() => {
+    if (!httpJsonBody) return { [DYNAMIC_INPUT_KEY]: dynamicInput };
+    httpJsonBody = replaceVariable(httpJsonBody, concatVariables);
+    try {
+      const jsonParse = JSON.parse(httpJsonBody);
+      const removeSignJson = removeUndefinedSign(jsonParse);
+      return { [DYNAMIC_INPUT_KEY]: dynamicInput, ...removeSignJson };
+    } catch (error) {
+      console.log(error);
+      return Promise.reject(`Invalid JSON body: ${httpJsonBody}`);
+    }
+  })();
+  // console.log(params, requestBody, headers);
+
+  try {
+    const { formatResponse, rawResponse } = await fetchData({
+      method: httpMethod,
+      url: httpReqUrl,
+      headers,
+      body: requestBody,
+      params
+    });
+
+    // format output value type
+    const results: Record<string, any> = {};
+    for (const key in formatResponse) {
+      const output = outputs.find((item) => item.key === key);
+      if (!output) continue;
+      results[key] = valueTypeFormat(formatResponse[key], output.valueType);
+    }
+
+    return {
+      responseData: {
+        price: 0,
+        params: Object.keys(params).length > 0 ? params : undefined,
+        body: Object.keys(requestBody).length > 0 ? requestBody : undefined,
+        headers: Object.keys(headers).length > 0 ? headers : undefined,
+        httpResult: rawResponse
+      },
+      ...results
+    };
+  } catch (error) {
+    return {
+      [ModuleOutputKeyEnum.failed]: true,
+      responseData: {
+        price: 0,
+        params: Object.keys(params).length > 0 ? params : undefined,
+        body: Object.keys(requestBody).length > 0 ? requestBody : undefined,
+        headers: Object.keys(headers).length > 0 ? headers : undefined,
+        httpResult: { error }
+      }
+    };
+  }
+};
+
+async function fetchData({
+  method,
+  url,
+  headers,
+  body,
+  params
+}: {
+  method: string;
+  url: string;
+  headers: Record<string, any>;
+  body: Record<string, any>;
+  params: Record<string, any>;
+}): Promise<Record<string, any>> {
+  const { data: response } = await axios<Record<string, any>>({
+    method,
+    baseURL: `http://${SERVICE_LOCAL_HOST}`,
+    url,
+    headers: {
+      'Content-Type': 'application/json',
+      ...headers
+    },
+    params: params,
+    data: method === 'POST' ? body : {}
+  });
+
+  /*
+    parse the json:
+    {
+      user: {
+        name: 'xxx',
+        age: 12
+      },
+      list: [
+        {
+          name: 'xxx',
+          age: 50
+        },
+        [{ test: 22 }]
+      ],
+      psw: 'xxx'
+    }
+
+    result: {
+      'user': { name: 'xxx', age: 12 },
+      'user.name': 'xxx',
+      'user.age': 12,
+      'list': [ { name: 'xxx', age: 50 }, [ [Object] ] ],
+      'list[0]': { name: 'xxx', age: 50 },
+      'list[0].name': 'xxx',
+      'list[0].age': 50,
+      'list[1]': [ { test: 22 } ],
+      'list[1][0]': { test: 22 },
+      'list[1][0].test': 22,
+      'psw': 'xxx'
+    }
+  */
+  const parseJson = (obj: Record<string, any>, prefix = '') => {
+    let result: Record<string, any> = {};
+
+    if (Array.isArray(obj)) {
+      for (let i = 0; i < obj.length; i++) {
+        result[`${prefix}[${i}]`] = obj[i];
+
+        if (Array.isArray(obj[i])) {
+          result = {
+            ...result,
+            ...parseJson(obj[i], `${prefix}[${i}]`)
+          };
+        } else if (typeof obj[i] === 'object') {
+          result = {
+            ...result,
+            ...parseJson(obj[i], `${prefix}[${i}].`)
+          };
+        }
+      }
+    } else if (typeof obj == 'object') {
+      for (const key in obj) {
+        result[`${prefix}${key}`] = obj[key];
+
+        if (Array.isArray(obj[key])) {
+          result = {
+            ...result,
+            ...parseJson(obj[key], `${prefix}${key}`)
+          };
+        } else if (typeof obj[key] === 'object') {
+          result = {
+            ...result,
+            ...parseJson(obj[key], `${prefix}${key}.`)
+          };
+        }
+      }
+    }
+
+    return result;
+  };
+
+  return {
+    formatResponse: parseJson(response),
+    rawResponse: response
+  };
+}
+
+function replaceVariable(text: string, obj: Record<string, any>) {
+  for (const [key, value] of Object.entries(obj)) {
+    if (value === undefined) {
+      text = text.replace(new RegExp(`{{${key}}}`, 'g'), UNDEFINED_SIGN);
+    } else {
+      const replacement = JSON.stringify(value);
+      const unquotedReplacement =
+        replacement.startsWith('"') && replacement.endsWith('"')
+          ? replacement.slice(1, -1)
+          : replacement;
+      text = text.replace(new RegExp(`{{${key}}}`, 'g'), unquotedReplacement);
+    }
+  }
+  return text || '';
+}
+function removeUndefinedSign(obj: Record<string, any>) {
+  for (const key in obj) {
+    if (obj[key] === UNDEFINED_SIGN) {
+      obj[key] = undefined;
+    } else if (Array.isArray(obj[key])) {
+      obj[key] = obj[key].map((item: any) => {
+        if (item === UNDEFINED_SIGN) {
+          return undefined;
+        } else if (typeof item === 'object') {
+          removeUndefinedSign(item);
+        }
+        return item;
+      });
+    } else if (typeof obj[key] === 'object') {
+      removeUndefinedSign(obj[key]);
+    }
+  }
+  return obj;
+}

projects/app/src/service/moduleDispatch/utils.ts

@@ -9,16 +9,6 @@ export const getHistories = (history?: ChatItemType[] | number, histories: ChatI
   return [];
 };
 
-export const flatDynamicParams = (params: Record<string, any>) => {
-  const dynamicParams = params[DYNAMIC_INPUT_KEY];
-  if (!dynamicParams) return params;
-  return {
-    ...params,
-    ...dynamicParams,
-    [DYNAMIC_INPUT_KEY]: undefined
-  };
-};
-
 /* value type format */
 export const valueTypeFormat = (value: any, type?: `${ModuleIOValueTypeEnum}`) => {
   if (value === undefined) return;

projects/app/src/service/support/wallet/bill/push.ts

@@ -149,14 +149,14 @@ export const pushGenerateVectorBill = ({
       list: [
         {
           moduleName: 'wallet.moduleName.index',
-          amount: total,
+          amount: totalVector,
           model: vectorModelName,
           charsLength
         },
         ...(extensionModel !== undefined
           ? [
               {
-                moduleName: extensionModelName,
+                moduleName: 'core.module.template.Query extension',
                 amount: extensionTotal,
                 model: extensionModelName,
                 inputTokens: extensionInputTokens,