V4.14.4 features (#6036)

* feat: add query optimize and bill (#6021) * add query optimize and bill * perf: query extension * fix: embe model * remove log * remove log * fix: test --------- Co-authored-by: xxyyh <2289112474@qq> Co-authored-by: archer <545436317@qq.com> * feat: notice (#6013) * feat: record user's language * feat: notice points/dataset indexes; support count limit; update docker-compose.yml * fix: ts error * feat: send auth code i18n * chore: dataset notice limit * chore: adjust * fix: ts * fix: countLimit race condition; i18n en-prefix locale fallback to en --------- Co-authored-by: archer <545436317@qq.com> * perf: comment * perf: send inform code * fix: type error (#6029) * feat: add ip region for chat logs (#6010) * feat: add ip region for chat logs * refactor: use Geolite2.mmdb * fix: export chat logs * fix: return location directly * test: add unit test * perf: log show ip data * adjust commercial plans (#6008) * plan frontend * plan limit * coupon * discount coupon * fix * type * fix audit * type * plan name * legacy plan * track * feat: add discount coupon * fix * fix discount coupon * openapi * type * type * env * api type * fix * fix: simple agent plugin input & agent dashboard card (#6034) * refactor: remove gridfs (#6031) * fix: replace gridfs multer operations with s3 compatible ops * wip: s3 features * refactor: remove gridfs * fix * perf: mock test * doc * doc * doc * fix: test * fix: s3 * fix: mock s3 * remove invalid config * fix: init query extension * initv4144 (#6037) * chore: initv4144 * fix * version * fix: new plans (#6039) * fix: new plans * qr modal tip * fix: buffer raw text filename (#6040) * fix: initv4144 (#6041) * fix: pay refresh (#6042) * fix: migration shell * rename collection * clear timerlock * clear timerlock * perf: faq * perf: bill schema * fix: openapi * doc * fix: share var render * feat: delete dataset queue * plan usage display (#6043) * plan usage display * text * fix * fix: ts * perf: remove invalid code * perf: init shell * doc * perf: rename field * perf: avatar presign * init * custom plan text (#6045) * fix plans * fix * fixed * computed --------- Co-authored-by: archer <545436317@qq.com> * init shell * plan text & price page back button (#6046) * init * index * delete dataset * delete dataset * perf: delete dataset * init --------- Co-authored-by: YeYuheng <57035043+YYH211@users.noreply.github.com> Co-authored-by: xxyyh <2289112474@qq> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: Roy <whoeverimf5@gmail.com> Co-authored-by: heheer <heheer@sealos.io>
2026-05-07 01:02:55 +08:00 · 2025-12-08 01:44:15 +08:00
parent 9d72f238c0
commit 2ccb5b50c6
247 changed files with 7342 additions and 3819 deletions
@@ -41,9 +41,6 @@ import { i18nT } from '../../../../../web/i18n/utils';
 import { postTextCensor } from '../../../chat/postTextCensor';
 import { createLLMResponse } from '../../../ai/llm/request';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
-import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
-import { getFileS3Key } from '../../../../common/s3/utils';
-import { addDays } from 'date-fns';

 export type ChatProps = ModuleDispatchProps<
  AIChatNodeProps & {
@@ -307,7 +304,6 @@ async function filterDatasetQuote({
      : '';

  return {
-    // datasetQuoteText: replaceDatasetQuoteTextWithJWT(datasetQuoteText, addDays(new Date(), 90))
    datasetQuoteText
  };
 }
@@ -1,7 +1,4 @@
-import {
-  type DispatchNodeResponseType,
-  type DispatchNodeResultType
-} from '@fastgpt/global/core/workflow/runtime/type.d';
+import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
 import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
 import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
@@ -117,7 +114,7 @@ export async function dispatchDatasetSearch(
      return emptyResult;
    }

-    // get vector
+    // Get vector model
    const vectorModel = getEmbeddingModel(
      (await MongoDataset.findById(datasets[0].datasetId, 'vectorModel').lean())?.vectorModel
    );
@@ -165,7 +162,7 @@ export async function dispatchDatasetSearch(

    // count bill results
    const nodeDispatchUsages: ChatNodeUsageType[] = [];
-    // vector
+    // 1. Search vector
    const { totalPoints: embeddingTotalPoints, modelName: embeddingModelName } =
      formatModelChars2Points({
        model: vectorModel.model,
@@ -177,96 +174,98 @@ export async function dispatchDatasetSearch(
      model: embeddingModelName,
      inputTokens: embeddingTokens
    });
-    // Rerank
-    const { totalPoints: reRankTotalPoints, modelName: reRankModelName } = formatModelChars2Points({
-      model: rerankModelData?.model,
-      inputTokens: reRankInputTokens
-    });
+    // 2. Rerank
    if (usingReRank) {
+      const { totalPoints: reRankTotalPoints, modelName: reRankModelName } =
+        formatModelChars2Points({
+          model: rerankModelData?.model,
+          inputTokens: reRankInputTokens
+        });
      nodeDispatchUsages.push({
        totalPoints: reRankTotalPoints,
-        moduleName: node.name,
+        moduleName: i18nT('account_usage:rerank'),
        model: reRankModelName,
        inputTokens: reRankInputTokens
      });
    }
-    // Query extension
-    (() => {
-      if (queryExtensionResult) {
-        const { totalPoints, modelName } = formatModelChars2Points({
-          model: queryExtensionResult.model,
-          inputTokens: queryExtensionResult.inputTokens,
-          outputTokens: queryExtensionResult.outputTokens
-        });
-        nodeDispatchUsages.push({
-          totalPoints,
-          moduleName: i18nT('common:core.module.template.Query extension'),
-          model: modelName,
-          inputTokens: queryExtensionResult.inputTokens,
-          outputTokens: queryExtensionResult.outputTokens
-        });
-        return {
-          totalPoints
-        };
-      }
-      return {
-        totalPoints: 0
-      };
-    })();
-    // Deep search
-    (() => {
-      if (deepSearchResult) {
-        const { totalPoints, modelName } = formatModelChars2Points({
-          model: deepSearchResult.model,
-          inputTokens: deepSearchResult.inputTokens,
-          outputTokens: deepSearchResult.outputTokens
-        });
-        nodeDispatchUsages.push({
-          totalPoints,
-          moduleName: i18nT('common:deep_rag_search'),
-          model: modelName,
-          inputTokens: deepSearchResult.inputTokens,
-          outputTokens: deepSearchResult.outputTokens
-        });
-        return {
-          totalPoints
-        };
-      }
-      return {
-        totalPoints: 0
-      };
-    })();
+    // 3. Query extension
+    if (queryExtensionResult) {
+      const { totalPoints: llmPoints, modelName: llmModelName } = formatModelChars2Points({
+        model: queryExtensionResult.llmModel,
+        inputTokens: queryExtensionResult.inputTokens,
+        outputTokens: queryExtensionResult.outputTokens
+      });
+      nodeDispatchUsages.push({
+        totalPoints: llmPoints,
+        moduleName: i18nT('common:core.module.template.Query extension'),
+        model: llmModelName,
+        inputTokens: queryExtensionResult.inputTokens,
+        outputTokens: queryExtensionResult.outputTokens
+      });

+      const { totalPoints: embeddingPoints, modelName: embeddingModelName } =
+        formatModelChars2Points({
+          model: queryExtensionResult.embeddingModel,
+          inputTokens: queryExtensionResult.embeddingTokens
+        });
+      nodeDispatchUsages.push({
+        totalPoints: embeddingPoints,
+        moduleName: `${i18nT('account_usage:ai.query_extension_embedding')}`,
+        model: embeddingModelName,
+        inputTokens: queryExtensionResult.embeddingTokens,
+        outputTokens: 0
+      });
+    }
+    // 4. Deep search
+    if (deepSearchResult) {
+      const { totalPoints, modelName } = formatModelChars2Points({
+        model: deepSearchResult.model,
+        inputTokens: deepSearchResult.inputTokens,
+        outputTokens: deepSearchResult.outputTokens
+      });
+      nodeDispatchUsages.push({
+        totalPoints,
+        moduleName: i18nT('common:deep_rag_search'),
+        model: modelName,
+        inputTokens: deepSearchResult.inputTokens,
+        outputTokens: deepSearchResult.outputTokens
+      });
+    }
    const totalPoints = nodeDispatchUsages.reduce((acc, item) => acc + item.totalPoints, 0);

-    const responseData: DispatchNodeResponseType & { totalPoints: number } = {
-      totalPoints,
-      query: userChatInput,
-      embeddingModel: vectorModel.name,
-      embeddingTokens,
-      similarity: usingSimilarityFilter ? similarity : undefined,
-      limit,
-      searchMode,
-      embeddingWeight:
-        searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
-      // Rerank
-      ...(searchUsingReRank && {
-        rerankModel: rerankModelData?.name,
-        rerankWeight: rerankWeight,
-        reRankInputTokens
-      }),
-      searchUsingReRank,
-      // Results
-      quoteList: searchRes,
-      queryExtensionResult,
-      deepSearchResult
-    };
-
    return {
      data: {
        quoteQA: searchRes
      },
-      [DispatchNodeResponseKeyEnum.nodeResponse]: responseData,
+      [DispatchNodeResponseKeyEnum.nodeResponse]: {
+        totalPoints,
+        query: userChatInput,
+        embeddingModel: vectorModel.name,
+        embeddingTokens,
+        similarity: usingSimilarityFilter ? similarity : undefined,
+        limit,
+        searchMode,
+        embeddingWeight:
+          searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
+        // Rerank
+        ...(searchUsingReRank && {
+          rerankModel: rerankModelData?.name,
+          rerankWeight: rerankWeight,
+          reRankInputTokens
+        }),
+        searchUsingReRank,
+        queryExtensionResult: queryExtensionResult
+          ? {
+              model: queryExtensionResult.llmModel,
+              inputTokens: queryExtensionResult.inputTokens,
+              outputTokens: queryExtensionResult.outputTokens,
+              query: queryExtensionResult.query
+            }
+          : undefined,
+        deepSearchResult,
+        // Results
+        quoteList: searchRes
+      },
      nodeDispatchUsages,
      [DispatchNodeResponseKeyEnum.toolResponses]:
        searchRes.length > 0
@@ -3,13 +3,12 @@ import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
 import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import { getLLMModel } from '../../../../core/ai/model';
+import { getLLMModel, getEmbeddingModel } from '../../../../core/ai/model';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
 import { queryExtension } from '../../../../core/ai/functions/queryExtension';
 import { getHistories } from '../utils';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
-import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';

 type Props = ModuleDispatchProps<{
  [NodeInputKeyEnum.aiModel]: string;
@@ -31,23 +30,39 @@ export const dispatchQueryExtension = async ({
  }

  const queryExtensionModel = getLLMModel(model);
+  const embeddingModel = getEmbeddingModel();
  const chatHistories = getHistories(history, histories);

-  const { extensionQueries, inputTokens, outputTokens } = await queryExtension({
+  const {
+    extensionQueries,
+    inputTokens,
+    outputTokens,
+    embeddingTokens,
+    llmModel,
+    embeddingModel: useEmbeddingModel
+  } = await queryExtension({
    chatBg: systemPrompt,
    query: userChatInput,
    histories: chatHistories,
-    model: queryExtensionModel.model
+    llmModel: queryExtensionModel.model,
+    embeddingModel: embeddingModel.model
  });

  extensionQueries.unshift(userChatInput);

-  const { totalPoints, modelName } = formatModelChars2Points({
-    model: queryExtensionModel.model,
+  const { totalPoints: llmPoints, modelName: llmModelName } = formatModelChars2Points({
+    model: llmModel,
    inputTokens,
    outputTokens
  });

+  const { totalPoints: embeddingPoints, modelName: embeddingModelName } = formatModelChars2Points({
+    model: useEmbeddingModel,
+    inputTokens: embeddingTokens
+  });
+
+  const totalPoints = llmPoints + embeddingPoints;
+
  const set = new Set<string>();
  const filterSameQueries = extensionQueries.filter((item) => {
    // 删除所有的标点符号与空格等，只对文本进行比较
@@ -63,19 +78,27 @@ export const dispatchQueryExtension = async ({
    },
    [DispatchNodeResponseKeyEnum.nodeResponse]: {
      totalPoints,
-      model: modelName,
+      model: llmModelName,
      inputTokens,
      outputTokens,
+      embeddingTokens,
      query: userChatInput,
      textOutput: JSON.stringify(filterSameQueries)
    },
    [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [
      {
        moduleName: node.name,
-        totalPoints,
-        model: modelName,
+        totalPoints: llmPoints,
+        model: llmModelName,
        inputTokens,
        outputTokens
+      },
+      {
+        moduleName: `${node.name} - Embedding`,
+        totalPoints: embeddingPoints,
+        model: embeddingModelName,
+        inputTokens: embeddingTokens,
+        outputTokens: 0
      }
    ]
  };
@@ -10,18 +10,17 @@ import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/f
 import { readS3FileContentByBuffer } from '../../../../common/file/read/utils';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
-import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { addLog } from '../../../../common/system/log';
-import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
-import { addDays, addMinutes } from 'date-fns';
+import { addDays } from 'date-fns';
 import { getNodeErrResponse } from '../utils';
 import { isInternalAddress } from '../../../../common/system/utils';
-import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
+import { replaceS3KeyToPreviewUrl } from '../../../dataset/utils';
 import { getFileS3Key } from '../../../../common/s3/utils';
 import { S3ChatSource } from '../../../../common/s3/sources/chat';
-import path from 'path';
+import path from 'node:path';
 import { S3Buckets } from '../../../../common/s3/constants';
 import { S3Sources } from '../../../../common/s3/type';
+import { getS3DatasetSource } from '../../../../common/s3/sources/dataset';

 type Props = ModuleDispatchProps<{
  [NodeInputKeyEnum.fileUrlList]: string[];
@@ -176,12 +175,15 @@ export const getFileContentFromLinks = async ({
    parseUrlList
      .map(async (url) => {
        // Get from buffer
-        const fileBuffer = await getRawTextBuffer(url);
-        if (fileBuffer) {
+        const rawTextBuffer = await getS3DatasetSource().getRawTextBuffer({
+          sourceId: url,
+          customPdfParse
+        });
+        if (rawTextBuffer) {
          return formatResponseObject({
-            filename: fileBuffer.sourceName || url,
+            filename: rawTextBuffer.filename || url,
            url,
-            content: fileBuffer.text
+            content: rawTextBuffer.text
          });
        }

@@ -264,14 +266,14 @@ export const getFileContentFromLinks = async ({
            usageId
          });

-          const replacedText = replaceDatasetQuoteTextWithJWT(rawText, addDays(new Date(), 90));
+          const replacedText = replaceS3KeyToPreviewUrl(rawText, addDays(new Date(), 90));

          // Add to buffer
-          addRawTextBuffer({
+          getS3DatasetSource().addRawTextBuffer({
            sourceId: url,
            sourceName: filename,
            text: replacedText,
-            expiredTime: addMinutes(new Date(), 20)
+            customPdfParse
          });

          return formatResponseObject({ filename, url, content: replacedText });