Update docs and response tag in share page (#694)

* perf: chunk index show * share response * perf: vector query * web printFinger * remove log * fix: bucket name * perf: training schema * perf: sort index
2025-07-23 05:12:39 +00:00 · 2024-01-05 18:02:37 +08:00
parent 331d18c88f
commit 3f088bce6a
23 changed files with 242 additions and 181 deletions
--- a/docSite/content/docs/commercial/intro.md
+++ b/docSite/content/docs/commercial/intro.md
@@ -33,7 +33,7 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独

 ## 商业版软件价格

-FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)
+FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)

 **共有服务**

@@ -46,8 +46,7 @@ FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。
 {{< table "table-hover table-striped-columns" >}}
 | 部署方式 | 特有服务 | 上线时长 | 价格 |
 | ---- | ---- | ---- | ---- |
-| Sealos公有云部署 |  1. 6个版本的升级服务。<br>2. 赠送 8000 元 Sealos 云资源额度。 | 半天 | 10000元/年 | 
-| Sealos全托管 |  1. 有效期内免费升级。<br>2. 免运维服务&数据库。<br>3. 赠送 10000 元 Sealos 云资源额度。 |  半天  | 3000元起/月（3个月起）<br>或<br>30000元起/年  |
+| Sealos全托管 |  1. 有效期内免费升级。<br>2. 免运维服务&数据库。 |  半天  | 3000元起/月（3个月起）<br>或<br>30000元起/年  |
 | 自有服务器-单机版 |  1. 6个版本的升级服务。 | 14天内 |  60000元/套（不限时长） | 
 | 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）|
 {{< /table >}}
--- a/docSite/content/docs/development/openapi/auth.md
+++ b/docSite/content/docs/development/openapi/auth.md
@@ -25,7 +25,7 @@ FastGPT 的 API Key **有 2 类**，一类是全局通用的 key (无法直接

 |        通用key               |           应用特定 key            |
 | --------------------- | --------------------- |
-| ![](/imgs/fastgpt-api2.png) | ![](/imgs/fastgpt-api.png) |
+| ![](/imgs/fastgpt-api2.jpg) | ![](/imgs/fastgpt-api.jpg) |

 ## 基本配置

--- a/packages/global/common/file/constants.ts
+++ b/packages/global/common/file/constants.ts
@@ -1,5 +1,10 @@
 export enum BucketNameEnum {
  dataset = 'dataset'
 }
+export const bucketNameMap = {
+  [BucketNameEnum.dataset]: {
+    label: 'common.file.bucket.dataset'
+  }
+};

 export const FileBaseUrl = '/api/common/file/read';
--- a/packages/global/common/file/read/index.ts
+++ b/packages/global/common/file/read/index.ts
@@ -20,13 +20,13 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer })

    const viewport = page.getViewport({ scale: 1 });
    const pageHeight = viewport.height;
-    const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内
-    const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内
+    const headerThreshold = pageHeight * 0.95;
+    const footerThreshold = pageHeight * 0.05;

    const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
      return (
        !token.transform ||
-        (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
+        (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
      );
    });

--- a/packages/service/common/file/upload/multer.ts
+++ b/packages/service/common/file/upload/multer.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { customAlphabet } from 'nanoid';
 import multer from 'multer';
 import path from 'path';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import fs from 'fs';

 const nanoid = customAlphabet('1234567890abcdef', 12);
@@ -45,6 +45,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
            return reject(error);
          }

+          // check bucket name
+          const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+          if (bucketName && !bucketNameMap[bucketName]) {
+            return reject('BucketName is invalid');
+          }
+
          resolve({
            ...req.body,
            files:
--- a/packages/service/common/vectorStore/controller.ts
+++ b/packages/service/common/vectorStore/controller.ts
@@ -48,7 +48,7 @@ export const updateDatasetDataVector = async ({
  // get vector
  const { vectors, tokens } = await getVectorsByText({
    model,
-    input: [query]
+    input: query
  });

  await getVectorObj().update({
--- a/packages/service/core/ai/embedding/index.ts
+++ b/packages/service/core/ai/embedding/index.ts
@@ -2,7 +2,7 @@ import { getAIApi } from '../config';

 export type GetVectorProps = {
  model: string;
-  input: string | string[];
+  input: string;
 };

 // text to vector
@@ -10,24 +10,13 @@ export async function getVectorsByText({
  model = 'text-embedding-ada-002',
  input
 }: GetVectorProps) {
-  if (typeof input === 'string' && !input) {
+  if (!input) {
    return Promise.reject({
      code: 500,
      message: 'input is empty'
    });
-  } else if (Array.isArray(input)) {
-    for (let i = 0; i < input.length; i++) {
-      if (!input[i]) {
-        return Promise.reject({
-          code: 500,
-          message: 'input array is empty'
-        });
-      }
-    }
-  }
-  if (typeof input === 'string') {
-    input = [input];
  }
+
  try {
    // 获取 chatAPI
    const ai = getAIApi();
@@ -36,7 +25,7 @@ export async function getVectorsByText({
    const result = await ai.embeddings
      .create({
        model,
-        input
+        input: [input]
      })
      .then(async (res) => {
        if (!res.data) {
@@ -47,6 +36,7 @@ export async function getVectorsByText({
          // @ts-ignore
          return Promise.reject(res.data?.err?.message || 'Embedding API Error');
        }
+
        return {
          tokens: res.usage.total_tokens || 0,
          vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
--- a/packages/service/core/dataset/training/schema.ts
+++ b/packages/service/core/dataset/training/schema.ts
@@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({
 });

 try {
+  TrainingDataSchema.index({ weight: -1 });
  TrainingDataSchema.index({ lockTime: 1 });
  TrainingDataSchema.index({ datasetId: 1 });
  TrainingDataSchema.index({ collectionId: 1 });
--- a/packages/web/common/system/utils.ts
+++ b/packages/web/common/system/utils.ts
@@ -0,0 +1,9 @@
+import FingerprintJS from '@fingerprintjs/fingerprintjs';
+
+const fpPromise = FingerprintJS.load();
+
+export const getUserFingerprint = async () => {
+  const fp = await fpPromise;
+  const result = await fp.get();
+  console.log(result.visitorId);
+};
--- a/packages/web/package.json
+++ b/packages/web/package.json
@@ -2,9 +2,6 @@
  "name": "@fastgpt/web",
  "version": "1.0.0",
  "dependencies": {
-    "@fastgpt/global": "workspace:*",
-    "joplin-turndown-plugin-gfm": "^1.0.12",
-    "turndown": "^7.1.2",
    "@chakra-ui/anatomy": "^2.2.1",
    "@chakra-ui/icons": "^2.1.1",
    "@chakra-ui/next-js": "^2.1.5",
@@ -13,16 +10,20 @@
    "@chakra-ui/system": "^2.6.1",
    "@emotion/react": "^11.11.1",
    "@emotion/styled": "^11.11.0",
+    "@fastgpt/global": "workspace:*",
+    "@fingerprintjs/fingerprintjs": "^4.2.1",
    "@monaco-editor/react": "^4.6.0",
+    "i18next": "^22.5.1",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "next-i18next": "^13.3.0",
    "react": "18.2.0",
    "react-dom": "18.2.0",
-    "i18next": "^22.5.1",
-    "next-i18next": "^13.3.0",
-    "react-i18next": "^12.3.1"
+    "react-i18next": "^12.3.1",
+    "turndown": "^7.1.2"
  },
  "devDependencies": {
-    "@types/turndown": "^5.0.4",
    "@types/react": "18.2.0",
-    "@types/react-dom": "18.2.0"
+    "@types/react-dom": "18.2.0",
+    "@types/turndown": "^5.0.4"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -163,6 +163,9 @@ importers:
      '@fastgpt/global':
        specifier: workspace:*
        version: link:../global
+      '@fingerprintjs/fingerprintjs':
+        specifier: ^4.2.1
+        version: registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1
      '@monaco-editor/react':
        specifier: ^4.6.0
        version: registry.npmmirror.com/@monaco-editor/react@4.6.0(monaco-editor@0.45.0)(react-dom@18.2.0)(react@18.2.0)
@@ -3836,6 +3839,14 @@ packages:
      - supports-color
    dev: true

+  registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1:
+    resolution: {integrity: sha512-uW+GVUNTgCXbVPEbgnbf5Aor22e1dyYR0JRwdUiZBaikfxr7KlhV9y0aahA1FB99fEeQVvhCEvTcPIFSYTy9Pw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@fingerprintjs/fingerprintjs/-/fingerprintjs-4.2.1.tgz}
+    name: '@fingerprintjs/fingerprintjs'
+    version: 4.2.1
+    dependencies:
+      tslib: registry.npmmirror.com/tslib@2.6.2
+    dev: false
+
  registry.npmmirror.com/@humanwhocodes/config-array@0.11.13:
    resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz}
    name: '@humanwhocodes/config-array'
--- a/projects/app/public/locales/en/common.json
+++ b/projects/app/public/locales/en/common.json
@@ -522,9 +522,13 @@
        },
        "score": {
          "embedding": "Embedding",
+          "embedding desc": "",
          "fullText": "Full text",
+          "fullText desc": "",
          "reRank": "ReRank",
-          "rrf": "RRF Merge"
+          "reRank desc": "",
+          "rrf": "RRF Merge",
+          "rrf desc": ""
        },
        "search mode": "Search Mode"
      },
@@ -874,8 +878,6 @@
    "QPM": "QPM",
    "QPM Tips": "The maximum number of queries per IP address per minute",
    "QPM is empty": "QPM is empty",
-    "Response Detail": "Quote",
-    "Response Detail tips": "Whether detailed data such as references to be returned",
    "token auth": "Token Auth",
    "token auth Tips": "Identity verification server address. If this value is set, the server will be specified to send a request for identity verification before each session",
    "token auth use cases": "Review the authentication instructions"
@@ -903,6 +905,12 @@
    "Update Your Plugin": "Update Plugin"
  },
  "support": {
+    "outlink": {
+      "share": {
+        "Response Quote": "Show Quote",
+        "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document."
+      }
+    },
    "user": {
      "Price": "Price",
      "auth": {
--- a/projects/app/public/locales/zh/common.json
+++ b/projects/app/public/locales/zh/common.json
@@ -878,8 +878,6 @@
    "QPM": "",
    "QPM Tips": "每个 IP 每分钟最多提问多少次",
    "QPM is empty": "QPM 不能为空",
-    "Response Detail": "返回详情",
-    "Response Detail tips": "是否需要返回详情（引用内容，调用时间等，不会返回预设提示词和完整上下文）",
    "token auth": "身份验证",
    "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会向指定服务器发送一个请求，进行身份校验",
    "token auth use cases": "查看身份验证使用说明"
@@ -907,6 +905,12 @@
    "Update Your Plugin": "更新插件"
  },
  "support": {
+    "outlink": {
+      "share": {
+        "Response Quote": "返回引用",
+        "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档"
+      }
+    },
    "user": {
      "Price": "计费标准",
      "auth": {
--- a/projects/app/src/components/ChatBox/ResponseTags.tsx
+++ b/projects/app/src/components/ChatBox/ResponseTags.tsx
@@ -133,6 +133,7 @@ const ResponseTags = ({
          </Flex>
        </>
      )}
+      {!isShare && (
        <Flex alignItems={'center'} mt={3} flexWrap={'wrap'}>
          {quoteList.length > 0 && (
            <MyTooltip label="查看引用">
@@ -180,7 +181,8 @@ const ResponseTags = ({
              {t('core.chat.response.Read complete response')}
            </Tag>
          </MyTooltip>
-
+        </Flex>
+      )}
      {!!quoteModalData && (
        <QuoteModal
          {...quoteModalData}
@@ -192,13 +194,8 @@ const ResponseTags = ({
        <ContextModal context={contextModalData} onClose={() => setContextModalData(undefined)} />
      )}
      {isOpenWholeModal && (
-          <WholeResponseModal
-            response={responseData}
-            isShare={isShare}
-            onClose={onCloseWholeModal}
-          />
+        <WholeResponseModal response={responseData} isShare={isShare} onClose={onCloseWholeModal} />
      )}
-      </Flex>
    </>
  );
 };
--- a/projects/app/src/components/core/dataset/QuoteItem.tsx
+++ b/projects/app/src/components/core/dataset/QuoteItem.tsx
@@ -126,6 +126,8 @@ const QuoteItem = ({
      >
        <Flex alignItems={'center'} mb={3}>
          {score?.primaryScore && (
+            <>
+              {canViewSource ? (
                <MyTooltip label={t(SearchScoreTypeMap[score.primaryScore.type]?.desc)}>
                  <Flex
                    px={'12px'}
@@ -140,7 +142,12 @@ const QuoteItem = ({
                    fontSize={'sm'}
                  >
                    <Box>#{score.primaryScore.index + 1}</Box>
-                <Box borderRightColor={'primary.700'} borderRightWidth={'1px'} h={'14px'} mx={2} />
+                    <Box
+                      borderRightColor={'primary.700'}
+                      borderRightWidth={'1px'}
+                      h={'14px'}
+                      mx={2}
+                    />
                    <Box>
                      {t(SearchScoreTypeMap[score.primaryScore.type]?.label)}
                      {SearchScoreTypeMap[score.primaryScore.type]?.showScore
@@ -149,8 +156,26 @@ const QuoteItem = ({
                    </Box>
                  </Flex>
                </MyTooltip>
+              ) : (
+                <Flex
+                  px={'12px'}
+                  py={'1px'}
+                  mr={4}
+                  borderRadius={'md'}
+                  color={'primary.700'}
+                  bg={'primary.50'}
+                  borderWidth={'1px'}
+                  borderColor={'primary.200'}
+                  alignItems={'center'}
+                  fontSize={'sm'}
+                >
+                  <Box>#{score.primaryScore.index + 1}</Box>
+                </Flex>
              )}
-          {score.secondaryScore.map((item, i) => (
+            </>
+          )}
+          {canViewSource &&
+            score.secondaryScore.map((item, i) => (
              <MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc)}>
                <Box fontSize={'xs'} mr={3}>
                  <Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}>
--- a/projects/app/src/global/core/dataset/type.d.ts
+++ b/projects/app/src/global/core/dataset/type.d.ts
@@ -28,5 +28,6 @@ export type DatasetDataListItemType = {
  collectionId: string;
  q: string; // embedding content
  a: string; // bonus content
+  chunkIndex?: number;
  indexes: DatasetDataSchemaType['indexes'];
 };
--- a/projects/app/src/pages/api/core/dataset/data/list.ts
+++ b/projects/app/src/pages/api/core/dataset/data/list.ts
@@ -32,7 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    };

    const [data, total] = await Promise.all([
-      MongoDatasetData.find(match, '_id datasetId collectionId q a indexes')
+      MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex indexes')
        .sort({ chunkIndex: 1, updateTime: -1 })
        .skip((pageNum - 1) * pageSize)
        .limit(pageSize)
--- a/projects/app/src/pages/api/v1/embeddings.ts
+++ b/projects/app/src/pages/api/v1/embeddings.ts
@@ -10,6 +10,7 @@ import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';

 type Props = GetVectorProps & {
+  input: string | string[];
  billId?: string;
 };

@@ -22,6 +23,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      throw new Error('input is nor array or string');
    }

+    const query = Array.isArray(input) ? input[0] : input;
+
    const { teamId, tmbId, apikey, authType } = await authCert({
      req,
      authToken: true,
@@ -30,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex

    await authTeamBalance(teamId);

-    const { tokens, vectors } = await getVectorsByText({ input, model });
+    const { tokens, vectors } = await getVectorsByText({ input: query, model });

    jsonRes(res, {
      data: {
--- a/projects/app/src/pages/app/detail/components/OutLink/Share.tsx
+++ b/projects/app/src/pages/app/detail/components/OutLink/Share.tsx
@@ -96,7 +96,7 @@ const Share = ({ appId }: { appId: string }) => {
            <Tr>
              <Th>名称</Th>
              <Th>金额消耗</Th>
-              <Th>返回详情</Th>
+              <Th>返回引用</Th>
              {feConfigs?.isPlus && (
                <>
                  <Th>IP限流（人/分钟）</Th>
@@ -373,8 +373,8 @@ function EditLinkModal({

        <Flex alignItems={'center'} mt={4}>
          <Flex flex={'0 0 90px'} alignItems={'center'}>
-            {t('outlink.Response Detail')}
-            <MyTooltip label={t('outlink.Response Detail tips' || '')}>
+            {t('support.outlink.share.Response Quote')}
+            <MyTooltip label={t('support.outlink.share.Response Quote tips' || '')}>
              <QuestionOutlineIcon ml={1} />
            </MyTooltip>
          </Flex>
--- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx
+++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx
@@ -307,7 +307,7 @@ const DataCard = () => {
          >
            <Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
              <Box border={theme.borders.base} px={2} fontSize={'sm'} mr={1} borderRadius={'md'}>
-                # {index + 1}
+                # {item.chunkIndex ?? '-'}
              </Box>
              <Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
                ID:{item._id}
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -295,7 +295,7 @@ export async function searchDatasetData(props: {
  const embeddingRecall = async ({ query, limit }: { query: string; limit: number }) => {
    const { vectors, tokens } = await getVectorsByText({
      model,
-      input: [query]
+      input: query
    });

    const { results } = await recallFromVectorStore({
--- a/projects/app/src/service/events/generateVector.ts
+++ b/projects/app/src/service/events/generateVector.ts
@@ -26,6 +26,8 @@ export async function generateVector(): Promise<any> {
  if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
  global.vectorQueueLen++;

+  const start = Date.now();
+
  // get training data
  const {
    data,
@@ -154,6 +156,8 @@ export async function generateVector(): Promise<any> {
    await MongoDatasetTraining.findByIdAndDelete(data._id);
    reduceQueue();
    generateVector();
+
+    console.log(`embedding finished, time: ${Date.now() - start}ms`);
  } catch (err: any) {
    reduceQueue(true);
    // log
--- a/projects/app/src/utils/service/core/chat/index.ts
+++ b/projects/app/src/utils/service/core/chat/index.ts
@@ -1,19 +1,16 @@
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
+import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';

-export function selectShareResponse({
+export const selectShareResponse = ({
  responseData = []
 }: {
  responseData?: ChatHistoryItemResType[];
-}) {
-  const filedList = [
-    'moduleType',
-    'moduleName',
-    'moduleLogo',
-    'runningTime',
-    'quoteList',
-    'question'
-  ];
-  return responseData.map((item) => {
+}) => {
+  const filedList = ['quoteList', 'moduleType'];
+  const filterModuleTypeList: any[] = [FlowNodeTypeEnum.chatNode];
+  return responseData
+    .filter((item) => filterModuleTypeList.includes(item.moduleType))
+    .map((item) => {
      const obj: Record<string, any> = {};
      for (let key in item) {
        if (filedList.includes(key)) {
@@ -23,4 +20,4 @@ export function selectShareResponse({
      }
      return obj as ChatHistoryItemResType;
    });
-}
+};