Update docs and response tag in share page (#694)

* perf: chunk index show

* share response

* perf: vector query

* web printFinger

* remove log

* fix: bucket name

* perf: training schema

* perf: sort index

packages/global/common/file/constants.ts

@@ -1,5 +1,10 @@
 export enum BucketNameEnum {
   dataset = 'dataset'
 }
+export const bucketNameMap = {
+  [BucketNameEnum.dataset]: {
+    label: 'common.file.bucket.dataset'
+  }
+};
 
 export const FileBaseUrl = '/api/common/file/read';

packages/global/common/file/read/index.ts

@@ -20,13 +20,13 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer })
 
     const viewport = page.getViewport({ scale: 1 });
     const pageHeight = viewport.height;
-    const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内
-    const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内
+    const headerThreshold = pageHeight * 0.95;
+    const footerThreshold = pageHeight * 0.05;
 
     const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
       return (
         !token.transform ||
-        (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
+        (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
       );
     });
 

packages/service/common/file/upload/multer.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { customAlphabet } from 'nanoid';
 import multer from 'multer';
 import path from 'path';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import fs from 'fs';
 
 const nanoid = customAlphabet('1234567890abcdef', 12);
@@ -45,6 +45,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
             return reject(error);
           }
 
+          // check bucket name
+          const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+          if (bucketName && !bucketNameMap[bucketName]) {
+            return reject('BucketName is invalid');
+          }
+
           resolve({
             ...req.body,
             files:

packages/service/common/vectorStore/controller.ts

@@ -48,7 +48,7 @@ export const updateDatasetDataVector = async ({
   // get vector
   const { vectors, tokens } = await getVectorsByText({
     model,
-    input: [query]
+    input: query
   });
 
   await getVectorObj().update({

packages/service/core/ai/embedding/index.ts

@@ -2,7 +2,7 @@ import { getAIApi } from '../config';
 
 export type GetVectorProps = {
   model: string;
-  input: string | string[];
+  input: string;
 };
 
 // text to vector
@@ -10,24 +10,13 @@ export async function getVectorsByText({
   model = 'text-embedding-ada-002',
   input
 }: GetVectorProps) {
-  if (typeof input === 'string' && !input) {
+  if (!input) {
     return Promise.reject({
       code: 500,
       message: 'input is empty'
     });
-  } else if (Array.isArray(input)) {
-    for (let i = 0; i < input.length; i++) {
-      if (!input[i]) {
-        return Promise.reject({
-          code: 500,
-          message: 'input array is empty'
-        });
-      }
-    }
-  }
-  if (typeof input === 'string') {
-    input = [input];
   }
+
   try {
     // 获取 chatAPI
     const ai = getAIApi();
@@ -36,7 +25,7 @@ export async function getVectorsByText({
     const result = await ai.embeddings
       .create({
         model,
-        input
+        input: [input]
       })
       .then(async (res) => {
         if (!res.data) {
@@ -47,6 +36,7 @@ export async function getVectorsByText({
           // @ts-ignore
           return Promise.reject(res.data?.err?.message || 'Embedding API Error');
         }
+
         return {
           tokens: res.usage.total_tokens || 0,
           vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))

packages/service/core/dataset/training/schema.ts

@@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({
 });
 
 try {
+  TrainingDataSchema.index({ weight: -1 });
   TrainingDataSchema.index({ lockTime: 1 });
   TrainingDataSchema.index({ datasetId: 1 });
   TrainingDataSchema.index({ collectionId: 1 });

packages/web/common/system/utils.ts

@@ -0,0 +1,9 @@
+import FingerprintJS from '@fingerprintjs/fingerprintjs';
+
+const fpPromise = FingerprintJS.load();
+
+export const getUserFingerprint = async () => {
+  const fp = await fpPromise;
+  const result = await fp.get();
+  console.log(result.visitorId);
+};

packages/web/package.json

@@ -2,9 +2,6 @@
   "name": "@fastgpt/web",
   "version": "1.0.0",
   "dependencies": {
-    "@fastgpt/global": "workspace:*",
-    "joplin-turndown-plugin-gfm": "^1.0.12",
-    "turndown": "^7.1.2",
     "@chakra-ui/anatomy": "^2.2.1",
     "@chakra-ui/icons": "^2.1.1",
     "@chakra-ui/next-js": "^2.1.5",
@@ -13,16 +10,20 @@
     "@chakra-ui/system": "^2.6.1",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
+    "@fastgpt/global": "workspace:*",
+    "@fingerprintjs/fingerprintjs": "^4.2.1",
     "@monaco-editor/react": "^4.6.0",
+    "i18next": "^22.5.1",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "next-i18next": "^13.3.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "i18next": "^22.5.1",
-    "next-i18next": "^13.3.0",
-    "react-i18next": "^12.3.1"
+    "react-i18next": "^12.3.1",
+    "turndown": "^7.1.2"
   },
   "devDependencies": {
-    "@types/turndown": "^5.0.4",
     "@types/react": "18.2.0",
-    "@types/react-dom": "18.2.0"
+    "@types/react-dom": "18.2.0",
+    "@types/turndown": "^5.0.4"
   }
 }