diff --git a/docSite/content/docs/commercial/intro.md b/docSite/content/docs/commercial/intro.md index ca018f5aa..353c89452 100644 --- a/docSite/content/docs/commercial/intro.md +++ b/docSite/content/docs/commercial/intro.md @@ -33,7 +33,7 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独 ## 商业版软件价格 -FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud) +FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud) **共有服务** @@ -46,8 +46,7 @@ FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。 {{< table "table-hover table-striped-columns" >}} | 部署方式 | 特有服务 | 上线时长 | 价格 | | ---- | ---- | ---- | ---- | -| Sealos公有云部署 | 1. 6个版本的升级服务。
2. 赠送 8000 元 Sealos 云资源额度。 | 半天 | 10000元/年 | -| Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。
3. 赠送 10000 元 Sealos 云资源额度。 | 半天 | 3000元起/月（3个月起）
或
30000元起/年 | +| Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 3000元起/月（3个月起）
或
30000元起/年 | | 自有服务器-单机版 | 1. 6个版本的升级服务。 | 14天内 | 60000元/套（不限时长） | | 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）| {{< /table >}} diff --git a/docSite/content/docs/development/openapi/auth.md b/docSite/content/docs/development/openapi/auth.md index 47674405d..cea420825 100644 --- a/docSite/content/docs/development/openapi/auth.md +++ b/docSite/content/docs/development/openapi/auth.md @@ -25,7 +25,7 @@ FastGPT 的 API Key **有 2 类**，一类是全局通用的 key (无法直接 | 通用key | 应用特定 key | | --------------------- | --------------------- | -| ![](/imgs/fastgpt-api2.png) | ![](/imgs/fastgpt-api.png) | +| ![](/imgs/fastgpt-api2.jpg) | ![](/imgs/fastgpt-api.jpg) | ## 基本配置 diff --git a/packages/global/common/file/constants.ts b/packages/global/common/file/constants.ts index a5e42b2f4..540404822 100644 --- a/packages/global/common/file/constants.ts +++ b/packages/global/common/file/constants.ts @@ -1,5 +1,10 @@ export enum BucketNameEnum { dataset = 'dataset' } +export const bucketNameMap = { + [BucketNameEnum.dataset]: { + label: 'common.file.bucket.dataset' + } +}; export const FileBaseUrl = '/api/common/file/read'; diff --git a/packages/global/common/file/read/index.ts b/packages/global/common/file/read/index.ts index 302ff8b3a..3c4e149d6 100644 --- a/packages/global/common/file/read/index.ts +++ b/packages/global/common/file/read/index.ts @@ -20,13 +20,13 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer }) const viewport = page.getViewport({ scale: 1 }); const pageHeight = viewport.height; - const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内 - const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内 + const headerThreshold = pageHeight * 0.95; + const footerThreshold = pageHeight * 0.05; const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => { return ( !token.transform || - (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold) + (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold) ); }); diff --git a/packages/service/common/file/upload/multer.ts b/packages/service/common/file/upload/multer.ts index 4530ce975..e74303cb4 100644 --- a/packages/service/common/file/upload/multer.ts +++ b/packages/service/common/file/upload/multer.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { customAlphabet } from 'nanoid'; import multer from 'multer'; import path from 'path'; -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; +import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants'; import fs from 'fs'; const nanoid = customAlphabet('1234567890abcdef', 12); @@ -45,6 +45,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { return reject(error); } + // check bucket name + const bucketName = req.body?.bucketName as `${BucketNameEnum}`; + if (bucketName && !bucketNameMap[bucketName]) { + return reject('BucketName is invalid'); + } + resolve({ ...req.body, files: diff --git a/packages/service/common/vectorStore/controller.ts b/packages/service/common/vectorStore/controller.ts index 0307c2da4..9d9ef5660 100644 --- a/packages/service/common/vectorStore/controller.ts +++ b/packages/service/common/vectorStore/controller.ts @@ -48,7 +48,7 @@ export const updateDatasetDataVector = async ({ // get vector const { vectors, tokens } = await getVectorsByText({ model, - input: [query] + input: query }); await getVectorObj().update({ diff --git a/packages/service/core/ai/embedding/index.ts b/packages/service/core/ai/embedding/index.ts index 88edfa74b..1b617179a 100644 --- a/packages/service/core/ai/embedding/index.ts +++ b/packages/service/core/ai/embedding/index.ts @@ -2,7 +2,7 @@ import { getAIApi } from '../config'; export type GetVectorProps = { model: string; - input: string | string[]; + input: string; }; // text to vector @@ -10,24 +10,13 @@ export async function getVectorsByText({ model = 'text-embedding-ada-002', input }: GetVectorProps) { - if (typeof input === 'string' && !input) { + if (!input) { return Promise.reject({ code: 500, message: 'input is empty' }); - } else if (Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - if (!input[i]) { - return Promise.reject({ - code: 500, - message: 'input array is empty' - }); - } - } - } - if (typeof input === 'string') { - input = [input]; } + try { // 获取 chatAPI const ai = getAIApi(); @@ -36,7 +25,7 @@ export async function getVectorsByText({ const result = await ai.embeddings .create({ model, - input + input: [input] }) .then(async (res) => { if (!res.data) { @@ -47,6 +36,7 @@ export async function getVectorsByText({ // @ts-ignore return Promise.reject(res.data?.err?.message || 'Embedding API Error'); } + return { tokens: res.usage.total_tokens || 0, vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding))) diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 8a22fabc8..ab1840382 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({ }); try { + TrainingDataSchema.index({ weight: -1 }); TrainingDataSchema.index({ lockTime: 1 }); TrainingDataSchema.index({ datasetId: 1 }); TrainingDataSchema.index({ collectionId: 1 }); diff --git a/packages/web/common/system/utils.ts b/packages/web/common/system/utils.ts new file mode 100644 index 000000000..07150fa50 --- /dev/null +++ b/packages/web/common/system/utils.ts @@ -0,0 +1,9 @@ +import FingerprintJS from '@fingerprintjs/fingerprintjs'; + +const fpPromise = FingerprintJS.load(); + +export const getUserFingerprint = async () => { + const fp = await fpPromise; + const result = await fp.get(); + console.log(result.visitorId); +}; diff --git a/packages/web/package.json b/packages/web/package.json index a5657245e..be7f3c759 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -2,9 +2,6 @@ "name": "@fastgpt/web", "version": "1.0.0", "dependencies": { - "@fastgpt/global": "workspace:*", - "joplin-turndown-plugin-gfm": "^1.0.12", - "turndown": "^7.1.2", "@chakra-ui/anatomy": "^2.2.1", "@chakra-ui/icons": "^2.1.1", "@chakra-ui/next-js": "^2.1.5", @@ -13,16 +10,20 @@ "@chakra-ui/system": "^2.6.1", "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", + "@fastgpt/global": "workspace:*", + "@fingerprintjs/fingerprintjs": "^4.2.1", "@monaco-editor/react": "^4.6.0", + "i18next": "^22.5.1", + "joplin-turndown-plugin-gfm": "^1.0.12", + "next-i18next": "^13.3.0", "react": "18.2.0", "react-dom": "18.2.0", - "i18next": "^22.5.1", - "next-i18next": "^13.3.0", - "react-i18next": "^12.3.1" + "react-i18next": "^12.3.1", + "turndown": "^7.1.2" }, "devDependencies": { - "@types/turndown": "^5.0.4", "@types/react": "18.2.0", - "@types/react-dom": "18.2.0" + "@types/react-dom": "18.2.0", + "@types/turndown": "^5.0.4" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6179bf34e..10aa86a18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,6 +163,9 @@ importers: '@fastgpt/global': specifier: workspace:* version: link:../global + '@fingerprintjs/fingerprintjs': + specifier: ^4.2.1 + version: registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1 '@monaco-editor/react': specifier: ^4.6.0 version: registry.npmmirror.com/@monaco-editor/react@4.6.0(monaco-editor@0.45.0)(react-dom@18.2.0)(react@18.2.0) @@ -3836,6 +3839,14 @@ packages: - supports-color dev: true + registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1: + resolution: {integrity: sha512-uW+GVUNTgCXbVPEbgnbf5Aor22e1dyYR0JRwdUiZBaikfxr7KlhV9y0aahA1FB99fEeQVvhCEvTcPIFSYTy9Pw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@fingerprintjs/fingerprintjs/-/fingerprintjs-4.2.1.tgz} + name: '@fingerprintjs/fingerprintjs' + version: 4.2.1 + dependencies: + tslib: registry.npmmirror.com/tslib@2.6.2 + dev: false + registry.npmmirror.com/@humanwhocodes/config-array@0.11.13: resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz} name: '@humanwhocodes/config-array' diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index d9ed7176c..aabb97f5c 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -522,9 +522,13 @@ }, "score": { "embedding": "Embedding", + "embedding desc": "", "fullText": "Full text", + "fullText desc": "", "reRank": "ReRank", - "rrf": "RRF Merge" + "reRank desc": "", + "rrf": "RRF Merge", + "rrf desc": "" }, "search mode": "Search Mode" }, @@ -874,8 +878,6 @@ "QPM": "QPM", "QPM Tips": "The maximum number of queries per IP address per minute", "QPM is empty": "QPM is empty", - "Response Detail": "Quote", - "Response Detail tips": "Whether detailed data such as references to be returned", "token auth": "Token Auth", "token auth Tips": "Identity verification server address. If this value is set, the server will be specified to send a request for identity verification before each session", "token auth use cases": "Review the authentication instructions" @@ -903,6 +905,12 @@ "Update Your Plugin": "Update Plugin" }, "support": { + "outlink": { + "share": { + "Response Quote": "Show Quote", + "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document." + } + }, "user": { "Price": "Price", "auth": { diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 30062da3f..97bc85651 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -878,8 +878,6 @@ "QPM": "", "QPM Tips": "每个 IP 每分钟最多提问多少次", "QPM is empty": "QPM 不能为空", - "Response Detail": "返回详情", - "Response Detail tips": "是否需要返回详情（引用内容，调用时间等，不会返回预设提示词和完整上下文）", "token auth": "身份验证", "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会向指定服务器发送一个请求，进行身份校验", "token auth use cases": "查看身份验证使用说明" @@ -907,6 +905,12 @@ "Update Your Plugin": "更新插件" }, "support": { + "outlink": { + "share": { + "Response Quote": "返回引用", + "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档" + } + }, "user": { "Price": "计费标准", "auth": { diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx index 62e7cc410..3a5cf85a6 100644 --- a/projects/app/src/components/ChatBox/ResponseTags.tsx +++ b/projects/app/src/components/ChatBox/ResponseTags.tsx @@ -133,72 +133,69 @@ const ResponseTags = ({ )} - - {quoteList.length > 0 && ( - - setQuoteModalData({ rawSearch: quoteList })} - > - {quoteList.length}条引用 + {!isShare && ( + + {quoteList.length > 0 && ( + + setQuoteModalData({ rawSearch: quoteList })} + > + {quoteList.length}条引用 + + + )} + {chatAccount === 1 && ( + <> + {historyPreview.length > 0 && ( + + setContextModalData(historyPreview)} + > + {historyPreview.length}条上下文 + + + )} + + )} + {chatAccount > 1 && ( + + 多组 AI 对话 + + )} + + {isPc && runningTime > 0 && ( + + + {runningTime}s + + + )} + + + {t('core.chat.response.Read complete response')} - )} - {chatAccount === 1 && ( - <> - {historyPreview.length > 0 && ( - - setContextModalData(historyPreview)} - > - {historyPreview.length}条上下文 - - - )} - - )} - {chatAccount > 1 && ( - - 多组 AI 对话 - - )} - - {isPc && runningTime > 0 && ( - - - {runningTime}s - - - )} - - - {t('core.chat.response.Read complete response')} - - - - {!!quoteModalData && ( - setQuoteModalData(undefined)} - /> - )} - {!!contextModalData && ( - setContextModalData(undefined)} /> - )} - {isOpenWholeModal && ( - - )} - + + )} + {!!quoteModalData && ( + setQuoteModalData(undefined)} + /> + )} + {!!contextModalData && ( + setContextModalData(undefined)} /> + )} + {isOpenWholeModal && ( + + )} ); }; diff --git a/projects/app/src/components/core/dataset/QuoteItem.tsx b/projects/app/src/components/core/dataset/QuoteItem.tsx index 13d67bb79..bdc61b643 100644 --- a/projects/app/src/components/core/dataset/QuoteItem.tsx +++ b/projects/app/src/components/core/dataset/QuoteItem.tsx @@ -126,63 +126,88 @@ const QuoteItem = ({ > {score?.primaryScore && ( - - - #{score.primaryScore.index + 1} - - - {t(SearchScoreTypeMap[score.primaryScore.type]?.label)} - {SearchScoreTypeMap[score.primaryScore.type]?.showScore - ? ` ${score.primaryScore.value.toFixed(4)}` - : ''} - - - - )} - {score.secondaryScore.map((item, i) => ( - - - - + {canViewSource ? ( + + - #{item.index + 1} - - - {t(SearchScoreTypeMap[item.type]?.label)}: {item.value.toFixed(4)} - - - - {SearchScoreTypeMap[item.type]?.showScore && ( - #{score.primaryScore.index + 1} + - )} + + {t(SearchScoreTypeMap[score.primaryScore.type]?.label)} + {SearchScoreTypeMap[score.primaryScore.type]?.showScore + ? ` ${score.primaryScore.value.toFixed(4)}` + : ''} + + + + ) : ( + + #{score.primaryScore.index + 1} + + )} + + )} + {canViewSource && + score.secondaryScore.map((item, i) => ( + + + + + #{item.index + 1} + + + {t(SearchScoreTypeMap[item.type]?.label)}: {item.value.toFixed(4)} + + + + {SearchScoreTypeMap[item.type]?.showScore && ( + + )} + - - - ))} + + ))} diff --git a/projects/app/src/global/core/dataset/type.d.ts b/projects/app/src/global/core/dataset/type.d.ts index e576acc96..bd810cd2f 100644 --- a/projects/app/src/global/core/dataset/type.d.ts +++ b/projects/app/src/global/core/dataset/type.d.ts @@ -28,5 +28,6 @@ export type DatasetDataListItemType = { collectionId: string; q: string; // embedding content a: string; // bonus content + chunkIndex?: number; indexes: DatasetDataSchemaType['indexes']; }; diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index 088c20839..b500fde74 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -32,7 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }; const [data, total] = await Promise.all([ - MongoDatasetData.find(match, '_id datasetId collectionId q a indexes') + MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex indexes') .sort({ chunkIndex: 1, updateTime: -1 }) .skip((pageNum - 1) * pageSize) .limit(pageSize) diff --git a/projects/app/src/pages/api/v1/embeddings.ts b/projects/app/src/pages/api/v1/embeddings.ts index 4937570f2..1bb57bbd1 100644 --- a/projects/app/src/pages/api/v1/embeddings.ts +++ b/projects/app/src/pages/api/v1/embeddings.ts @@ -10,6 +10,7 @@ import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools'; import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools'; type Props = GetVectorProps & { + input: string | string[]; billId?: string; }; @@ -22,6 +23,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex throw new Error('input is nor array or string'); } + const query = Array.isArray(input) ? input[0] : input; + const { teamId, tmbId, apikey, authType } = await authCert({ req, authToken: true, @@ -30,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex await authTeamBalance(teamId); - const { tokens, vectors } = await getVectorsByText({ input, model }); + const { tokens, vectors } = await getVectorsByText({ input: query, model }); jsonRes(res, { data: { diff --git a/projects/app/src/pages/app/detail/components/OutLink/Share.tsx b/projects/app/src/pages/app/detail/components/OutLink/Share.tsx index 3b938e79b..124651060 100644 --- a/projects/app/src/pages/app/detail/components/OutLink/Share.tsx +++ b/projects/app/src/pages/app/detail/components/OutLink/Share.tsx @@ -96,7 +96,7 @@ const Share = ({ appId }: { appId: string }) => { 名称 金额消耗 - 返回详情 + 返回引用 {feConfigs?.isPlus && ( <> IP限流（人/分钟） @@ -373,8 +373,8 @@ function EditLinkModal({ - {t('outlink.Response Detail')} - + {t('support.outlink.share.Response Quote')} + diff --git a/projects/app/src/pages/dataset/detail/components/DataCard.tsx b/projects/app/src/pages/dataset/detail/components/DataCard.tsx index 8dd601bb6..285ffe3e0 100644 --- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx @@ -307,7 +307,7 @@ const DataCard = () => { > - # {index + 1} + # {item.chunkIndex ?? '-'} ID:{item._id} diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 0c031215a..839519946 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -295,7 +295,7 @@ export async function searchDatasetData(props: { const embeddingRecall = async ({ query, limit }: { query: string; limit: number }) => { const { vectors, tokens } = await getVectorsByText({ model, - input: [query] + input: query }); const { results } = await recallFromVectorStore({ diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts index ec4296eeb..217f76d85 100644 --- a/projects/app/src/service/events/generateVector.ts +++ b/projects/app/src/service/events/generateVector.ts @@ -26,6 +26,8 @@ export async function generateVector(): Promise { if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return; global.vectorQueueLen++; + const start = Date.now(); + // get training data const { data, @@ -154,6 +156,8 @@ export async function generateVector(): Promise { await MongoDatasetTraining.findByIdAndDelete(data._id); reduceQueue(); generateVector(); + + console.log(`embedding finished, time: ${Date.now() - start}ms`); } catch (err: any) { reduceQueue(true); // log diff --git a/projects/app/src/utils/service/core/chat/index.ts b/projects/app/src/utils/service/core/chat/index.ts index d88c5d1e7..2c0d13127 100644 --- a/projects/app/src/utils/service/core/chat/index.ts +++ b/projects/app/src/utils/service/core/chat/index.ts @@ -1,26 +1,23 @@ import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; -export function selectShareResponse({ +export const selectShareResponse = ({ responseData = [] }: { responseData?: ChatHistoryItemResType[]; -}) { - const filedList = [ - 'moduleType', - 'moduleName', - 'moduleLogo', - 'runningTime', - 'quoteList', - 'question' - ]; - return responseData.map((item) => { - const obj: Record = {}; - for (let key in item) { - if (filedList.includes(key)) { - // @ts-ignore - obj[key] = item[key]; +}) => { + const filedList = ['quoteList', 'moduleType']; + const filterModuleTypeList: any[] = [FlowNodeTypeEnum.chatNode]; + return responseData + .filter((item) => filterModuleTypeList.includes(item.moduleType)) + .map((item) => { + const obj: Record = {}; + for (let key in item) { + if (filedList.includes(key)) { + // @ts-ignore + obj[key] = item[key]; + } } - } - return obj as ChatHistoryItemResType; - }); -} + return obj as ChatHistoryItemResType; + }); +};