diff --git a/.gitignore b/.gitignore index 1497f1f3d..4d05069b9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,4 @@ docSite/.vercel *.local.* -# jetbrains .idea/ diff --git a/dev.md b/dev.md new file mode 100644 index 000000000..c400cc6ce --- /dev/null +++ b/dev.md @@ -0,0 +1,17 @@ +# 打包命令 + +```sh +# Build image, not proxy +docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app . + +# build image with proxy +docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao . +``` + +# Pg 常用索引 + +```sql +CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id); +CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id); +CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id); +``` \ No newline at end of file diff --git a/files/deploy/fastgpt/clash/Country.mmdb b/files/deploy/fastgpt/clash/Country.mmdb deleted file mode 100644 index c668d31db..000000000 Binary files a/files/deploy/fastgpt/clash/Country.mmdb and /dev/null differ diff --git a/files/deploy/fastgpt/clash/clash-linux-amd64-v3 b/files/deploy/fastgpt/clash/clash-linux-amd64-v3 deleted file mode 100644 index c1dd3a33e..000000000 Binary files a/files/deploy/fastgpt/clash/clash-linux-amd64-v3 and /dev/null differ diff --git a/files/deploy/fastgpt/clash/config.yaml b/files/deploy/fastgpt/clash/config.yaml deleted file mode 100644 index 5a8a90c90..000000000 --- a/files/deploy/fastgpt/clash/config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -mixed-port: 7890 -allow-lan: false -bind-address: '*' -mode: rule -log-level: warning -dns: - enable: true - ipv6: false - nameserver: - - 8.8.8.8 - - 8.8.4.4 - cache-size: 400 -proxies: - -proxy-groups: - - { - name: '♻️ 自动选择', - type: url-test, - proxies: - [ - 香港V02×1.5, - ABC, - 印度01, - 台湾03, - 新加坡02, - 新加坡03, - 日本01, - 日本02, - 新加坡01, - 美国01, - 美国02, - 台湾01, - 台湾02 - ], - url: 'https://api.openai.com', - interval: 3600 - } -rules: - - 'DOMAIN-SUFFIX,google.com,♻️ 自动选择' - - 'DOMAIN-SUFFIX,ai.fastgpt.in,♻️ 自动选择' - - 'DOMAIN-SUFFIX,openai.com,♻️ 自动选择' - - 'DOMAIN-SUFFIX,api.openai.com,♻️ 自动选择' - - 'MATCH,DIRECT' diff --git a/files/deploy/fastgpt/clash/proxy.sh b/files/deploy/fastgpt/clash/proxy.sh deleted file mode 100644 index fcfa4b1a1..000000000 --- a/files/deploy/fastgpt/clash/proxy.sh +++ /dev/null @@ -1,18 +0,0 @@ -export ALL_PROXY=socks5://127.0.0.1:7891 -export http_proxy=http://127.0.0.1:7890 -export https_proxy=http://127.0.0.1:7890 -export HTTP_PROXY=http://127.0.0.1:7890 -export HTTPS_PROXY=http://127.0.0.1:7890 - -OLD_PROCESS=$(pgrep clash) -if [ ! -z "$OLD_PROCESS" ]; then - echo "Killing old process: $OLD_PROCESS" - kill $OLD_PROCESS -fi -sleep 2 - -cd /root/fastgpt/clash/fast -rm -f ./nohup.out || true -rm -f ./cache.db || true -nohup ./clash-linux-amd64-v3 -d ./ & -echo "Restart clash fast" diff --git a/files/deploy/fastgpt/clash/stop.sh b/files/deploy/fastgpt/clash/stop.sh deleted file mode 100644 index 989e5a2bb..000000000 --- a/files/deploy/fastgpt/clash/stop.sh +++ /dev/null @@ -1,10 +0,0 @@ -export ALL_PROXY='' -export http_proxy='' -export https_proxy='' -export HTTP_PROXY='' -export HTTPS_PROXY='' -OLD_PROCESS=$(pgrep clash) -if [ ! -z "$OLD_PROCESS" ]; then - echo "Killing old process: $OLD_PROCESS" - kill $OLD_PROCESS -fi diff --git a/packages/global/common/file/api.d.ts b/packages/global/common/file/api.d.ts index b1ae998dc..7e05c5f93 100644 --- a/packages/global/common/file/api.d.ts +++ b/packages/global/common/file/api.d.ts @@ -1,9 +1,15 @@ -export type UploadImgProps = { - base64Img: string; +import { MongoImageTypeEnum } from './image/constants'; + +export type preUploadImgProps = { + type: `${MongoImageTypeEnum}`; + expiredTime?: Date; metadata?: Record; shareId?: string; }; +export type UploadImgProps = preUploadImgProps & { + base64Img: string; +}; export type UrlFetchParams = { urlList: string[]; @@ -11,6 +17,7 @@ export type UrlFetchParams = { }; export type UrlFetchResponse = { url: string; + title: string; content: string; selector?: string; }[]; diff --git a/packages/global/common/file/image/constants.ts b/packages/global/common/file/image/constants.ts new file mode 100644 index 000000000..7136f8da4 --- /dev/null +++ b/packages/global/common/file/image/constants.ts @@ -0,0 +1,52 @@ +export const imageBaseUrl = '/api/system/img/'; + +export enum MongoImageTypeEnum { + systemAvatar = 'systemAvatar', + appAvatar = 'appAvatar', + pluginAvatar = 'pluginAvatar', + datasetAvatar = 'datasetAvatar', + userAvatar = 'userAvatar', + teamAvatar = 'teamAvatar', + + chatImage = 'chatImage', + docImage = 'docImage' +} +export const mongoImageTypeMap = { + [MongoImageTypeEnum.systemAvatar]: { + label: 'common.file.type.appAvatar', + unique: true + }, + [MongoImageTypeEnum.appAvatar]: { + label: 'common.file.type.appAvatar', + unique: true + }, + [MongoImageTypeEnum.pluginAvatar]: { + label: 'common.file.type.pluginAvatar', + unique: true + }, + [MongoImageTypeEnum.datasetAvatar]: { + label: 'common.file.type.datasetAvatar', + unique: true + }, + [MongoImageTypeEnum.userAvatar]: { + label: 'common.file.type.userAvatar', + unique: true + }, + [MongoImageTypeEnum.teamAvatar]: { + label: 'common.file.type.teamAvatar', + unique: true + }, + + [MongoImageTypeEnum.chatImage]: { + label: 'common.file.type.chatImage', + unique: false + }, + [MongoImageTypeEnum.docImage]: { + label: 'common.file.type.docImage', + unique: false + } +}; + +export const uniqueImageTypeList = Object.entries(mongoImageTypeMap) + .filter(([key, value]) => value.unique) + .map(([key]) => key as `${MongoImageTypeEnum}`); diff --git a/packages/global/common/file/image/type.d.ts b/packages/global/common/file/image/type.d.ts new file mode 100644 index 000000000..79228d3d5 --- /dev/null +++ b/packages/global/common/file/image/type.d.ts @@ -0,0 +1,11 @@ +import { MongoImageTypeEnum } from './constants'; + +export type MongoImageSchemaType = { + teamId: string; + binary: Buffer; + createTime: Date; + expiredTime?: Date; + type: `${MongoImageTypeEnum}`; + + metadata?: { fileId?: string }; +}; diff --git a/packages/global/common/math/date.ts b/packages/global/common/math/date.ts new file mode 100644 index 000000000..86985b515 --- /dev/null +++ b/packages/global/common/math/date.ts @@ -0,0 +1,10 @@ +// The number of days left in the month is calculated as 30 days per month, and less than 1 day is calculated as 1 day +export const getMonthRemainingDays = () => { + const now = new Date(); + const year = now.getFullYear(); + const month = now.getMonth(); + const date = now.getDate(); + const days = new Date(year, month + 1, 0).getDate(); + const remainingDays = days - date; + return remainingDays + 1; +}; diff --git a/packages/global/common/string/markdown.ts b/packages/global/common/string/markdown.ts index b52a76b26..32299f2dd 100644 --- a/packages/global/common/string/markdown.ts +++ b/packages/global/common/string/markdown.ts @@ -15,10 +15,10 @@ export const simpleMarkdownText = (rawText: string) => { return `[${cleanedLinkText}](${url})`; }); - // replace special \.* …… - const reg1 = /\\([-.!`_(){}\[\]])/g; + // replace special #\.* …… + const reg1 = /\\([#`!*()+-_\[\]{}\\.])/g; if (reg1.test(rawText)) { - rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1'); + rawText = rawText.replace(reg1, '$1'); } // replace \\n @@ -45,24 +45,26 @@ export const uploadMarkdownBase64 = async ({ uploadImgController }: { rawText: string; - uploadImgController: (base64: string) => Promise; + uploadImgController?: (base64: string) => Promise; }) => { - // match base64, upload and replace it - const base64Regex = /data:image\/.*;base64,([^\)]+)/g; - const base64Arr = rawText.match(base64Regex) || []; - // upload base64 and replace it - await Promise.all( - base64Arr.map(async (base64Img) => { - try { - const str = await uploadImgController(base64Img); + if (uploadImgController) { + // match base64, upload and replace it + const base64Regex = /data:image\/.*;base64,([^\)]+)/g; + const base64Arr = rawText.match(base64Regex) || []; + // upload base64 and replace it + await Promise.all( + base64Arr.map(async (base64Img) => { + try { + const str = await uploadImgController(base64Img); - rawText = rawText.replace(base64Img, str); - } catch (error) { - rawText = rawText.replace(base64Img, ''); - rawText = rawText.replace(/!\[.*\]\(\)/g, ''); - } - }) - ); + rawText = rawText.replace(base64Img, str); + } catch (error) { + rawText = rawText.replace(base64Img, ''); + rawText = rawText.replace(/!\[.*\]\(\)/g, ''); + } + }) + ); + } // Remove white space on both sides of the picture const trimReg = /(!\[.*\]\(.*\))\s*/g; @@ -70,5 +72,20 @@ export const uploadMarkdownBase64 = async ({ rawText = rawText.replace(trimReg, '$1'); } - return simpleMarkdownText(rawText); + return rawText; +}; + +export const markdownProcess = async ({ + rawText, + uploadImgController +}: { + rawText: string; + uploadImgController?: (base64: string) => Promise; +}) => { + const imageProcess = await uploadMarkdownBase64({ + rawText, + uploadImgController + }); + + return simpleMarkdownText(imageProcess); }; diff --git a/packages/global/common/string/tiktoken/index.ts b/packages/global/common/string/tiktoken/index.ts index 4770ba268..2f6a3e4d3 100644 --- a/packages/global/common/string/tiktoken/index.ts +++ b/packages/global/common/string/tiktoken/index.ts @@ -33,6 +33,12 @@ export function countPromptTokens( ) { const enc = getTikTokenEnc(); const text = `${role}\n${prompt}`; + + // too large a text will block the thread + if (text.length > 15000) { + return text.length * 1.7; + } + try { const encodeText = enc.encode(text); return encodeText.length + role.length; // 补充 role 估算值 diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index fc80c93e5..5f9bd137a 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -1,3 +1,4 @@ import dayjs from 'dayjs'; -export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm'); +export const formatTime2YMDHM = (time?: Date) => + time ? dayjs(time).format('YYYY-MM-DD HH:mm') : ''; diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index 485062b2d..2fa177ac1 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -1,4 +1,5 @@ import crypto from 'crypto'; +import { customAlphabet } from 'nanoid'; /* check string is a web link */ export function strIsLink(str?: string) { @@ -36,3 +37,7 @@ export function replaceVariable(text: string, obj: Record { + return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)(); +}; diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index 2b78c82fd..2dd489f0a 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -51,6 +51,10 @@ export type FastGPTFeConfigsType = { favicon?: string; customApiDomain?: string; customSharePageDomain?: string; + subscription?: { + datasetStoreFreeSize?: number; + datasetStorePrice?: number; + }; }; export type SystemEnvType = { @@ -63,4 +67,5 @@ export type SystemEnvType = { declare global { var feConfigs: FastGPTFeConfigsType; var systemEnv: SystemEnvType; + var systemInitd: boolean; } diff --git a/packages/global/core/chat/constants.ts b/packages/global/core/chat/constants.ts index 6474e7402..43f650dc8 100644 --- a/packages/global/core/chat/constants.ts +++ b/packages/global/core/chat/constants.ts @@ -31,16 +31,16 @@ export enum ChatSourceEnum { } export const ChatSourceMap = { [ChatSourceEnum.test]: { - name: 'chat.logs.test' + name: 'core.chat.logs.test' }, [ChatSourceEnum.online]: { - name: 'chat.logs.online' + name: 'core.chat.logs.online' }, [ChatSourceEnum.share]: { - name: 'chat.logs.share' + name: 'core.chat.logs.share' }, [ChatSourceEnum.api]: { - name: 'chat.logs.api' + name: 'core.chat.logs.api' } }; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 0a482b543..0b3f18697 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -1,5 +1,5 @@ import { DatasetDataIndexItemType, DatasetSchemaType } from './type'; -import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; import type { LLMModelItemType } from '../ai/model.d'; /* ================= dataset ===================== */ @@ -16,21 +16,38 @@ export type DatasetUpdateBody = { }; /* ================= collection ===================== */ -export type CreateDatasetCollectionParams = { +export type DatasetCollectionChunkMetadataType = { + trainingType?: `${TrainingModeEnum}`; + chunkSize?: number; + chunkSplitter?: string; + qaPrompt?: string; +}; +export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; parentId?: string; name: string; type: `${DatasetCollectionTypeEnum}`; - trainingType?: `${DatasetCollectionTrainingModeEnum}`; - chunkSize?: number; fileId?: string; rawLink?: string; - qaPrompt?: string; rawTextLength?: number; hashRawText?: string; metadata?: Record; }; +export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { + datasetId: string; + parentId?: string; + metadata?: Record; +}; +export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + name: string; + text: string; +}; +export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + link: string; + chunkSplitter?: string; +}; + /* ================= data ===================== */ export type PgSearchRawType = { id: string; diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constant.ts index fb451b0e2..8c6aaf38d 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constant.ts @@ -53,23 +53,7 @@ export const DatasetCollectionTypeMap = { name: 'core.dataset.link' }, [DatasetCollectionTypeEnum.virtual]: { - name: 'core.dataset.Virtual File' - } -}; -export enum DatasetCollectionTrainingModeEnum { - manual = 'manual', - chunk = 'chunk', - qa = 'qa' -} -export const DatasetCollectionTrainingTypeMap = { - [DatasetCollectionTrainingModeEnum.manual]: { - label: 'core.dataset.collection.training.type manual' - }, - [DatasetCollectionTrainingModeEnum.chunk]: { - label: 'core.dataset.collection.training.type chunk' - }, - [DatasetCollectionTrainingModeEnum.qa]: { - label: 'core.dataset.collection.training.type qa' + name: 'core.dataset.Manual collection' } }; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 0c05982cc..8e99af6dd 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -42,11 +42,15 @@ export type DatasetCollectionSchemaType = { type: `${DatasetCollectionTypeEnum}`; createTime: Date; updateTime: Date; + trainingType: `${TrainingModeEnum}`; chunkSize: number; + chunkSplitter?: string; + qaPrompt?: string; + fileId?: string; rawLink?: string; - qaPrompt?: string; + rawTextLength?: number; hashRawText?: string; metadata?: { diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts index da848ad5a..577d009b4 100644 --- a/packages/global/core/dataset/utils.ts +++ b/packages/global/core/dataset/utils.ts @@ -1,4 +1,4 @@ -import { DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; import { getFileIcon } from '../../common/file/icon'; import { strIsLink } from '../../common/string/tools'; @@ -55,3 +55,8 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin dataId }; } + +export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => { + if (mode === TrainingModeEnum.qa) return data.length * 20; + return data.length; +}; diff --git a/packages/global/package.json b/packages/global/package.json index 7e57ff034..cce0af6d6 100644 --- a/packages/global/package.json +++ b/packages/global/package.json @@ -7,7 +7,7 @@ "encoding": "^0.1.13", "js-tiktoken": "^1.0.7", "openai": "4.23.0", - "pdfjs-dist": "^4.0.269", + "nanoid": "^4.0.1", "timezones-list": "^3.0.2" }, "devDependencies": { diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts index 7a937d8cf..4fff5930a 100644 --- a/packages/global/support/user/team/type.d.ts +++ b/packages/global/support/user/team/type.d.ts @@ -9,7 +9,6 @@ export type TeamSchema = { createTime: Date; balance: number; maxSize: number; - lastDatasetBillTime: Date; limit: { lastExportDatasetTime: Date; lastWebsiteSyncTime: Date; diff --git a/packages/global/support/wallet/bill/constants.ts b/packages/global/support/wallet/bill/constants.ts index 319bd5ff8..6d30230bb 100644 --- a/packages/global/support/wallet/bill/constants.ts +++ b/packages/global/support/wallet/bill/constants.ts @@ -7,7 +7,7 @@ export enum BillSourceEnum { api = 'api', shareLink = 'shareLink', training = 'training', - datasetStore = 'datasetStore' + datasetExpand = 'datasetExpand' } export const BillSourceMap: Record<`${BillSourceEnum}`, string> = { @@ -15,5 +15,5 @@ export const BillSourceMap: Record<`${BillSourceEnum}`, string> = { [BillSourceEnum.api]: 'Api', [BillSourceEnum.shareLink]: '免登录链接', [BillSourceEnum.training]: '数据训练', - [BillSourceEnum.datasetStore]: '知识库存储' + [BillSourceEnum.datasetExpand]: '知识库扩容' }; diff --git a/packages/global/support/wallet/sub/api.d.ts b/packages/global/support/wallet/sub/api.d.ts new file mode 100644 index 000000000..ce0b9e681 --- /dev/null +++ b/packages/global/support/wallet/sub/api.d.ts @@ -0,0 +1,4 @@ +export type SubDatasetSizeParams = { + size: number; + renew: boolean; +}; diff --git a/packages/global/support/wallet/sub/constants.ts b/packages/global/support/wallet/sub/constants.ts new file mode 100644 index 000000000..6e03e56ca --- /dev/null +++ b/packages/global/support/wallet/sub/constants.ts @@ -0,0 +1,37 @@ +export enum SubTypeEnum { + datasetStore = 'datasetStore' +} + +export const subTypeMap = { + [SubTypeEnum.datasetStore]: { + label: 'support.user.team.subscription.type.datasetStore' + } +}; + +export enum SubModeEnum { + month = 'month', + year = 'year' +} + +export const subModeMap = { + [SubModeEnum.month]: { + label: 'support.user.team.subscription.mode.month' + }, + [SubModeEnum.year]: { + label: 'support.user.team.subscription.mode.year' + } +}; + +export enum SubStatusEnum { + active = 'active', + expired = 'expired' +} + +export const subStatusMap = { + [SubStatusEnum.active]: { + label: 'support.user.team.subscription.status.active' + }, + [SubStatusEnum.expired]: { + label: 'support.user.team.subscription.status.expired' + } +}; diff --git a/packages/global/support/wallet/sub/type.d.ts b/packages/global/support/wallet/sub/type.d.ts new file mode 100644 index 000000000..31e174a7e --- /dev/null +++ b/packages/global/support/wallet/sub/type.d.ts @@ -0,0 +1,12 @@ +import { SubModeEnum, SubStatusEnum, SubTypeEnum } from './constants'; + +export type TeamSubSchema = { + teamId: string; + type: `${SubTypeEnum}`; + mode: `${SubModeEnum}`; + status: `${SubStatusEnum}`; + renew: boolean; + startTime: Date; + expiredTime: Date; + datasetStoreAmount?: number; +}; diff --git a/packages/service/common/file/constants.ts b/packages/service/common/file/constants.ts new file mode 100644 index 000000000..a3449c4be --- /dev/null +++ b/packages/service/common/file/constants.ts @@ -0,0 +1,6 @@ +import path from 'path'; + +export const tmpFileDirPath = + process.env.NODE_ENV === 'production' ? '/app/tmp' : path.join(process.cwd(), 'tmp'); + +export const previewMaxCharCount = 3000; diff --git a/packages/service/common/file/image/constant.ts b/packages/service/common/file/image/constant.ts deleted file mode 100644 index 518900c30..000000000 --- a/packages/service/common/file/image/constant.ts +++ /dev/null @@ -1 +0,0 @@ -export const imageBaseUrl = '/api/system/img/'; diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index 7cfe44828..c8da371c8 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -1,5 +1,5 @@ import { UploadImgProps } from '@fastgpt/global/common/file/api'; -import { imageBaseUrl } from './constant'; +import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; import { MongoImage } from './schema'; export function getMongoImgUrl(id: string) { @@ -8,10 +8,13 @@ export function getMongoImgUrl(id: string) { export const maxImgSize = 1024 * 1024 * 12; export async function uploadMongoImg({ + type, base64Img, teamId, expiredTime, - metadata + metadata, + + shareId }: UploadImgProps & { teamId: string; }) { @@ -20,12 +23,16 @@ export async function uploadMongoImg({ } const base64Data = base64Img.split(',')[1]; + const binary = Buffer.from(base64Data, 'base64'); const { _id } = await MongoImage.create({ + type, teamId, - binary: Buffer.from(base64Data, 'base64'), + binary, expiredTime: expiredTime, - metadata + metadata, + + shareId }); return getMongoImgUrl(String(_id)); diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index b401be0e2..0b4ff832a 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -1,5 +1,7 @@ import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { connectionMongo, type Model } from '../../mongo'; +import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d'; +import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants'; const { Schema, model, models } = connectionMongo; const ImageSchema = new Schema({ @@ -12,12 +14,18 @@ const ImageSchema = new Schema({ type: Date, default: () => new Date() }, - binary: { - type: Buffer - }, expiredTime: { type: Date }, + binary: { + type: Buffer + }, + type: { + type: String, + enum: Object.keys(mongoImageTypeMap), + required: true + }, + metadata: { type: Object } @@ -25,14 +33,13 @@ const ImageSchema = new Schema({ try { ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); + ImageSchema.index({ type: 1 }); + ImageSchema.index({ teamId: 1 }); } catch (error) { console.log(error); } -export const MongoImage: Model<{ - teamId: string; - binary: Buffer; - metadata?: { fileId?: string }; -}> = models['image'] || model('image', ImageSchema); +export const MongoImage: Model = + models['image'] || model('image', ImageSchema); MongoImage.syncIndexes(); diff --git a/packages/service/common/file/load/pdf.ts b/packages/service/common/file/load/pdf.ts new file mode 100644 index 000000000..4cbb4673d --- /dev/null +++ b/packages/service/common/file/load/pdf.ts @@ -0,0 +1,68 @@ +import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'; +// @ts-ignore +import('pdfjs-dist/legacy/build/pdf.worker.min.mjs'); +import { ReadFileParams } from './type'; + +type TokenType = { + str: string; + dir: string; + width: number; + height: number; + transform: number[]; + fontName: string; + hasEOL: boolean; +}; + +export const readPdfFile = async ({ path }: ReadFileParams) => { + const readPDFPage = async (doc: any, pageNo: number) => { + const page = await doc.getPage(pageNo); + const tokenizedText = await page.getTextContent(); + + const viewport = page.getViewport({ scale: 1 }); + const pageHeight = viewport.height; + const headerThreshold = pageHeight * 0.95; + const footerThreshold = pageHeight * 0.05; + + const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => { + return ( + !token.transform || + (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold) + ); + }); + + // concat empty string 'hasEOL' + for (let i = 0; i < pageTexts.length; i++) { + const item = pageTexts[i]; + if (item.str === '' && pageTexts[i - 1]) { + pageTexts[i - 1].hasEOL = item.hasEOL; + pageTexts.splice(i, 1); + i--; + } + } + + page.cleanup(); + + return pageTexts + .map((token) => { + const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str); + + return paragraphEnd ? `${token.str}\n` : token.str; + }) + .join(''); + }; + + const loadingTask = pdfjs.getDocument(path); + const doc = await loadingTask.promise; + + const pageTextPromises = []; + for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) { + pageTextPromises.push(readPDFPage(doc, pageNo)); + } + const pageTexts = await Promise.all(pageTextPromises); + + loadingTask.destroy(); + + return { + rawText: pageTexts.join('') + }; +}; diff --git a/packages/service/common/file/load/type.d.ts b/packages/service/common/file/load/type.d.ts new file mode 100644 index 000000000..ffb9f3ee0 --- /dev/null +++ b/packages/service/common/file/load/type.d.ts @@ -0,0 +1,18 @@ +export type ReadFileParams = { + preview: boolean; + teamId: string; + path: string; + metadata?: Record; +}; + +export type ReadFileResponse = { + rawText: string; +}; + +export type ReadFileBufferItemType = ReadFileParams & { + rawText: string; +}; + +declare global { + var readFileBuffers: ReadFileBufferItemType[]; +} diff --git a/packages/service/common/file/load/utils.ts b/packages/service/common/file/load/utils.ts new file mode 100644 index 000000000..760bc610e --- /dev/null +++ b/packages/service/common/file/load/utils.ts @@ -0,0 +1,50 @@ +import { readPdfFile } from './pdf'; +import { readDocFle } from './word'; +import { ReadFileBufferItemType, ReadFileParams } from './type'; + +global.readFileBuffers = global.readFileBuffers || []; + +const bufferMaxSize = 200; + +export const pushFileReadBuffer = (params: ReadFileBufferItemType) => { + global.readFileBuffers.push(params); + + if (global.readFileBuffers.length > bufferMaxSize) { + global.readFileBuffers.shift(); + } +}; +export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) => + global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId); + +export const readFileContent = async (params: ReadFileParams) => { + const { path } = params; + + const buffer = getReadFileBuffer(params); + + if (buffer) { + return buffer; + } + + const extension = path?.split('.')?.pop()?.toLowerCase() || ''; + + const { rawText } = await (async () => { + switch (extension) { + case 'pdf': + return readPdfFile(params); + case 'docx': + return readDocFle(params); + default: + return Promise.reject('Only support .pdf, .docx'); + } + })(); + + pushFileReadBuffer({ + ...params, + rawText + }); + + return { + ...params, + rawText + }; +}; diff --git a/packages/service/common/file/load/word.ts b/packages/service/common/file/load/word.ts new file mode 100644 index 000000000..8842a6fd9 --- /dev/null +++ b/packages/service/common/file/load/word.ts @@ -0,0 +1,22 @@ +import mammoth from 'mammoth'; +import { htmlToMarkdown } from '../../string/markdown'; +import { ReadFileParams } from './type'; +/** + * read docx to markdown + */ +export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => { + try { + const { value: html } = await mammoth.convertToHtml({ + path + }); + + const md = await htmlToMarkdown(html); + + return { + rawText: md + }; + } catch (error) { + console.log('error doc read:', error); + return Promise.reject('Can not read doc file, please convert to PDF'); + } +}; diff --git a/packages/service/common/file/upload/multer.ts b/packages/service/common/file/multer.ts similarity index 80% rename from packages/service/common/file/upload/multer.ts rename to packages/service/common/file/multer.ts index e74303cb4..62ff53313 100644 --- a/packages/service/common/file/upload/multer.ts +++ b/packages/service/common/file/multer.ts @@ -1,11 +1,9 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { customAlphabet } from 'nanoid'; import multer from 'multer'; import path from 'path'; import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants'; -import fs from 'fs'; - -const nanoid = customAlphabet('1234567890abcdef', 12); +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import { tmpFileDirPath } from './constants'; type FileType = { fieldname: string; @@ -17,7 +15,9 @@ type FileType = { size: number; }; -export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { +const expiredTime = 30 * 60 * 1000; + +export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { maxSize *= 1024 * 1024; class UploadModel { uploader = multer({ @@ -26,9 +26,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { }, preservePath: true, storage: multer.diskStorage({ - filename: (_req, file, cb) => { + // destination: (_req, _file, cb) => { + // cb(null, tmpFileDirPath); + // }, + filename: async (req, file, cb) => { const { ext } = path.parse(decodeURIComponent(file.originalname)); - cb(null, nanoid() + ext); + cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`); } }) }).any(); @@ -75,14 +78,4 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { } return new UploadModel(); -} - -export const removeFilesByPaths = (paths: string[]) => { - paths.forEach((path) => { - fs.unlink(path, (err) => { - if (err) { - console.error(err); - } - }); - }); }; diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts new file mode 100644 index 000000000..641aed8f9 --- /dev/null +++ b/packages/service/common/file/utils.ts @@ -0,0 +1,33 @@ +import fs from 'fs'; +import { tmpFileDirPath } from './constants'; + +export const removeFilesByPaths = (paths: string[]) => { + paths.forEach((path) => { + fs.unlink(path, (err) => { + if (err) { + console.error(err); + } + }); + }); +}; + +/* cron job. check expired tmp files */ +export const checkExpiredTmpFiles = () => { + // get all file name + const files = fs.readdirSync(tmpFileDirPath).map((name) => { + const timestampStr = name.split('-')[0]; + const expiredTimestamp = timestampStr ? Number(timestampStr) : 0; + + return { + filename: name, + expiredTimestamp, + path: `${tmpFileDirPath}/${name}` + }; + }); + + // count expiredFiles + const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now()); + + // remove expiredFiles + removeFilesByPaths(expiredFiles.map((item) => item.path)); +}; diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 44a6c1344..722e77c46 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -50,8 +50,11 @@ export const cheerioToHtml = ({ .get() .join('\n'); + const title = $('head title').text() || $('h1:first').text() || fetchUrl; + return { html, + title, usedSelector }; }; @@ -70,7 +73,7 @@ export const urlsFetch = async ({ }); const $ = cheerio.load(fetchRes.data); - const { html, usedSelector } = cheerioToHtml({ + const { title, html, usedSelector } = cheerioToHtml({ fetchUrl: url, $, selector @@ -79,6 +82,7 @@ export const urlsFetch = async ({ return { url, + title, content: md, selector: usedSelector }; @@ -87,6 +91,7 @@ export const urlsFetch = async ({ return { url, + title: '', content: '', selector: '' }; diff --git a/packages/service/common/string/markdown.ts b/packages/service/common/string/markdown.ts index fd292d861..21c0987cd 100644 --- a/packages/service/common/string/markdown.ts +++ b/packages/service/common/string/markdown.ts @@ -15,7 +15,9 @@ export const htmlToMarkdown = (html?: string | null) => worker.on('message', (md: string) => { worker.terminate(); - resolve(simpleMarkdownText(md)); + let rawText = simpleMarkdownText(md); + + resolve(rawText); }); worker.on('error', (err) => { worker.terminate(); diff --git a/packages/service/common/system/cron.ts b/packages/service/common/system/cron.ts new file mode 100644 index 000000000..7b5e32513 --- /dev/null +++ b/packages/service/common/system/cron.ts @@ -0,0 +1,6 @@ +import nodeCron from 'node-cron'; + +export const setCron = (time: string, cb: () => void) => { + // second minute hour day month week + return nodeCron.schedule(time, cb); +}; diff --git a/packages/service/common/system/log.ts b/packages/service/common/system/log.ts index 4035c4b19..ed7a2fd8a 100644 --- a/packages/service/common/system/log.ts +++ b/packages/service/common/system/log.ts @@ -49,6 +49,7 @@ export const addLog = { }, error(msg: string, error?: any) { this.log('error', msg, { + message: error?.message, stack: error?.stack, ...(error?.config && { config: { diff --git a/packages/service/common/vectorStore/controller.d.ts b/packages/service/common/vectorStore/controller.d.ts index 3ddd01fa0..671230299 100644 --- a/packages/service/common/vectorStore/controller.d.ts +++ b/packages/service/common/vectorStore/controller.d.ts @@ -2,6 +2,8 @@ export type DeleteDatasetVectorProps = { id?: string; datasetIds?: string[]; collectionIds?: string[]; + + collectionId?: string; dataIds?: string[]; }; diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index 0828c31f7..a2e044d6f 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -101,14 +101,19 @@ export const deleteDatasetDataVector = async ( retry?: number; } ): Promise => { - const { id, datasetIds, collectionIds, dataIds, retry = 2 } = props; + const { id, datasetIds, collectionIds, collectionId, dataIds, retry = 2 } = props; const where = await (() => { if (id) return `id=${id}`; if (datasetIds) return `dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})`; - if (collectionIds) + if (collectionIds) { return `collection_id IN (${collectionIds.map((id) => `'${String(id)}'`).join(',')})`; - if (dataIds) return `data_id IN (${dataIds.map((id) => `'${String(id)}'`).join(',')})`; + } + if (collectionId && dataIds) { + return `collection_id='${String(collectionId)}' and data_id IN (${dataIds + .map((id) => `'${String(id)}'`) + .join(',')})`; + } return Promise.reject('deleteDatasetData: no where'); })(); diff --git a/packages/service/core/ai/embedding/index.ts b/packages/service/core/ai/embedding/index.ts index 1b617179a..2b785ab0a 100644 --- a/packages/service/core/ai/embedding/index.ts +++ b/packages/service/core/ai/embedding/index.ts @@ -32,7 +32,7 @@ export async function getVectorsByText({ return Promise.reject('Embedding API 404'); } if (!res?.data?.[0]?.embedding) { - console.log(res?.data); + console.log(res); // @ts-ignore return Promise.reject(res.data?.err?.message || 'Embedding API Error'); } diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index beed3af23..aefbd7d01 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -2,8 +2,7 @@ import { connectionMongo, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type'; import { ChatRoleMap } from '@fastgpt/global/core/chat/constants'; -import { customAlphabet } from 'nanoid'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24); +import { getNanoid } from '@fastgpt/global/common/string/tools'; import { TeamCollectionName, TeamMemberCollectionName @@ -13,24 +12,6 @@ import { userCollectionName } from '../../support/user/schema'; import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; const ChatItemSchema = new Schema({ - dataId: { - type: String, - require: true, - default: () => nanoid() - }, - appId: { - type: Schema.Types.ObjectId, - ref: appCollectionName, - required: true - }, - chatId: { - type: String, - require: true - }, - userId: { - type: Schema.Types.ObjectId, - ref: userCollectionName - }, teamId: { type: Schema.Types.ObjectId, ref: TeamCollectionName, @@ -41,6 +22,24 @@ const ChatItemSchema = new Schema({ ref: TeamMemberCollectionName, required: true }, + userId: { + type: Schema.Types.ObjectId, + ref: userCollectionName + }, + chatId: { + type: String, + require: true + }, + dataId: { + type: String, + require: true, + default: () => getNanoid(22) + }, + appId: { + type: Schema.Types.ObjectId, + ref: appCollectionName, + required: true + }, time: { type: Date, default: () => new Date() @@ -80,10 +79,11 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ dataId: -1 }); + ChatItemSchema.index({ teamId: 1 }); ChatItemSchema.index({ time: -1 }); ChatItemSchema.index({ appId: 1 }); ChatItemSchema.index({ chatId: 1 }); + ChatItemSchema.index({ obj: 1 }); ChatItemSchema.index({ userGoodFeedback: 1 }); ChatItemSchema.index({ userBadFeedback: 1 }); ChatItemSchema.index({ customFeedbacks: 1 }); diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 954344517..d0e7c14d2 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,7 +1,4 @@ -import { - DatasetCollectionTrainingModeEnum, - DatasetCollectionTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; @@ -12,11 +9,15 @@ export async function createOneCollection({ parentId, datasetId, type, - trainingType = DatasetCollectionTrainingModeEnum.manual, + + trainingType = TrainingModeEnum.chunk, chunkSize = 0, + chunkSplitter, + qaPrompt, + fileId, rawLink, - qaPrompt, + hashRawText, rawTextLength, metadata = {}, @@ -30,11 +31,15 @@ export async function createOneCollection({ datasetId, name, type, + trainingType, chunkSize, + chunkSplitter, + qaPrompt, + fileId, rawLink, - qaPrompt, + rawTextLength, hashRawText, metadata @@ -74,7 +79,7 @@ export function createDefaultCollection({ datasetId, parentId, type: DatasetCollectionTypeEnum.virtual, - trainingType: DatasetCollectionTrainingModeEnum.manual, + trainingType: TrainingModeEnum.chunk, chunkSize: 0, updateTime: new Date('2099') }); diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 02ba9a576..f2b5f8def 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,10 +1,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; -import { - DatasetCollectionTrainingTypeMap, - DatasetCollectionTypeMap -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant'; import { DatasetCollectionName } from '../schema'; import { TeamCollectionName, @@ -56,15 +53,23 @@ const DatasetCollectionSchema = new Schema({ type: Date, default: () => new Date() }, + trainingType: { type: String, - enum: Object.keys(DatasetCollectionTrainingTypeMap), + enum: Object.keys(TrainingTypeMap), required: true }, chunkSize: { type: Number, required: true }, + chunkSplitter: { + type: String + }, + qaPrompt: { + type: String + }, + fileId: { type: Schema.Types.ObjectId, ref: 'dataset.files' @@ -72,9 +77,6 @@ const DatasetCollectionSchema = new Schema({ rawLink: { type: String }, - qaPrompt: { - type: String - }, rawTextLength: { type: Number @@ -89,8 +91,9 @@ const DatasetCollectionSchema = new Schema({ }); try { + DatasetCollectionSchema.index({ teamId: 1 }); DatasetCollectionSchema.index({ datasetId: 1 }); - DatasetCollectionSchema.index({ datasetId: 1, parentId: 1 }); + DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, parentId: 1 }); DatasetCollectionSchema.index({ updateTime: -1 }); DatasetCollectionSchema.index({ hashRawText: -1 }); } catch (error) { diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 6918f0376..225e01b22 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -4,7 +4,7 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { MongoDatasetTraining } from '../training/schema'; import { urlsFetch } from '../../../common/string/cheerio'; -import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { hashStr } from '@fastgpt/global/common/string/tools'; /** @@ -92,8 +92,12 @@ export const getCollectionAndRawText = async ({ return Promise.reject('Collection not found'); } - const rawText = await (async () => { - if (newRawText) return newRawText; + const { title, rawText } = await (async () => { + if (newRawText) + return { + title: '', + rawText: newRawText + }; // link if (col.type === DatasetCollectionTypeEnum.link && col.rawLink) { // crawl new data @@ -102,12 +106,18 @@ export const getCollectionAndRawText = async ({ selector: col.datasetId?.websiteConfig?.selector || col?.metadata?.webPageSelector }); - return result[0].content; + return { + title: result[0].title, + rawText: result[0].content + }; } // file - return ''; + return { + title: '', + rawText: '' + }; })(); const hashRawText = hashStr(rawText); @@ -115,6 +125,7 @@ export const getCollectionAndRawText = async ({ return { collection: col, + title, rawText, isSameRawText }; @@ -135,6 +146,7 @@ export const reloadCollectionChunks = async ({ rawText?: string; }) => { const { + title, rawText: newRawText, collection: col, isSameRawText @@ -154,6 +166,11 @@ export const reloadCollectionChunks = async ({ }); // insert to training queue + const model = await (() => { + if (col.trainingType === TrainingModeEnum.chunk) return col.datasetId.vectorModel; + if (col.trainingType === TrainingModeEnum.qa) return col.datasetId.agentModel; + return Promise.reject('Training model error'); + })(); await MongoDatasetTraining.insertMany( chunks.map((item, i) => ({ teamId: col.teamId, @@ -163,7 +180,7 @@ export const reloadCollectionChunks = async ({ billId, mode: col.trainingType, prompt: '', - model: col.datasetId.vectorModel, + model, q: item, a: '', chunkIndex: i @@ -172,6 +189,7 @@ export const reloadCollectionChunks = async ({ // update raw text await MongoDatasetCollection.findByIdAndUpdate(col._id, { + ...(title && { name: title }), rawTextLength: newRawText.length, hashRawText: hashStr(newRawText) }); diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index bc0f84daa..e1af0ad0b 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -75,7 +75,13 @@ export async function delCollectionRelevantData({ /** * delete one data by mongoDataId */ -export async function delDatasetDataByDataId(mongoDataId: string) { - await deleteDatasetDataVector({ dataIds: [mongoDataId] }); +export async function delDatasetDataByDataId({ + collectionId, + mongoDataId +}: { + collectionId: string; + mongoDataId: string; +}) { + await deleteDatasetDataVector({ collectionId, dataIds: [mongoDataId] }); await MongoDatasetData.findByIdAndDelete(mongoDataId); } diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index f79b730b5..8681cc9fc 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -85,12 +85,13 @@ const DatasetDataSchema = new Schema({ }); try { + DatasetDataSchema.index({ teamId: 1 }); DatasetDataSchema.index({ datasetId: 1 }); DatasetDataSchema.index({ collectionId: 1 }); DatasetDataSchema.index({ updateTime: -1 }); + DatasetDataSchema.index({ collectionId: 1, q: 1, a: 1 }); // full text index DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' }); - DatasetDataSchema.index({ inited: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 039ead527..827487a23 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -92,7 +92,7 @@ const DatasetSchema = new Schema({ }); try { - DatasetSchema.index({ userId: 1 }); + DatasetSchema.index({ teamId: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index ab1840382..8d7742763 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({ }); try { + TrainingDataSchema.index({ teamId: 1 }); TrainingDataSchema.index({ weight: -1 }); TrainingDataSchema.index({ lockTime: 1 }); TrainingDataSchema.index({ datasetId: 1 }); diff --git a/packages/service/package.json b/packages/service/package.json index d1405aa31..baa926fb5 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -3,17 +3,19 @@ "version": "1.0.0", "dependencies": { "@fastgpt/global": "workspace:*", - "cookie": "^0.5.0", - "encoding": "^0.1.13", - "jsonwebtoken": "^9.0.2", - "mongoose": "^7.0.2", - "nanoid": "^4.0.1", - "dayjs": "^1.11.7", - "next": "13.5.2", - "multer": "1.4.5-lts.1", "axios": "^1.5.1", "cheerio": "1.0.0-rc.12", + "cookie": "^0.5.0", + "dayjs": "^1.11.7", + "encoding": "^0.1.13", + "jsonwebtoken": "^9.0.2", + "mammoth": "^1.6.0", + "mongoose": "^7.0.2", + "multer": "1.4.5-lts.1", + "next": "13.5.2", "nextjs-cors": "^2.1.2", + "node-cron": "^3.0.3", + "pdfjs-dist": "^4.0.269", "pg": "^8.10.0", "tunnel": "^0.0.6" }, @@ -21,6 +23,7 @@ "@types/cookie": "^0.5.2", "@types/jsonwebtoken": "^9.0.3", "@types/multer": "^1.4.10", + "@types/node-cron": "^3.0.11", "@types/pg": "^8.6.6", "@types/tunnel": "^0.0.4" } diff --git a/packages/service/support/openapi/tools.ts b/packages/service/support/openapi/tools.ts index 852e17342..b6bedee1e 100644 --- a/packages/service/support/openapi/tools.ts +++ b/packages/service/support/openapi/tools.ts @@ -1,18 +1,22 @@ import { MongoOpenApi } from './schema'; -export async function updateApiKeyUsedTime(id: string) { - await MongoOpenApi.findByIdAndUpdate(id, { +export function updateApiKeyUsedTime(id: string) { + MongoOpenApi.findByIdAndUpdate(id, { lastUsedTime: new Date() + }).catch((err) => { + console.log('update apiKey used time error', err); }); } -export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) { - await MongoOpenApi.findOneAndUpdate( +export function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) { + MongoOpenApi.findOneAndUpdate( { apiKey: apikey }, { $inc: { usage } } - ); + ).catch((err) => { + console.log('update apiKey usage error', err); + }); } diff --git a/packages/service/support/outLink/tools.ts b/packages/service/support/outLink/tools.ts index c4e7e7370..3b9afa32d 100644 --- a/packages/service/support/outLink/tools.ts +++ b/packages/service/support/outLink/tools.ts @@ -9,17 +9,15 @@ export const updateOutLinkUsage = async ({ shareId: string; total: number; }) => { - try { - await MongoOutLink.findOneAndUpdate( - { shareId }, - { - $inc: { total }, - lastTime: new Date() - } - ); - } catch (err) { + MongoOutLink.findOneAndUpdate( + { shareId }, + { + $inc: { total }, + lastTime: new Date() + } + ).catch((err) => { console.log('update shareChat error', err); - } + }); }; export const pushResult2Remote = async ({ diff --git a/packages/service/support/permission/limit/dataset.ts b/packages/service/support/permission/limit/dataset.ts new file mode 100644 index 000000000..a214465e2 --- /dev/null +++ b/packages/service/support/permission/limit/dataset.ts @@ -0,0 +1,20 @@ +import { getVectorCountByTeamId } from '../../../common/vectorStore/controller'; +import { getTeamDatasetValidSub } from '../../wallet/sub/utils'; + +export const checkDatasetLimit = async ({ + teamId, + freeSize = Infinity, + insertLen = 0 +}: { + teamId: string; + freeSize?: number; + insertLen?: number; +}) => { + const { maxSize } = await getTeamDatasetValidSub({ teamId, freeSize }); + const usedSize = await getVectorCountByTeamId(teamId); + + if (usedSize + insertLen >= maxSize) { + return Promise.reject(`数据库容量已满，无法继续添加。可以在账号页面进行扩容。`); + } + return; +}; diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index 3001854ab..eaeb9fba9 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -30,9 +30,6 @@ const TeamSchema = new Schema({ type: Number, default: 5 }, - lastDatasetBillTime: { - type: Date - }, limit: { lastExportDatasetTime: { type: Date diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index e5822efc7..25c8b00c1 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -54,6 +54,7 @@ const BillSchema = new Schema({ try { BillSchema.index({ teamId: 1 }); BillSchema.index({ tmbId: 1 }); + BillSchema.index({ tmbId: 1, time: 1 }); BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 }); } catch (error) { console.log(error); diff --git a/packages/service/support/wallet/sub/schema.ts b/packages/service/support/wallet/sub/schema.ts new file mode 100644 index 000000000..2f2cc3357 --- /dev/null +++ b/packages/service/support/wallet/sub/schema.ts @@ -0,0 +1,55 @@ +import { connectionMongo, type Model } from '../../../common/mongo'; +const { Schema, model, models } = connectionMongo; +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { subModeMap, subStatusMap, subTypeMap } from '@fastgpt/global/support/wallet/sub/constants'; +import type { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type'; + +export const subCollectionName = 'team.subscription'; + +const SubSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + type: { + type: String, + enum: Object.keys(subTypeMap), + required: true + }, + mode: { + type: String, + enum: Object.keys(subModeMap), + required: true + }, + status: { + type: String, + enum: Object.keys(subStatusMap), + required: true + }, + renew: { + type: Boolean, + default: true + }, + startTime: { + type: Date + }, + expiredTime: { + type: Date + }, + datasetStoreAmount: { + type: Number + } +}); + +try { + SubSchema.index({ teamId: 1 }); + SubSchema.index({ status: 1 }); + SubSchema.index({ type: 1 }); + SubSchema.index({ expiredTime: -1 }); +} catch (error) { + console.log(error); +} + +export const MongoTeamSub: Model = + models[subCollectionName] || model(subCollectionName, SubSchema); diff --git a/packages/service/support/wallet/sub/utils.ts b/packages/service/support/wallet/sub/utils.ts new file mode 100644 index 000000000..8abd4834d --- /dev/null +++ b/packages/service/support/wallet/sub/utils.ts @@ -0,0 +1,31 @@ +import { SubStatusEnum } from '@fastgpt/global/support/wallet/sub/constants'; +import { MongoTeamSub } from './schema'; + +/* get team dataset size */ +export const getTeamDatasetValidSub = async ({ + teamId, + freeSize = Infinity +}: { + teamId: string; + freeSize?: number; +}) => { + const sub = await MongoTeamSub.findOne({ + teamId, + status: SubStatusEnum.active + }) + .sort({ + expiredTime: -1 + }) + .lean(); + + const maxSize = (() => { + if (!sub || !sub.datasetStoreAmount) return freeSize; + + return sub.datasetStoreAmount + freeSize; + })(); + + return { + maxSize, + sub + }; +}; diff --git a/packages/service/type.d.ts b/packages/service/type.d.ts new file mode 100644 index 000000000..0f9435125 --- /dev/null +++ b/packages/service/type.d.ts @@ -0,0 +1,3 @@ +declare global { + var defaultTeamDatasetLimit: number; +} diff --git a/packages/web/common/file/img.ts b/packages/web/common/file/img.ts index 8f994eb72..cdffbc3bd 100644 --- a/packages/web/common/file/img.ts +++ b/packages/web/common/file/img.ts @@ -4,15 +4,13 @@ export type CompressImgProps = { maxSize?: number; }; -export const compressBase64ImgAndUpload = ({ +export const compressBase64Img = ({ base64Img, maxW = 1080, maxH = 1080, - maxSize = 1024 * 500, // 300kb - uploadController + maxSize = 1024 * 500 // 500kb }: CompressImgProps & { base64Img: string; - uploadController: (base64: string) => Promise; }) => { return new Promise((resolve, reject) => { const fileType = @@ -54,12 +52,7 @@ export const compressBase64ImgAndUpload = ({ return reject('图片太大了'); } - try { - const src = await uploadController(compressedDataUrl); - resolve(src); - } catch (error) { - reject(error); - } + resolve(compressedDataUrl); }; img.onerror = reject; }); diff --git a/packages/web/common/file/read.ts b/packages/web/common/file/read.ts deleted file mode 100644 index 5069be7b9..000000000 --- a/packages/web/common/file/read.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { uploadMarkdownBase64 } from '@fastgpt/global/common/string/markdown'; -import { htmlStr2Md } from '../string/markdown'; -/** - * read file raw text - */ -export const readFileRawText = (file: File) => { - return new Promise((resolve: (_: string) => void, reject) => { - try { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result as string); - }; - reader.onerror = (err) => { - console.log('error txt read:', err); - reject('Read file error'); - }; - reader.readAsText(file); - } catch (error) { - reject(error); - } - }); -}; - -export const readMdFile = async ({ - file, - uploadImgController -}: { - file: File; - uploadImgController: (base64: string) => Promise; -}) => { - const md = await readFileRawText(file); - const rawText = await uploadMarkdownBase64({ - rawText: md, - uploadImgController - }); - return rawText; -}; - -export const readHtmlFile = async ({ - file, - uploadImgController -}: { - file: File; - uploadImgController: (base64: string) => Promise; -}) => { - const md = htmlStr2Md(await readFileRawText(file)); - const rawText = await uploadMarkdownBase64({ - rawText: md, - uploadImgController - }); - - return rawText; -}; diff --git a/packages/web/common/file/read/html.ts b/packages/web/common/file/read/html.ts new file mode 100644 index 000000000..04eb911ab --- /dev/null +++ b/packages/web/common/file/read/html.ts @@ -0,0 +1,21 @@ +import { htmlStr2Md } from '../../string/markdown'; +import { readFileRawText } from './rawText'; +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; + +export const readHtmlFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const { rawText } = await readFileRawText(file); + const md = htmlStr2Md(rawText); + + const simpleMd = await markdownProcess({ + rawText: md, + uploadImgController + }); + + return { rawText: rawText }; +}; diff --git a/packages/web/common/file/read/index.ts b/packages/web/common/file/read/index.ts new file mode 100644 index 000000000..d5e4c6ad7 --- /dev/null +++ b/packages/web/common/file/read/index.ts @@ -0,0 +1,46 @@ +import { loadFile2Buffer } from '../utils'; +import { readHtmlFile } from './html'; +import { readMdFile } from './md'; +import { readPdfFile } from './pdf'; +import { readFileRawText } from './rawText'; +import { readWordFile } from './word'; + +export const readFileRawContent = async ({ + file, + uploadBase64Controller +}: { + file: File; + uploadBase64Controller?: (base64: string) => Promise; +}): Promise<{ + rawText: string; +}> => { + const extension = file?.name?.split('.')?.pop()?.toLowerCase(); + + switch (extension) { + case 'txt': + return readFileRawText(file); + case 'md': + return readMdFile({ + file, + uploadImgController: uploadBase64Controller + }); + case 'html': + return readHtmlFile({ + file, + uploadImgController: uploadBase64Controller + }); + case 'pdf': + const pdf = await loadFile2Buffer({ file }); + return readPdfFile({ pdf }); + case 'docx': + return readWordFile({ + file, + uploadImgController: uploadBase64Controller + }); + + default: + return { + rawText: '' + }; + } +}; diff --git a/packages/web/common/file/read/md.ts b/packages/web/common/file/read/md.ts new file mode 100644 index 000000000..5df750c92 --- /dev/null +++ b/packages/web/common/file/read/md.ts @@ -0,0 +1,17 @@ +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; +import { readFileRawText } from './rawText'; + +export const readMdFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const { rawText: md } = await readFileRawText(file); + const simpleMd = await markdownProcess({ + rawText: md, + uploadImgController + }); + return { rawText: simpleMd }; +}; diff --git a/packages/global/common/file/read/index.ts b/packages/web/common/file/read/pdf.ts similarity index 84% rename from packages/global/common/file/read/index.ts rename to packages/web/common/file/read/pdf.ts index 3c4e149d6..2e7ac97eb 100644 --- a/packages/global/common/file/read/index.ts +++ b/packages/web/common/file/read/pdf.ts @@ -1,18 +1,18 @@ /* read file to txt */ import * as pdfjsLib from 'pdfjs-dist'; -export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer }) => { - pdfjsLib.GlobalWorkerOptions.workerSrc = '/js/pdf.worker.js'; +type TokenType = { + str: string; + dir: string; + width: number; + height: number; + transform: number[]; + fontName: string; + hasEOL: boolean; +}; - type TokenType = { - str: string; - dir: string; - width: number; - height: number; - transform: number[]; - fontName: string; - hasEOL: boolean; - }; +export const readPdfFile = async ({ pdf }: { pdf: ArrayBuffer }) => { + pdfjsLib.GlobalWorkerOptions.workerSrc = '/js/pdf.worker.js'; const readPDFPage = async (doc: any, pageNo: number) => { const page = await doc.getPage(pageNo); @@ -58,5 +58,7 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer }) } const pageTexts = await Promise.all(pageTextPromises); - return pageTexts.join(''); + return { + rawText: pageTexts.join('') + }; }; diff --git a/packages/web/common/file/read/rawText.ts b/packages/web/common/file/read/rawText.ts new file mode 100644 index 000000000..6a9e4faeb --- /dev/null +++ b/packages/web/common/file/read/rawText.ts @@ -0,0 +1,22 @@ +/** + * read file raw text + */ +export const readFileRawText = (file: File) => { + return new Promise<{ rawText: string }>((resolve, reject) => { + try { + const reader = new FileReader(); + reader.onload = () => { + resolve({ + rawText: reader.result as string + }); + }; + reader.onerror = (err) => { + console.log('error txt read:', err); + reject('Read file error'); + }; + reader.readAsText(file); + } catch (error) { + reject(error); + } + }); +}; diff --git a/packages/web/common/file/read/word.ts b/packages/web/common/file/read/word.ts new file mode 100644 index 000000000..24f93c789 --- /dev/null +++ b/packages/web/common/file/read/word.ts @@ -0,0 +1,28 @@ +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; +import { htmlStr2Md } from '../../string/markdown'; +import { loadFile2Buffer } from '../utils'; +import mammoth from 'mammoth'; + +export const readWordFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const buffer = await loadFile2Buffer({ file }); + + const { value: html } = await mammoth.convertToHtml({ + arrayBuffer: buffer + }); + const md = htmlStr2Md(html); + + const rawText = await markdownProcess({ + rawText: md, + uploadImgController: uploadImgController + }); + + return { + rawText + }; +}; diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts new file mode 100644 index 000000000..74d16a226 --- /dev/null +++ b/packages/web/common/file/utils.ts @@ -0,0 +1,31 @@ +import { getErrText } from '@fastgpt/global/common/error/utils'; + +export const loadFile2Buffer = ({ file, onError }: { file: File; onError?: (err: any) => void }) => + new Promise((resolve, reject) => { + try { + let reader = new FileReader(); + reader.readAsArrayBuffer(file); + reader.onload = async ({ target }) => { + if (!target?.result) { + onError?.('Load file error'); + return reject('Load file error'); + } + try { + resolve(target.result as ArrayBuffer); + } catch (err) { + console.log(err, 'Load file error'); + onError?.(err); + + reject(getErrText(err, 'Load file error')); + } + }; + reader.onerror = (err) => { + console.log(err, 'Load file error'); + onError?.(err); + + reject(getErrText(err, 'Load file error')); + }; + } catch (error) { + reject('The browser does not support file content reading'); + } + }); diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index eead911dc..15a935dd6 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -1,7 +1,6 @@ // @ts-nocheck export const iconPaths = { - chat: () => import('./icons/chat.svg'), chatSend: () => import('./icons/chatSend.svg'), closeSolid: () => import('./icons/closeSolid.svg'), collectionLight: () => import('./icons/collectionLight.svg'), diff --git a/packages/web/components/common/Icon/icons/chat.svg b/packages/web/components/common/Icon/icons/chat.svg deleted file mode 100644 index c83e51a52..000000000 --- a/packages/web/components/common/Icon/icons/chat.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/courseLight.svg b/packages/web/components/common/Icon/icons/common/courseLight.svg index d8da2e2cb..f2499c882 100644 --- a/packages/web/components/common/Icon/icons/common/courseLight.svg +++ b/packages/web/components/common/Icon/icons/common/courseLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/gitLight.svg b/packages/web/components/common/Icon/icons/common/gitLight.svg index dc921e04d..589704833 100644 --- a/packages/web/components/common/Icon/icons/common/gitLight.svg +++ b/packages/web/components/common/Icon/icons/common/gitLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg b/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg index 516bf45ba..58780e79a 100644 --- a/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg +++ b/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg b/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg index e53209d80..dc9d8da06 100644 --- a/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg +++ b/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/aiFill.svg b/packages/web/components/common/Icon/icons/core/app/aiFill.svg index cc478d8d1..e1c47f193 100644 --- a/packages/web/components/common/Icon/icons/core/app/aiFill.svg +++ b/packages/web/components/common/Icon/icons/core/app/aiFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/aiLight.svg b/packages/web/components/common/Icon/icons/core/app/aiLight.svg index 016b34de4..ac7dd776d 100644 --- a/packages/web/components/common/Icon/icons/core/app/aiLight.svg +++ b/packages/web/components/common/Icon/icons/core/app/aiLight.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg index 08f301463..dc84253cb 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatLight.svg b/packages/web/components/common/Icon/icons/core/chat/chatLight.svg index aa20dd8d3..17bfab514 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatLight.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatLight.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg index c49518384..a4cddf788 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg b/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg index 0cba56612..e8554594a 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg b/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg index b784f35dd..4114aec8f 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg index cd6efe529..570bde86c 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg @@ -1,14 +1,5 @@ - - - - + - + d="M9.85672 2.01367H4.14326C3.70589 2.01366 3.3426 2.01366 3.04618 2.03788C2.73757 2.06309 2.45044 2.11744 2.1797 2.25539C1.76241 2.468 1.42315 2.80727 1.21053 3.22455C1.07258 3.49529 1.01824 3.78242 0.993023 4.09103C0.968804 4.38745 0.968811 4.75074 0.968819 5.18812V7.63287C0.968811 8.07025 0.968804 8.43354 0.993023 8.72996C1.01824 9.03857 1.07258 9.3257 1.21053 9.59644C1.42315 10.0137 1.76241 10.353 2.1797 10.5656C2.45044 10.7036 2.73757 10.7579 3.04618 10.7831C3.34257 10.8073 3.70583 10.8073 4.14316 10.8073H6.41666V11.8198H4.82086C4.49869 11.8198 4.23752 12.081 4.23752 12.4031C4.23752 12.7253 4.49869 12.9865 4.82086 12.9865H9.17913C9.5013 12.9865 9.76246 12.7253 9.76246 12.4031C9.76246 12.081 9.5013 11.8198 9.17913 11.8198H7.58333V10.8073H9.85671C10.294 10.8073 10.6574 10.8073 10.9538 10.7831C11.2624 10.7579 11.5495 10.7036 11.8203 10.5656C12.2376 10.353 12.5768 10.0137 12.7895 9.59644C12.9274 9.3257 12.9818 9.03857 13.007 8.72996C13.0312 8.43355 13.0312 8.07027 13.0312 7.63292V5.1881C13.0312 4.75075 13.0312 4.38744 13.007 4.09103C12.9818 3.78242 12.9274 3.49529 12.7895 3.22455C12.5768 2.80727 12.2376 2.468 11.8203 2.25539C11.5495 2.11744 11.2624 2.06309 10.9538 2.03788C10.6574 2.01366 10.2941 2.01366 9.85672 2.01367Z" + fill="#8A95A7" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/user/userFill.svg b/packages/web/components/common/Icon/icons/support/user/userFill.svg index e9f4226c5..8c17a6310 100644 --- a/packages/web/components/common/Icon/icons/support/user/userFill.svg +++ b/packages/web/components/common/Icon/icons/support/user/userFill.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/user/userLight.svg b/packages/web/components/common/Icon/icons/support/user/userLight.svg index 9147dba37..4f3177913 100644 --- a/packages/web/components/common/Icon/icons/support/user/userLight.svg +++ b/packages/web/components/common/Icon/icons/support/user/userLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/package.json b/packages/web/package.json index be7f3c759..119b245f6 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -13,9 +13,11 @@ "@fastgpt/global": "workspace:*", "@fingerprintjs/fingerprintjs": "^4.2.1", "@monaco-editor/react": "^4.6.0", + "mammoth": "^1.6.0", "i18next": "^22.5.1", "joplin-turndown-plugin-gfm": "^1.0.12", "next-i18next": "^13.3.0", + "pdfjs-dist": "^4.0.269", "react": "18.2.0", "react-dom": "18.2.0", "react-i18next": "^12.3.1", diff --git a/packages/web/tsconfig.json b/packages/web/tsconfig.json index d57aec33f..78e8afa67 100644 --- a/packages/web/tsconfig.json +++ b/packages/web/tsconfig.json @@ -15,9 +15,6 @@ "jsx": "preserve", "incremental": true, "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"], "exclude": ["node_modules","./components/common/Icon/constants.ts"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10aa86a18..eaa3df1c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,12 +47,12 @@ importers: js-tiktoken: specifier: ^1.0.7 version: registry.npmmirror.com/js-tiktoken@1.0.7 + nanoid: + specifier: ^4.0.1 + version: registry.npmmirror.com/nanoid@4.0.1 openai: specifier: 4.23.0 version: registry.npmmirror.com/openai@4.23.0(encoding@0.1.13) - pdfjs-dist: - specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) timezones-list: specifier: ^3.0.2 version: registry.npmmirror.com/timezones-list@3.0.2 @@ -96,21 +96,27 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2 + mammoth: + specifier: ^1.6.0 + version: registry.npmmirror.com/mammoth@1.6.0 mongoose: specifier: ^7.0.2 version: registry.npmmirror.com/mongoose@7.0.2 multer: specifier: 1.4.5-lts.1 version: registry.npmmirror.com/multer@1.4.5-lts.1 - nanoid: - specifier: ^4.0.1 - version: registry.npmmirror.com/nanoid@4.0.1 next: specifier: 13.5.2 version: registry.npmmirror.com/next@13.5.2(@babel/core@7.23.7)(react-dom@18.2.0)(react@18.2.0)(sass@1.58.3) nextjs-cors: specifier: ^2.1.2 version: registry.npmmirror.com/nextjs-cors@2.1.2(next@13.5.2) + node-cron: + specifier: ^3.0.3 + version: registry.npmmirror.com/node-cron@3.0.3 + pdfjs-dist: + specifier: ^4.0.269 + version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) pg: specifier: ^8.10.0 version: registry.npmmirror.com/pg@8.10.0 @@ -127,6 +133,9 @@ importers: '@types/multer': specifier: ^1.4.10 version: registry.npmmirror.com/@types/multer@1.4.10 + '@types/node-cron': + specifier: ^3.0.11 + version: registry.npmmirror.com/@types/node-cron@3.0.11 '@types/pg': specifier: ^8.6.6 version: registry.npmmirror.com/@types/pg@8.6.6 @@ -175,9 +184,15 @@ importers: joplin-turndown-plugin-gfm: specifier: ^1.0.12 version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12 + mammoth: + specifier: ^1.6.0 + version: registry.npmmirror.com/mammoth@1.6.0 next-i18next: specifier: ^13.3.0 version: registry.npmmirror.com/next-i18next@13.3.0(i18next@22.5.1)(next@13.5.2)(react-i18next@12.3.1)(react@18.2.0) + pdfjs-dist: + specifier: ^4.0.269 + version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) react: specifier: 18.2.0 version: registry.npmmirror.com/react@18.2.0 @@ -287,9 +302,6 @@ importers: lodash: specifier: ^4.17.21 version: registry.npmmirror.com/lodash@4.17.21 - mammoth: - specifier: ^1.6.0 - version: registry.npmmirror.com/mammoth@1.6.0 mermaid: specifier: ^10.2.3 version: registry.npmmirror.com/mermaid@10.2.3 @@ -5107,6 +5119,12 @@ packages: '@types/express': registry.npmmirror.com/@types/express@4.17.21 dev: true + registry.npmmirror.com/@types/node-cron@3.0.11: + resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node-cron/-/node-cron-3.0.11.tgz} + name: '@types/node-cron' + version: 3.0.11 + dev: true + registry.npmmirror.com/@types/node-fetch@2.6.10: resolution: {integrity: sha512-PPpPK6F9ALFTn59Ka3BaL+qGuipRfxNE8qVgkp0bVixeiR2c2/L+IVOiBdu9JhhT22sWnQEp6YyHGI2b2+CMcA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node-fetch/-/node-fetch-2.6.10.tgz} name: '@types/node-fetch' @@ -11087,6 +11105,15 @@ packages: next: registry.npmmirror.com/next@13.5.2(@babel/core@7.23.7)(react-dom@18.2.0)(react@18.2.0)(sass@1.58.3) dev: false + registry.npmmirror.com/node-cron@3.0.3: + resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-cron/-/node-cron-3.0.3.tgz} + name: node-cron + version: 3.0.3 + engines: {node: '>=6.0.0'} + dependencies: + uuid: registry.npmmirror.com/uuid@8.3.2 + dev: false + registry.npmmirror.com/node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-domexception/-/node-domexception-1.0.0.tgz} name: node-domexception @@ -13768,6 +13795,13 @@ packages: engines: {node: '>= 0.4.0'} dev: false + registry.npmmirror.com/uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz} + name: uuid + version: 8.3.2 + hasBin: true + dev: false + registry.npmmirror.com/uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/uuid/-/uuid-9.0.1.tgz} name: uuid diff --git a/projects/app/.gitignore b/projects/app/.gitignore deleted file mode 100644 index 4b3390ce7..000000000 --- a/projects/app/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# dependencies -node_modules/ -# next.js -.next/ -out/ -# production -build/ - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# local env files -.env*.local - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts -platform.json -testApi/ -local/ -.husky/ -data/*.local.* \ No newline at end of file diff --git a/projects/app/package.json b/projects/app/package.json index a16b52696..baaaa052c 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -37,7 +37,6 @@ "jschardet": "^3.0.0", "jsonwebtoken": "^9.0.2", "lodash": "^4.17.21", - "mammoth": "^1.6.0", "mermaid": "^10.2.3", "nanoid": "^4.0.1", "next": "13.5.2", diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 940dbc8b4..3b3ba828e 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -57,45 +57,6 @@ "Title is required": "Title is required" } }, - "chat": { - "Admin Mark Content": "Corrected response", - "Confirm to clear history": "Confirm to clear history?", - "Confirm to clear share chat history": " Are you sure to delete all chats?", - "Converting to text": "Converting to text...", - "Exit Chat": "Exit", - "Feedback Failed": "Feedback Failed", - "Feedback Mark": "Mark", - "Feedback Modal": "Feedback", - "Feedback Modal Tip": "Enter what you find unsatisfactory", - "Feedback Submit": "Submit", - "Feedback Success": "Feedback Success", - "Feedback Update Failed": "Feedback Update Failed", - "History": "History", - "Mark": "Mark", - "Mark Description": "The annotation feature is currently in beta. \n\n After clicking Add annotation, you need to select a knowledge base in order to store annotation data. You can use this feature to quickly annotate questions and expected answers to guide the model to the next answer. At present, the annotation function, like other data in the knowledge base, is affected by the model, which does not mean that the annotation meets 100% expectations. The \n\n annotation data is only unidirectional synchronization with the knowledge base. If the knowledge base modifies the annotation data, the annotation data displayed in the log cannot be synchronized", - "Mark Description Title": "Mark Description", - "New Chat": "New Chat", - "Fresh Chat": "New Chat", - "Question Guide Tips": "I guess what you're asking is", - "Quote": "Quote", - "Pin": "Pin", - "Unpin": "Unpin", - "Read Mark Description": "Read mark description", - "Select Mark Kb": "Select Dataset", - "Select Mark Kb Desc": "Select a dataset to store the expected answers", - "You need to a chat app": "You need to a chat app", - "Failed to initialize chat": "Failed to initialize chat", - "Custom History Title": "Custom history title", - "Custom History Title Description": "If set to empty, chat history will be followed automatically.", - "History Amount": "{{amount}} records", - "logs": { - "api": "API", - "online": "Online Chat", - "share": "Share", - "test": "Test Chat " - }, - "retry": "Retry" - }, "common": { "Add": "Add", "Add New": "Add", @@ -161,6 +122,7 @@ "Set Avatar": "Set Avatar", "Set Name": "Make a nice name", "Status": "Status", + "Submit success": "Update Success", "Team": "Team", "Test": "Test", "Time": "Time", @@ -170,6 +132,7 @@ "Update Success": "Update Success", "Update Successful": "Update Successful", "Update Time": "Update Time", + "Update success": "Update success", "Upload File Failed": "Upload File Failed", "Username": "UserName", "Website": "Website", @@ -188,6 +151,7 @@ "Common Tip": "No data" }, "error": { + "Update error": "Update error", "unKnow": "There was an accident" }, "export": "", @@ -298,17 +262,46 @@ } }, "chat": { + "Admin Mark Content": "Corrected response", "Audio Speech Error": "Audio Speech Error", + "Confirm to clear history": "Confirm to clear history?", + "Confirm to clear share chat history": " Are you sure to delete all chats?", + "Converting to text": "Converting to text...", + "Custom History Title": "Custom history title", + "Custom History Title Description": "If set to empty, chat history will be followed automatically.", + "Exit Chat": "Exit", + "Failed to initialize chat": "Failed to initialize chat", + "Feedback Failed": "Feedback Failed", + "Feedback Mark": "Mark", + "Feedback Modal": "Feedback", + "Feedback Modal Tip": "Enter what you find unsatisfactory", + "Feedback Submit": "Submit", + "Feedback Success": "Feedback Success", + "Feedback Update Failed": "Feedback Update Failed", + "History": "History", + "History Amount": "{{amount}} records", + "Mark": "Mark", + "Mark Description": "The annotation feature is currently in beta. \n\n After clicking Add annotation, you need to select a knowledge base in order to store annotation data. You can use this feature to quickly annotate questions and expected answers to guide the model to the next answer. At present, the annotation function, like other data in the knowledge base, is affected by the model, which does not mean that the annotation meets 100% expectations. The \n\n annotation data is only unidirectional synchronization with the knowledge base. If the knowledge base modifies the annotation data, the annotation data displayed in the log cannot be synchronized", + "Mark Description Title": "Mark Description", + "New Chat": "New Chat", + "Pin": "Pin", + "Question Guide Tips": "I guess what you're asking is", + "Quote": "Quote", "Quote Amount": "Dataset Quote:{{amount}}", + "Read Mark Description": "Read mark description", "Record": "Speech", "Restart": "Restart", "Select File": "Select file", "Select Image": "Select Image", + "Select Mark Kb": "Select Dataset", + "Select Mark Kb Desc": "Select a dataset to store the expected answers", "Send Message": "Send Message", "Speaking": "I'm listening...", "Start Chat": "Start Chat", "Stop Speak": "Stop Speak", "Type a message": "Input problem", + "Unpin": "Unpin", + "You need to a chat app": "You need to a chat app", "error": { "Chat error": "Chat error", "Messages empty": "Interface content is empty, maybe the text is too long ~", @@ -322,6 +315,12 @@ "No Content": "The user did not fill in the specific feedback content", "Read User dislike": "User dislike\nClick to view content" }, + "logs": { + "api": "API", + "online": "Online Chat", + "share": "Share", + "test": "Test Chat " + }, "markdown": { "Edit Question": "Edit Question", "Quick Question": "Ask the question immediately", @@ -365,6 +364,7 @@ "search using reRank": "ReRank", "text output": "Text Output" }, + "retry": "Retry", "tts": { "Stop Speech": "Stop" } @@ -383,9 +383,11 @@ "Delete Website Tips": "Confirm to delete the website", "Empty Dataset": "", "Empty Dataset Tips": "There is no knowledge base yet, go create one!", + "File collection": "File collection", "Folder Dataset": "Folder", "Go Dataset": "To Dataset", "Intro Placeholder": "This dataset has not yet been introduced~", + "Manual collection": "Manual collection", "My Dataset": "My Dataset", "Name": "Name", "Quote Length": "Quote Length", @@ -395,7 +397,6 @@ "Set Website Config": "Configuring Website", "Similarity": "Similarity", "Sync Time": "Update Time", - "Virtual File": "Virtual File", "Website Dataset": "Website Sync", "Website Dataset Desc": "Web site synchronization allows you to build a knowledge base directly from a web link", "collection": { @@ -455,7 +456,8 @@ "Total Amount": "{{total}} Chunks", "data is deleted": "Data is deleted", "get data error": "Get data error", - "id": "Data ID" + "id": "Data ID", + "unit": "pieces" }, "error": { "Start Sync Failed": "Start Sync Failed", @@ -704,7 +706,7 @@ "Confirm to delete the data": "Confirm to delete the data?", "Confirm to delete the file": "Are you sure to delete the file and all its data?", "Create Folder": "Create Folder", - "Create Virtual File": "Virtual File", + "Create manual collection": "Manual collection", "Delete Dataset Error": "Delete dataset failed", "Edit Folder": "Edit Folder", "Export": "Export", @@ -715,6 +717,7 @@ "Files": "{{total}} Files", "Folder Name": "Input folder name", "Insert Data": "Insert", + "Manual collection Tip": "Manual Collections allow you to create a custom container to hold data", "Manual Data": "Manual Data", "Manual Input": "Manual Input", "Manual Mark": "Manual Mark", @@ -727,7 +730,6 @@ "System Data Queue": "Data Queue", "Training Name": "Dataset Training", "Upload Time": "Upload Time", - "Virtual File Tip": "Virtual files allow you to create a custom container to hold data", "collections": { "Click to view file": "View File Data", "Click to view folder": "To Folder", @@ -735,7 +737,7 @@ "Confirm to delete the folder": "Are you sure to delete this folder and all its contents?", "Create And Import": "Create/Import", "Create Training Data": "Training-{{filename}}", - "Create Virtual File Success": "Create Virtual File Success", + "Create manual collection Success": "Create manual collection Success", "Data Amount": "Data Amount", "Select Collection": "Select Collection", "Select One Collection To Store": "Select the collection to store" @@ -918,6 +920,9 @@ "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document." } }, + "subscription": { + "Cancel subscription": "Cancel" + }, "user": { "Price": "Price", "auth": { @@ -927,6 +932,21 @@ "Github": "Github", "Google": "Google", "Provider error": "Login exception, please try again" + }, + "team": { + "Dataset usage": "Dataset usage" + } + }, + "wallet": { + "Buy more": "Buy more", + "Confirm pay": "Confirm pay", + "Pay error": "Pay error", + "Pay success": "Pay success", + "subscription": { + "Current dataset store": "Current dataset store subscription", + "Dataset store": "Dataset store size", + "Dataset store price tip": "Deduct it from the account balance on the 1st of each month", + "Expand size": "Expand size" } } }, @@ -1069,4 +1089,4 @@ "qa": "QA Generation" } } -} \ No newline at end of file +} diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index a8d68971c..9b0512844 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -57,45 +57,6 @@ "Title is required": "模块名不能为空" } }, - "chat": { - "Admin Mark Content": "纠正后的回复", - "Confirm to clear history": "确认清空该应用的在线聊天记录？分享和 API 调用的记录不会被清空。", - "Confirm to clear share chat history": "确认删除所有聊天记录？", - "Converting to text": "正在转换为文本...", - "Exit Chat": "退出聊天", - "Feedback Failed": "提交反馈异常", - "Feedback Mark": "标注", - "Feedback Modal": "结果反馈", - "Feedback Modal Tip": "输入你觉得回答不满意的地方", - "Feedback Submit": "提交反馈", - "Feedback Success": "反馈成功!", - "Feedback Update Failed": "更新反馈状态失败", - "History": "记录", - "Mark": "标注预期回答", - "Mark Description": "当前标注功能为测试版。\n\n点击添加标注后，需要选择一个知识库，以便存储标注数据。你可以通过该功能快速的标注问题和预期回答，以便引导模型下次的回答。\n\n目前，标注功能同知识库其他数据一样，受模型的影响，不代表标注后 100% 符合预期。\n\n标注数据仅单向与知识库同步，如果知识库修改了该标注数据，日志展示的标注数据无法同步", - "Mark Description Title": "标注功能介绍", - "New Chat": "新对话", - "Fresh Chat": "新的对话", - "Question Guide Tips": "猜你想问", - "Quote": "引用", - "Pin": "置顶", - "Unpin": "取消置顶", - "Read Mark Description": "查看标注功能介绍", - "Select Mark Kb": "选择知识库", - "Select Mark Kb Desc": "选择一个知识库存储预期答案", - "You need to a chat app": "你需要创建一个应用", - "Failed to initialize chat": "初始化聊天失败", - "Custom History Title": "自定义历史记录标题", - "Custom History Title Description": "如果设置为空，会自动跟随聊天记录。", - "History Amount": "{{amount}}条记录", - "logs": { - "api": "API 调用", - "online": "在线使用", - "share": "外部链接调用", - "test": "测试" - }, - "retry": "重新生成" - }, "common": { "Add": "添加", "Add New": "新增", @@ -161,6 +122,7 @@ "Set Avatar": "点击设置头像", "Set Name": "取个名字", "Status": "状态", + "Submit success": "提交成功", "Team": "团队", "Test": "测试", "Time": "时间", @@ -170,6 +132,7 @@ "Update Success": "更新成功", "Update Successful": "更新成功", "Update Time": "更新时间", + "Update success": "更新成功", "Upload File Failed": "上传文件失败", "Username": "用户名", "Website": "网站", @@ -188,6 +151,7 @@ "Common Tip": "没有什么数据噢~" }, "error": { + "Update error": "更新失败", "unKnow": "出现了点意外~" }, "export": "", @@ -298,17 +262,46 @@ } }, "chat": { + "Admin Mark Content": "纠正后的回复", "Audio Speech Error": "语音播报异常", + "Confirm to clear history": "确认清空该应用的在线聊天记录？分享和 API 调用的记录不会被清空。", + "Confirm to clear share chat history": "确认删除所有聊天记录？", + "Converting to text": "正在转换为文本...", + "Custom History Title": "自定义历史记录标题", + "Custom History Title Description": "如果设置为空，会自动跟随聊天记录。", + "Exit Chat": "退出聊天", + "Failed to initialize chat": "初始化聊天失败", + "Feedback Failed": "提交反馈异常", + "Feedback Mark": "标注", + "Feedback Modal": "结果反馈", + "Feedback Modal Tip": "输入你觉得回答不满意的地方", + "Feedback Submit": "提交反馈", + "Feedback Success": "反馈成功!", + "Feedback Update Failed": "更新反馈状态失败", + "History": "记录", + "History Amount": "{{amount}}条记录", + "Mark": "标注预期回答", + "Mark Description": "当前标注功能为测试版。\n\n点击添加标注后，需要选择一个知识库，以便存储标注数据。你可以通过该功能快速的标注问题和预期回答，以便引导模型下次的回答。\n\n目前，标注功能同知识库其他数据一样，受模型的影响，不代表标注后 100% 符合预期。\n\n标注数据仅单向与知识库同步，如果知识库修改了该标注数据，日志展示的标注数据无法同步", + "Mark Description Title": "标注功能介绍", + "New Chat": "新对话", + "Pin": "置顶", + "Question Guide Tips": "猜你想问", + "Quote": "引用", "Quote Amount": "知识库引用({{amount}}条)", + "Read Mark Description": "查看标注功能介绍", "Record": "语音输入", "Restart": "重开对话", "Select File": "选择文件", "Select Image": "选择图片", + "Select Mark Kb": "选择知识库", + "Select Mark Kb Desc": "选择一个知识库存储预期答案", "Send Message": "发送", "Speaking": "我在听，请说...", "Start Chat": "开始对话", "Stop Speak": "停止录音", "Type a message": "输入问题", + "Unpin": "取消置顶", + "You need to a chat app": "你需要创建一个应用", "error": { "Chat error": "对话出现异常", "Messages empty": "接口内容为空，可能文本超长了~", @@ -322,6 +315,12 @@ "No Content": "用户没有填写具体反馈内容", "Read User dislike": "用户表示反对\n点击查看内容" }, + "logs": { + "api": "API 调用", + "online": "在线使用", + "share": "外部链接调用", + "test": "测试" + }, "markdown": { "Edit Question": "编辑问题", "Quick Question": "点我立即提问", @@ -365,6 +364,7 @@ "search using reRank": "结果重排", "text output": "文本输出" }, + "retry": "重新生成", "tts": { "Stop Speech": "停止" } @@ -383,9 +383,11 @@ "Delete Website Tips": "确认删除该站点？", "Empty Dataset": "", "Empty Dataset Tips": "还没有知识库，快去创建一个吧！", + "File collection": "文件数据集", "Folder Dataset": "文件夹", "Go Dataset": "前往知识库", "Intro Placeholder": "这个知识库还没有介绍~", + "Manual collection": "手动数据集", "My Dataset": "我的知识库", "Name": "知识库名称", "Quote Length": "引用内容长度", @@ -395,7 +397,6 @@ "Set Website Config": "开始配置网站信息", "Similarity": "相关度", "Sync Time": "最后更新时间", - "Virtual File": "虚拟文件", "Website Dataset": "Web 站点同步", "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", "collection": { @@ -455,7 +456,8 @@ "Total Amount": "{{total}} 组", "data is deleted": "该数据已被删除", "get data error": "获取数据异常", - "id": "数据ID" + "id": "数据ID", + "unit": "条" }, "error": { "Start Sync Failed": "开始同步失败", @@ -704,7 +706,7 @@ "Confirm to delete the data": "确认删除该数据？", "Confirm to delete the file": "确认删除该文件及其所有数据？", "Create Folder": "创建文件夹", - "Create Virtual File": "创建虚拟文件", + "Create manual collection": "创建手动数据集", "Delete Dataset Error": "删除知识库异常", "Edit Folder": "编辑文件夹", "Export": "导出", @@ -715,6 +717,7 @@ "Files": "文件: {{total}}个", "Folder Name": "输入文件夹名称", "Insert Data": "插入", + "Manual collection Tip": "手动数据集允许创建一个空的容器装入数据", "Manual Data": "手动录入", "Manual Input": "手动录入", "Manual Mark": "手动标注", @@ -727,7 +730,6 @@ "System Data Queue": "排队长度", "Training Name": "数据训练", "Upload Time": "上传时间", - "Virtual File Tip": "虚拟文件允许创建一个自定义的容器装入数据", "collections": { "Click to view file": "点击查看文件详情", "Click to view folder": "进入目录", @@ -735,7 +737,7 @@ "Confirm to delete the folder": "确认删除该文件夹及里面所有内容？", "Create And Import": "新建/导入", "Create Training Data": "文件训练-{{filename}}", - "Create Virtual File Success": "创建虚拟文件成功", + "Create manual collection Success": "创建手动数据集成功", "Data Amount": "数据总量", "Select Collection": "选择文件", "Select One Collection To Store": "选择一个文件进行存储" @@ -918,6 +920,9 @@ "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档" } }, + "subscription": { + "Cancel subscription": "取消订阅" + }, "user": { "Price": "计费标准", "auth": { @@ -927,6 +932,21 @@ "Github": "Github 登录", "Google": "Google 登录", "Provider error": "登录异常，请重试" + }, + "team": { + "Dataset usage": "知识库容量" + } + }, + "wallet": { + "Buy more": "扩容", + "Confirm pay": "支付确认", + "Pay error": "支付失败", + "Pay success": "支付成功", + "subscription": { + "Current dataset store": "当前额外容量", + "Dataset store": "知识库容量", + "Dataset store price tip": "每月1号从账号余额里扣除", + "Expand size": "扩大容量" } } }, @@ -1069,4 +1089,4 @@ "qa": "QA 拆分" } } -} \ No newline at end of file +} diff --git a/projects/app/src/components/ChatBox/FeedbackModal.tsx b/projects/app/src/components/ChatBox/FeedbackModal.tsx index 163b01092..a29cfba5a 100644 --- a/projects/app/src/components/ChatBox/FeedbackModal.tsx +++ b/projects/app/src/components/ChatBox/FeedbackModal.tsx @@ -40,8 +40,8 @@ const FeedbackModal = ({ onSuccess() { onSuccess(ref.current?.value || t('core.chat.feedback.No Content')); }, - successToast: t('chat.Feedback Success'), - errorToast: t('chat.Feedback Failed') + successToast: t('core.chat.Feedback Success'), + errorToast: t('core.chat.Feedback Failed') }); return ( @@ -49,17 +49,17 @@ const FeedbackModal = ({ isOpen={true} onClose={onClose} iconSrc="/imgs/modal/badAnswer.svg" - title={t('chat.Feedback Modal')} + title={t('core.chat.Feedback Modal')} > - + diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx index 351156790..39a727435 100644 --- a/projects/app/src/components/ChatBox/MessageInput.tsx +++ b/projects/app/src/components/ChatBox/MessageInput.tsx @@ -8,10 +8,11 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import { useRouter } from 'next/router'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { compressImgFileAndUpload } from '@/web/common/file/controller'; -import { useToast } from '@/web/common/hooks/useToast'; import { customAlphabet } from 'nanoid'; import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants'; import { addDays } from 'date-fns'; +import { useRequest } from '@/web/common/hooks/useRequest'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); enum FileTypeEnum { @@ -45,7 +46,6 @@ const MessageInput = ({ resetInputVal: (val: string) => void; }) => { const { shareId } = useRouter().query as { shareId?: string }; - const { toast } = useToast(); const { isSpeaking, isTransCription, @@ -68,17 +68,18 @@ const MessageInput = ({ maxCount: 10 }); - const uploadFile = useCallback( - async (file: FileItemType) => { + const { mutate: uploadFile } = useRequest({ + mutationFn: async (file: FileItemType) => { if (file.type === FileTypeEnum.image) { try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.chatImage, file: file.rawFile, maxW: 4329, maxH: 4329, maxSize: 1024 * 1024 * 5, // 30 day expired. - expiredTime: addDays(new Date(), 30), + expiredTime: addDays(new Date(), 7), shareId }); setFileList((state) => @@ -94,16 +95,13 @@ const MessageInput = ({ } catch (error) { setFileList((state) => state.filter((item) => item.id !== file.id)); console.log(error); - - toast({ - status: 'error', - title: t('common.Upload File Failed') - }); + return Promise.reject(error); } } }, - [shareId, t, toast] - ); + errorToast: t('common.Upload File Failed') + }); + const onSelectFile = useCallback( async (files: File[]) => { if (!files || files.length === 0) { @@ -219,7 +217,7 @@ ${images.map((img) => JSON.stringify({ src: img.src })).join('\n')} visibility={isSpeaking && isTransCription ? 'visible' : 'hidden'} > - {t('chat.Converting to text')} + {t('core.chat.Converting to text')} {/* file preview */} diff --git a/projects/app/src/components/ChatBox/QuoteModal.tsx b/projects/app/src/components/ChatBox/QuoteModal.tsx index b058943e8..3299b607b 100644 --- a/projects/app/src/components/ChatBox/QuoteModal.tsx +++ b/projects/app/src/components/ChatBox/QuoteModal.tsx @@ -5,7 +5,7 @@ import MyModal from '../MyModal'; import { useTranslation } from 'next-i18next'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import QuoteItem from '../core/dataset/QuoteItem'; -import { RawSourceText } from '@/pages/dataset/detail/components/InputDataModal'; +import RawSourceBox from '../core/dataset/RawSourceBox'; const QuoteModal = ({ rawSearch = [], @@ -46,7 +46,7 @@ const QuoteModal = ({ title={ {metadata ? ( - + ) : ( <>{t('core.chat.Quote Amount', { amount: rawSearch.length })} )} diff --git a/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx b/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx index b9218e084..7ff0ee333 100644 --- a/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx +++ b/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx @@ -19,7 +19,7 @@ const ReadFeedbackModal = ({ isOpen={true} onClose={onClose} iconSrc="/imgs/modal/readFeedback.svg" - title={t('chat.Feedback Modal')} + title={t('core.chat.Feedback Modal')} > {content} diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx index 8a1a1d6aa..6665f74e0 100644 --- a/projects/app/src/components/ChatBox/ResponseTags.tsx +++ b/projects/app/src/components/ChatBox/ResponseTags.tsx @@ -92,7 +92,7 @@ const ResponseTags = ({ <> {sourceList.length > 0 && ( <> - + {sourceList.map((item) => ( diff --git a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx index db67d9b7b..425aa1af8 100644 --- a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx +++ b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx @@ -46,7 +46,7 @@ const SelectMarkCollection = ({ paths={paths} onClose={onClose} setParentId={setParentId} - tips={t('chat.Select Mark Kb Desc')} + tips={t('core.chat.Select Mark Kb Desc')} > { - if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) { + if ( + !data.q || + !adminMarkData.datasetId || + !adminMarkData.collectionId || + !data.dataId + ) { return onClose(); } onSuccess({ - dataId: data.id, + dataId: data.dataId, datasetId: adminMarkData.datasetId, collectionId: adminMarkData.collectionId, q: data.q, diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx index ef09f5724..9c57b2f88 100644 --- a/projects/app/src/components/ChatBox/index.tsx +++ b/projects/app/src/components/ChatBox/index.tsx @@ -910,14 +910,15 @@ const ChatBox = ( )} {/* admin mark content */} {showMarkIcon && item.adminFeedback && ( - + - {`${item.adminFeedback.q || ''}${ - item.adminFeedback.a ? `\n${item.adminFeedback.a}` : '' - }`} + + {item.adminFeedback.q} + {item.adminFeedback.a} + )} @@ -996,6 +997,7 @@ const ChatBox = ( setAdminMarkData={(e) => setAdminMarkData({ ...e, chatItemId: adminMarkData.chatItemId })} onClose={() => setAdminMarkData(undefined)} onSuccess={(adminFeedback) => { + console.log(adminMarkData); if (!appId || !chatId || !adminMarkData.chatItemId) return; updateChatAdminFeedback({ appId, @@ -1003,6 +1005,7 @@ const ChatBox = ( chatItemId: adminMarkData.chatItemId, ...adminFeedback }); + // update dom setChatHistory((state) => state.map((chatItem) => @@ -1234,7 +1237,7 @@ function ChatController({ {!!onDelete && ( <> {onRetry && ( - + ))} {!!onMark && ( - + { { label: t('navbar.Chat'), icon: 'core/chat/chatLight', - activeIcon: 'chatcore/dataset/chatFillFill', + activeIcon: 'core/chat/chatFill', link: `/chat?appId=${lastChatAppId}&chatId=${lastChatId}`, activeLink: ['/chat'] }, @@ -77,6 +77,12 @@ const Navbar = ({ unread }: { unread: number }) => { h: '58px', borderRadius: 'md' }; + const hoverStyle: LinkProps = { + _hover: { + bg: 'myGray.05', + color: 'primary.600' + } + }; return ( { @@ -161,10 +168,11 @@ const Navbar = ({ unread }: { unread: number }) => { @@ -177,8 +185,9 @@ const Navbar = ({ unread }: { unread: number }) => { href="https://github.com/labring/FastGPT" target={'_blank'} {...itemStyles} + {...hoverStyle} mt={0} - color={'#9096a5'} + color={'myGray.500'} > diff --git a/projects/app/src/components/Markdown/chat/QuestionGuide.tsx b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx index 26a2f2650..e187bb3d8 100644 --- a/projects/app/src/components/Markdown/chat/QuestionGuide.tsx +++ b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx @@ -24,7 +24,7 @@ const QuestionGuide = ({ text }: { text: string }) => { return questionGuides.length > 0 ? ( - + {questionGuides.map((text) => ( { const [isLoading, setIsLoading] = useState(true); const [succeed, setSucceed] = useState(false); const { isOpen, onOpen, onClose } = useDisclosure(); + const [scale, setScale] = useState(1); + + const handleWheel: WheelEventHandler = (e) => { + setScale((prevScale) => { + const newScale = prevScale + e.deltaY * 0.5 * -0.01; + if (newScale < 0.5) return 0.5; + if (newScale > 10) return 10; + return newScale; + }); + }; + return ( { { fallbackSrc={'/imgs/errImg.png'} fallbackStrategy={'onError'} objectFit={'contain'} + onWheel={handleWheel} /> diff --git a/projects/app/src/components/Markdown/index.module.scss b/projects/app/src/components/Markdown/index.module.scss index 552151dec..684e63fde 100644 --- a/projects/app/src/components/Markdown/index.module.scss +++ b/projects/app/src/components/Markdown/index.module.scss @@ -343,7 +343,6 @@ margin: 10px 0; } .markdown { - text-align: justify; tab-size: 4; word-spacing: normal; width: 100%; diff --git a/projects/app/src/components/Markdown/index.tsx b/projects/app/src/components/Markdown/index.tsx index c413a9a8a..2c51ac8e9 100644 --- a/projects/app/src/components/Markdown/index.tsx +++ b/projects/app/src/components/Markdown/index.tsx @@ -112,6 +112,17 @@ function A({ children, ...props }: any) { } const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: boolean }) => { + const components = useMemo( + () => ({ + img: Image, + pre: 'div', + p: (pProps: any) =>

, + code: Code, + a: A + }), + [] + ); + const formatSource = source .replace(/\\n/g, '\n ') .replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2') @@ -124,13 +135,7 @@ const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: `} remarkPlugins={[RemarkMath, RemarkGfm, RemarkBreaks]} rehypePlugins={[RehypeKatex]} - components={{ - img: Image, - pre: 'div', - p: (pProps) =>

, - code: Code, - a: A - }} + components={components} linkTarget={'_blank'} > {formatSource} diff --git a/projects/app/src/components/Select/SelectAiModel.tsx b/projects/app/src/components/Select/SelectAiModel.tsx index 28045e95f..3e7f1c947 100644 --- a/projects/app/src/components/Select/SelectAiModel.tsx +++ b/projects/app/src/components/Select/SelectAiModel.tsx @@ -4,18 +4,19 @@ import MySelect, { type SelectProps } from './index'; import { useTranslation } from 'next-i18next'; import dynamic from 'next/dynamic'; import { useDisclosure } from '@chakra-ui/react'; +import { feConfigs } from '@/web/common/system/staticData'; const PriceBox = dynamic(() => import('@/components/support/wallet/Price')); const SelectAiModel = ({ list, ...props }: SelectProps) => { const { t } = useTranslation(); - const expandList = useMemo( - () => - list.concat({ - label: t('support.user.Price'), - value: 'price' - }), - [list, t] - ); + const expandList = useMemo(() => { + return feConfigs.show_pay + ? list.concat({ + label: t('support.user.Price'), + value: 'price' + }) + : list; + }, [list, t]); const { isOpen: isOpenPriceBox, diff --git a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx new file mode 100644 index 000000000..12e8e0046 --- /dev/null +++ b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx @@ -0,0 +1,30 @@ +import { Box, Flex, FlexProps } from '@chakra-ui/react'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useTranslation } from 'next-i18next'; +import React from 'react'; +import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant'; + +const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => { + const { t } = useTranslation(); + + const item = DatasetTypeMap[type]; + + return ( + + + {t(item.label)} + + ); +}; + +export default DatasetTypeTag; diff --git a/projects/app/src/components/core/dataset/QuoteItem.tsx b/projects/app/src/components/core/dataset/QuoteItem.tsx index bdc61b643..feb22152e 100644 --- a/projects/app/src/components/core/dataset/QuoteItem.tsx +++ b/projects/app/src/components/core/dataset/QuoteItem.tsx @@ -1,9 +1,6 @@ import React, { useMemo, useState } from 'react'; import { Box, Flex, Link, Progress } from '@chakra-ui/react'; -import { - type InputDataType, - RawSourceText -} from '@/pages/dataset/detail/components/InputDataModal'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type.d'; import NextLink from 'next/link'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -11,9 +8,6 @@ import { useTranslation } from 'next-i18next'; import MyTooltip from '@/components/MyTooltip'; import dynamic from 'next/dynamic'; import MyBox from '@/components/common/MyBox'; -import { getDatasetDataItemById } from '@/web/core/dataset/api'; -import { useRequest } from '@/web/common/hooks/useRequest'; -import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constant'; const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal')); @@ -58,17 +52,7 @@ const QuoteItem = ({ linkToDataset?: boolean; }) => { const { t } = useTranslation(); - const [editInputData, setEditInputData] = useState(); - - const { mutate: onclickEdit, isLoading } = useRequest({ - mutationFn: async (id: string) => { - return getDatasetDataItemById(id); - }, - onSuccess(data: DatasetDataItemType) { - setEditInputData(data); - }, - errorToast: t('core.dataset.data.get data error') - }); + const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); const score = useMemo(() => { if (!Array.isArray(quoteItem.score)) { @@ -114,7 +98,6 @@ const QuoteItem = ({ return ( <> - + {score?.primaryScore && ( <> {canViewSource ? ( @@ -132,7 +115,6 @@ const QuoteItem = ({ ( - + #{item.index + 1} @@ -223,7 +205,7 @@ const QuoteItem = ({ {quoteItem.q.length + (quoteItem.a?.length || 0)} - onclickEdit(quoteItem.id)} + onClick={() => + setEditInputData({ + dataId: quoteItem.id, + collectionId: quoteItem.collectionId + }) + } /> @@ -271,7 +258,7 @@ const QuoteItem = ({ )} - {editInputData && editInputData.id && ( + {editInputData && ( setEditInputData(undefined)} onSuccess={() => { @@ -280,7 +267,7 @@ const QuoteItem = ({ onDelete={() => { console.log('删除引用成功'); }} - defaultValue={editInputData} + dataId={editInputData.dataId} collectionId={editInputData.collectionId} /> )} diff --git a/projects/app/src/components/core/dataset/RawSourceBox.tsx b/projects/app/src/components/core/dataset/RawSourceBox.tsx new file mode 100644 index 000000000..3c8855385 --- /dev/null +++ b/projects/app/src/components/core/dataset/RawSourceBox.tsx @@ -0,0 +1,68 @@ +import React, { useMemo } from 'react'; +import { Box, BoxProps, Image } from '@chakra-ui/react'; +import { useToast } from '@/web/common/hooks/useToast'; +import { getErrText } from '@fastgpt/global/common/error/utils'; +import MyTooltip from '@/components/MyTooltip'; +import { useTranslation } from 'next-i18next'; +import { getFileAndOpen } from '@/web/core/dataset/utils'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; + +type Props = BoxProps & { + sourceName?: string; + sourceId?: string; + canView?: boolean; +}; + +const RawSourceBox = ({ sourceId, sourceName = '', canView = true, ...props }: Props) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { setLoading } = useSystemStore(); + + const canPreview = useMemo(() => !!sourceId && canView, [canView, sourceId]); + + const icon = useMemo(() => getSourceNameIcon({ sourceId, sourceName }), [sourceId, sourceName]); + + return ( + + { + setLoading(true); + try { + await getFileAndOpen(sourceId as string); + } catch (error) { + toast({ + title: t(getErrText(error, 'error.fileNotFound')), + status: 'error' + }); + } + setLoading(false); + } + } + : {})} + {...props} + > + + + {sourceName || t('common.UnKnow Source')} + + + + ); +}; + +export default RawSourceBox; diff --git a/projects/app/src/components/core/dataset/SelectModal.tsx b/projects/app/src/components/core/dataset/SelectModal.tsx index ab333d3c8..e3557a823 100644 --- a/projects/app/src/components/core/dataset/SelectModal.tsx +++ b/projects/app/src/components/core/dataset/SelectModal.tsx @@ -38,7 +38,7 @@ const DatasetSelectContainer = ({ parentId: path.parentId, parentName: path.parentName }))} - FirstPathDom={t('chat.Select Mark Kb')} + FirstPathDom={t('core.chat.Select Mark Kb')} onClick={(e) => { setParentId(e); }} diff --git a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx index de6407a12..4583a07ee 100644 --- a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx +++ b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx @@ -12,6 +12,7 @@ import MyTooltip from '@/components/MyTooltip'; import Avatar from '@/components/Avatar'; import { postCreateTeam, putUpdateTeam } from '@/web/support/user/team/api'; import { CreateTeamProps } from '@fastgpt/global/support/user/team/controller.d'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; export type FormDataType = CreateTeamProps & { id?: string; @@ -50,6 +51,7 @@ function EditModal({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.teamAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/components/support/wallet/Price.tsx b/projects/app/src/components/support/wallet/Price.tsx index 344869244..6182da4b1 100644 --- a/projects/app/src/components/support/wallet/Price.tsx +++ b/projects/app/src/components/support/wallet/Price.tsx @@ -17,14 +17,6 @@ import Markdown from '@/components/Markdown'; const Price = ({ onClose }: { onClose: () => void }) => { const list = [ - { - title: '知识库存储', - describe: '', - md: ` -| 计费项 | 价格(￥) | -| --- | --- | -| 知识库索引数量 | 0/1000条/天 |` - }, { title: '对话模型', describe: '', diff --git a/projects/app/src/components/support/wallet/SubDatasetModal.tsx b/projects/app/src/components/support/wallet/SubDatasetModal.tsx new file mode 100644 index 000000000..de50f862d --- /dev/null +++ b/projects/app/src/components/support/wallet/SubDatasetModal.tsx @@ -0,0 +1,171 @@ +import React, { useState } from 'react'; + +import MyModal from '@/components/MyModal'; +import { useTranslation } from 'next-i18next'; +import { + Box, + Flex, + ModalBody, + NumberInput, + NumberInputField, + NumberInputStepper, + NumberIncrementStepper, + NumberDecrementStepper, + ModalFooter, + Button +} from '@chakra-ui/react'; +import { useQuery } from '@tanstack/react-query'; +import { getTeamDatasetValidSub, postExpandTeamDatasetSub } from '@/web/support/wallet/sub/api'; +import Markdown from '@/components/Markdown'; +import MyTooltip from '@/components/MyTooltip'; +import { QuestionOutlineIcon } from '@chakra-ui/icons'; +import { useConfirm } from '@/web/common/hooks/useConfirm'; +import { getMonthRemainingDays } from '@fastgpt/global/common/math/date'; +import { useRequest } from '@/web/common/hooks/useRequest'; +import { useRouter } from 'next/router'; +import { feConfigs } from '@/web/common/system/staticData'; +import { useToast } from '@/web/common/hooks/useToast'; +import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; +import MySelect from '@/components/Select'; + +const SubDatasetModal = ({ onClose }: { onClose: () => void }) => { + const datasetStoreFreeSize = feConfigs?.subscription?.datasetStoreFreeSize || 0; + const datasetStorePrice = feConfigs?.subscription?.datasetStorePrice || 0; + + const { t } = useTranslation(); + const { toast } = useToast(); + const router = useRouter(); + const { ConfirmModal, openConfirm } = useConfirm({}); + const [datasetSize, setDatasetSize] = useState(0); + const [isRenew, setIsRenew] = useState('false'); + + const isExpand = datasetSize > 0; + + const { data: datasetSub } = useQuery(['getTeamDatasetValidSub'], getTeamDatasetValidSub, { + onSuccess(res) { + setIsRenew(`${res?.sub?.renew}`); + } + }); + + const { mutate, isLoading } = useRequest({ + mutationFn: () => postExpandTeamDatasetSub({ size: datasetSize, renew: isRenew === 'true' }), + onSuccess(res) { + if (isExpand) { + router.reload(); + } else { + onClose(); + } + }, + successToast: isExpand ? t('support.wallet.Pay success') : t('common.Update success'), + errorToast: isExpand ? t('support.wallet.Pay error') : t('common.error.Update error') + }); + + return ( + + + <> + + {t('support.user.Price')} + + + + + + + + {t('support.wallet.subscription.Current dataset store')}: + + {datasetSub?.sub?.datasetStoreAmount || 0} + {t('core.dataset.data.unit')} + + + {datasetSub?.sub?.expiredTime && ( + + 到期时间: + {formatTime2YMDHM(datasetSub?.sub?.expiredTime)} + + )} + + + 是否续订: + + + + {t('support.wallet.subscription.Expand size')} + + { + setDatasetSize(Number(e)); + }} + > + + + + + + + 000{t('core.dataset.data.unit')} + + + + + + + + + + + ); +}; + +export default SubDatasetModal; diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index ed6c41052..6cad6a2f8 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -1,5 +1,5 @@ import { - DatasetCollectionTrainingModeEnum, + TrainingModeEnum, DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index b4fd2a042..5ec3c0180 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -30,7 +30,7 @@ export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & { export type PushDatasetDataProps = { collectionId: string; data: PushDatasetDataChunkProps[]; - mode: `${TrainingModeEnum}`; + trainingMode: `${TrainingModeEnum}`; prompt?: string; billId?: string; }; diff --git a/projects/app/src/pages/account/components/Info.tsx b/projects/app/src/pages/account/components/Info.tsx index 7c4f4d478..b2f1743ce 100644 --- a/projects/app/src/pages/account/components/Info.tsx +++ b/projects/app/src/pages/account/components/Info.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useRef } from 'react'; +import React, { useCallback, useMemo, useRef } from 'react'; import { Box, Flex, @@ -8,7 +8,8 @@ import { Divider, Select, Input, - Link + Link, + Progress } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; import { UserUpdateParams } from '@/types/user'; @@ -22,7 +23,6 @@ import { compressImgFileAndUpload } from '@/web/common/file/controller'; import { feConfigs, systemVersion } from '@/web/common/system/staticData'; import { useTranslation } from 'next-i18next'; import { timezoneList } from '@fastgpt/global/common/time/timezone'; -import Loading from '@/components/Loading'; import Avatar from '@/components/Avatar'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@/components/MyTooltip'; @@ -32,20 +32,14 @@ import MySelect from '@/components/Select'; import { formatStorePrice2Read } from '@fastgpt/global/support/wallet/bill/tools'; import { putUpdateMemberName } from '@/web/support/user/team/api'; import { getDocPath } from '@/web/common/system/doc'; +import { getTeamDatasetValidSub } from '@/web/support/wallet/sub/api'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu')); -const PayModal = dynamic(() => import('./PayModal'), { - loading: () => , - ssr: false -}); -const UpdatePswModal = dynamic(() => import('./UpdatePswModal'), { - loading: () => , - ssr: false -}); -const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal'), { - loading: () => , - ssr: false -}); +const PayModal = dynamic(() => import('./PayModal')); +const UpdatePswModal = dynamic(() => import('./UpdatePswModal')); +const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal')); +const SubDatasetModal = dynamic(() => import('@/components/support/wallet/SubDatasetModal')); const UserInfo = () => { const theme = useTheme(); @@ -69,6 +63,11 @@ const UserInfo = () => { onOpen: onOpenUpdatePsw } = useDisclosure(); const { isOpen: isOpenOpenai, onClose: onCloseOpenai, onOpen: onOpenOpenai } = useDisclosure(); + const { + isOpen: isOpenSubDatasetModal, + onClose: onCloseSubDatasetModal, + onOpen: onOpenSubDatasetModal + } = useDisclosure(); const { File, onOpen: onOpenSelectFile } = useSelectFile({ fileType: '.jpg,.png', @@ -97,6 +96,7 @@ const UserInfo = () => { if (!file || !userInfo) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.userAvatar, file, maxW: 300, maxH: 300 @@ -122,6 +122,27 @@ const UserInfo = () => { } }); + const { data: datasetSub = { maxSize: 0, usedSize: 0 } } = useQuery( + ['getTeamDatasetValidSub'], + getTeamDatasetValidSub + ); + const datasetUsageMap = useMemo(() => { + const rate = datasetSub.usedSize / datasetSub.maxSize; + + const colorScheme = (() => { + if (rate < 0.5) return 'green'; + if (rate < 0.8) return 'yellow'; + return 'red'; + })(); + + return { + colorScheme, + value: rate * 100, + maxSize: datasetSub.maxSize, + usedSize: datasetSub.usedSize + }; + }, [datasetSub.maxSize, datasetSub.usedSize]); + return ( { {t('user.Change')} - - - - {t('user.team.Balance')}:  + {feConfigs.isPlus && ( + <> + + + + {t('user.team.Balance')}:  + + + {formatStorePrice2Read(userInfo?.team?.balance).toFixed(3)} 元 + + {feConfigs?.show_pay && userInfo?.team?.canWrite && ( + + )} + - - {formatStorePrice2Read(userInfo?.team?.balance).toFixed(3)} 元 + + + + {t('support.user.team.Dataset usage')}: {datasetUsageMap.usedSize}/ + {datasetSub.maxSize} + + + + + + - {feConfigs?.show_pay && userInfo?.team?.canWrite && ( - - )} - - + + )} + {feConfigs?.docUrl && ( { onClose={onCloseOpenai} /> )} + {isOpenSubDatasetModal && } ); }; -export default UserInfo; +export default React.memo(UserInfo); diff --git a/projects/app/src/pages/account/components/InformTable.tsx b/projects/app/src/pages/account/components/InformTable.tsx index c79964d10..960230053 100644 --- a/projects/app/src/pages/account/components/InformTable.tsx +++ b/projects/app/src/pages/account/components/InformTable.tsx @@ -46,12 +46,12 @@ const BillTable = () => { }} > - {item.title} + {item.title} {formatTimeToChatTime(item.time)} - + {item.content} {!item.read && ( diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index d28762451..76101589d 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller'; -import { getUploadModel, removeFilesByPaths } from '@fastgpt/service/common/file/upload/multer'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; /** * Creates the multer uploader @@ -16,12 +16,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< let filePaths: string[] = []; try { + const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); + const { files, bucketName, metadata } = await upload.doUpload(req, res); filePaths = files.map((file) => file.path); await connectToDatabase(); - const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); if (!bucketName) { throw new Error('bucketName is empty'); @@ -53,8 +54,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< error }); } - - removeFilesByPaths(filePaths); } export const config = { diff --git a/projects/app/src/pages/api/common/file/uploadImage.ts b/projects/app/src/pages/api/common/file/uploadImage.ts index 65bc8a573..6e58af019 100644 --- a/projects/app/src/pages/api/common/file/uploadImage.ts +++ b/projects/app/src/pages/api/common/file/uploadImage.ts @@ -8,15 +8,13 @@ import { UploadImgProps } from '@fastgpt/global/common/file/api'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { base64Img, expiredTime, metadata, shareId } = req.body as UploadImgProps; + const { shareId, ...body } = req.body as UploadImgProps; const { teamId } = await authCertOrShareId({ req, shareId, authToken: true }); const data = await uploadMongoImg({ teamId, - base64Img, - expiredTime, - metadata + ...body }); jsonRes(res, { data }); diff --git a/projects/app/src/pages/api/common/system/getInitData.ts b/projects/app/src/pages/api/common/system/getInitData.ts index 4878e8165..8eb660f35 100644 --- a/projects/app/src/pages/api/common/system/getInitData.ts +++ b/projects/app/src/pages/api/common/system/getInitData.ts @@ -59,39 +59,44 @@ const defaultFeConfigs: FastGPTFeConfigsType = { }; export async function getInitConfig() { + if (global.systemInitd) return; + global.systemInitd = true; + try { - if (global.feConfigs) return; await connectToDatabase(); - initGlobal(); - await initSystemConfig(); + await Promise.all([ + initGlobal(), + initSystemConfig(), + getSimpleModeTemplates(), + getSystemVersion(), + getSystemPlugin() + ]); + + console.log({ + simpleModeTemplates: global.simpleModeTemplates, + communityPlugins: global.communityPlugins + }); } catch (error) { console.error('Load init config error', error); + global.systemInitd = false; if (!global.feConfigs) { exit(1); } } - await getSimpleModeTemplates(); +} - getSystemVersion(); - getSystemPlugin(); +export function initGlobal() { + if (global.communityPlugins) return; - console.log({ - feConfigs: global.feConfigs, - systemEnv: global.systemEnv, - chatModels: global.chatModels, - qaModels: global.qaModels, - cqModels: global.cqModels, - extractModels: global.extractModels, - qgModels: global.qgModels, - vectorModels: global.vectorModels, - reRankModels: global.reRankModels, - audioSpeechModels: global.audioSpeechModels, - whisperModel: global.whisperModel, - simpleModeTemplates: global.simpleModeTemplates, - communityPlugins: global.communityPlugins - }); + global.communityPlugins = []; + global.simpleModeTemplates = []; + global.qaQueueLen = global.qaQueueLen ?? 0; + global.vectorQueueLen = global.vectorQueueLen ?? 0; + // init tikToken + getTikTokenEnc(); + initHttpAgent(); } export async function initSystemConfig() { @@ -137,19 +142,24 @@ export async function initSystemConfig() { global.reRankModels = config.reRankModels; global.audioSpeechModels = config.audioSpeechModels; global.whisperModel = config.whisperModel; -} -export function initGlobal() { - global.communityPlugins = []; - global.simpleModeTemplates = []; - global.qaQueueLen = global.qaQueueLen ?? 0; - global.vectorQueueLen = global.vectorQueueLen ?? 0; - // init tikToken - getTikTokenEnc(); - initHttpAgent(); + console.log({ + feConfigs: global.feConfigs, + systemEnv: global.systemEnv, + chatModels: global.chatModels, + qaModels: global.qaModels, + cqModels: global.cqModels, + extractModels: global.extractModels, + qgModels: global.qgModels, + vectorModels: global.vectorModels, + reRankModels: global.reRankModels, + audioSpeechModels: global.audioSpeechModels, + whisperModel: global.whisperModel + }); } export function getSystemVersion() { + if (global.systemVersion) return; try { if (process.env.NODE_ENV === 'development') { global.systemVersion = process.env.npm_package_version || '0.0.0'; diff --git a/projects/app/src/pages/api/common/system/refreshConfig.ts b/projects/app/src/pages/api/common/system/refreshConfig.ts deleted file mode 100644 index e877c27e2..000000000 --- a/projects/app/src/pages/api/common/system/refreshConfig.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { initSystemConfig } from './getInitData'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - await authCert({ req, authRoot: true }); - await initSystemConfig(); - - console.log(`refresh config`); - console.log({ - chatModels: global.chatModels, - qaModels: global.qaModels, - cqModels: global.cqModels, - extractModels: global.extractModels, - qgModels: global.qgModels, - vectorModels: global.vectorModels, - reRankModels: global.reRankModels, - audioSpeechModels: global.audioSpeechModels, - whisperModel: global.whisperModel, - feConfigs: global.feConfigs, - systemEnv: global.systemEnv - }); - } catch (error) { - console.log(error); - } - jsonRes(res); -} diff --git a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts index 6d7c0749b..d1655e738 100644 --- a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts +++ b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts @@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + chatId, dataId: chatItemId }, { diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts b/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts new file mode 100644 index 000000000..7a4f74807 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts @@ -0,0 +1,88 @@ +/* + Create one dataset collection +*/ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; +import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; +import { getQAModel, getVectorModel } from '@/service/core/ai/model'; +import { reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils'; +import { startQueue } from '@/service/utils/tools'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { + link, + trainingType = TrainingModeEnum.chunk, + chunkSize = 512, + chunkSplitter, + qaPrompt, + ...body + } = req.body as LinkCreateDatasetCollectionParams; + + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId: body.datasetId, + per: 'w' + }); + + // 1. check dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(trainingType, new Array(10)) + }); + + // 2. create collection + const collectionId = await createOneCollection({ + ...body, + name: link, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.link, + + trainingType, + chunkSize, + chunkSplitter, + qaPrompt, + + rawLink: link + }); + + // 3. create bill and start sync + const { billId } = await createTrainingBill({ + teamId, + tmbId, + appName: 'core.dataset.collection.Sync Collection', + billSource: BillSourceEnum.training, + vectorModel: getVectorModel(dataset.vectorModel).name, + agentModel: getQAModel(dataset.agentModel).name + }); + await reloadCollectionChunks({ + collectionId, + tmbId, + billId + }); + + startQueue(); + + jsonRes(res, { + data: { collectionId } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts b/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts new file mode 100644 index 000000000..6e131c4dc --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts @@ -0,0 +1,90 @@ +/* + Create one dataset collection +*/ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { hashStr } from '@fastgpt/global/common/string/tools'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { + text, + trainingType = TrainingModeEnum.chunk, + chunkSize = 512, + chunkSplitter, + qaPrompt, + ...body + } = req.body as TextCreateDatasetCollectionParams; + + const { teamId, tmbId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId: body.datasetId, + per: 'w' + }); + + // 1. split text to chunks + const { chunks } = splitText2Chunks({ + text, + chunkLen: chunkSize, + overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0, + customReg: chunkSplitter ? [chunkSplitter] : [], + countTokens: false + }); + + // 2. check dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(trainingType, chunks) + }); + + // 3. create collection + const collectionId = await createOneCollection({ + ...body, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.virtual, + + trainingType, + chunkSize, + chunkSplitter, + qaPrompt, + + hashRawText: hashStr(text), + rawTextLength: text.length + }); + + // 4. push chunks to training queue + const insertResults = await pushDataToDatasetCollection({ + teamId, + tmbId, + collectionId, + trainingMode: trainingType, + data: chunks.map((text, index) => ({ + q: text, + chunkIndex: index + })) + }); + + jsonRes(res, { + data: { collectionId, results: insertResults } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/collection/create.ts b/projects/app/src/pages/api/core/dataset/collection/create.ts index 140d34bb8..a889f6c80 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create.ts @@ -5,7 +5,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; -import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; @@ -14,13 +13,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await connectToDatabase(); const body = req.body as CreateDatasetCollectionParams; - // auth. not visitor and dataset is public - const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true }); - await authDataset({ + const { teamId, tmbId } = await authDataset({ req, authToken: true, + authApiKey: true, datasetId: body.datasetId, - per: 'r' + per: 'w' }); jsonRes(res, { diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 778b1b3da..9544badec 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -4,13 +4,12 @@ import { connectToDatabase } from '@/service/mongo'; import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils'; import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId } = req.query as { collectionId: string }; + const { id: collectionId } = req.query as { id: string }; if (!collectionId) { throw new Error('CollectionIdId is required'); @@ -19,6 +18,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await authDatasetCollection({ req, authToken: true, + authApiKey: true, collectionId, per: 'w' }); diff --git a/projects/app/src/pages/api/core/dataset/collection/detail.ts b/projects/app/src/pages/api/core/dataset/collection/detail.ts index 737eb6b24..8c3aa084a 100644 --- a/projects/app/src/pages/api/core/dataset/collection/detail.ts +++ b/projects/app/src/pages/api/core/dataset/collection/detail.ts @@ -22,6 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const { collection, canWrite } = await authDatasetCollection({ req, authToken: true, + authApiKey: true, collectionId: id, per: 'r' }); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 153113184..4ffb1e94c 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -11,7 +11,6 @@ import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant import { startQueue } from '@/service/utils/tools'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; -import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -27,12 +26,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< simple = false } = req.body as GetDatasetCollectionsProps; searchText = searchText?.replace(/'/g, ''); + pageSize = Math.min(pageSize, 30); // auth dataset and get my role - const { tmbId } = await authDataset({ req, authToken: true, datasetId, per: 'r' }); - const { canWrite } = await authUserRole({ req, authToken: true }); + const { teamId, tmbId, canWrite } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'r' + }); const match = { + teamId: new Types.ObjectId(teamId), datasetId: new Types.ObjectId(datasetId), parentId: parentId ? new Types.ObjectId(parentId) : null, ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), @@ -85,9 +91,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } }, - { $project: { _id: 1 } } + { $count: 'count' } ], - as: 'trainings' + as: 'trainingCount' } }, // count collection total data @@ -103,9 +109,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } }, - { $project: { _id: 1 } } + { $count: 'count' } ], - as: 'datas' + as: 'dataCount' } }, { @@ -117,10 +123,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< type: 1, status: 1, updateTime: 1, - dataAmount: { $size: '$datas' }, - trainingAmount: { $size: '$trainings' }, fileId: 1, - rawLink: 1 + rawLink: 1, + dataAmount: { + $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] + }, + trainingAmount: { + $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] + } } }, { @@ -144,7 +154,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ); if (data.find((item) => item.trainingAmount > 0)) { - startQueue(1); + startQueue(); } // count collections diff --git a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts index 505e6d6b3..b7a1b9042 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts @@ -38,7 +38,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< return Promise.reject(DatasetErrEnum.unLinkCollection); } - const { rawText, isSameRawText } = await getCollectionAndRawText({ + const { title, rawText, isSameRawText } = await getCollectionAndRawText({ collection }); @@ -68,7 +68,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< tmbId: collection.tmbId, parentId: collection.parentId, datasetId: collection.datasetId._id, - name: collection.name, + name: title || collection.name, type: collection.type, trainingType: collection.trainingType, chunkSize: collection.chunkSize, diff --git a/projects/app/src/pages/api/core/dataset/collection/update.ts b/projects/app/src/pages/api/core/dataset/collection/update.ts index ac29bd7e3..f45a3c164 100644 --- a/projects/app/src/pages/api/core/dataset/collection/update.ts +++ b/projects/app/src/pages/api/core/dataset/collection/update.ts @@ -16,7 +16,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } // 凭证校验 - await authDatasetCollection({ req, authToken: true, collectionId: id, per: 'w' }); + await authDatasetCollection({ + req, + authToken: true, + authApiKey: true, + collectionId: id, + per: 'w' + }); const updateFields: Record = { ...(parentId !== undefined && { parentId: parentId || null }), diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index cd37214ce..d7533443d 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -16,12 +16,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< type, avatar, vectorModel = global.vectorModels[0].model, - agentModel + agentModel = global.qaModels[0].model } = req.body as CreateDatasetParams; - // 凭证校验 + // auth const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true }); + // check model valid + const vectorModelStore = global.vectorModels.find((item) => item.model === vectorModel); + const agentModelStore = global.qaModels.find((item) => item.model === agentModel); + if (!vectorModelStore || !agentModelStore) { + throw new Error('vectorModel or qaModel is invalid'); + } + + // check limit + const authCount = await MongoDataset.countDocuments({ + teamId, + type: DatasetTypeEnum.dataset + }); + if (authCount >= 50) { + throw new Error('每个团队上限 50 个知识库'); + } + const { _id } = await MongoDataset.create({ name, teamId, diff --git a/projects/app/src/pages/api/core/dataset/data/delete.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts index ab4a7d75a..5bf4df034 100644 --- a/projects/app/src/pages/api/core/dataset/data/delete.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -8,8 +8,8 @@ import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/contr export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { dataId } = req.query as { - dataId: string; + const { id: dataId } = req.query as { + id: string; }; if (!dataId) { @@ -17,9 +17,18 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } // 凭证校验 - await authDatasetData({ req, authToken: true, dataId, per: 'w' }); + const { datasetData } = await authDatasetData({ + req, + authToken: true, + authApiKey: true, + dataId, + per: 'w' + }); - await delDatasetDataByDataId(dataId); + await delDatasetDataByDataId({ + collectionId: datasetData.collectionId, + mongoDataId: dataId + }); jsonRes(res, { data: 'success' diff --git a/projects/app/src/pages/api/core/dataset/data/detail.ts b/projects/app/src/pages/api/core/dataset/data/detail.ts index f734eb3c5..57f93c29f 100644 --- a/projects/app/src/pages/api/core/dataset/data/detail.ts +++ b/projects/app/src/pages/api/core/dataset/data/detail.ts @@ -13,12 +13,18 @@ export type Response = { export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { dataId } = req.query as { - dataId: string; + const { id: dataId } = req.query as { + id: string; }; // 凭证校验 - const { datasetData } = await authDatasetData({ req, authToken: true, dataId, per: 'r' }); + const { datasetData } = await authDatasetData({ + req, + authToken: true, + authApiKey: true, + dataId, + per: 'r' + }); jsonRes(res, { data: datasetData diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index fbf3f411e..0a794bfe3 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -16,6 +16,7 @@ import { authTeamBalance } from '@/service/support/permission/auth/bill'; import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; import { InsertOneDatasetDataProps } from '@/global/core/dataset/api'; import { simpleText } from '@fastgpt/global/common/string/tools'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -39,6 +40,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: 1 + }); + // auth collection and get dataset const [ { diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index b500fde74..e014ec7a4 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -17,8 +17,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< collectionId } = req.body as GetDatasetDataListProps; + pageSize = Math.min(pageSize, 30); + // 凭证校验 - await authDatasetCollection({ req, authToken: true, collectionId, per: 'r' }); + await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' }); searchText = searchText.replace(/'/g, ''); @@ -32,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }; const [data, total] = await Promise.all([ - MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex indexes') + MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') .sort({ chunkIndex: 1, updateTime: -1 }) .skip((pageNum - 1) * pageSize) .limit(pageSize) diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index a43823dbf..5e05a7fde 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -2,38 +2,30 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; -import { startQueue } from '@/service/utils/tools'; -import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import type { PushDataResponse } from '@/global/core/api/datasetRes.d'; import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; -import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; -import { getQAModel, getVectorModel } from '@/service/core/ai/model'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; -import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; -import { simpleText } from '@fastgpt/global/common/string/tools'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId, data, mode = TrainingModeEnum.chunk } = req.body as PushDatasetDataProps; + const { collectionId, data } = req.body as PushDatasetDataProps; if (!collectionId || !Array.isArray(data)) { throw new Error('collectionId or data is empty'); } - if (!TrainingTypeMap[mode]) { - throw new Error(`Mode is not ${Object.keys(TrainingTypeMap).join(', ')}`); - } - if (data.length > 200) { throw new Error('Data is too long, max 200'); } // 凭证校验 - const { teamId, tmbId } = await authDatasetCollection({ + const { teamId, tmbId, collection } = await authDatasetCollection({ req, authToken: true, authApiKey: true, @@ -41,6 +33,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); + // auth dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(collection.trainingType, data) + }); + jsonRes(res, { data: await pushDataToDatasetCollection({ ...req.body, @@ -56,141 +55,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } }); -export async function pushDataToDatasetCollection({ - teamId, - tmbId, - collectionId, - data, - mode, - prompt, - billId -}: { - teamId: string; - tmbId: string; -} & PushDatasetDataProps): Promise { - const { datasetId, model, maxToken, weight } = await checkModelValid({ - mode, - collectionId - }); - - // format q and a, remove empty char - data.forEach((item) => { - item.q = simpleText(item.q); - item.a = simpleText(item.a); - - item.indexes = item.indexes - ?.map((index) => { - return { - ...index, - text: simpleText(index.text) - }; - }) - .filter(Boolean); - }); - - // filter repeat or equal content - const set = new Set(); - const filterResult: Record = { - success: [], - overToken: [], - repeat: [], - error: [] - }; - - data.forEach((item) => { - if (!item.q) { - filterResult.error.push(item); - return; - } - - const text = item.q + item.a; - - // count q token - const token = countPromptTokens(item.q); - - if (token > maxToken) { - filterResult.overToken.push(item); - return; - } - - if (set.has(text)) { - console.log('repeat', item); - filterResult.repeat.push(item); - } else { - filterResult.success.push(item); - set.add(text); - } - }); - - // 插入记录 - const insertRes = await MongoDatasetTraining.insertMany( - filterResult.success.map((item, i) => ({ - teamId, - tmbId, - datasetId, - collectionId, - billId, - mode, - prompt, - model, - q: item.q, - a: item.a, - chunkIndex: item.chunkIndex ?? i, - weight: weight ?? 0, - indexes: item.indexes - })) - ); - - insertRes.length > 0 && startQueue(); - delete filterResult.success; - - return { - insertLen: insertRes.length, - ...filterResult - }; -} - -export async function checkModelValid({ - mode, - collectionId -}: { - mode: `${TrainingModeEnum}`; - collectionId: string; -}) { - const { - datasetId: { _id: datasetId, vectorModel, agentModel } - } = await getCollectionWithDataset(collectionId); - - if (mode === TrainingModeEnum.chunk) { - if (!collectionId) return Promise.reject(`CollectionId is empty`); - const vectorModelData = getVectorModel(vectorModel); - if (!vectorModelData) { - return Promise.reject(`Model ${vectorModel} is inValid`); - } - - return { - datasetId, - maxToken: vectorModelData.maxToken * 1.5, - model: vectorModelData.model, - weight: vectorModelData.weight - }; - } - - if (mode === TrainingModeEnum.qa) { - const qaModelData = getQAModel(agentModel); - if (!qaModelData) { - return Promise.reject(`Model ${agentModel} is inValid`); - } - return { - datasetId, - maxToken: qaModelData.maxContext * 0.8, - model: qaModelData.model, - weight: 0 - }; - } - return Promise.reject(`Mode ${mode} is inValid`); -} - export const config = { api: { bodyParser: { diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index 05bdbbd6b..41279637a 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -11,7 +11,7 @@ import { UpdateDatasetDataProps } from '@/global/core/dataset/api'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id, q = '', a, indexes } = req.body as UpdateDatasetDataProps; + const { id, q = '', a, indexes = [] } = req.body as UpdateDatasetDataProps; // auth data permission const { @@ -23,6 +23,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } = await authDatasetData({ req, authToken: true, + authApiKey: true, dataId: id, per: 'w' }); diff --git a/projects/app/src/pages/api/core/dataset/detail.ts b/projects/app/src/pages/api/core/dataset/detail.ts index 399a887c0..7e936a622 100644 --- a/projects/app/src/pages/api/core/dataset/detail.ts +++ b/projects/app/src/pages/api/core/dataset/detail.ts @@ -20,6 +20,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const { dataset, canWrite, isOwner } = await authDataset({ req, authToken: true, + authApiKey: true, datasetId, per: 'r' }); diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index d5a411f32..9ebea8e33 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -15,7 +15,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({ req, - authToken: true + authToken: true, + authApiKey: true }); const datasets = await MongoDataset.find({ diff --git a/projects/app/src/pages/api/plusApi/[...path].ts b/projects/app/src/pages/api/plusApi/[...path].ts index 58a275ead..761fc3ba7 100644 --- a/projects/app/src/pages/api/plusApi/[...path].ts +++ b/projects/app/src/pages/api/plusApi/[...path].ts @@ -3,14 +3,11 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { request } from '@fastgpt/service/common/api/plusRequest'; import type { Method } from 'axios'; import { setCookie } from '@fastgpt/service/support/permission/controller'; -import { getInitConfig } from '../common/system/getInitData'; -import { FastGPTProUrl } from '@fastgpt/service/common/system/constants'; +import { connectToDatabase } from '@/service/mongo'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - if (!FastGPTProUrl) { - await getInitConfig(); - } + await connectToDatabase(); const method = (req.method || 'POST') as Method; const { path = [], ...query } = req.query as any; diff --git a/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts b/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts new file mode 100644 index 000000000..d55c9ddaa --- /dev/null +++ b/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts @@ -0,0 +1,39 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getTeamDatasetValidSub } from '@fastgpt/service/support/wallet/sub/utils'; +import { getVectorCountByTeamId } from '@fastgpt/service/common/vectorStore/controller'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + // 凭证校验 + const { teamId } = await authCert({ + req, + authToken: true + }); + + const [{ sub, maxSize }, usedSize] = await Promise.all([ + getTeamDatasetValidSub({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize + }), + getVectorCountByTeamId(teamId) + ]); + + jsonRes(res, { + data: { + sub, + maxSize, + usedSize + } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts index f7f86513e..22fd91dce 100644 --- a/projects/app/src/pages/api/v1/audio/transcriptions.ts +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -2,7 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { getUploadModel, removeFilesByPaths } from '@fastgpt/service/common/file/upload/multer'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; +import { removeFilesByPaths } from '@fastgpt/service/common/file/utils'; import fs from 'fs'; import { getAIApi } from '@fastgpt/service/core/ai/config'; import { pushWhisperBill } from '@/service/support/wallet/bill/push'; diff --git a/projects/app/src/pages/api/v1/embeddings.ts b/projects/app/src/pages/api/v1/embeddings.ts index 1bb57bbd1..4371c2609 100644 --- a/projects/app/src/pages/api/v1/embeddings.ts +++ b/projects/app/src/pages/api/v1/embeddings.ts @@ -35,19 +35,17 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { tokens, vectors } = await getVectorsByText({ input: query, model }); - jsonRes(res, { - data: { - object: 'list', - data: vectors.map((item, index) => ({ - object: 'embedding', - index: index, - embedding: item - })), - model, - usage: { - prompt_tokens: tokens, - total_tokens: tokens - } + res.json({ + object: 'list', + data: vectors.map((item, index) => ({ + object: 'embedding', + index: index, + embedding: item + })), + model, + usage: { + prompt_tokens: tokens, + total_tokens: tokens } }); diff --git a/projects/app/src/pages/app/detail/components/InfoModal.tsx b/projects/app/src/pages/app/detail/components/InfoModal.tsx index c7a957a7b..6786ea4b8 100644 --- a/projects/app/src/pages/app/detail/components/InfoModal.tsx +++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx @@ -22,6 +22,7 @@ import MyModal from '@/components/MyModal'; import { useAppStore } from '@/web/core/app/store/useAppStore'; import PermissionRadio from '@/components/support/permission/Radio'; import { useTranslation } from 'next-i18next'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const InfoModal = ({ defaultApp, @@ -45,7 +46,6 @@ const InfoModal = ({ setValue, getValues, formState: { errors }, - reset, handleSubmit } = useForm({ defaultValues: defaultApp @@ -102,6 +102,7 @@ const InfoModal = ({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.appAvatar, file, maxW: 300, maxH: 300 @@ -187,4 +188,4 @@ const InfoModal = ({ ); }; -export default InfoModal; +export default React.memo(InfoModal); diff --git a/projects/app/src/pages/app/detail/components/Logs.tsx b/projects/app/src/pages/app/detail/components/Logs.tsx index 674190b96..06e843069 100644 --- a/projects/app/src/pages/app/detail/components/Logs.tsx +++ b/projects/app/src/pages/app/detail/components/Logs.tsx @@ -81,7 +81,7 @@ const Logs = ({ appId }: { appId: string }) => { cursor={'pointer'} onClick={onOpenMarkDesc} > - {t('chat.Read Mark Description')} + {t('core.chat.Read Mark Description')} @@ -202,9 +202,9 @@ const Logs = ({ appId }: { appId: string }) => { - {t('chat.Mark Description')} + {t('core.chat.Mark Description')} ); diff --git a/projects/app/src/pages/app/list/component/CreateModal.tsx b/projects/app/src/pages/app/list/component/CreateModal.tsx index 12a2e28c1..2ccac40d2 100644 --- a/projects/app/src/pages/app/list/component/CreateModal.tsx +++ b/projects/app/src/pages/app/list/component/CreateModal.tsx @@ -26,6 +26,7 @@ import Avatar from '@/components/Avatar'; import MyTooltip from '@/components/MyTooltip'; import MyModal from '@/components/MyModal'; import { useTranslation } from 'next-i18next'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; type FormType = { avatar: string; @@ -59,6 +60,7 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: ( if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.appAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/pages/chat/components/ChatHeader.tsx b/projects/app/src/pages/chat/components/ChatHeader.tsx index 3db3bdbc4..694da5767 100644 --- a/projects/app/src/pages/chat/components/ChatHeader.tsx +++ b/projects/app/src/pages/chat/components/ChatHeader.tsx @@ -35,7 +35,7 @@ const ChatHeader = ({ () => chatContentReplaceBlock(history[history.length - 2]?.value)?.slice(0, 8) || appName || - t('chat.New Chat'), + t('core.chat.New Chat'), [appName, history] ); @@ -56,8 +56,8 @@ const ChatHeader = ({ {history.length === 0 - ? t('chat.Fresh Chat') - : t('chat.History Amount', { amount: history.length })} + ? t('core.chat.New Chat') + : t('core.chat.History Amount', { amount: history.length })} {!!chatModels && chatModels.length > 0 && ( diff --git a/projects/app/src/pages/chat/components/ChatHistorySlider.tsx b/projects/app/src/pages/chat/components/ChatHistorySlider.tsx index 96dc5281c..ecc9f9efd 100644 --- a/projects/app/src/pages/chat/components/ChatHistorySlider.tsx +++ b/projects/app/src/pages/chat/components/ChatHistorySlider.tsx @@ -74,18 +74,20 @@ const ChatHistorySlider = ({ // custom title edit const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({ - title: t('chat.Custom History Title'), - placeholder: t('chat.Custom History Title Description') + title: t('core.chat.Custom History Title'), + placeholder: t('core.chat.Custom History Title Description') }); const { openConfirm, ConfirmModal } = useConfirm({ content: isShare - ? t('chat.Confirm to clear share chat history') - : t('chat.Confirm to clear history') + ? t('core.chat.Confirm to clear share chat history') + : t('core.chat.Confirm to clear history') }); const concatHistory = useMemo( () => - !activeChatId ? [{ id: activeChatId, title: t('chat.New Chat') }].concat(history) : history, + !activeChatId + ? [{ id: activeChatId, title: t('core.chat.New Chat') }].concat(history) + : history, [activeChatId, history, t] ); @@ -144,7 +146,7 @@ const ChatHistorySlider = ({ mr={2} list={[ { label: 'App', id: TabEnum.app }, - { label: 'chat.History', id: TabEnum.history } + { label: t('core.chat.History'), id: TabEnum.history } ]} activeId={currentTab} onChange={(e) => setCurrentTab(e as `${TabEnum}`)} @@ -160,7 +162,7 @@ const ChatHistorySlider = ({ overflow={'hidden'} onClick={() => onChangeChat()} > - {t('chat.New Chat')} + {t('core.chat.New Chat')} {(isPc || isShare) && ( @@ -240,7 +242,7 @@ const ChatHistorySlider = ({ }} > - {item.top ? t('chat.Unpin') : t('chat.Pin')} + {item.top ? t('core.chat.Unpin') : t('core.chat.Pin')} )} {onSetCustomTitle && ( @@ -336,7 +338,7 @@ const ChatHistorySlider = ({ borderRadius={'50%'} aria-label={''} /> - {t('chat.Exit Chat')} + {t('core.chat.Exit Chat')} )} diff --git a/projects/app/src/pages/chat/components/SliderApps.tsx b/projects/app/src/pages/chat/components/SliderApps.tsx index 5960a46f7..596be623f 100644 --- a/projects/app/src/pages/chat/components/SliderApps.tsx +++ b/projects/app/src/pages/chat/components/SliderApps.tsx @@ -35,7 +35,7 @@ const SliderApps = ({ appId }: { appId: string }) => { borderRadius={'50%'} aria-label={''} /> - {t('chat.Exit Chat')} + {t('core.chat.Exit Chat')} diff --git a/projects/app/src/pages/chat/components/ToolMenu.tsx b/projects/app/src/pages/chat/components/ToolMenu.tsx index d033fb766..d568e4bfe 100644 --- a/projects/app/src/pages/chat/components/ToolMenu.tsx +++ b/projects/app/src/pages/chat/components/ToolMenu.tsx @@ -15,7 +15,7 @@ const ToolMenu = ({ history }: { history: ChatItemType[] }) => { () => [ { icon: 'core/chat/chatLight', - label: t('chat.New Chat'), + label: t('core.chat.New Chat'), onClick: () => { router.replace({ query: { diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index 897e5db28..7f4b21f9b 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -86,7 +86,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { const newTitle = chatContentReplaceBlock(prompts[0].content).slice(0, 20) || prompts[1]?.value?.slice(0, 20) || - t('chat.New Chat'); + t('core.chat.New Chat'); // new chat if (completionChatId !== chatId) { @@ -166,7 +166,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { setLastChatAppId(''); setLastChatId(''); toast({ - title: getErrText(e, t('chat.Failed to initialize chat')), + title: getErrText(e, t('core.chat.Failed to initialize chat')), status: 'error' }); if (e?.code === 501) { @@ -210,7 +210,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { if (apps.length === 0) { toast({ status: 'error', - title: t('chat.You need to a chat app') + title: t('core.chat.You need to a chat app') }); router.replace('/app/list'); } else { diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index 72ca51d20..0fd6c3939 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -88,7 +88,7 @@ const OutLink = ({ const newTitle = chatContentReplaceBlock(prompts[0].content).slice(0, 20) || prompts[1]?.value?.slice(0, 20) || - t('chat.New Chat'); + t('core.chat.New Chat'); // new chat if (completionChatId !== chatId) { diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx index 7e6352022..6fe48d4e5 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx @@ -43,7 +43,7 @@ import EmptyTip from '@/components/EmptyTip'; import { FolderAvatarSrc, DatasetCollectionTypeEnum, - DatasetCollectionTrainingModeEnum, + TrainingModeEnum, DatasetTypeEnum, DatasetTypeMap, DatasetStatusEnum, @@ -63,6 +63,7 @@ import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSyncResultEnum } from '../../../../../../../packages/global/core/dataset/constant'; +import MyBox from '@/components/common/MyBox'; const FileImportModal = dynamic(() => import('./Import/ImportModal'), {}); const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {}); @@ -100,8 +101,8 @@ const CollectionCard = () => { } = useDisclosure(); const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = useEditTitle({ - title: t('dataset.Create Virtual File'), - tip: t('dataset.Virtual File Tip'), + title: t('dataset.Create manual collection'), + tip: t('dataset.Manual collection Tip'), canEmpty: false }); @@ -186,7 +187,7 @@ const CollectionCard = () => { name: string; type: `${DatasetCollectionTypeEnum}`; callback?: (id: string) => void; - trainingType?: `${DatasetCollectionTrainingModeEnum}`; + trainingType?: `${TrainingModeEnum}`; rawLink?: string; chunkSize?: number; }) => { @@ -224,7 +225,7 @@ const CollectionCard = () => { const { mutate: onDelCollection, isLoading: isDeleting } = useRequest({ mutationFn: (collectionId: string) => { return delDatasetCollectionById({ - collectionId + id: collectionId }); }, onSuccess() { @@ -296,501 +297,522 @@ const CollectionCard = () => { }, [parentId]); return ( - - - - ({ - parentId: path.parentId, - parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName - }))} - FirstPathDom={ - <> - - {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) - - {datasetDetail?.websiteConfig?.url && ( - - {t('core.dataset.website.Base Url')}: - - {datasetDetail.websiteConfig.url} - - - )} - - } - onClick={(e) => { - router.replace({ - query: { - ...router.query, - parentId: e - } - }); - }} - /> - - - {isPc && ( - - + + + + + ({ + parentId: path.parentId, + parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName + }))} + FirstPathDom={ + <> + + {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) + + {datasetDetail?.websiteConfig?.url && ( + + {t('core.dataset.website.Base Url')}: + + {datasetDetail.websiteConfig.url} + + + )} + } - w={['100%', '250px']} - size={['sm', 'md']} - placeholder={t('common.Search') || ''} - value={searchText} - onChange={(e) => { - setSearchText(e.target.value); - debounceRefetch(); - }} - onBlur={() => { - if (searchText === lastSearch.current) return; - getData(1); - }} - onKeyDown={(e) => { - if (searchText === lastSearch.current) return; - if (e.key === 'Enter') { - getData(1); - } + onClick={(e) => { + router.replace({ + query: { + ...router.query, + parentId: e + } + }); }} /> - - )} - {datasetDetail?.type === DatasetTypeEnum.dataset && ( - <> - {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - - - {t('dataset.collections.Create And Import')} - - + + + {isPc && ( + + } - menuList={[ - { - child: ( - - {''} - {t('Folder')} + onChange={(e) => { + setSearchText(e.target.value); + debounceRefetch(); + }} + onBlur={() => { + if (searchText === lastSearch.current) return; + getData(1); + }} + onKeyDown={(e) => { + if (searchText === lastSearch.current) return; + if (e.key === 'Enter') { + getData(1); + } + }} + /> + + )} + {datasetDetail?.type === DatasetTypeEnum.dataset && ( + <> + {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + + + + {t('dataset.collections.Create And Import')} - ), - onClick: () => setEditFolderData({}) - }, - { - child: ( - - {''} - {t('dataset.Create Virtual File')} - - ), - onClick: () => { - onOpenCreateVirtualFileModal({ - defaultVal: '', - onSuccess: (name) => { - onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); + + } + menuList={[ + { + child: ( + + {''} + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + child: ( + + {''} + {t('core.dataset.Manual collection')} + + ), + onClick: () => { + onOpenCreateVirtualFileModal({ + defaultVal: '', + onSuccess: (name) => { + onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); + } + }); + } + }, + { + child: ( + + {''} + {t('core.dataset.File collection')} + + ), + onClick: onOpenFileImportModal + } + ]} + /> + )} + + )} + {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( + <> + {datasetDetail?.websiteConfig?.url ? ( + + {datasetDetail.status === DatasetStatusEnum.active && ( + + )} + {datasetDetail.status === DatasetStatusEnum.syncing && ( + + + + {t('core.dataset.status.syncing')} + + + )} + + ) : ( + + )} + + )} + + + + + + + + + + + + + + + {formatCollections.map((collection, index) => ( + { + setDragStartId(collection._id); + }} + onDragOver={(e) => { + e.preventDefault(); + const targetId = e.currentTarget.getAttribute('data-drag-id'); + if (!targetId) return; + DatasetCollectionTypeEnum.folder && setDragTargetId(targetId); + }} + onDragLeave={(e) => { + e.preventDefault(); + setDragTargetId(undefined); + }} + onDrop={async (e) => { + e.preventDefault(); + if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return; + // update parentId + try { + await putDatasetCollectionById({ + id: dragStartId, + parentId: dragTargetId + }); + getData(pageNum); + } catch (error) {} + setDragTargetId(undefined); + }} + title={ + collection.type === DatasetCollectionTypeEnum.folder + ? t('dataset.collections.Click to view folder') + : t('dataset.collections.Click to view file') + } + onClick={() => { + if (collection.type === DatasetCollectionTypeEnum.folder) { + router.replace({ + query: { + ...router.query, + parentId: collection._id + } + }); + } else { + router.replace({ + query: { + ...router.query, + collectionId: collection._id, + currentTab: TabEnum.dataCard } }); } - }, - { - child: ( - - - {t('dataset.File Input')} - - ), - onClick: onOpenFileImportModal - } - ]} - /> - )} - - )} - {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( - <> - {datasetDetail?.websiteConfig?.url ? ( - - {datasetDetail.status === DatasetStatusEnum.active && ( - - )} - {datasetDetail.status === DatasetStatusEnum.syncing && ( - - - - {t('core.dataset.status.syncing')} - - - )} - - ) : ( - - )} - - )} - - - -
+ # + + {t('common.Name')} + + {t('dataset.collections.Data Amount')} + + {t('core.dataset.Sync Time')} + + {t('common.Status')} + +
- - - - - - - - - - - {formatCollections.map((collection, index) => ( - { - setDragStartId(collection._id); - }} - onDragOver={(e) => { - e.preventDefault(); - const targetId = e.currentTarget.getAttribute('data-drag-id'); - if (!targetId) return; - DatasetCollectionTypeEnum.folder && setDragTargetId(targetId); - }} - onDragLeave={(e) => { - e.preventDefault(); - setDragTargetId(undefined); - }} - onDrop={async (e) => { - e.preventDefault(); - if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return; - // update parentId - try { - await putDatasetCollectionById({ - id: dragStartId, - parentId: dragTargetId - }); - getData(pageNum); - } catch (error) {} - setDragTargetId(undefined); - }} - title={ - collection.type === DatasetCollectionTypeEnum.folder - ? t('dataset.collections.Click to view folder') - : t('dataset.collections.Click to view file') - } - onClick={() => { - if (collection.type === DatasetCollectionTypeEnum.folder) { - router.replace({ - query: { - ...router.query, - parentId: collection._id - } - }); - } else { - router.replace({ - query: { - ...router.query, - collectionId: collection._id, - currentTab: TabEnum.dataCard - } - }); - } - }} - > - - - - - - + + + + + - - ))} - -
#{t('common.Name')}{t('dataset.collections.Data Amount')}{t('core.dataset.Sync Time')}{t('common.Status')} -
{index + 1} - - {''} - - - {collection.name} - - - - {collection.dataAmount || '-'}{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')} - - {t(collection.statusText)} - - e.stopPropagation()}> - {collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - + {index + 1} + + {''} + + + {collection.name} + + + + {collection.dataAmount || '-'}{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')} + + {t(collection.statusText)} + + e.stopPropagation()}> + {collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + - - } - menuList={[ - ...(collection.type === DatasetCollectionTypeEnum.link - ? [ - { - child: ( - - - {t('core.dataset.collection.Sync')} - - ), - onClick: () => - openSyncConfirm(() => { - onclickStartSync(collection._id); - })() + _hover={{ + color: 'primary.500', + '& .icon': { + bg: 'myGray.200' } - ] - : []), - { - child: ( - - - {t('Move')} - - ), - onClick: () => setMoveCollectionData({ collectionId: collection._id }) - }, - { - child: ( - - - {t('Rename')} - - ), - onClick: () => - onOpenEditTitleModal({ - defaultVal: collection.name, - onSuccess: (newName) => { - onUpdateCollectionName({ - collectionId: collection._id, - name: newName - }); - } - }) - }, - { - child: ( - - - {t('common.Delete')} - - ), - onClick: () => - openDeleteConfirm( - () => { - onDelCollection(collection._id); - }, - undefined, - collection.type === DatasetCollectionTypeEnum.folder - ? t('dataset.collections.Confirm to delete the folder') - : t('dataset.Confirm to delete the file') - )() - } - ]} - /> - )} -
- {total > pageSize && ( - - - - )} - {total === 0 && ( - - {datasetDetail.status === DatasetStatusEnum.syncing && ( - <>{t('core.dataset.status.syncing')} - )} - {datasetDetail.status === DatasetStatusEnum.active && ( - <> - {!datasetDetail?.websiteConfig?.url ? ( - <> - {t('core.dataset.collection.Website Empty Tip')} - {', '} - - {t('core.dataset.collection.Click top config website')} - - - ) : ( - <>{t('core.dataset.website.UnValid Website Tip')} - )} - - )} - - ) - } + + + } + menuList={[ + ...(collection.type === DatasetCollectionTypeEnum.link + ? [ + { + child: ( + + + {t('core.dataset.collection.Sync')} + + ), + onClick: () => + openSyncConfirm(() => { + onclickStartSync(collection._id); + })() + } + ] + : []), + { + child: ( + + + {t('Move')} + + ), + onClick: () => setMoveCollectionData({ collectionId: collection._id }) + }, + { + child: ( + + + {t('Rename')} + + ), + onClick: () => + onOpenEditTitleModal({ + defaultVal: collection.name, + onSuccess: (newName) => { + onUpdateCollectionName({ + collectionId: collection._id, + name: newName + }); + } + }) + }, + { + child: ( + + + {t('common.Delete')} + + ), + onClick: () => + openDeleteConfirm( + () => { + onDelCollection(collection._id); + }, + undefined, + collection.type === DatasetCollectionTypeEnum.folder + ? t('dataset.collections.Confirm to delete the folder') + : t('dataset.Confirm to delete the file') + )() + } + ]} + /> + )} + + + ))} + + + {total > pageSize && ( + + + + )} + {total === 0 && ( + + {datasetDetail.status === DatasetStatusEnum.syncing && ( + <>{t('core.dataset.status.syncing')} + )} + {datasetDetail.status === DatasetStatusEnum.active && ( + <> + {!datasetDetail?.websiteConfig?.url ? ( + <> + {t('core.dataset.collection.Website Empty Tip')} + {', '} + + {t('core.dataset.collection.Click top config website')} + + + ) : ( + <>{t('core.dataset.website.UnValid Website Tip')} + )} + + )} + + ) + } + /> + )} + + + + + + + {isOpenFileImportModal && ( + { + getData(1); + onCloseFileImportModal(); + }} + onClose={onCloseFileImportModal} /> )} - - - - - - - - {isOpenFileImportModal && ( - { - getData(1); - onCloseFileImportModal(); - }} - onClose={onCloseFileImportModal} - /> - )} - {!!editFolderData && ( - setEditFolderData(undefined)} - editCallback={async (name) => { - try { - if (editFolderData.id) { - await putDatasetCollectionById({ - id: editFolderData.id, - name - }); - getData(pageNum); - } else { - onCreateCollection({ - name, - type: DatasetCollectionTypeEnum.folder - }); + {!!editFolderData && ( + setEditFolderData(undefined)} + editCallback={async (name) => { + try { + if (editFolderData.id) { + await putDatasetCollectionById({ + id: editFolderData.id, + name + }); + getData(pageNum); + } else { + onCreateCollection({ + name, + type: DatasetCollectionTypeEnum.folder + }); + } + } catch (error) { + return Promise.reject(error); } - } catch (error) { - return Promise.reject(error); - } - }} - isEdit={!!editFolderData.id} - name={editFolderData.name} - /> - )} - {!!moveCollectionData && ( - setMoveCollectionData(undefined)} - onSuccess={async ({ parentId }) => { - await putDatasetCollectionById({ - id: moveCollectionData.collectionId, - parentId - }); - getData(pageNum); - setMoveCollectionData(undefined); - toast({ - status: 'success', - title: t('common.folder.Move Success') - }); - }} - /> - )} - {isOpenWebsiteModal && ( - - )} - + }} + isEdit={!!editFolderData.id} + name={editFolderData.name} + /> + )} + {!!moveCollectionData && ( + setMoveCollectionData(undefined)} + onSuccess={async ({ parentId }) => { + await putDatasetCollectionById({ + id: moveCollectionData.collectionId, + parentId + }); + getData(pageNum); + setMoveCollectionData(undefined); + toast({ + status: 'success', + title: t('common.folder.Move Success') + }); + }} + /> + )} + {isOpenWebsiteModal && ( + + )} + + ); }; diff --git a/projects/app/src/pages/dataset/detail/components/DataCard.tsx b/projects/app/src/pages/dataset/detail/components/DataCard.tsx index 285ffe3e0..45b6f215a 100644 --- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx @@ -32,16 +32,17 @@ import { useRouter } from 'next/router'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyInput from '@/components/MyInput'; import { useLoading } from '@/web/common/hooks/useLoading'; -import InputDataModal, { RawSourceText, type InputDataType } from '../components/InputDataModal'; +import InputDataModal from '../components/InputDataModal'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import type { DatasetDataListItemType } from '@/global/core/dataset/type.d'; import { TabEnum } from '..'; import { useUserStore } from '@/web/support/user/useUserStore'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; -import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { DatasetCollectionTypeMap, - DatasetCollectionTrainingTypeMap + TrainingModeEnum, + TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; @@ -90,7 +91,7 @@ const DataCard = () => { } }); - const [editInputData, setEditInputData] = useState(); + const [editDataId, setEditDataId] = useState(); // get first page data const getFirstData = useCallback( @@ -154,7 +155,7 @@ const DataCard = () => { }, { label: t('core.dataset.collection.metadata.Training Type'), - value: t(DatasetCollectionTrainingTypeMap[collection.trainingType]?.label) + value: t(TrainingTypeMap[collection.trainingType]?.label) }, { label: t('core.dataset.collection.metadata.Chunk Size'), @@ -193,7 +194,7 @@ const DataCard = () => { /> - { size={['sm', 'md']} onClick={() => { if (!collection) return; - setEditInputData({ - q: '', - indexes: [getDefaultIndex({ dataId: `${Date.now()}` })] - }); + setEditDataId(''); }} > {t('dataset.Insert Data')} @@ -297,12 +295,7 @@ const DataCard = () => { }} onClick={() => { if (!collection) return; - setEditInputData({ - id: item._id, - q: item.q, - a: item.a, - indexes: item.indexes - }); + setEditDataId(item._id); }} > @@ -424,11 +417,11 @@ const DataCard = () => { )} - {editInputData !== undefined && collection && ( + {editDataId !== undefined && collection && ( setEditInputData(undefined)} + dataId={editDataId} + onClose={() => setEditDataId(undefined)} onSuccess={() => getData(pageNum)} onDelete={() => getData(pageNum)} /> diff --git a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx index f5b5588d1..834c54cff 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx @@ -4,14 +4,8 @@ import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useToast } from '@/web/common/hooks/useToast'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { simpleText } from '@fastgpt/global/common/string/tools'; -import { - fileDownload, - readCsvContent, - readPdfContent, - readDocContent -} from '@/web/common/file/utils'; -import { readFileRawText, readMdFile, readHtmlFile } from '@fastgpt/web/common/file/read'; -import { getUploadMdImgController, uploadFiles } from '@/web/common/file/controller'; +import { fileDownload, readCsvContent } from '@/web/common/file/utils'; +import { getUploadBase64ImgController, uploadFiles } from '@/web/common/file/controller'; import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react'; import React, { DragEvent, useCallback, useState } from 'react'; import { useTranslation } from 'next-i18next'; @@ -25,6 +19,8 @@ import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d'; import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d'; +import { readFileRawContent } from '@fastgpt/web/common/file/read/index'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal')); @@ -168,36 +164,22 @@ const FileSelect = ({ } // parse and upload files - let text = await (async () => { - switch (extension) { - case 'txt': - return readFileRawText(file); - case 'md': - return readMdFile({ - file, - uploadImgController: (base64Img) => - getUploadMdImgController({ base64Img, metadata: { fileId } }) - }); - case 'html': - return readHtmlFile({ - file, - uploadImgController: (base64Img) => - getUploadMdImgController({ base64Img, metadata: { fileId } }) - }); - case 'pdf': - return readPdfContent(file); - case 'docx': - return readDocContent(file, { + let { rawText } = await readFileRawContent({ + file, + uploadBase64Controller: (base64Img) => + getUploadBase64ImgController({ + base64Img, + type: MongoImageTypeEnum.docImage, + metadata: { fileId - }); - } - return ''; - })(); + } + }) + }); - if (text) { - text = simpleText(text); + if (rawText) { + rawText = simpleText(rawText); const { chunks, tokens } = splitText2Chunks({ - text, + text: rawText, chunkLen, overlapRatio, customReg: customSplitChar ? [customSplitChar] : [] @@ -207,7 +189,7 @@ const FileSelect = ({ id: nanoid(), filename: file.name, icon, - rawText: text, + rawText, tokens, type: DatasetCollectionTypeEnum.file, fileId, diff --git a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx index 1848e947b..5b9a08535 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx @@ -10,10 +10,7 @@ const CsvImport = dynamic(() => import('./Csv'), {}); import MyModal from '@/components/MyModal'; import Provider from './Provider'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; -import { - DatasetCollectionTrainingModeEnum, - TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; export enum ImportTypeEnum { chunk = 'chunk', @@ -46,24 +43,21 @@ const ImportData = ({ chunkOverlapRatio: 0.2, inputPrice: vectorModel?.inputPrice || 0, outputPrice: 0, - mode: TrainingModeEnum.chunk, - collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk + collectionTrainingType: TrainingModeEnum.chunk }, [ImportTypeEnum.qa]: { defaultChunkLen: agentModel?.maxContext * 0.55 || 8000, chunkOverlapRatio: 0, inputPrice: agentModel?.inputPrice || 0, outputPrice: agentModel?.outputPrice || 0, - mode: TrainingModeEnum.qa, - collectionTrainingType: DatasetCollectionTrainingModeEnum.qa + collectionTrainingType: TrainingModeEnum.qa }, [ImportTypeEnum.csv]: { defaultChunkLen: 0, chunkOverlapRatio: 0, inputPrice: vectorModel?.inputPrice || 0, outputPrice: 0, - mode: TrainingModeEnum.chunk, - collectionTrainingType: DatasetCollectionTrainingModeEnum.manual + collectionTrainingType: TrainingModeEnum.chunk } }; return map[importType]; diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx index 246d74539..def0a6ada 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx @@ -16,10 +16,7 @@ import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { hashStr } from '@fastgpt/global/common/string/tools'; import { useToast } from '@/web/common/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; -import { - DatasetCollectionTrainingModeEnum, - TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { Box, Flex, Image, useTheme } from '@chakra-ui/react'; import { CloseIcon } from '@chakra-ui/icons'; import DeleteIcon, { hoverDeleteStyles } from '@fastgpt/web/components/common/Icon/delete'; @@ -104,7 +101,6 @@ const Provider = ({ parentId, inputPrice, outputPrice, - mode, collectionTrainingType, vectorModel, agentModel, @@ -118,8 +114,7 @@ const Provider = ({ parentId: string; inputPrice: number; outputPrice: number; - mode: `${TrainingModeEnum}`; - collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`; + collectionTrainingType: `${TrainingModeEnum}`; vectorModel: string; agentModel: string; defaultChunkLen: number; @@ -147,14 +142,14 @@ const Provider = ({ const totalTokens = useMemo(() => files.reduce((sum, file) => sum + file.tokens, 0), [files]); const price = useMemo(() => { - if (mode === TrainingModeEnum.qa) { + if (collectionTrainingType === TrainingModeEnum.qa) { const inputTotal = totalTokens * inputPrice; const outputTotal = totalTokens * 0.5 * outputPrice; return formatModelPrice2Read(inputTotal + outputTotal); } return formatModelPrice2Read(totalTokens * inputPrice); - }, [inputPrice, mode, outputPrice, totalTokens]); + }, [collectionTrainingType, inputPrice, outputPrice, totalTokens]); /* start upload data @@ -169,7 +164,7 @@ const Provider = ({ for await (const file of files) { // create training bill const billId = await postCreateTrainingBill({ - name: t('dataset.collections.Create Training Data', { filename: file.filename }), + name: file.filename, vectorModel, agentModel }); @@ -180,11 +175,15 @@ const Provider = ({ parentId, name: file.filename, type: file.type, + + trainingType: collectionTrainingType, + chunkSize: chunkLen, + chunkSplitter: customSplitChar, + qaPrompt: collectionTrainingType === TrainingModeEnum.qa ? prompt : '', + fileId: file.fileId, rawLink: file.rawLink, - chunkSize: chunkLen, - trainingType: collectionTrainingType, - qaPrompt: mode === TrainingModeEnum.qa ? prompt : '', + rawTextLength: file.rawText.length, hashRawText: hashStr(file.rawText), metadata: file.metadata @@ -195,8 +194,8 @@ const Provider = ({ const { insertLen } = await chunksUpload({ collectionId, billId, + trainingMode: collectionTrainingType, chunks, - mode, onUploading: (insertLen) => { setSuccessChunks((state) => state + insertLen); }, diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index d0cc85823..787f2c772 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -1,10 +1,9 @@ -import React, { useCallback, useState, useMemo } from 'react'; +import React, { useState, useMemo } from 'react'; import { useRouter } from 'next/router'; import { Box, Flex, Button, IconButton, Input, Textarea } from '@chakra-ui/react'; import { DeleteIcon } from '@chakra-ui/icons'; import { delDatasetById } from '@/web/core/dataset/api'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; -import { useToast } from '@/web/common/hooks/useToast'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { useConfirm } from '@/web/common/hooks/useConfirm'; import { useForm } from 'react-hook-form'; @@ -17,6 +16,7 @@ import PermissionRadio from '@/components/support/permission/Radio'; import MySelect from '@/components/Select'; import { qaModelList } from '@/web/common/system/staticData'; import { useRequest } from '@/web/common/hooks/useRequest'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const Info = ({ datasetId }: { datasetId: string }) => { const { t } = useTranslation(); @@ -70,6 +70,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { const file = e[0]; if (!file) return Promise.resolve(null); return compressImgFileAndUpload({ + type: MongoImageTypeEnum.datasetAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx index 538560414..a2cef7f4f 100644 --- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx @@ -1,44 +1,38 @@ import React, { useMemo, useState } from 'react'; -import { Box, Flex, Button, Textarea, BoxProps, Image, useTheme, Grid } from '@chakra-ui/react'; +import { Box, Flex, Button, Textarea, useTheme, Grid } from '@chakra-ui/react'; import { useFieldArray, useForm } from 'react-hook-form'; import { postInsertData2Dataset, putDatasetDataById, delOneDatasetDataById, - getDatasetCollectionById + getDatasetCollectionById, + getDatasetDataItemById } from '@/web/core/dataset/api'; import { useToast } from '@/web/common/hooks/useToast'; -import { getErrText } from '@fastgpt/global/common/error/utils'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyModal from '@/components/MyModal'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'next-i18next'; -import { getFileAndOpen } from '@/web/core/dataset/utils'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useRequest } from '@/web/common/hooks/useRequest'; import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { useConfirm } from '@/web/common/hooks/useConfirm'; -import { getDefaultIndex, getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; -import { feConfigs, vectorModelList } from '@/web/common/system/staticData'; +import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; +import { vectorModelList } from '@/web/common/system/staticData'; import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant'; import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type'; import SideTabs from '@/components/SideTabs'; -import { useLoading } from '@/web/common/hooks/useLoading'; import DeleteIcon from '@fastgpt/web/components/common/Icon/delete'; import { defaultCollectionDetail } from '@/constants/dataset'; import { getDocPath } from '@/web/common/system/doc'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; +import MyBox from '@/components/common/MyBox'; +import { getErrText } from '@fastgpt/global/common/error/utils'; -export type RawSourceTextProps = BoxProps & { - sourceName?: string; - sourceId?: string; - canView?: boolean; -}; export type InputDataType = { - id?: string; q: string; - a?: string; + a: string; indexes: (Omit & { dataId?: string; // pg data id })[]; @@ -53,26 +47,25 @@ enum TabEnum { const InputDataModal = ({ collectionId, + dataId, defaultValue, onClose, onSuccess, onDelete }: { collectionId: string; - defaultValue: InputDataType; + dataId?: string; + defaultValue?: { q: string; a?: string }; onClose: () => void; - onSuccess: (data: InputDataType) => void; + onSuccess: (data: InputDataType & { dataId: string }) => void; onDelete?: () => void; }) => { const { t } = useTranslation(); const theme = useTheme(); const { toast } = useToast(); - const { Loading } = useLoading(); const [currentTab, setCurrentTab] = useState(TabEnum.content); - const { register, handleSubmit, reset, control } = useForm({ - defaultValues: defaultValue - }); + const { register, handleSubmit, reset, control } = useForm(); const { fields: indexes, append: appendIndexes, @@ -89,14 +82,15 @@ const InputDataModal = ({ id: TabEnum.index, icon: 'kbTest' }, - ...(defaultValue.id + ...(dataId ? [{ label: t('dataset.data.edit.Delete'), id: TabEnum.delete, icon: 'delete' }] : []), { label: t('dataset.data.edit.Course'), id: TabEnum.doc, icon: 'common/courseLight' } ]; const { ConfirmModal, openConfirm } = useConfirm({ - content: t('dataset.data.Delete Tip') + content: t('dataset.data.Delete Tip'), + type: 'delete' }); const { data: collection = defaultCollectionDetail } = useQuery( @@ -105,6 +99,37 @@ const InputDataModal = ({ return getDatasetCollectionById(collectionId); } ); + const { isFetching: isFetchingData } = useQuery( + ['getDatasetDataItemById', dataId], + () => { + if (dataId) return getDatasetDataItemById(dataId); + return null; + }, + { + onSuccess(res) { + if (res) { + reset({ + q: res.q, + a: res.a, + indexes: res.indexes + }); + } else if (defaultValue) { + reset({ + q: defaultValue.q, + a: defaultValue.a, + indexes: [getDefaultIndex({ dataId: `${Date.now()}` })] + }); + } + }, + onError(err) { + toast({ + status: 'error', + title: getErrText(err) + }); + onClose(); + } + } + ); const maxToken = useMemo(() => { const vectorModel = @@ -130,7 +155,7 @@ const InputDataModal = ({ const data = { ...e }; - data.id = await postInsertData2Dataset({ + const dataId = await postInsertData2Dataset({ collectionId: collection._id, q: e.q, a: e.a, @@ -140,7 +165,10 @@ const InputDataModal = ({ ) }); - return data; + return { + ...data, + dataId + }; }, successToast: t('dataset.data.Input Success Tip'), onSuccess(e) { @@ -158,17 +186,18 @@ const InputDataModal = ({ // update const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({ mutationFn: async (e: InputDataType) => { - if (!e.id) return e; + if (!dataId) return e; // not exactly same await putDatasetDataById({ - id: e.id, - q: e.q, - a: e.a, - indexes: e.indexes + id: dataId, + ...e }); - return e; + return { + dataId, + ...e + }; }, successToast: t('dataset.data.Update Success Tip'), errorToast: t('common.error.unKnow'), @@ -180,8 +209,8 @@ const InputDataModal = ({ // delete const { mutate: onDeleteData, isLoading: isDeleting } = useRequest({ mutationFn: () => { - if (!onDelete || !defaultValue.id) return Promise.resolve(null); - return delOneDatasetDataById(defaultValue.id); + if (!onDelete || !dataId) return Promise.resolve(null); + return delOneDatasetDataById(dataId); }, onSuccess() { if (!onDelete) return; @@ -192,13 +221,16 @@ const InputDataModal = ({ errorToast: t('common.error.unKnow') }); - const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]); + const isLoading = useMemo( + () => isImporting || isUpdating || isFetchingData || isDeleting, + [isImporting, isUpdating, isFetchingData, isDeleting] + ); return ( - + - {currentTab === TabEnum.content && ( - <>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')} + <>{dataId ? t('dataset.data.Update Data') : t('dataset.data.Input Data')} )} {currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}} @@ -351,82 +383,24 @@ const InputDataModal = ({ )} - - + - ); }; -export default InputDataModal; - -export function RawSourceText({ - sourceId, - sourceName = '', - canView = true, - ...props -}: RawSourceTextProps) { - const { t } = useTranslation(); - const { toast } = useToast(); - const { setLoading } = useSystemStore(); - - const canPreview = useMemo(() => !!sourceId && canView, [canView, sourceId]); - - const icon = useMemo(() => getSourceNameIcon({ sourceId, sourceName }), [sourceId, sourceName]); - - return ( - - { - setLoading(true); - try { - await getFileAndOpen(sourceId as string); - } catch (error) { - toast({ - title: t(getErrText(error, 'error.fileNotFound')), - status: 'error' - }); - } - setLoading(false); - } - } - : {})} - {...props} - > - - - {sourceName || t('common.UnKnow Source')} - - - - ); -} +export default React.memo(InputDataModal); diff --git a/projects/app/src/pages/dataset/detail/components/Test.tsx b/projects/app/src/pages/dataset/detail/components/Test.tsx index bb97919e2..1040659f4 100644 --- a/projects/app/src/pages/dataset/detail/components/Test.tsx +++ b/projects/app/src/pages/dataset/detail/components/Test.tsx @@ -233,7 +233,7 @@ const Test = ({ datasetId }: { datasetId: string }) => { h={'100%'} resize={'none'} variant={'unstyled'} - maxLength={datasetDetail.vectorModel.maxToken} + maxLength={datasetDetail.vectorModel?.maxToken} placeholder={t('core.dataset.test.Test Text Placeholder')} onFocus={() => setIsFocus(true)} {...register('inputText', { @@ -314,7 +314,7 @@ const Test = ({ datasetId }: { datasetId: string }) => { {/* result show */} - + @@ -384,6 +384,9 @@ const TestHistories = React.memo(function TestHistories({ }} cursor={'pointer'} fontSize={'sm'} + {...(item.id === datasetTestItem?.id && { + bg: 'primary.50' + })} onClick={() => setDatasetTestItem(item)} > diff --git a/projects/app/src/pages/dataset/detail/index.tsx b/projects/app/src/pages/dataset/detail/index.tsx index ad7f7ca02..16d037838 100644 --- a/projects/app/src/pages/dataset/detail/index.tsx +++ b/projects/app/src/pages/dataset/detail/index.tsx @@ -16,8 +16,6 @@ import { serviceSideProps } from '@/web/common/utils/i18n'; import { useTranslation } from 'next-i18next'; import { getTrainingQueueLen } from '@/web/core/dataset/api'; import MyTooltip from '@/components/MyTooltip'; -import { QuestionOutlineIcon } from '@chakra-ui/icons'; -import { feConfigs } from '@/web/common/system/staticData'; import Script from 'next/script'; import CollectionCard from './components/CollectionCard'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; @@ -29,6 +27,7 @@ import { } from '@fastgpt/global/core/dataset/constant'; import { useConfirm } from '@/web/common/hooks/useConfirm'; import { useRequest } from '@/web/common/hooks/useRequest'; +import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag'; const DataCard = dynamic(() => import('./components/DataCard'), { ssr: false @@ -150,50 +149,47 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T {isPc ? ( - - - - {datasetDetail.name} - - - {DatasetTypeMap[datasetDetail.type] && ( - - - {t(DatasetTypeMap[datasetDetail.type]?.label)} - {datasetDetail.type === DatasetTypeEnum.websiteDataset && - datasetDetail.status === DatasetStatusEnum.active && ( - - - openConfirmSync( - onUpdateDatasetWebsiteConfig, - undefined, - t('core.dataset.website.Confirm Create Tips') - )() - } - /> - - )} + + + + + {datasetDetail.name} + - )} + {DatasetTypeMap[datasetDetail.type] && ( + + + {datasetDetail.type === DatasetTypeEnum.websiteDataset && + datasetDetail.status === DatasetStatusEnum.active && ( + + + openConfirmSync( + onUpdateDatasetWebsiteConfig, + undefined, + t('core.dataset.website.Confirm Create Tips') + )() + } + /> + + )} + + )} + - + {t('core.dataset.training.Agent queue')}({agentTrainingMap.tip}) @@ -229,6 +225,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T /> + void; parentId?: string }) => { const { t } = useTranslation(); @@ -49,6 +50,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.datasetAvatar, file, maxW: 300, maxH: 300 @@ -62,7 +64,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st }); } }, - [setValue, toast] + [setValue, t, toast] ); /* create a new kb and router to it */ diff --git a/projects/app/src/pages/dataset/list/index.tsx b/projects/app/src/pages/dataset/list/index.tsx index 522f7fe7b..c5832433e 100644 --- a/projects/app/src/pages/dataset/list/index.tsx +++ b/projects/app/src/pages/dataset/list/index.tsx @@ -22,7 +22,7 @@ import { putDatasetById, postCreateDataset } from '@/web/core/dataset/api'; -import { checkTeamExportDatasetLimit } from '@/web/support/user/api'; +import { checkTeamExportDatasetLimit } from '@/web/support/user/team/api'; import { useTranslation } from 'next-i18next'; import Avatar from '@/components/Avatar'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -44,6 +44,7 @@ import PermissionIconText from '@/components/support/permission/IconText'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { DatasetItemType } from '@fastgpt/global/core/dataset/type'; import ParentPaths from '@/components/common/ParentPaths'; +import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag'; const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const MoveModal = dynamic(() => import('./component/MoveModal'), { ssr: false }); @@ -409,8 +410,9 @@ const Kb = () => { - - {t(dataset.label)} + {dataset.type !== DatasetTypeEnum.folder && ( + + )} ))} diff --git a/projects/app/src/pages/plugin/list/component/EditModal.tsx b/projects/app/src/pages/plugin/list/component/EditModal.tsx index 60fc6af53..b9d152e0c 100644 --- a/projects/app/src/pages/plugin/list/component/EditModal.tsx +++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx @@ -17,6 +17,7 @@ import { useConfirm } from '@/web/common/hooks/useConfirm'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { CreateOnePluginParams } from '@fastgpt/global/core/plugin/controller'; import { customAlphabet } from 'nanoid'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); export type FormType = CreateOnePluginParams & { @@ -92,6 +93,7 @@ const CreateModal = ({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.pluginAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts new file mode 100644 index 000000000..33434c505 --- /dev/null +++ b/projects/app/src/service/common/system/cron.ts @@ -0,0 +1,18 @@ +import { initSystemConfig } from '@/pages/api/common/system/getInitData'; +import { generateQA } from '@/service/events/generateQA'; +import { generateVector } from '@/service/events/generateVector'; +import { setCron } from '@fastgpt/service/common/system/cron'; + +export const setUpdateSystemConfigCron = () => { + setCron('*/5 * * * *', () => { + initSystemConfig(); + console.log('refresh system config'); + }); +}; + +export const setTrainingQueueCron = () => { + setCron('*/3 * * * *', () => { + generateVector(); + generateQA(); + }); +}; diff --git a/projects/app/src/service/core/ai/rerank.ts b/projects/app/src/service/core/ai/rerank.ts index 4b1cf781a..5ced9f110 100644 --- a/projects/app/src/service/core/ai/rerank.ts +++ b/projects/app/src/service/core/ai/rerank.ts @@ -27,7 +27,7 @@ export function reRankRecall({ query, inputs }: PostReRankProps) { return data; }) .catch((err) => { - console.log(err); + console.log('rerank error:', err); return []; }); diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 839519946..eacd5fda3 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -14,7 +14,8 @@ import { DatasetDataIndexTypeEnum, DatasetSearchModeEnum, DatasetSearchModeMap, - SearchScoreTypeEnum + SearchScoreTypeEnum, + TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; import { jiebaSplit } from '@/service/common/string/jieba'; @@ -27,7 +28,173 @@ import { } from '@fastgpt/global/core/dataset/type'; import { reRankRecall } from '../../ai/rerank'; import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; -import { hashStr } from '@fastgpt/global/common/string/tools'; +import { hashStr, simpleText } from '@fastgpt/global/common/string/tools'; +import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; +import type { PushDataResponse } from '@/global/core/api/datasetRes'; +import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; +import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; +import { startQueue } from '@/service/utils/tools'; +import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; +import { getQAModel, getVectorModel } from '../../ai/model'; +import { delay } from '@fastgpt/global/common/system/utils'; + +export async function pushDataToDatasetCollection({ + teamId, + tmbId, + collectionId, + data, + prompt, + billId, + trainingMode +}: { + teamId: string; + tmbId: string; +} & PushDatasetDataProps): Promise { + const checkModelValid = async ({ collectionId }: { collectionId: string }) => { + const { + datasetId: { _id: datasetId, vectorModel, agentModel } + } = await getCollectionWithDataset(collectionId); + + if (trainingMode === TrainingModeEnum.chunk) { + if (!collectionId) return Promise.reject(`CollectionId is empty`); + const vectorModelData = getVectorModel(vectorModel); + if (!vectorModelData) { + return Promise.reject(`Model ${vectorModel} is inValid`); + } + + return { + datasetId, + maxToken: vectorModelData.maxToken * 1.5, + model: vectorModelData.model, + weight: vectorModelData.weight + }; + } + + if (trainingMode === TrainingModeEnum.qa) { + const qaModelData = getQAModel(agentModel); + if (!qaModelData) { + return Promise.reject(`Model ${agentModel} is inValid`); + } + return { + datasetId, + maxToken: qaModelData.maxContext * 0.8, + model: qaModelData.model, + weight: 0 + }; + } + return Promise.reject(`Mode ${trainingMode} is inValid`); + }; + + const { datasetId, model, maxToken, weight } = await checkModelValid({ + collectionId + }); + + // format q and a, remove empty char + data.forEach((item) => { + item.q = simpleText(item.q); + item.a = simpleText(item.a); + + item.indexes = item.indexes + ?.map((index) => { + return { + ...index, + text: simpleText(index.text) + }; + }) + .filter(Boolean); + }); + + // filter repeat or equal content + const set = new Set(); + const filterResult: Record = { + success: [], + overToken: [], + repeat: [], + error: [] + }; + + data.forEach((item) => { + if (!item.q) { + filterResult.error.push(item); + return; + } + + const text = item.q + item.a; + + // count q token + const token = countPromptTokens(item.q); + + if (token > maxToken) { + filterResult.overToken.push(item); + return; + } + + if (set.has(text)) { + console.log('repeat', item); + filterResult.repeat.push(item); + } else { + filterResult.success.push(item); + set.add(text); + } + }); + + // 插入记录 + const insertData = async (dataList: PushDatasetDataChunkProps[], retry = 3): Promise => { + try { + const results = await MongoDatasetTraining.insertMany( + dataList.map((item, i) => ({ + teamId, + tmbId, + datasetId, + collectionId, + billId, + mode: trainingMode, + prompt, + model, + q: item.q, + a: item.a, + chunkIndex: item.chunkIndex ?? i, + weight: weight ?? 0, + indexes: item.indexes + })) + ); + await delay(500); + return results.length; + } catch (error) { + if (retry > 0) { + await delay(1000); + return insertData(dataList, retry - 1); + } + return Promise.reject(error); + } + }; + + let insertLen = 0; + const chunkSize = 50; + const chunkList = filterResult.success.reduce( + (acc, cur) => { + const lastChunk = acc[acc.length - 1]; + if (lastChunk.length < chunkSize) { + lastChunk.push(cur); + } else { + acc.push([cur]); + } + return acc; + }, + [[]] as PushDatasetDataChunkProps[][] + ); + for await (const chunks of chunkList) { + insertLen += await insertData(chunks); + } + + startQueue(); + delete filterResult.success; + + return { + insertLen, + ...filterResult + }; +} /* insert data. * 1. create data id @@ -439,7 +606,9 @@ export async function searchDatasetData(props: { })) }); - if (!Array.isArray(results)) return []; + if (!Array.isArray(results)) { + return []; + } // add new score to data const mergeResult = results @@ -457,7 +626,6 @@ export async function searchDatasetData(props: { return mergeResult; } catch (error) { - usingReRank = false; return []; } }; diff --git a/projects/app/src/service/events/generateQA.ts b/projects/app/src/service/events/generateQA.ts index d4c58aaa8..cb3684a9f 100644 --- a/projects/app/src/service/events/generateQA.ts +++ b/projects/app/src/service/events/generateQA.ts @@ -8,20 +8,15 @@ import { addLog } from '@fastgpt/service/common/system/log'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { Prompt_AgentQA } from '@/global/core/prompt/agent'; -import { pushDataToDatasetCollection } from '@/pages/api/core/dataset/data/pushData'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { authTeamBalance } from '../support/permission/auth/bill'; import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; -const reduceQueue = (retry = false) => { +const reduceQueue = () => { global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; - if (global.qaQueueLen === 0 && retry) { - setTimeout(() => { - generateQA(); - }, 60000); - } return global.vectorQueueLen === 0; }; @@ -144,11 +139,11 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`; teamId: data.teamId, tmbId: data.tmbId, collectionId: data.collectionId, + trainingMode: TrainingModeEnum.chunk, data: qaArr.map((item) => ({ ...item, chunkIndex: data.chunkIndex })), - mode: TrainingModeEnum.chunk, billId: data.billId }); @@ -178,7 +173,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`; reduceQueue(); generateQA(); } catch (err: any) { - reduceQueue(true); + reduceQueue(); // log if (err?.response) { addLog.info('openai error: 生成QA错误', { diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts index 217f76d85..831aad971 100644 --- a/projects/app/src/service/events/generateVector.ts +++ b/projects/app/src/service/events/generateVector.ts @@ -9,15 +9,9 @@ import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller'; -const reduceQueue = (retry = false) => { +const reduceQueue = () => { global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0; - if (global.vectorQueueLen === 0 && retry) { - setTimeout(() => { - generateVector(); - }, 60000); - } - return global.vectorQueueLen === 0; }; @@ -159,7 +153,7 @@ export async function generateVector(): Promise { console.log(`embedding finished, time: ${Date.now() - start}ms`); } catch (err: any) { - reduceQueue(true); + reduceQueue(); // log if (err?.response) { addLog.info('openai error: 生成向量错误', { diff --git a/projects/app/src/service/moduleDispatch/chat/oneapi.ts b/projects/app/src/service/moduleDispatch/chat/oneapi.ts index 6f8623263..36ffeca5b 100644 --- a/projects/app/src/service/moduleDispatch/chat/oneapi.ts +++ b/projects/app/src/service/moduleDispatch/chat/oneapi.ts @@ -214,7 +214,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise { return connectMongo({ beforeHook: () => {}, - afterHook: () => { + afterHook: async () => { initVectorStore(); // start queue startQueue(); - return initRootUser(); + // init system config + getInitConfig(); + + // cron + setUpdateSystemConfigCron(); + setTrainingQueueCron(); + + initRootUser(); } }); } diff --git a/projects/app/src/service/utils/chat/saveChat.ts b/projects/app/src/service/utils/chat/saveChat.ts index 78ef20da2..3b6d25048 100644 --- a/projects/app/src/service/utils/chat/saveChat.ts +++ b/projects/app/src/service/utils/chat/saveChat.ts @@ -60,7 +60,6 @@ export async function saveChat({ })) ) ]; - console.log(metadataUpdate); const title = chatContentReplaceBlock(content[0].value).slice(0, 20) || diff --git a/projects/app/src/service/utils/tools.ts b/projects/app/src/service/utils/tools.ts index b51118419..207dc4274 100644 --- a/projects/app/src/service/utils/tools.ts +++ b/projects/app/src/service/utils/tools.ts @@ -2,20 +2,9 @@ import { generateQA } from '../events/generateQA'; import { generateVector } from '../events/generateVector'; /* start task */ -export const startQueue = (limit?: number) => { +export const startQueue = () => { if (!global.systemEnv) return; - if (limit) { - for (let i = 0; i < limit; i++) { - generateVector(); - generateQA(); - } - return; - } - for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) { - generateQA(); - } - for (let i = 0; i < global.systemEnv.vectorMaxProcess; i++) { - generateVector(); - } + generateQA(); + generateVector(); }; diff --git a/projects/app/src/web/common/file/controller.ts b/projects/app/src/web/common/file/controller.ts index 89773e777..c45161aa1 100644 --- a/projects/app/src/web/common/file/controller.ts +++ b/projects/app/src/web/common/file/controller.ts @@ -1,10 +1,8 @@ import { postUploadImg, postUploadFiles } from '@/web/common/file/api'; import { UploadImgProps } from '@fastgpt/global/common/file/api'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { - compressBase64ImgAndUpload as compressBase64ImgAndUploadControl, - type CompressImgProps -} from '@fastgpt/web/common/file/img'; +import { preUploadImgProps } from '@fastgpt/global/common/file/api'; +import { compressBase64Img, type CompressImgProps } from '@fastgpt/web/common/file/img'; /** * upload file to mongo gridfs @@ -34,57 +32,45 @@ export const uploadFiles = ({ }); }; -export const getUploadMdImgController = ({ - base64Img, - metadata -}: { - base64Img: string; - metadata: Record; -}) => - compressBase64ImgAndUpload({ - base64Img, +export const getUploadBase64ImgController = (props: CompressImgProps & UploadImgProps) => + compressBase64Img({ maxW: 4000, maxH: 4000, maxSize: 1024 * 1024 * 5, - metadata + ...props }); /** * compress image. response base64 * @param maxSize The max size of the compressed image */ -export const compressBase64ImgAndUpload = ({ - expiredTime, - metadata, - shareId, - ...props -}: UploadImgProps & CompressImgProps) => { - return compressBase64ImgAndUploadControl({ - ...props, - uploadController: (base64Img) => - postUploadImg({ - shareId, - base64Img, - expiredTime, - metadata - }) - }); -}; -export const compressImgFileAndUpload = async ({ - file, +export const compressBase64ImgAndUpload = async ({ + base64Img, maxW, maxH, maxSize, - expiredTime, - shareId -}: { - file: File; - maxW?: number; - maxH?: number; - maxSize?: number; - expiredTime?: Date; - shareId?: string; -}) => { + ...props +}: UploadImgProps & CompressImgProps) => { + const compressUrl = await compressBase64Img({ + base64Img, + maxW, + maxH, + maxSize + }); + + return postUploadImg({ + ...props, + base64Img: compressUrl + }); +}; + +export const compressImgFileAndUpload = async ({ + file, + ...props +}: preUploadImgProps & + CompressImgProps & { + file: File; + }) => { const reader = new FileReader(); reader.readAsDataURL(file); @@ -94,16 +80,12 @@ export const compressImgFileAndUpload = async ({ }; reader.onerror = (err) => { console.log(err); - reject('压缩图片异常'); + reject('Load image error'); }; }); return compressBase64ImgAndUpload({ base64Img, - maxW, - maxH, - maxSize, - expiredTime, - shareId + ...props }); }; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index d71617eb7..23dcafc8f 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -1,80 +1,5 @@ -import mammoth from 'mammoth'; import Papa from 'papaparse'; -import { compressBase64ImgAndUpload } from './controller'; -import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown'; -import { htmlStr2Md } from '@fastgpt/web/common/string/markdown'; -import { readPdfFile } from '@fastgpt/global/common/file/read/index'; -import { readFileRawText } from '@fastgpt/web/common/file/read'; - -/** - * read pdf to raw text - */ -export const readPdfContent = (file: File) => - new Promise((resolve, reject) => { - try { - let reader = new FileReader(); - reader.readAsArrayBuffer(file); - reader.onload = async (event) => { - if (!event?.target?.result) return reject('解析 PDF 失败'); - try { - const content = await readPdfFile({ pdf: event.target.result }); - - resolve(content); - } catch (err) { - console.log(err, 'pdf load error'); - reject('解析 PDF 失败'); - } - }; - reader.onerror = (err) => { - console.log(err, 'pdf load error'); - reject('解析 PDF 失败'); - }; - } catch (error) { - reject('浏览器不支持文件内容读取'); - } - }); - -/** - * read docx to markdown - */ -export const readDocContent = (file: File, metadata: Record) => - new Promise((resolve, reject) => { - try { - const reader = new FileReader(); - reader.readAsArrayBuffer(file); - reader.onload = async ({ target }) => { - if (!target?.result) return reject('读取 doc 文件失败'); - try { - const buffer = target.result as ArrayBuffer; - const { value: html } = await mammoth.convertToHtml({ - arrayBuffer: buffer - }); - const md = htmlStr2Md(html); - - const rawText = await uploadMarkdownBase64(md, metadata); - - resolve(rawText); - } catch (error) { - window.umami?.track('wordReadError', { - err: error?.toString() - }); - console.log('error doc read:', error); - - reject('读取 doc 文件失败, 请转换成 PDF'); - } - }; - reader.onerror = (err) => { - window.umami?.track('wordReadError', { - err: err?.toString() - }); - console.log('error doc read:', err); - - reject('读取 doc 文件失败'); - }; - } catch (error) { - reject('浏览器不支持文件内容读取'); - } - }); +import { readFileRawText } from '@fastgpt/web/common/file/read/rawText'; /** * read csv to json @@ -85,7 +10,7 @@ export const readDocContent = (file: File, metadata: Record) => */ export const readCsvContent = async (file: File) => { try { - const textArr = await readFileRawText(file); + const { rawText: textArr } = await readFileRawText(file); const csvArr = Papa.parse(textArr).data as string[][]; if (csvArr.length === 0) { throw new Error('csv 解析失败'); @@ -99,44 +24,6 @@ export const readCsvContent = async (file: File) => { } }; -/** - * format markdown - * 1. upload base64 - * 2. replace \ - */ -export const uploadMarkdownBase64 = async (rawText: string = '', metadata: Record) => { - // match base64, upload and replace it - const base64Regex = /data:image\/.*;base64,([^\)]+)/g; - const base64Arr = rawText.match(base64Regex) || []; - // upload base64 and replace it - await Promise.all( - base64Arr.map(async (base64Img) => { - try { - const str = await compressBase64ImgAndUpload({ - base64Img, - maxW: 4329, - maxH: 4329, - maxSize: 1024 * 1024 * 5, - metadata - }); - - rawText = rawText.replace(base64Img, str); - } catch (error) { - rawText = rawText.replace(base64Img, ''); - rawText = rawText.replace(/!\[.*\]\(\)/g, ''); - } - }) - ); - - // Remove white space on both sides of the picture - const trimReg = /(!\[.*\]\(.*\))\s*/g; - if (trimReg.test(rawText)) { - rawText = rawText.replace(trimReg, '$1'); - } - - return simpleMarkdownText(rawText); -}; - /** * file download by text */ diff --git a/projects/app/src/web/common/hooks/useConfirm.tsx b/projects/app/src/web/common/hooks/useConfirm.tsx index f856616ea..ec01215bf 100644 --- a/projects/app/src/web/common/hooks/useConfirm.tsx +++ b/projects/app/src/web/common/hooks/useConfirm.tsx @@ -105,7 +105,7 @@ export const useConfirm = (props?: { )}