diff --git a/deploy/docker/docker-compose-milvus.yml b/deploy/docker/docker-compose-milvus.yml index f875d61ec..e54f43dc8 100644 --- a/deploy/docker/docker-compose-milvus.yml +++ b/deploy/docker/docker-compose-milvus.yml @@ -110,6 +110,18 @@ services: # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程 wait $$! + redis: + image: redis:7.2-alpine + container_name: redis + # ports: + # - 6379:6379 + networks: + - fastgpt + restart: always + command: | + redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction + volumes: + - ./redis/data:/data # fastgpt sandbox: @@ -157,6 +169,8 @@ services: # zilliz 连接参数 - MILVUS_ADDRESS=http://milvusStandalone:19530 - MILVUS_TOKEN=none + # Redis 地址 + - REDIS_URL=redis://default:mypassword@redis:6379 # sandbox 地址 - SANDBOX_URL=http://sandbox:3000 # 日志等级: debug, info, warn, error diff --git a/deploy/docker/docker-compose-pgvector.yml b/deploy/docker/docker-compose-pgvector.yml index 0a21d9c29..c3f2db44a 100644 --- a/deploy/docker/docker-compose-pgvector.yml +++ b/deploy/docker/docker-compose-pgvector.yml @@ -69,6 +69,19 @@ services: # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程 wait $$! + redis: + image: redis:7.2-alpine + container_name: redis + # ports: + # - 6379:6379 + networks: + - fastgpt + restart: always + command: | + redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction + volumes: + - ./redis/data:/data + # fastgpt sandbox: container_name: sandbox @@ -114,6 +127,8 @@ services: - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin # pg 连接参数 - PG_URL=postgresql://username:password@pg:5432/postgres + # Redis 连接参数 + - REDIS_URL=redis://default:mypassword@redis:6379 # sandbox 地址 - SANDBOX_URL=http://sandbox:3000 # 日志等级: debug, info, warn, error diff --git a/deploy/docker/docker-compose-zilliz.yml b/deploy/docker/docker-compose-zilliz.yml index 8e2c0bb7a..47e54a104 100644 --- a/deploy/docker/docker-compose-zilliz.yml +++ b/deploy/docker/docker-compose-zilliz.yml @@ -51,6 +51,19 @@ services: # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程 wait $$! + redis: + image: redis:7.2-alpine + container_name: redis + # ports: + # - 6379:6379 + networks: + - fastgpt + restart: always + command: | + redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction + volumes: + - ./redis/data:/data + sandbox: container_name: sandbox image: ghcr.io/labring/fastgpt-sandbox:v4.9.3 # git @@ -92,6 +105,8 @@ services: - FILE_TOKEN_KEY=filetoken # MongoDB 连接参数. 用户名myusername,密码mypassword。 - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin + # Redis 连接参数 + - REDIS_URI=redis://default:mypassword@redis:6379 # zilliz 连接参数 - MILVUS_ADDRESS=zilliz_cloud_address - MILVUS_TOKEN=zilliz_cloud_token diff --git a/docSite/assets/imgs/sealos-redis1.png b/docSite/assets/imgs/sealos-redis1.png new file mode 100644 index 000000000..240f30786 Binary files /dev/null and b/docSite/assets/imgs/sealos-redis1.png differ diff --git a/docSite/assets/imgs/sealos-redis2.png b/docSite/assets/imgs/sealos-redis2.png new file mode 100644 index 000000000..990ef3243 Binary files /dev/null and b/docSite/assets/imgs/sealos-redis2.png differ diff --git a/docSite/assets/imgs/sealos-redis3.png b/docSite/assets/imgs/sealos-redis3.png new file mode 100644 index 000000000..c0ff867fc Binary files /dev/null and b/docSite/assets/imgs/sealos-redis3.png differ diff --git a/docSite/content/zh-cn/docs/development/upgrading/494.md b/docSite/content/zh-cn/docs/development/upgrading/494.md index 0bfc9b020..ed1e166f7 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/494.md +++ b/docSite/content/zh-cn/docs/development/upgrading/494.md @@ -7,11 +7,44 @@ toc: true weight: 796 --- +## 升级指南 + +### 1. 做好数据备份 + +### 1. 安装 Redis + +* docker 部署的用户，参考最新的 `docker-compose.yml` 文件增加 Redis 配置。增加一个 redis 容器，并配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。 +* Sealos 部署的用户，在数据库里新建一个`redis`数据库，并复制`内网地址的 connection` 作为 `redis` 的链接串。然后配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。 + +| | | | +| --- | --- | --- | +| ![](/imgs/sealos-redis1.png) | ![](/imgs/sealos-redis2.png) | ![](/imgs/sealos-redis3.png) | + +### 2. 更新镜像 tag + + +### 3. 执行升级脚本 + +该脚本仅需商业版用户执行。 + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv494' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +**脚本功能** + +1. 更新站点同步定时器 ## 🚀 新增内容 1. 集合数据训练状态展示 2. SMTP 发送邮件插件 +3. BullMQ 消息队列。 +4. 站点同步支持配置训练参数。 ## 🐛 修复 diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 40e696b2d..abb5db927 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -15,7 +15,6 @@ export type DatasetUpdateBody = { name?: string; avatar?: string; intro?: string; - status?: DatasetSchemaType['status']; agentModel?: string; vlmModel?: string; @@ -26,6 +25,7 @@ export type DatasetUpdateBody = { apiServer?: DatasetSchemaType['apiServer']; yuqueServer?: DatasetSchemaType['yuqueServer']; feishuServer?: DatasetSchemaType['feishuServer']; + chunkSettings?: DatasetSchemaType['chunkSettings']; // sync schedule autoSync?: boolean; @@ -141,7 +141,6 @@ export type PushDatasetDataChunkProps = { export type PostWebsiteSyncParams = { datasetId: string; - billId: string; }; export type PushDatasetDataProps = { diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 627129835..8631b65b8 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -50,7 +50,8 @@ export const DatasetTypeMap = { export enum DatasetStatusEnum { active = 'active', - syncing = 'syncing' + syncing = 'syncing', + waiting = 'waiting' } export const DatasetStatusMap = { [DatasetStatusEnum.active]: { @@ -58,6 +59,9 @@ export const DatasetStatusMap = { }, [DatasetStatusEnum.syncing]: { label: i18nT('common:core.dataset.status.syncing') + }, + [DatasetStatusEnum.waiting]: { + label: i18nT('common:core.dataset.status.waiting') } }; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 67bde78fb..734f938f1 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -17,6 +17,20 @@ import { SourceMemberType } from 'support/user/type'; import { DatasetDataIndexTypeEnum } from './data/constants'; import { ChunkSettingModeEnum } from './constants'; +export type ChunkSettingsType = { + trainingType: DatasetCollectionDataProcessModeEnum; + autoIndexes?: boolean; + imageIndex?: boolean; + + chunkSettingMode?: ChunkSettingModeEnum; + chunkSplitMode?: DataChunkSplitModeEnum; + + chunkSize?: number; + indexSize?: number; + chunkSplitter?: string; + qaPrompt?: string; +}; + export type DatasetSchemaType = { _id: string; parentId?: string; @@ -29,7 +43,6 @@ export type DatasetSchemaType = { name: string; intro: string; type: `${DatasetTypeEnum}`; - status: `${DatasetStatusEnum}`; vectorModel: string; agentModel: string; @@ -39,14 +52,16 @@ export type DatasetSchemaType = { url: string; selector: string; }; + + chunkSettings?: ChunkSettingsType; + inheritPermission: boolean; apiServer?: APIFileServer; feishuServer?: FeishuServer; yuqueServer?: YuqueServer; - autoSync?: boolean; - // abandon + autoSync?: boolean; externalReadUrl?: string; defaultPermission?: number; }; @@ -193,6 +208,7 @@ export type DatasetListItemType = { }; export type DatasetItemType = Omit & { + status: `${DatasetStatusEnum}`; vectorModel: EmbeddingModelItemType; agentModel: LLMModelItemType; vlmModel?: LLMModelItemType; diff --git a/packages/service/common/bullmq/index.ts b/packages/service/common/bullmq/index.ts new file mode 100644 index 000000000..3d093bac5 --- /dev/null +++ b/packages/service/common/bullmq/index.ts @@ -0,0 +1,74 @@ +import { ConnectionOptions, Processor, Queue, QueueOptions, Worker, WorkerOptions } from 'bullmq'; +import { addLog } from '../system/log'; +import { newQueueRedisConnection, newWorkerRedisConnection } from '../redis'; + +const defaultWorkerOpts: Omit = { + removeOnComplete: { + count: 0 // Delete jobs immediately on completion + }, + removeOnFail: { + count: 0 // Delete jobs immediately on failure + } +}; + +export enum QueueNames { + websiteSync = 'websiteSync' +} + +export const queues = (() => { + if (!global.queues) { + global.queues = new Map(); + } + return global.queues; +})(); +export const workers = (() => { + if (!global.workers) { + global.workers = new Map(); + } + return global.workers; +})(); + +export function getQueue( + name: QueueNames, + opts?: Omit +): Queue { + // check if global.queues has the queue + const queue = queues.get(name); + if (queue) { + return queue as Queue; + } + const newQueue = new Queue(name.toString(), { + connection: newQueueRedisConnection(), + ...opts + }); + + // default error handler, to avoid unhandled exceptions + newQueue.on('error', (error) => { + addLog.error(`MQ Queue [${name}]: ${error.message}`, error); + }); + queues.set(name, newQueue); + return newQueue; +} + +export function getWorker( + name: QueueNames, + processor: Processor, + opts?: Omit +): Worker { + const worker = workers.get(name); + if (worker) { + return worker as Worker; + } + + const newWorker = new Worker(name.toString(), processor, { + connection: newWorkerRedisConnection(), + ...defaultWorkerOpts, + ...opts + }); + // default error handler, to avoid unhandled exceptions + newWorker.on('error', (error) => { + addLog.error(`MQ Worker [${name}]: ${error.message}`, error); + }); + workers.set(name, newWorker); + return newWorker; +} diff --git a/packages/service/common/bullmq/type.d.ts b/packages/service/common/bullmq/type.d.ts new file mode 100644 index 000000000..723675b27 --- /dev/null +++ b/packages/service/common/bullmq/type.d.ts @@ -0,0 +1,7 @@ +import { Queue, Worker } from 'bullmq'; +import { QueueNames } from './index'; + +declare global { + var queues: Map | undefined; + var workers: Map | undefined; +} diff --git a/packages/service/common/redis/index.ts b/packages/service/common/redis/index.ts new file mode 100644 index 000000000..5e88a4ba1 --- /dev/null +++ b/packages/service/common/redis/index.ts @@ -0,0 +1,27 @@ +import Redis from 'ioredis'; + +const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379'; + +export function newQueueRedisConnection() { + const redis = new Redis(REDIS_URL); + redis.on('connect', () => { + console.log('Redis connected'); + }); + redis.on('error', (error) => { + console.error('Redis connection error', error); + }); + return redis; +} + +export function newWorkerRedisConnection() { + const redis = new Redis(REDIS_URL, { + maxRetriesPerRequest: null + }); + redis.on('connect', () => { + console.log('Redis connected'); + }); + redis.on('error', (error) => { + console.error('Redis connection error', error); + }); + return redis; +} diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 44e5d07da..14c1c0bcd 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,6 +1,7 @@ import { DatasetCollectionTypeEnum, - DatasetCollectionDataProcessModeEnum + DatasetCollectionDataProcessModeEnum, + DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; @@ -104,7 +105,8 @@ export const createCollectionAndInsertData = async ({ hashRawText: hashStr(rawText), rawTextLength: rawText.length, nextSyncTime: (() => { - if (!dataset.autoSync) return undefined; + // ignore auto collections sync for website datasets + if (!dataset.autoSync && dataset.type === DatasetTypeEnum.websiteDataset) return undefined; if ( [DatasetCollectionTypeEnum.link, DatasetCollectionTypeEnum.apiFile].includes( createCollectionParams.type diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 9522c69f2..1b1ceb913 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,13 +1,8 @@ import { connectionMongo, getMongoModel } from '../../../common/mongo'; -const { Schema, model, models } = connectionMongo; +const { Schema } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; -import { - DatasetCollectionTypeMap, - DatasetCollectionDataProcessModeEnum, - ChunkSettingModeEnum, - DataChunkSplitModeEnum -} from '@fastgpt/global/core/dataset/constants'; -import { DatasetCollectionName } from '../schema'; +import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; +import { ChunkSettings, DatasetCollectionName } from '../schema'; import { TeamCollectionName, TeamMemberCollectionName @@ -90,25 +85,7 @@ const DatasetCollectionSchema = new Schema({ customPdfParse: Boolean, // Chunk settings - imageIndex: Boolean, - autoIndexes: Boolean, - trainingType: { - type: String, - enum: Object.values(DatasetCollectionDataProcessModeEnum) - }, - chunkSettingMode: { - type: String, - enum: Object.values(ChunkSettingModeEnum) - }, - chunkSplitMode: { - type: String, - enum: Object.values(DataChunkSplitModeEnum) - }, - chunkSize: Number, - chunkSplitter: String, - - indexSize: Number, - qaPrompt: String + ...ChunkSettings }); DatasetCollectionSchema.virtual('dataset', { diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index 06be050a9..6cda361ac 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -9,6 +9,8 @@ import { deleteDatasetDataVector } from '../../common/vectorStore/controller'; import { MongoDatasetDataText } from './data/dataTextSchema'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { retryFn } from '@fastgpt/global/common/system/utils'; +import { removeWebsiteSyncJobScheduler } from './websiteSync'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 22f79fd25..43bbe3a89 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -1,7 +1,8 @@ import { getMongoModel, Schema } from '../../common/mongo'; import { - DatasetStatusEnum, - DatasetStatusMap, + ChunkSettingModeEnum, + DataChunkSplitModeEnum, + DatasetCollectionDataProcessModeEnum, DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants'; @@ -13,6 +14,28 @@ import type { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d'; export const DatasetCollectionName = 'datasets'; +export const ChunkSettings = { + imageIndex: Boolean, + autoIndexes: Boolean, + trainingType: { + type: String, + enum: Object.values(DatasetCollectionDataProcessModeEnum) + }, + chunkSettingMode: { + type: String, + enum: Object.values(ChunkSettingModeEnum) + }, + chunkSplitMode: { + type: String, + enum: Object.values(DataChunkSplitModeEnum) + }, + chunkSize: Number, + chunkSplitter: String, + + indexSize: Number, + qaPrompt: String +}; + const DatasetSchema = new Schema({ parentId: { type: Schema.Types.ObjectId, @@ -40,11 +63,6 @@ const DatasetSchema = new Schema({ required: true, default: DatasetTypeEnum.dataset }, - status: { - type: String, - enum: Object.keys(DatasetStatusMap), - default: DatasetStatusEnum.active - }, avatar: { type: String, default: '/icon/logo.svg' @@ -84,6 +102,9 @@ const DatasetSchema = new Schema({ } } }, + chunkSettings: { + type: ChunkSettings + }, inheritPermission: { type: Boolean, default: true @@ -98,9 +119,8 @@ const DatasetSchema = new Schema({ type: Object }, - autoSync: Boolean, - // abandoned + autoSync: Boolean, externalReadUrl: { type: String }, diff --git a/packages/service/core/dataset/websiteSync/index.ts b/packages/service/core/dataset/websiteSync/index.ts new file mode 100644 index 000000000..d0a02bb53 --- /dev/null +++ b/packages/service/core/dataset/websiteSync/index.ts @@ -0,0 +1,80 @@ +import { Processor } from 'bullmq'; +import { getQueue, getWorker, QueueNames } from '../../../common/bullmq'; +import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants'; + +export type WebsiteSyncJobData = { + datasetId: string; +}; + +export const websiteSyncQueue = getQueue(QueueNames.websiteSync, { + defaultJobOptions: { + attempts: 3, // retry 3 times + backoff: { + type: 'exponential', + delay: 1000 // delay 1 second between retries + } + } +}); +export const getWebsiteSyncWorker = (processor: Processor) => { + return getWorker(QueueNames.websiteSync, processor, { + removeOnFail: { + age: 15 * 24 * 60 * 60, // Keep up to 15 days + count: 1000 // Keep up to 1000 jobs + }, + concurrency: 1 // Set worker to process only 1 job at a time + }); +}; + +export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => { + const datasetId = String(data.datasetId); + // deduplication: make sure only 1 job + return websiteSyncQueue.add(datasetId, data, { deduplication: { id: datasetId } }); +}; + +export const getWebsiteSyncDatasetStatus = async (datasetId: string) => { + const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId); + if (!jobId) { + return DatasetStatusEnum.active; + } + const job = await websiteSyncQueue.getJob(jobId); + if (!job) { + return DatasetStatusEnum.active; + } + + const jobState = await job.getState(); + + if (['waiting-children', 'waiting'].includes(jobState)) { + return DatasetStatusEnum.waiting; + } + if (jobState === 'active') { + return DatasetStatusEnum.syncing; + } + + return DatasetStatusEnum.active; +}; + +// Scheduler setting +const repeatDuration = 24 * 60 * 60 * 1000; // every day +export const upsertWebsiteSyncJobScheduler = (data: WebsiteSyncJobData, startDate?: number) => { + const datasetId = String(data.datasetId); + + return websiteSyncQueue.upsertJobScheduler( + datasetId, + { + every: repeatDuration, + startDate: startDate || new Date().getTime() + repeatDuration // First run tomorrow + }, + { + name: datasetId, + data + } + ); +}; + +export const getWebsiteSyncJobScheduler = (datasetId: string) => { + return websiteSyncQueue.getJobScheduler(String(datasetId)); +}; + +export const removeWebsiteSyncJobScheduler = (datasetId: string) => { + return websiteSyncQueue.removeJobScheduler(String(datasetId)); +}; diff --git a/packages/service/package.json b/packages/service/package.json index 8bf1784ed..622a8a9cd 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -7,6 +7,7 @@ "@xmldom/xmldom": "^0.8.10", "@zilliz/milvus2-sdk-node": "2.4.2", "axios": "^1.8.2", + "bullmq": "^5.44.0", "chalk": "^5.3.0", "cheerio": "1.0.0-rc.12", "cookie": "^0.7.1", @@ -18,6 +19,7 @@ "file-type": "^19.0.0", "form-data": "^4.0.0", "iconv-lite": "^0.6.3", + "ioredis": "^5.6.0", "joplin-turndown-plugin-gfm": "^1.0.12", "json5": "^2.2.3", "jsonpath-plus": "^10.3.0", diff --git a/packages/web/components/common/Icon/icons/change.svg b/packages/web/components/common/Icon/icons/change.svg index 8ab5546eb..546b49d89 100644 --- a/packages/web/components/common/Icon/icons/change.svg +++ b/packages/web/components/common/Icon/icons/change.svg @@ -1,9 +1,9 @@ - - - + + + diff --git a/packages/web/components/common/Input/NumberInput/index.tsx b/packages/web/components/common/Input/NumberInput/index.tsx index fb817d8e5..cde19e69c 100644 --- a/packages/web/components/common/Input/NumberInput/index.tsx +++ b/packages/web/components/common/Input/NumberInput/index.tsx @@ -26,23 +26,43 @@ const MyNumberInput = (props: Props) => { { - if (!onBlur) return; const numE = Number(e.target.value); - if (isNaN(numE)) { - // @ts-ignore - onBlur(''); - } else { - onBlur(numE); + if (onBlur) { + if (isNaN(numE)) { + // @ts-ignore + onBlur(''); + } else { + onBlur(numE); + } + } + if (register && name) { + const event = { + target: { + name, + value: numE + } + }; + register(name).onBlur(event); } }} onChange={(e) => { - if (!onChange) return; const numE = Number(e); - if (isNaN(numE)) { - // @ts-ignore - onChange(''); - } else { - onChange(numE); + if (onChange) { + if (isNaN(numE)) { + // @ts-ignore + onChange(''); + } else { + onChange(numE); + } + } + if (register && name) { + const event = { + target: { + name, + value: numE + } + }; + register(name).onChange(event); } }} > diff --git a/packages/web/components/common/Tag/index.tsx b/packages/web/components/common/Tag/index.tsx index 7236f424b..b62974555 100644 --- a/packages/web/components/common/Tag/index.tsx +++ b/packages/web/components/common/Tag/index.tsx @@ -1,5 +1,5 @@ import React, { useMemo } from 'react'; -import { Box, Flex, type FlexProps } from '@chakra-ui/react'; +import { Box, BoxProps, Flex, type FlexProps } from '@chakra-ui/react'; type ColorSchemaType = 'white' | 'blue' | 'green' | 'red' | 'yellow' | 'gray' | 'purple' | 'adora'; @@ -8,6 +8,7 @@ export type TagProps = FlexProps & { colorSchema?: ColorSchemaType; type?: 'fill' | 'borderFill' | 'borderSolid'; showDot?: boolean; + DotStyles?: BoxProps; }; const colorMap: Record< @@ -60,7 +61,14 @@ const colorMap: Record< } }; -const MyTag = ({ children, colorSchema = 'blue', type = 'fill', showDot, ...props }: TagProps) => { +const MyTag = ({ + children, + colorSchema = 'blue', + type = 'fill', + showDot, + DotStyles, + ...props +}: TagProps) => { const theme = useMemo(() => { return colorMap[colorSchema]; }, [colorSchema]); @@ -81,7 +89,9 @@ const MyTag = ({ children, colorSchema = 'blue', type = 'fill', showDot, ...prop bg={type !== 'borderSolid' ? theme.bg : 'transparent'} {...props} > - {showDot && } + {showDot && ( + + )} {children} ); diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index f729cca3e..3498a8549 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -512,7 +512,7 @@ "core.dataset.Query extension intro": "Enabling the question optimization function can improve the accuracy of Dataset searches during continuous conversations. After enabling this function, when performing Dataset searches, the AI will complete the missing information of the question based on the conversation history.", "core.dataset.Quote Length": "Quote Content Length", "core.dataset.Read Dataset": "View Dataset Details", - "core.dataset.Set Website Config": "Start Configuring Website Information", + "core.dataset.Set Website Config": "Start Configuring", "core.dataset.Start export": "Export Started", "core.dataset.Table collection": "Table Dataset", "core.dataset.Text collection": "Text Dataset", @@ -528,7 +528,6 @@ "core.dataset.collection.Website Empty Tip": "No Website Associated Yet", "core.dataset.collection.Website Link": "Website Address", "core.dataset.collection.id": "Collection ID", - "core.dataset.collection.metadata.Chunk Size": "Chunk Size", "core.dataset.collection.metadata.Createtime": "Creation Time", "core.dataset.collection.metadata.Raw text length": "Raw Text Length", "core.dataset.collection.metadata.Updatetime": "Update Time", @@ -630,6 +629,7 @@ "core.dataset.search.search mode": "Search Method", "core.dataset.status.active": "Ready", "core.dataset.status.syncing": "Syncing", + "core.dataset.status.waiting": "Waiting", "core.dataset.test.Batch test": "Batch Test", "core.dataset.test.Batch test Placeholder": "Select a CSV File", "core.dataset.test.Search Test": "Search Test", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 3eae9589a..1a3a36091 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -7,6 +7,7 @@ "auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.", "auto_training_queue": "Enhanced index queueing", "chunk_max_tokens": "max_tokens", + "chunk_size": "Block size", "close_auto_sync": "Are you sure you want to turn off automatic sync?", "collection.Create update time": "Creation/Update Time", "collection.Training type": "Training", @@ -70,6 +71,7 @@ "image_auto_parse": "Automatic image indexing", "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes", "image_training_queue": "Queue of image processing", + "immediate_sync": "Immediate Synchronization", "import.Auto mode Estimated Price Tips": "The text understanding model needs to be called, which requires more points: {{price}} points/1K tokens", "import.Embedding Estimated Price Tips": "Only use the index model and consume a small amount of AI points: {{price}} points/1K tokens", "import_confirm": "Confirm upload", @@ -86,6 +88,7 @@ "keep_image": "Keep the picture", "move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.", "open_auto_sync": "After scheduled synchronization is turned on, the system will try to synchronize the collection from time to time every day. During the collection synchronization period, the collection data will not be searched.", + "params_config": "Config", "params_setting": "Parameter settings", "pdf_enhance_parse": "PDF enhancement analysis", "pdf_enhance_parse_price": "{{price}} points/page", @@ -144,6 +147,7 @@ "vllm_model": "Image understanding model", "website_dataset": "Website Sync", "website_dataset_desc": "Website sync allows you to build a Dataset directly using a web link.", + "website_info": "Website Information", "yuque_dataset": "Yuque Dataset", "yuque_dataset_config": "Yuque Dataset Config", "yuque_dataset_desc": "Can build a dataset using Yuque documents by configuring permissions, without secondary storage" diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index 245f06b7c..1f68a9b57 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -515,7 +515,7 @@ "core.dataset.Query extension intro": "开启问题优化功能，可以提高提高连续对话时，知识库搜索的精度。开启该功能后，在进行知识库搜索时，会根据对话记录，利用 AI 补全问题缺失的信息。", "core.dataset.Quote Length": "引用内容长度", "core.dataset.Read Dataset": "查看知识库详情", - "core.dataset.Set Website Config": "开始配置网站信息", + "core.dataset.Set Website Config": "开始配置", "core.dataset.Start export": "已开始导出", "core.dataset.Table collection": "表格数据集", "core.dataset.Text collection": "文本数据集", @@ -531,7 +531,6 @@ "core.dataset.collection.Website Empty Tip": "还没有关联网站", "core.dataset.collection.Website Link": "Web 站点地址", "core.dataset.collection.id": "集合 ID", - "core.dataset.collection.metadata.Chunk Size": "分割大小", "core.dataset.collection.metadata.Createtime": "创建时间", "core.dataset.collection.metadata.Raw text length": "原文长度", "core.dataset.collection.metadata.Updatetime": "更新时间", @@ -633,6 +632,7 @@ "core.dataset.search.search mode": "搜索方式", "core.dataset.status.active": "已就绪", "core.dataset.status.syncing": "同步中", + "core.dataset.status.waiting": "排队中", "core.dataset.test.Batch test": "批量测试", "core.dataset.test.Batch test Placeholder": "选择一个 CSV 文件", "core.dataset.test.Search Test": "搜索测试", @@ -1291,4 +1291,4 @@ "yes": "是", "yesterday": "昨天", "yesterday_detail_time": "昨天 {{time}}" -} \ No newline at end of file +} diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index 53a07dfda..2db1f2330 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -7,6 +7,7 @@ "auto_indexes_tips": "通过大模型进行额外索引生成，提高语义丰富度，提高检索的精度。", "auto_training_queue": "增强索引排队", "chunk_max_tokens": "分块上限", + "chunk_size": "分块大小", "close_auto_sync": "确认关闭自动同步功能？", "collection.Create update time": "创建/更新时间", "collection.Training type": "训练模式", @@ -70,6 +71,7 @@ "image_auto_parse": "图片自动索引", "image_auto_parse_tips": "调用 VLM 自动标注文档里的图片，并生成额外的检索索引", "image_training_queue": "图片处理排队", + "immediate_sync": "立即同步", "import.Auto mode Estimated Price Tips": "需调用文本理解模型，需要消耗较多AI 积分：{{price}} 积分/1K tokens", "import.Embedding Estimated Price Tips": "仅使用索引模型，消耗少量 AI 积分：{{price}} 积分/1K tokens", "import_confirm": "确认上传", @@ -86,6 +88,7 @@ "keep_image": "保留图片", "move.hint": "移动后，所选知识库/文件夹将继承新文件夹的权限设置，原先的权限设置失效。", "open_auto_sync": "开启定时同步后，系统将会每天不定时尝试同步集合，集合同步期间，会出现无法搜索到该集合数据现象。", + "params_config": "配置", "params_setting": "参数设置", "pdf_enhance_parse": "PDF增强解析", "pdf_enhance_parse_price": "{{price}}积分/页", @@ -145,6 +148,7 @@ "vllm_model": "图片理解模型", "website_dataset": "Web 站点同步", "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", + "website_info": "网站信息", "yuque_dataset": "语雀知识库", "yuque_dataset_config": "配置语雀知识库", "yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储" diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index 955858e7d..bd820fb05 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -511,7 +511,7 @@ "core.dataset.Query extension intro": "開啟問題最佳化功能，可以提高連續對話時知識庫搜尋的準確度。開啟此功能後，在進行知識庫搜尋時，系統會根據對話記錄，利用 AI 補充問題中缺少的資訊。", "core.dataset.Quote Length": "引用內容長度", "core.dataset.Read Dataset": "檢視知識庫詳細資料", - "core.dataset.Set Website Config": "開始設定網站資訊", + "core.dataset.Set Website Config": "開始設定", "core.dataset.Start export": "已開始匯出", "core.dataset.Table collection": "表格資料集", "core.dataset.Text collection": "文字資料集", @@ -527,7 +527,6 @@ "core.dataset.collection.Website Empty Tip": "還沒有關聯網站", "core.dataset.collection.Website Link": "網站網址", "core.dataset.collection.id": "集合 ID", - "core.dataset.collection.metadata.Chunk Size": "分割大小", "core.dataset.collection.metadata.Createtime": "建立時間", "core.dataset.collection.metadata.Raw text length": "原始文字長度", "core.dataset.collection.metadata.Updatetime": "更新時間", @@ -629,6 +628,7 @@ "core.dataset.search.search mode": "搜索方式", "core.dataset.status.active": "已就緒", "core.dataset.status.syncing": "同步中", + "core.dataset.status.waiting": "排队中", "core.dataset.test.Batch test": "批次測試", "core.dataset.test.Batch test Placeholder": "選擇一個 CSV 檔案", "core.dataset.test.Search Test": "搜尋測試", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index 7bc6c16c2..c744ddc9d 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -7,6 +7,7 @@ "auto_indexes_tips": "通過大模型進行額外索引生成，提高語義豐富度，提高檢索的精度。", "auto_training_queue": "增強索引排隊", "chunk_max_tokens": "分塊上限", + "chunk_size": "分塊大小", "close_auto_sync": "確認關閉自動同步功能？", "collection.Create update time": "建立／更新時間", "collection.Training type": "分段模式", @@ -70,6 +71,7 @@ "image_auto_parse": "圖片自動索引", "image_auto_parse_tips": "調用 VLM 自動標註文檔裡的圖片，並生成額外的檢索索引", "image_training_queue": "圖片處理排隊", + "immediate_sync": "立即同步", "import.Auto mode Estimated Price Tips": "需呼叫文字理解模型，將消耗較多 AI 點數：{{price}} 點數 / 1K tokens", "import.Embedding Estimated Price Tips": "僅使用索引模型，消耗少量 AI 點數：{{price}} 點數 / 1K tokens", "import_confirm": "確認上傳", @@ -86,6 +88,7 @@ "keep_image": "保留圖片", "move.hint": "移動後，所選資料集／資料夾將繼承新資料夾的權限設定，原先的權限設定將失效。", "open_auto_sync": "開啟定時同步後，系統將每天不定時嘗試同步集合，集合同步期間，會出現無法搜尋到該集合資料現象。", + "params_config": "配置", "params_setting": "參數設置", "pdf_enhance_parse": "PDF增強解析", "pdf_enhance_parse_price": "{{price}}積分/頁", @@ -144,6 +147,7 @@ "vllm_model": "圖片理解模型", "website_dataset": "網站同步", "website_dataset_desc": "網站同步功能讓您可以直接使用網頁連結建立資料集", + "website_info": "網站資訊", "yuque_dataset": "語雀知識庫", "yuque_dataset_config": "配置語雀知識庫", "yuque_dataset_desc": "可通過配置語雀文檔權限，使用語雀文檔構建知識庫，文檔不會進行二次存儲" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ba6886bb..0df4b04ab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,9 @@ importers: axios: specifier: ^1.8.2 version: 1.8.3 + bullmq: + specifier: ^5.44.0 + version: 5.44.0 chalk: specifier: ^5.3.0 version: 5.4.1 @@ -202,6 +205,9 @@ importers: iconv-lite: specifier: ^0.6.3 version: 0.6.3 + ioredis: + specifier: ^5.6.0 + version: 5.6.0 joplin-turndown-plugin-gfm: specifier: ^1.0.12 version: 1.0.12 @@ -2044,6 +2050,9 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -2314,6 +2323,36 @@ packages: '@mongodb-js/saslprep@1.2.0': resolution: {integrity: sha512-+ywrb0AqkfaYuhHs6LxKWgqbh3I72EpEgESCw37o+9qPx9WTCkgDm2B+eMrwehGtHBWHFU4GXvnSCNiFhhausg==} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + '@napi-rs/wasm-runtime@0.2.7': resolution: {integrity: sha512-5yximcFK5FNompXfJFoWanu5l8v1hNGqNHh9du1xETp9HWk/B/PzvchX55WYOPaIeNglG8++68AAiauBAtbnzw==} @@ -4014,6 +4053,9 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bullmq@5.44.0: + resolution: {integrity: sha512-OnEtkuXyrUx2Jm5BpH92+ttrobblBdCbkhOe3OoR0hxZuAilI3mPWlwELslhfImRpDv8rK+C/0/VK7I8f3xIig==} + bundle-n-require@1.1.2: resolution: {integrity: sha512-bEk2jakVK1ytnZ9R2AAiZEeK/GxPUM8jvcRxHZXifZDMcjkI4EG/GlsJ2YGSVYT9y/p/gA9/0yDY8rCGsSU6Tg==} @@ -4248,6 +4290,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -5860,6 +5906,10 @@ packages: intersection-observer@0.12.2: resolution: {integrity: sha512-7m1vEcPCxXYI8HqnL8CKI6siDyD+eIWSwgB3DZA+ZTogxk9I4CDnj4wilt9x/+/QbHI4YG5YZNmC6458/e9Ktg==} + ioredis@5.6.0: + resolution: {integrity: sha512-tBZlIIWbndeWBWCXWZiqtOF/yxf6yZX3tAlTJ7nfo5jhd6dctNxF7QnYlZLZ1a0o0pDoen7CgZqO+zjNaFbJAg==} + engines: {node: '>=12.22.0'} + ip-address@9.0.5: resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} engines: {node: '>= 12'} @@ -6554,9 +6604,15 @@ packages: lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.isboolean@3.0.3: resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} @@ -7128,6 +7184,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.2: + resolution: {integrity: sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==} + mssql@11.0.1: resolution: {integrity: sha512-KlGNsugoT90enKlR8/G36H0kTxPthDhmtNUCwEHvgRza5Cjpjoj+P2X6eMpFUDN7pFrJZsKadL4x990G8RBE1w==} engines: {node: '>=18'} @@ -7260,6 +7323,10 @@ packages: encoding: optional: true + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-gyp@10.3.1: resolution: {integrity: sha512-Pp3nFHBThHzVtNY7U6JfPjvT/DTE8+o/4xKsLQtBoU+j2HLsGlhcfzflAoUreaJbNmYnX+LlLi0qjV8kpyO6xQ==} engines: {node: ^16.14.0 || >=18.0.0} @@ -8041,6 +8108,14 @@ packages: react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} @@ -8490,6 +8565,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + state-local@1.0.7: resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} @@ -11160,6 +11238,8 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} + '@ioredis/commands@1.2.0': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -11565,6 +11645,24 @@ snapshots: dependencies: sparse-bitfield: 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@napi-rs/wasm-runtime@0.2.7': dependencies: '@emnapi/core': 1.3.1 @@ -13456,6 +13554,18 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.44.0: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.6.0 + msgpackr: 1.11.2 + node-abort-controller: 3.1.1 + semver: 7.7.1 + tslib: 2.8.1 + uuid: 9.0.1 + transitivePeerDependencies: + - supports-color + bundle-n-require@1.1.2: dependencies: esbuild: 0.25.1 @@ -13713,6 +13823,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + co@4.6.0: {} collapse-white-space@1.0.6: {} @@ -14626,7 +14738,7 @@ snapshots: '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.2) eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0) + eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0) eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0) eslint-plugin-jsx-a11y: 6.10.2(eslint@8.56.0) eslint-plugin-react: 7.37.4(eslint@8.56.0) @@ -14646,7 +14758,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0): + eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.0 @@ -14661,14 +14773,14 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0))(eslint@8.56.0): + eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.2) eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0) + eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0) transitivePeerDependencies: - supports-color @@ -14683,7 +14795,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.56.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0))(eslint@8.56.0) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -15692,6 +15804,20 @@ snapshots: intersection-observer@0.12.2: {} + ioredis@5.6.0: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.4.0 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ip-address@9.0.5: dependencies: jsbn: 1.1.0 @@ -16558,8 +16684,12 @@ snapshots: lodash.debounce@4.0.8: {} + lodash.defaults@4.2.0: {} + lodash.includes@4.3.0: {} + lodash.isarguments@3.1.0: {} + lodash.isboolean@3.0.3: {} lodash.isinteger@4.0.4: {} @@ -17481,6 +17611,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.2: + optionalDependencies: + msgpackr-extract: 3.0.3 + mssql@11.0.1: dependencies: '@tediousjs/connection-string': 0.5.0 @@ -17624,6 +17770,11 @@ snapshots: optionalDependencies: encoding: 0.1.13 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.0.3 + optional: true + node-gyp@10.3.1: dependencies: env-paths: 2.2.1 @@ -18499,6 +18650,12 @@ snapshots: tiny-invariant: 1.3.3 victory-vendor: 36.9.2 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + redux@4.2.1: dependencies: '@babel/runtime': 7.26.10 @@ -19048,6 +19205,8 @@ snapshots: stackback@0.0.2: {} + standard-as-callback@2.1.0: {} + state-local@1.0.7: {} state-toggle@1.0.3: {} diff --git a/projects/app/.env.template b/projects/app/.env.template index 297b19b31..e6ccd8542 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -20,6 +20,8 @@ AIPROXY_API_TOKEN=xxxxx # 强制将图片转成 base64 传递给模型 MULTIPLE_DATA_TO_BASE64=true +# Redis URL +REDIS_URL=redis://default:password@127.0.0.1:6379 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。 MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin @@ -65,4 +67,4 @@ CHECK_INTERNAL_IP=false # # 日志来源ID前缀 # CHAT_LOG_SOURCE_ID_PREFIX=fastgpt- # 自定义跨域，不配置时，默认都允许跨域（逗号分割） -ALLOWED_ORIGINS= \ No newline at end of file +ALLOWED_ORIGINS= diff --git a/projects/app/src/instrumentation.ts b/projects/app/src/instrumentation.ts index 3abc0e04a..1681d5ba4 100644 --- a/projects/app/src/instrumentation.ts +++ b/projects/app/src/instrumentation.ts @@ -1,6 +1,6 @@ import { exit } from 'process'; -/* +/* Init system */ export async function register() { diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx index f43bf12bb..9d8061d56 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx @@ -1,19 +1,18 @@ import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; -import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react'; +import { Dispatch, ReactNode, SetStateAction, useState } from 'react'; import { useTranslation } from 'next-i18next'; import { createContext, useContextSelector } from 'use-context-selector'; -import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useDisclosure } from '@chakra-ui/react'; import { checkTeamWebSyncLimit } from '@/web/support/user/team/api'; -import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api'; import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api'; import dynamic from 'next/dynamic'; import { usePagination } from '@fastgpt/web/hooks/usePagination'; import { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; import { useRouter } from 'next/router'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { WebsiteConfigFormType } from './WebsiteConfig'; const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig')); @@ -66,7 +65,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => const router = useRouter(); const { parentId = '' } = router.query as { parentId: string }; - const { datasetDetail, datasetId, updateDataset } = useContextSelector( + const { datasetDetail, datasetId, updateDataset, loadDatasetDetail } = useContextSelector( DatasetPageContext, (v) => v ); @@ -75,30 +74,31 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({ content: t('dataset:start_sync_website_tip') }); + const syncWebsite = async () => { + await checkTeamWebSyncLimit(); + await postWebsiteSync({ datasetId: datasetId }); + await loadDatasetDetail(datasetId); + }; const { isOpen: isOpenWebsiteModal, onOpen: onOpenWebsiteModal, onClose: onCloseWebsiteModal } = useDisclosure(); - const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({ - mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => { - onCloseWebsiteModal(); - await checkTeamWebSyncLimit(); + const { runAsync: onUpdateDatasetWebsiteConfig } = useRequest2( + async (websiteConfig: WebsiteConfigFormType) => { await updateDataset({ id: datasetId, - websiteConfig, - status: DatasetStatusEnum.syncing + websiteConfig: websiteConfig.websiteConfig, + chunkSettings: websiteConfig.chunkSettings }); - const billId = await postCreateTrainingUsage({ - name: t('common:core.dataset.training.Website Sync'), - datasetId: datasetId - }); - await postWebsiteSync({ datasetId: datasetId, billId }); - - return; + await syncWebsite(); }, - errorToast: t('common:common.Update Failed') - }); + { + onSuccess() { + onCloseWebsiteModal(); + } + } + ); // collection list const [searchText, setSearchText] = useState(''); @@ -124,7 +124,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => }); const contextValue: CollectionPageContextType = { - openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig), + openWebSyncConfirm: openWebSyncConfirm(syncWebsite), onOpenWebsiteModal, searchText, @@ -149,10 +149,6 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => )} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx index d73c3a6a1..30c098e71 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx @@ -25,6 +25,9 @@ const EmptyCollectionTip = () => { {datasetDetail.status === DatasetStatusEnum.syncing && ( <>{t('common:core.dataset.status.syncing')} )} + {datasetDetail.status === DatasetStatusEnum.waiting && ( + <>{t('common:core.dataset.status.waiting')} + )} {datasetDetail.status === DatasetStatusEnum.active && ( <> {!datasetDetail?.websiteConfig?.url ? ( diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index 7fc193f40..672d68c3d 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -1,35 +1,23 @@ import React from 'react'; -import { - Box, - Flex, - MenuButton, - Button, - Link, - useTheme, - useDisclosure, - HStack -} from '@chakra-ui/react'; +import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react'; import { getDatasetCollectionPathById, postDatasetCollection, putDatasetCollectionById } from '@/web/core/dataset/api'; -import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyInput from '@/components/MyInput'; -import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useRouter } from 'next/router'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; import { DatasetCollectionTypeEnum, - TrainingModeEnum, DatasetTypeEnum, DatasetTypeMap, - DatasetStatusEnum, - DatasetCollectionDataProcessModeEnum + DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants'; import EditFolderModal, { useEditFolder } from '../../EditFolderModal'; import { TabEnum } from '../../../../pages/dataset/detail/index'; @@ -43,26 +31,35 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex import { useSystem } from '@fastgpt/web/hooks/useSystem'; import HeaderTagPopOver from './HeaderTagPopOver'; import MyBox from '@fastgpt/web/components/common/MyBox'; +import Icon from '@fastgpt/web/components/common/Icon'; +import MyTag from '@fastgpt/web/components/common/Tag/index'; const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); const Header = ({}: {}) => { const { t } = useTranslation(); - const theme = useTheme(); - const { feConfigs } = useSystemStore(); + const { isPc } = useSystem(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); const router = useRouter(); const { parentId = '' } = router.query as { parentId: string }; - const { isPc } = useSystem(); - const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } = - useContextSelector(CollectionPageContext, (v) => v); + const { + searchText, + setSearchText, + total, + getData, + pageNum, + onOpenWebsiteModal, + openWebSyncConfirm + } = useContextSelector(CollectionPageContext, (v) => v); - const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () => - getDatasetCollectionPathById(parentId) - ); + const { data: paths = [] } = useRequest2(() => getDatasetCollectionPathById(parentId), { + refreshDeps: [parentId], + manual: false + }); const { editFolderData, setEditFolderData } = useEditFolder(); const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = @@ -72,13 +69,14 @@ const Header = ({}: {}) => { canEmpty: false }); + // Import collection const { isOpen: isOpenFileSourceSelector, onOpen: onOpenFileSourceSelector, onClose: onCloseFileSourceSelector } = useDisclosure(); - const { runAsync: onCreateCollection, loading: onCreating } = useRequest2( + const { runAsync: onCreateCollection } = useRequest2( async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => { const id = await postDatasetCollection({ parentId, @@ -100,7 +98,7 @@ const Header = ({}: {}) => { const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset; return ( - + { {!isWebSite && } {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel as any)}({total}) + {/* Website sync */} {datasetDetail?.websiteConfig?.url && ( - {t('common:core.dataset.website.Base Url')}: + {t('common:core.dataset.website.Base Url')}: {datasetDetail.websiteConfig.url} @@ -171,12 +171,14 @@ const Header = ({}: {}) => { )} {/* Tag */} - {datasetDetail.permission.hasWritePer && feConfigs?.isPlus && } + {datasetDetail.type !== DatasetTypeEnum.websiteDataset && + datasetDetail.permission.hasWritePer && + feConfigs?.isPlus && } {/* diff collection button */} {datasetDetail.permission.hasWritePer && ( - + {datasetDetail?.type === DatasetTypeEnum.dataset && ( { onClick: () => { onOpenCreateVirtualFileModal({ defaultVal: '', - onSuccess: (name) => { - onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); - } + onSuccess: (name) => + onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }) }); } }, @@ -272,35 +273,60 @@ const Header = ({}: {}) => { {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( <> {datasetDetail?.websiteConfig?.url ? ( - + <> {datasetDetail.status === DatasetStatusEnum.active && ( - + + + + )} {datasetDetail.status === DatasetStatusEnum.syncing && ( - - - - {t('common:core.dataset.status.syncing')} - - + {t('common:core.dataset.status.syncing')} + )} - + {datasetDetail.status === DatasetStatusEnum.waiting && ( + + {t('common:core.dataset.status.waiting')} + + )} + ) : ( - )} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx index e66d4c658..374bf7ecf 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx @@ -1,110 +1,215 @@ -import React from 'react'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { useTranslation } from 'next-i18next'; -import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react'; import { strIsLink } from '@fastgpt/global/common/string/tools'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useForm } from 'react-hook-form'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { getDocPath } from '@/web/common/system/doc'; import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useMyStep } from '@fastgpt/web/hooks/useStep'; +import MyDivider from '@fastgpt/web/components/common/MyDivider'; +import React, { useRef } from 'react'; +import { + Box, + Link, + Input, + Button, + ModalBody, + ModalFooter, + Textarea, + Stack +} from '@chakra-ui/react'; +import { + DataChunkSplitModeEnum, + DatasetCollectionDataProcessModeEnum +} from '@fastgpt/global/core/dataset/constants'; +import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import CollectionChunkForm, { + collectionChunkForm2StoreChunkData, + type CollectionChunkFormType +} from '../Form/CollectionChunkForm'; +import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils'; +import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type'; -type FormType = { - url?: string | undefined; - selector?: string | undefined; +export type WebsiteConfigFormType = { + websiteConfig: { + url: string; + selector: string; + }; + chunkSettings: ChunkSettingsType; }; const WebsiteConfigModal = ({ onClose, - onSuccess, - defaultValue = { - url: '', - selector: '' - } + onSuccess }: { onClose: () => void; - onSuccess: (data: FormType) => void; - defaultValue?: FormType; + onSuccess: (data: WebsiteConfigFormType) => void; }) => { const { t } = useTranslation(); const { feConfigs } = useSystemStore(); const { toast } = useToast(); - const { register, handleSubmit } = useForm({ - defaultValues: defaultValue + const steps = [ + { + title: t('dataset:website_info') + }, + { + title: t('dataset:params_config') + } + ]; + + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const websiteConfig = datasetDetail.websiteConfig; + const chunkSettings = datasetDetail.chunkSettings; + + const { + register: websiteInfoForm, + handleSubmit: websiteInfoHandleSubmit, + getValues: websiteInfoGetValues + } = useForm({ + defaultValues: { + url: websiteConfig?.url || '', + selector: websiteConfig?.selector || '' + } }); - const isEdit = !!defaultValue.url; - const confirmTip = isEdit - ? t('common:core.dataset.website.Confirm Update Tips') - : t('common:core.dataset.website.Confirm Create Tips'); + + const isEdit = !!websiteConfig?.url; const { ConfirmModal, openConfirm } = useConfirm({ type: 'common' }); + const { activeStep, goToPrevious, goToNext, MyStep } = useMyStep({ + defaultStep: 0, + steps + }); + + const form = useForm({ + defaultValues: { + trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk, + imageIndex: chunkSettings?.imageIndex || false, + autoIndexes: chunkSettings?.autoIndexes || false, + + chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto, + chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size, + embeddingChunkSize: chunkSettings?.chunkSize || 2000, + qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel), + indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512, + + chunkSplitter: chunkSettings?.chunkSplitter || '', + qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description + } + }); + return ( - - - {t('common:core.dataset.website.Config Description')} - {feConfigs?.docUrl && ( - + + + + + {activeStep == 0 && ( + <> + - {t('common:common.course.Read Course')} - - )} - - - {t('common:core.dataset.website.Base Url')} - - - - - {t('common:core.dataset.website.Selector')}({t('common:common.choosable')}) - - - + {t('common:core.dataset.website.Config Description')} + {feConfigs?.docUrl && ( + + {t('common:common.course.Read Course')} + + )} + + + {t('common:core.dataset.website.Base Url')} + + + + + {t('common:core.dataset.website.Selector')}({t('common:common.choosable')}) + + + + + )} + {activeStep == 1 && } - - + {activeStep == 0 && ( + <> + + + + )} + {activeStep == 1 && ( + <> + + + + )} @@ -112,3 +217,42 @@ const WebsiteConfigModal = ({ }; export default WebsiteConfigModal; + +const PromptTextarea = ({ + defaultValue, + onChange, + onClose +}: { + defaultValue: string; + onChange: (e: string) => void; + onClose: () => void; +}) => { + const ref = useRef(null); + const { t } = useTranslation(); + + return ( + + +