website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

deploy/docker/docker-compose-milvus.yml

@@ -110,6 +110,18 @@ services:
 
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
 
   # fastgpt
   sandbox:
@@ -157,6 +169,8 @@ services:
       # zilliz 连接参数
       - MILVUS_ADDRESS=http://milvusStandalone:19530
       - MILVUS_TOKEN=none
+      # Redis 地址
+      - REDIS_URL=redis://default:mypassword@redis:6379
       # sandbox 地址
       - SANDBOX_URL=http://sandbox:3000
       # 日志等级: debug, info, warn, error

deploy/docker/docker-compose-pgvector.yml

@@ -69,6 +69,19 @@ services:
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
 
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
+
   # fastgpt
   sandbox:
     container_name: sandbox
@@ -114,6 +127,8 @@ services:
       - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
       # pg 连接参数
       - PG_URL=postgresql://username:password@pg:5432/postgres
+      # Redis 连接参数
+      - REDIS_URL=redis://default:mypassword@redis:6379
       # sandbox 地址
       - SANDBOX_URL=http://sandbox:3000
       # 日志等级: debug, info, warn, error

deploy/docker/docker-compose-zilliz.yml

@@ -51,6 +51,19 @@ services:
 
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
+
   sandbox:
     container_name: sandbox
     image: ghcr.io/labring/fastgpt-sandbox:v4.9.3 # git
@@ -92,6 +105,8 @@ services:
       - FILE_TOKEN_KEY=filetoken
       # MongoDB 连接参数. 用户名myusername,密码mypassword。
       - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
+      # Redis 连接参数
+      - REDIS_URI=redis://default:mypassword@redis:6379
       # zilliz 连接参数
       - MILVUS_ADDRESS=zilliz_cloud_address
       - MILVUS_TOKEN=zilliz_cloud_token

docSite/content/zh-cn/docs/development/upgrading/494.md

@@ -7,11 +7,44 @@ toc: true
 weight: 796
 ---
 
+## 升级指南
+
+### 1. 做好数据备份
+
+### 1. 安装 Redis
+
+* docker 部署的用户，参考最新的 `docker-compose.yml` 文件增加 Redis 配置。增加一个 redis 容器，并配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。
+* Sealos 部署的用户，在数据库里新建一个`redis`数据库，并复制`内网地址的 connection` 作为 `redis` 的链接串。然后配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。
+
+| | | |
+| --- | --- |  --- |
+| ![](/imgs/sealos-redis1.png) | ![](/imgs/sealos-redis2.png) |  ![](/imgs/sealos-redis3.png) |
+
+### 2. 更新镜像 tag
+
+
+### 3. 执行升级脚本
+
+该脚本仅需商业版用户执行。
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv494' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+**脚本功能**
+
+1. 更新站点同步定时器
 
 ## 🚀 新增内容
 
 1. 集合数据训练状态展示
 2. SMTP 发送邮件插件
+3. BullMQ 消息队列。
+4. 站点同步支持配置训练参数。
 
 ## 🐛 修复
 

packages/global/core/dataset/api.d.ts

@@ -15,7 +15,6 @@ export type DatasetUpdateBody = {
   name?: string;
   avatar?: string;
   intro?: string;
-  status?: DatasetSchemaType['status'];
 
   agentModel?: string;
   vlmModel?: string;
@@ -26,6 +25,7 @@ export type DatasetUpdateBody = {
   apiServer?: DatasetSchemaType['apiServer'];
   yuqueServer?: DatasetSchemaType['yuqueServer'];
   feishuServer?: DatasetSchemaType['feishuServer'];
+  chunkSettings?: DatasetSchemaType['chunkSettings'];
 
   // sync schedule
   autoSync?: boolean;
@@ -141,7 +141,6 @@ export type PushDatasetDataChunkProps = {
 
 export type PostWebsiteSyncParams = {
   datasetId: string;
-  billId: string;
 };
 
 export type PushDatasetDataProps = {

packages/global/core/dataset/constants.ts

@@ -50,7 +50,8 @@ export const DatasetTypeMap = {
 
 export enum DatasetStatusEnum {
   active = 'active',
-  syncing = 'syncing'
+  syncing = 'syncing',
+  waiting = 'waiting'
 }
 export const DatasetStatusMap = {
   [DatasetStatusEnum.active]: {
@@ -58,6 +59,9 @@ export const DatasetStatusMap = {
   },
   [DatasetStatusEnum.syncing]: {
     label: i18nT('common:core.dataset.status.syncing')
+  },
+  [DatasetStatusEnum.waiting]: {
+    label: i18nT('common:core.dataset.status.waiting')
   }
 };
 

packages/global/core/dataset/type.d.ts

@@ -17,6 +17,20 @@ import { SourceMemberType } from 'support/user/type';
 import { DatasetDataIndexTypeEnum } from './data/constants';
 import { ChunkSettingModeEnum } from './constants';
 
+export type ChunkSettingsType = {
+  trainingType: DatasetCollectionDataProcessModeEnum;
+  autoIndexes?: boolean;
+  imageIndex?: boolean;
+
+  chunkSettingMode?: ChunkSettingModeEnum;
+  chunkSplitMode?: DataChunkSplitModeEnum;
+
+  chunkSize?: number;
+  indexSize?: number;
+  chunkSplitter?: string;
+  qaPrompt?: string;
+};
+
 export type DatasetSchemaType = {
   _id: string;
   parentId?: string;
@@ -29,7 +43,6 @@ export type DatasetSchemaType = {
   name: string;
   intro: string;
   type: `${DatasetTypeEnum}`;
-  status: `${DatasetStatusEnum}`;
 
   vectorModel: string;
   agentModel: string;
@@ -39,14 +52,16 @@ export type DatasetSchemaType = {
     url: string;
     selector: string;
   };
+
+  chunkSettings?: ChunkSettingsType;
+
   inheritPermission: boolean;
   apiServer?: APIFileServer;
   feishuServer?: FeishuServer;
   yuqueServer?: YuqueServer;
 
-  autoSync?: boolean;
-
   // abandon
+  autoSync?: boolean;
   externalReadUrl?: string;
   defaultPermission?: number;
 };
@@ -193,6 +208,7 @@ export type DatasetListItemType = {
 };
 
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel' | 'vlmModel'> & {
+  status: `${DatasetStatusEnum}`;
   vectorModel: EmbeddingModelItemType;
   agentModel: LLMModelItemType;
   vlmModel?: LLMModelItemType;

packages/service/common/bullmq/index.ts

@@ -0,0 +1,74 @@
+import { ConnectionOptions, Processor, Queue, QueueOptions, Worker, WorkerOptions } from 'bullmq';
+import { addLog } from '../system/log';
+import { newQueueRedisConnection, newWorkerRedisConnection } from '../redis';
+
+const defaultWorkerOpts: Omit<ConnectionOptions, 'connection'> = {
+  removeOnComplete: {
+    count: 0 // Delete jobs immediately on completion
+  },
+  removeOnFail: {
+    count: 0 // Delete jobs immediately on failure
+  }
+};
+
+export enum QueueNames {
+  websiteSync = 'websiteSync'
+}
+
+export const queues = (() => {
+  if (!global.queues) {
+    global.queues = new Map<QueueNames, Queue>();
+  }
+  return global.queues;
+})();
+export const workers = (() => {
+  if (!global.workers) {
+    global.workers = new Map<QueueNames, Worker>();
+  }
+  return global.workers;
+})();
+
+export function getQueue<DataType, ReturnType = void>(
+  name: QueueNames,
+  opts?: Omit<QueueOptions, 'connection'>
+): Queue<DataType, ReturnType> {
+  // check if global.queues has the queue
+  const queue = queues.get(name);
+  if (queue) {
+    return queue as Queue<DataType, ReturnType>;
+  }
+  const newQueue = new Queue<DataType, ReturnType>(name.toString(), {
+    connection: newQueueRedisConnection(),
+    ...opts
+  });
+
+  // default error handler, to avoid unhandled exceptions
+  newQueue.on('error', (error) => {
+    addLog.error(`MQ Queue [${name}]: ${error.message}`, error);
+  });
+  queues.set(name, newQueue);
+  return newQueue;
+}
+
+export function getWorker<DataType, ReturnType = void>(
+  name: QueueNames,
+  processor: Processor<DataType, ReturnType>,
+  opts?: Omit<WorkerOptions, 'connection'>
+): Worker<DataType, ReturnType> {
+  const worker = workers.get(name);
+  if (worker) {
+    return worker as Worker<DataType, ReturnType>;
+  }
+
+  const newWorker = new Worker<DataType, ReturnType>(name.toString(), processor, {
+    connection: newWorkerRedisConnection(),
+    ...defaultWorkerOpts,
+    ...opts
+  });
+  // default error handler, to avoid unhandled exceptions
+  newWorker.on('error', (error) => {
+    addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
+  });
+  workers.set(name, newWorker);
+  return newWorker;
+}

packages/service/common/bullmq/type.d.ts

@@ -0,0 +1,7 @@
+import { Queue, Worker } from 'bullmq';
+import { QueueNames } from './index';
+
+declare global {
+  var queues: Map<QueueNames, Queue> | undefined;
+  var workers: Map<QueueNames, Worker> | undefined;
+}

packages/service/common/redis/index.ts

@@ -0,0 +1,27 @@
+import Redis from 'ioredis';
+
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
+
+export function newQueueRedisConnection() {
+  const redis = new Redis(REDIS_URL);
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+}
+
+export function newWorkerRedisConnection() {
+  const redis = new Redis(REDIS_URL, {
+    maxRetriesPerRequest: null
+  });
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+}

packages/service/core/dataset/collection/controller.ts

@@ -1,6 +1,7 @@
 import {
   DatasetCollectionTypeEnum,
-  DatasetCollectionDataProcessModeEnum
+  DatasetCollectionDataProcessModeEnum,
+  DatasetTypeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
 import { MongoDatasetCollection } from './schema';
@@ -104,7 +105,8 @@ export const createCollectionAndInsertData = async ({
       hashRawText: hashStr(rawText),
       rawTextLength: rawText.length,
       nextSyncTime: (() => {
-        if (!dataset.autoSync) return undefined;
+        // ignore auto collections sync for website datasets
+        if (!dataset.autoSync && dataset.type === DatasetTypeEnum.websiteDataset) return undefined;
         if (
           [DatasetCollectionTypeEnum.link, DatasetCollectionTypeEnum.apiFile].includes(
             createCollectionParams.type

packages/service/core/dataset/collection/schema.ts

@@ -1,13 +1,8 @@
 import { connectionMongo, getMongoModel } from '../../../common/mongo';
-const { Schema, model, models } = connectionMongo;
+const { Schema } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import {
+import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
-  DatasetCollectionTypeMap,
+import { ChunkSettings, DatasetCollectionName } from '../schema';
-  DatasetCollectionDataProcessModeEnum,
-  ChunkSettingModeEnum,
-  DataChunkSplitModeEnum
-} from '@fastgpt/global/core/dataset/constants';
-import { DatasetCollectionName } from '../schema';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
@@ -90,25 +85,7 @@ const DatasetCollectionSchema = new Schema({
   customPdfParse: Boolean,
 
   // Chunk settings
-  imageIndex: Boolean,
+  ...ChunkSettings
-  autoIndexes: Boolean,
-  trainingType: {
-    type: String,
-    enum: Object.values(DatasetCollectionDataProcessModeEnum)
-  },
-  chunkSettingMode: {
-    type: String,
-    enum: Object.values(ChunkSettingModeEnum)
-  },
-  chunkSplitMode: {
-    type: String,
-    enum: Object.values(DataChunkSplitModeEnum)
-  },
-  chunkSize: Number,
-  chunkSplitter: String,
-
-  indexSize: Number,
-  qaPrompt: String
 });
 
 DatasetCollectionSchema.virtual('dataset', {

packages/service/core/dataset/controller.ts

@@ -9,6 +9,8 @@ import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 import { MongoDatasetDataText } from './data/dataTextSchema';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { retryFn } from '@fastgpt/global/common/system/utils';
+import { removeWebsiteSyncJobScheduler } from './websiteSync';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */

packages/service/core/dataset/schema.ts

@@ -1,7 +1,8 @@
 import { getMongoModel, Schema } from '../../common/mongo';
 import {
-  DatasetStatusEnum,
+  ChunkSettingModeEnum,
-  DatasetStatusMap,
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap
 } from '@fastgpt/global/core/dataset/constants';
@@ -13,6 +14,28 @@ import type { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
 
 export const DatasetCollectionName = 'datasets';
 
+export const ChunkSettings = {
+  imageIndex: Boolean,
+  autoIndexes: Boolean,
+  trainingType: {
+    type: String,
+    enum: Object.values(DatasetCollectionDataProcessModeEnum)
+  },
+  chunkSettingMode: {
+    type: String,
+    enum: Object.values(ChunkSettingModeEnum)
+  },
+  chunkSplitMode: {
+    type: String,
+    enum: Object.values(DataChunkSplitModeEnum)
+  },
+  chunkSize: Number,
+  chunkSplitter: String,
+
+  indexSize: Number,
+  qaPrompt: String
+};
+
 const DatasetSchema = new Schema({
   parentId: {
     type: Schema.Types.ObjectId,
@@ -40,11 +63,6 @@ const DatasetSchema = new Schema({
     required: true,
     default: DatasetTypeEnum.dataset
   },
-  status: {
-    type: String,
-    enum: Object.keys(DatasetStatusMap),
-    default: DatasetStatusEnum.active
-  },
   avatar: {
     type: String,
     default: '/icon/logo.svg'
@@ -84,6 +102,9 @@ const DatasetSchema = new Schema({
       }
     }
   },
+  chunkSettings: {
+    type: ChunkSettings
+  },
   inheritPermission: {
     type: Boolean,
     default: true
@@ -98,9 +119,8 @@ const DatasetSchema = new Schema({
     type: Object
   },
 
-  autoSync: Boolean,
-
   // abandoned
+  autoSync: Boolean,
   externalReadUrl: {
     type: String
   },

packages/service/core/dataset/websiteSync/index.ts

@@ -0,0 +1,80 @@
+import { Processor } from 'bullmq';
+import { getQueue, getWorker, QueueNames } from '../../../common/bullmq';
+import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants';
+
+export type WebsiteSyncJobData = {
+  datasetId: string;
+};
+
+export const websiteSyncQueue = getQueue<WebsiteSyncJobData>(QueueNames.websiteSync, {
+  defaultJobOptions: {
+    attempts: 3, // retry 3 times
+    backoff: {
+      type: 'exponential',
+      delay: 1000 // delay 1 second between retries
+    }
+  }
+});
+export const getWebsiteSyncWorker = (processor: Processor<WebsiteSyncJobData>) => {
+  return getWorker<WebsiteSyncJobData>(QueueNames.websiteSync, processor, {
+    removeOnFail: {
+      age: 15 * 24 * 60 * 60, // Keep up to 15 days
+      count: 1000 // Keep up to 1000 jobs
+    },
+    concurrency: 1 // Set worker to process only 1 job at a time
+  });
+};
+
+export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => {
+  const datasetId = String(data.datasetId);
+  // deduplication: make sure only 1 job
+  return websiteSyncQueue.add(datasetId, data, { deduplication: { id: datasetId } });
+};
+
+export const getWebsiteSyncDatasetStatus = async (datasetId: string) => {
+  const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId);
+  if (!jobId) {
+    return DatasetStatusEnum.active;
+  }
+  const job = await websiteSyncQueue.getJob(jobId);
+  if (!job) {
+    return DatasetStatusEnum.active;
+  }
+
+  const jobState = await job.getState();
+
+  if (['waiting-children', 'waiting'].includes(jobState)) {
+    return DatasetStatusEnum.waiting;
+  }
+  if (jobState === 'active') {
+    return DatasetStatusEnum.syncing;
+  }
+
+  return DatasetStatusEnum.active;
+};
+
+// Scheduler setting
+const repeatDuration = 24 * 60 * 60 * 1000; // every day
+export const upsertWebsiteSyncJobScheduler = (data: WebsiteSyncJobData, startDate?: number) => {
+  const datasetId = String(data.datasetId);
+
+  return websiteSyncQueue.upsertJobScheduler(
+    datasetId,
+    {
+      every: repeatDuration,
+      startDate: startDate || new Date().getTime() + repeatDuration // First run tomorrow
+    },
+    {
+      name: datasetId,
+      data
+    }
+  );
+};
+
+export const getWebsiteSyncJobScheduler = (datasetId: string) => {
+  return websiteSyncQueue.getJobScheduler(String(datasetId));
+};
+
+export const removeWebsiteSyncJobScheduler = (datasetId: string) => {
+  return websiteSyncQueue.removeJobScheduler(String(datasetId));
+};

packages/service/package.json

@@ -7,6 +7,7 @@
     "@xmldom/xmldom": "^0.8.10",
     "@zilliz/milvus2-sdk-node": "2.4.2",
     "axios": "^1.8.2",
+    "bullmq": "^5.44.0",
     "chalk": "^5.3.0",
     "cheerio": "1.0.0-rc.12",
     "cookie": "^0.7.1",
@@ -18,6 +19,7 @@
     "file-type": "^19.0.0",
     "form-data": "^4.0.0",
     "iconv-lite": "^0.6.3",
+    "ioredis": "^5.6.0",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "json5": "^2.2.3",
     "jsonpath-plus": "^10.3.0",

packages/web/components/common/Icon/icons/change.svg

@@ -1,9 +1,9 @@
 <svg viewBox="0 0 13 12" fill="none" xmlns="http://www.w3.org/2000/svg">
 <g id="icon/line/change">
 <g id="Vector">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M3.23479 4.71964C3.23479 4.4435 3.45864 4.21964 3.73479 4.21964L11.0348 4.21964C11.3109 4.21964 11.5348 4.4435 11.5348 4.71964C11.5348 4.99579 11.3109 5.21964 11.0348 5.21964L3.73479 5.21964C3.45864 5.21964 3.23479 4.99579 3.23479 4.71964Z" fill="#3370FF"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M3.23479 4.71964C3.23479 4.4435 3.45864 4.21964 3.73479 4.21964L11.0348 4.21964C11.3109 4.21964 11.5348 4.4435 11.5348 4.71964C11.5348 4.99579 11.3109 5.21964 11.0348 5.21964L3.73479 5.21964C3.45864 5.21964 3.23479 4.99579 3.23479 4.71964Z"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M8.70133 2.38619C8.89659 2.19093 9.21317 2.19093 9.40843 2.38619L11.3883 4.36609C11.5836 4.56135 11.5836 4.87794 11.3883 5.0732C11.1931 5.26846 10.8765 5.26846 10.6812 5.0732L8.70133 3.0933C8.50607 2.89804 8.50607 2.58145 8.70133 2.38619Z" fill="#3370FF"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.70133 2.38619C8.89659 2.19093 9.21317 2.19093 9.40843 2.38619L11.3883 4.36609C11.5836 4.56135 11.5836 4.87794 11.3883 5.0732C11.1931 5.26846 10.8765 5.26846 10.6812 5.0732L8.70133 3.0933C8.50607 2.89804 8.50607 2.58145 8.70133 2.38619Z"/>
-<path d="M1.84361 6.81774C1.78456 6.84214 1.72923 6.87834 1.68124 6.92633C1.63324 6.97433 1.59704 7.02965 1.57264 7.08871C1.54825 7.1476 1.53479 7.21217 1.53479 7.27989C1.53479 7.34768 1.54828 7.41232 1.57273 7.47128C1.59639 7.52847 1.63112 7.58215 1.67692 7.62907C1.67852 7.63071 1.68013 7.63234 1.68176 7.63396L3.66114 9.61334C3.8564 9.8086 4.17298 9.8086 4.36824 9.61334C4.5635 9.41808 4.5635 9.10149 4.36824 8.90623L3.2419 7.77989L9.33479 7.77989C9.61093 7.77989 9.83479 7.55603 9.83479 7.27989C9.83479 7.00374 9.61093 6.77989 9.33479 6.77989H2.03479C2.03325 6.77989 2.03171 6.77989 2.03017 6.77991C1.96414 6.7805 1.90117 6.7939 1.84361 6.81774Z" fill="#3370FF"/>
+<path d="M1.84361 6.81774C1.78456 6.84214 1.72923 6.87834 1.68124 6.92633C1.63324 6.97433 1.59704 7.02965 1.57264 7.08871C1.54825 7.1476 1.53479 7.21217 1.53479 7.27989C1.53479 7.34768 1.54828 7.41232 1.57273 7.47128C1.59639 7.52847 1.63112 7.58215 1.67692 7.62907C1.67852 7.63071 1.68013 7.63234 1.68176 7.63396L3.66114 9.61334C3.8564 9.8086 4.17298 9.8086 4.36824 9.61334C4.5635 9.41808 4.5635 9.10149 4.36824 8.90623L3.2419 7.77989L9.33479 7.77989C9.61093 7.77989 9.83479 7.55603 9.83479 7.27989C9.83479 7.00374 9.61093 6.77989 9.33479 6.77989H2.03479C2.03325 6.77989 2.03171 6.77989 2.03017 6.77991C1.96414 6.7805 1.90117 6.7939 1.84361 6.81774Z"/>
 </g>
 </g>
 </svg>

packages/web/components/common/Input/NumberInput/index.tsx

@@ -26,23 +26,43 @@ const MyNumberInput = (props: Props) => {
     <NumberInput
       {...restProps}
       onBlur={(e) => {
-        if (!onBlur) return;
         const numE = Number(e.target.value);
-        if (isNaN(numE)) {
+        if (onBlur) {
-          // @ts-ignore
+          if (isNaN(numE)) {
-          onBlur('');
+            // @ts-ignore
-        } else {
+            onBlur('');
-          onBlur(numE);
+          } else {
+            onBlur(numE);
+          }
+        }
+        if (register && name) {
+          const event = {
+            target: {
+              name,
+              value: numE
+            }
+          };
+          register(name).onBlur(event);
         }
       }}
       onChange={(e) => {
-        if (!onChange) return;
         const numE = Number(e);
-        if (isNaN(numE)) {
+        if (onChange) {
-          // @ts-ignore
+          if (isNaN(numE)) {
-          onChange('');
+            // @ts-ignore
-        } else {
+            onChange('');
-          onChange(numE);
+          } else {
+            onChange(numE);
+          }
+        }
+        if (register && name) {
+          const event = {
+            target: {
+              name,
+              value: numE
+            }
+          };
+          register(name).onChange(event);
         }
       }}
     >

packages/web/components/common/Tag/index.tsx

@@ -1,5 +1,5 @@
 import React, { useMemo } from 'react';
-import { Box, Flex, type FlexProps } from '@chakra-ui/react';
+import { Box, BoxProps, Flex, type FlexProps } from '@chakra-ui/react';
 
 type ColorSchemaType = 'white' | 'blue' | 'green' | 'red' | 'yellow' | 'gray' | 'purple' | 'adora';
 
@@ -8,6 +8,7 @@ export type TagProps = FlexProps & {
   colorSchema?: ColorSchemaType;
   type?: 'fill' | 'borderFill' | 'borderSolid';
   showDot?: boolean;
+  DotStyles?: BoxProps;
 };
 
 const colorMap: Record<
@@ -60,7 +61,14 @@ const colorMap: Record<
   }
 };
 
-const MyTag = ({ children, colorSchema = 'blue', type = 'fill', showDot, ...props }: TagProps) => {
+const MyTag = ({
+  children,
+  colorSchema = 'blue',
+  type = 'fill',
+  showDot,
+  DotStyles,
+  ...props
+}: TagProps) => {
   const theme = useMemo(() => {
     return colorMap[colorSchema];
   }, [colorSchema]);
@@ -81,7 +89,9 @@ const MyTag = ({ children, colorSchema = 'blue', type = 'fill', showDot, ...prop
       bg={type !== 'borderSolid' ? theme.bg : 'transparent'}
       {...props}
     >
-      {showDot && <Box w={1.5} h={1.5} borderRadius={'md'} bg={theme.color} mr={1.5}></Box>}
+      {showDot && (
+        <Box w={1.5} h={1.5} borderRadius={'md'} bg={theme.color} mr={1.5} {...DotStyles}></Box>
+      )}
       {children}
     </Flex>
   );

packages/web/i18n/en/common.json

@@ -512,7 +512,7 @@
   "core.dataset.Query extension intro": "Enabling the question optimization function can improve the accuracy of Dataset searches during continuous conversations. After enabling this function, when performing Dataset searches, the AI will complete the missing information of the question based on the conversation history.",
   "core.dataset.Quote Length": "Quote Content Length",
   "core.dataset.Read Dataset": "View Dataset Details",
-  "core.dataset.Set Website Config": "Start Configuring Website Information",
+  "core.dataset.Set Website Config": "Start Configuring",
   "core.dataset.Start export": "Export Started",
   "core.dataset.Table collection": "Table Dataset",
   "core.dataset.Text collection": "Text Dataset",
@@ -528,7 +528,6 @@
   "core.dataset.collection.Website Empty Tip": "No Website Associated Yet",
   "core.dataset.collection.Website Link": "Website Address",
   "core.dataset.collection.id": "Collection ID",
-  "core.dataset.collection.metadata.Chunk Size": "Chunk Size",
   "core.dataset.collection.metadata.Createtime": "Creation Time",
   "core.dataset.collection.metadata.Raw text length": "Raw Text Length",
   "core.dataset.collection.metadata.Updatetime": "Update Time",
@@ -630,6 +629,7 @@
   "core.dataset.search.search mode": "Search Method",
   "core.dataset.status.active": "Ready",
   "core.dataset.status.syncing": "Syncing",
+  "core.dataset.status.waiting": "Waiting",
   "core.dataset.test.Batch test": "Batch Test",
   "core.dataset.test.Batch test Placeholder": "Select a CSV File",
   "core.dataset.test.Search Test": "Search Test",

packages/web/i18n/en/dataset.json

@@ -7,6 +7,7 @@
   "auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.",
   "auto_training_queue": "Enhanced index queueing",
   "chunk_max_tokens": "max_tokens",
+  "chunk_size": "Block size",
   "close_auto_sync": "Are you sure you want to turn off automatic sync?",
   "collection.Create update time": "Creation/Update Time",
   "collection.Training type": "Training",
@@ -70,6 +71,7 @@
   "image_auto_parse": "Automatic image indexing",
   "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
   "image_training_queue": "Queue of image processing",
+  "immediate_sync": "Immediate Synchronization",
   "import.Auto mode Estimated Price Tips": "The text understanding model needs to be called, which requires more points: {{price}} points/1K tokens",
   "import.Embedding Estimated Price Tips": "Only use the index model and consume a small amount of AI points: {{price}} points/1K tokens",
   "import_confirm": "Confirm upload",
@@ -86,6 +88,7 @@
   "keep_image": "Keep the picture",
   "move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.",
   "open_auto_sync": "After scheduled synchronization is turned on, the system will try to synchronize the collection from time to time every day. During the collection synchronization period, the collection data will not be searched.",
+  "params_config": "Config",
   "params_setting": "Parameter settings",
   "pdf_enhance_parse": "PDF enhancement analysis",
   "pdf_enhance_parse_price": "{{price}} points/page",
@@ -144,6 +147,7 @@
   "vllm_model": "Image understanding model",
   "website_dataset": "Website Sync",
   "website_dataset_desc": "Website sync allows you to build a Dataset directly using a web link.",
+  "website_info": "Website Information",
   "yuque_dataset": "Yuque Dataset",
   "yuque_dataset_config": "Yuque Dataset Config",
   "yuque_dataset_desc": "Can build a dataset using Yuque documents by configuring permissions, without secondary storage"

packages/web/i18n/zh-CN/common.json

@@ -515,7 +515,7 @@
   "core.dataset.Query extension intro": "开启问题优化功能，可以提高提高连续对话时，知识库搜索的精度。开启该功能后，在进行知识库搜索时，会根据对话记录，利用 AI 补全问题缺失的信息。",
   "core.dataset.Quote Length": "引用内容长度",
   "core.dataset.Read Dataset": "查看知识库详情",
-  "core.dataset.Set Website Config": "开始配置网站信息",
+  "core.dataset.Set Website Config": "开始配置",
   "core.dataset.Start export": "已开始导出",
   "core.dataset.Table collection": "表格数据集",
   "core.dataset.Text collection": "文本数据集",
@@ -531,7 +531,6 @@
   "core.dataset.collection.Website Empty Tip": "还没有关联网站",
   "core.dataset.collection.Website Link": "Web 站点地址",
   "core.dataset.collection.id": "集合 ID",
-  "core.dataset.collection.metadata.Chunk Size": "分割大小",
   "core.dataset.collection.metadata.Createtime": "创建时间",
   "core.dataset.collection.metadata.Raw text length": "原文长度",
   "core.dataset.collection.metadata.Updatetime": "更新时间",
@@ -633,6 +632,7 @@
   "core.dataset.search.search mode": "搜索方式",
   "core.dataset.status.active": "已就绪",
   "core.dataset.status.syncing": "同步中",
+  "core.dataset.status.waiting": "排队中",
   "core.dataset.test.Batch test": "批量测试",
   "core.dataset.test.Batch test Placeholder": "选择一个 CSV 文件",
   "core.dataset.test.Search Test": "搜索测试",
@@ -1291,4 +1291,4 @@
   "yes": "是",
   "yesterday": "昨天",
   "yesterday_detail_time": "昨天 {{time}}"
-}
+}

packages/web/i18n/zh-CN/dataset.json

@@ -7,6 +7,7 @@
   "auto_indexes_tips": "通过大模型进行额外索引生成，提高语义丰富度，提高检索的精度。",
   "auto_training_queue": "增强索引排队",
   "chunk_max_tokens": "分块上限",
+  "chunk_size": "分块大小",
   "close_auto_sync": "确认关闭自动同步功能？",
   "collection.Create update time": "创建/更新时间",
   "collection.Training type": "训练模式",
@@ -70,6 +71,7 @@
   "image_auto_parse": "图片自动索引",
   "image_auto_parse_tips": "调用 VLM 自动标注文档里的图片，并生成额外的检索索引",
   "image_training_queue": "图片处理排队",
+  "immediate_sync": "立即同步",
   "import.Auto mode Estimated Price Tips": "需调用文本理解模型，需要消耗较多AI 积分：{{price}} 积分/1K tokens",
   "import.Embedding Estimated Price Tips": "仅使用索引模型，消耗少量 AI 积分：{{price}} 积分/1K tokens",
   "import_confirm": "确认上传",
@@ -86,6 +88,7 @@
   "keep_image": "保留图片",
   "move.hint": "移动后，所选知识库/文件夹将继承新文件夹的权限设置，原先的权限设置失效。",
   "open_auto_sync": "开启定时同步后，系统将会每天不定时尝试同步集合，集合同步期间，会出现无法搜索到该集合数据现象。",
+  "params_config": "配置",
   "params_setting": "参数设置",
   "pdf_enhance_parse": "PDF增强解析",
   "pdf_enhance_parse_price": "{{price}}积分/页",
@@ -145,6 +148,7 @@
   "vllm_model": "图片理解模型",
   "website_dataset": "Web 站点同步",
   "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
+  "website_info": "网站信息",
   "yuque_dataset": "语雀知识库",
   "yuque_dataset_config": "配置语雀知识库",
   "yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储"

packages/web/i18n/zh-Hant/common.json

@@ -511,7 +511,7 @@
   "core.dataset.Query extension intro": "開啟問題最佳化功能，可以提高連續對話時知識庫搜尋的準確度。開啟此功能後，在進行知識庫搜尋時，系統會根據對話記錄，利用 AI 補充問題中缺少的資訊。",
   "core.dataset.Quote Length": "引用內容長度",
   "core.dataset.Read Dataset": "檢視知識庫詳細資料",
-  "core.dataset.Set Website Config": "開始設定網站資訊",
+  "core.dataset.Set Website Config": "開始設定",
   "core.dataset.Start export": "已開始匯出",
   "core.dataset.Table collection": "表格資料集",
   "core.dataset.Text collection": "文字資料集",
@@ -527,7 +527,6 @@
   "core.dataset.collection.Website Empty Tip": "還沒有關聯網站",
   "core.dataset.collection.Website Link": "網站網址",
   "core.dataset.collection.id": "集合 ID",
-  "core.dataset.collection.metadata.Chunk Size": "分割大小",
   "core.dataset.collection.metadata.Createtime": "建立時間",
   "core.dataset.collection.metadata.Raw text length": "原始文字長度",
   "core.dataset.collection.metadata.Updatetime": "更新時間",
@@ -629,6 +628,7 @@
   "core.dataset.search.search mode": "搜索方式",
   "core.dataset.status.active": "已就緒",
   "core.dataset.status.syncing": "同步中",
+  "core.dataset.status.waiting": "排队中",
   "core.dataset.test.Batch test": "批次測試",
   "core.dataset.test.Batch test Placeholder": "選擇一個 CSV 檔案",
   "core.dataset.test.Search Test": "搜尋測試",

packages/web/i18n/zh-Hant/dataset.json

@@ -7,6 +7,7 @@
   "auto_indexes_tips": "通過大模型進行額外索引生成，提高語義豐富度，提高檢索的精度。",
   "auto_training_queue": "增強索引排隊",
   "chunk_max_tokens": "分塊上限",
+  "chunk_size": "分塊大小",
   "close_auto_sync": "確認關閉自動同步功能？",
   "collection.Create update time": "建立／更新時間",
   "collection.Training type": "分段模式",
@@ -70,6 +71,7 @@
   "image_auto_parse": "圖片自動索引",
   "image_auto_parse_tips": "調用 VLM 自動標註文檔裡的圖片，並生成額外的檢索索引",
   "image_training_queue": "圖片處理排隊",
+  "immediate_sync": "立即同步",
   "import.Auto mode Estimated Price Tips": "需呼叫文字理解模型，將消耗較多 AI 點數：{{price}} 點數 / 1K tokens",
   "import.Embedding Estimated Price Tips": "僅使用索引模型，消耗少量 AI 點數：{{price}} 點數 / 1K tokens",
   "import_confirm": "確認上傳",
@@ -86,6 +88,7 @@
   "keep_image": "保留圖片",
   "move.hint": "移動後，所選資料集／資料夾將繼承新資料夾的權限設定，原先的權限設定將失效。",
   "open_auto_sync": "開啟定時同步後，系統將每天不定時嘗試同步集合，集合同步期間，會出現無法搜尋到該集合資料現象。",
+  "params_config": "配置",
   "params_setting": "參數設置",
   "pdf_enhance_parse": "PDF增強解析",
   "pdf_enhance_parse_price": "{{price}}積分/頁",
@@ -144,6 +147,7 @@
   "vllm_model": "圖片理解模型",
   "website_dataset": "網站同步",
   "website_dataset_desc": "網站同步功能讓您可以直接使用網頁連結建立資料集",
+  "website_info": "網站資訊",
   "yuque_dataset": "語雀知識庫",
   "yuque_dataset_config": "配置語雀知識庫",
   "yuque_dataset_desc": "可通過配置語雀文檔權限，使用語雀文檔構建知識庫，文檔不會進行二次存儲"

pnpm-lock.yaml

@@ -169,6 +169,9 @@ importers:
       axios:
         specifier: ^1.8.2
         version: 1.8.3
+      bullmq:
+        specifier: ^5.44.0
+        version: 5.44.0
       chalk:
         specifier: ^5.3.0
         version: 5.4.1
@@ -202,6 +205,9 @@ importers:
       iconv-lite:
         specifier: ^0.6.3
         version: 0.6.3
+      ioredis:
+        specifier: ^5.6.0
+        version: 5.6.0
       joplin-turndown-plugin-gfm:
         specifier: ^1.0.12
         version: 1.0.12
@@ -2044,6 +2050,9 @@ packages:
     resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==}
     deprecated: Use @eslint/object-schema instead
 
+  '@ioredis/commands@1.2.0':
+    resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
+
   '@isaacs/cliui@8.0.2':
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
@@ -2314,6 +2323,36 @@ packages:
   '@mongodb-js/saslprep@1.2.0':
     resolution: {integrity: sha512-+ywrb0AqkfaYuhHs6LxKWgqbh3I72EpEgESCw37o+9qPx9WTCkgDm2B+eMrwehGtHBWHFU4GXvnSCNiFhhausg==}
 
+  '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3':
+    resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==}
+    cpu: [arm64]
+    os: [darwin]
+
+  '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3':
+    resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==}
+    cpu: [x64]
+    os: [darwin]
+
+  '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3':
+    resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==}
+    cpu: [arm64]
+    os: [linux]
+
+  '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3':
+    resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==}
+    cpu: [arm]
+    os: [linux]
+
+  '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3':
+    resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==}
+    cpu: [x64]
+    os: [linux]
+
+  '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3':
+    resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==}
+    cpu: [x64]
+    os: [win32]
+
   '@napi-rs/wasm-runtime@0.2.7':
     resolution: {integrity: sha512-5yximcFK5FNompXfJFoWanu5l8v1hNGqNHh9du1xETp9HWk/B/PzvchX55WYOPaIeNglG8++68AAiauBAtbnzw==}
 
@@ -4014,6 +4053,9 @@ packages:
   buffer@6.0.3:
     resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
 
+  bullmq@5.44.0:
+    resolution: {integrity: sha512-OnEtkuXyrUx2Jm5BpH92+ttrobblBdCbkhOe3OoR0hxZuAilI3mPWlwELslhfImRpDv8rK+C/0/VK7I8f3xIig==}
+
   bundle-n-require@1.1.2:
     resolution: {integrity: sha512-bEk2jakVK1ytnZ9R2AAiZEeK/GxPUM8jvcRxHZXifZDMcjkI4EG/GlsJ2YGSVYT9y/p/gA9/0yDY8rCGsSU6Tg==}
 
@@ -4248,6 +4290,10 @@ packages:
     resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==}
     engines: {node: '>=6'}
 
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
   co@4.6.0:
     resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
     engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
@@ -5860,6 +5906,10 @@ packages:
   intersection-observer@0.12.2:
     resolution: {integrity: sha512-7m1vEcPCxXYI8HqnL8CKI6siDyD+eIWSwgB3DZA+ZTogxk9I4CDnj4wilt9x/+/QbHI4YG5YZNmC6458/e9Ktg==}
 
+  ioredis@5.6.0:
+    resolution: {integrity: sha512-tBZlIIWbndeWBWCXWZiqtOF/yxf6yZX3tAlTJ7nfo5jhd6dctNxF7QnYlZLZ1a0o0pDoen7CgZqO+zjNaFbJAg==}
+    engines: {node: '>=12.22.0'}
+
   ip-address@9.0.5:
     resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==}
     engines: {node: '>= 12'}
@@ -6554,9 +6604,15 @@ packages:
   lodash.debounce@4.0.8:
     resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==}
 
+  lodash.defaults@4.2.0:
+    resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
+
   lodash.includes@4.3.0:
     resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
 
+  lodash.isarguments@3.1.0:
+    resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
+
   lodash.isboolean@3.0.3:
     resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
 
@@ -7128,6 +7184,13 @@ packages:
   ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
 
+  msgpackr-extract@3.0.3:
+    resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==}
+    hasBin: true
+
+  msgpackr@1.11.2:
+    resolution: {integrity: sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==}
+
   mssql@11.0.1:
     resolution: {integrity: sha512-KlGNsugoT90enKlR8/G36H0kTxPthDhmtNUCwEHvgRza5Cjpjoj+P2X6eMpFUDN7pFrJZsKadL4x990G8RBE1w==}
     engines: {node: '>=18'}
@@ -7260,6 +7323,10 @@ packages:
       encoding:
         optional: true
 
+  node-gyp-build-optional-packages@5.2.2:
+    resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==}
+    hasBin: true
+
   node-gyp@10.3.1:
     resolution: {integrity: sha512-Pp3nFHBThHzVtNY7U6JfPjvT/DTE8+o/4xKsLQtBoU+j2HLsGlhcfzflAoUreaJbNmYnX+LlLi0qjV8kpyO6xQ==}
     engines: {node: ^16.14.0 || >=18.0.0}
@@ -8041,6 +8108,14 @@ packages:
       react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
       react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
 
+  redis-errors@1.2.0:
+    resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
+    engines: {node: '>=4'}
+
+  redis-parser@3.0.0:
+    resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
+    engines: {node: '>=4'}
+
   redux@4.2.1:
     resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==}
 
@@ -8490,6 +8565,9 @@ packages:
   stackback@0.0.2:
     resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
 
+  standard-as-callback@2.1.0:
+    resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
+
   state-local@1.0.7:
     resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==}
 
@@ -11160,6 +11238,8 @@ snapshots:
 
   '@humanwhocodes/object-schema@2.0.3': {}
 
+  '@ioredis/commands@1.2.0': {}
+
   '@isaacs/cliui@8.0.2':
     dependencies:
       string-width: 5.1.2
@@ -11565,6 +11645,24 @@ snapshots:
     dependencies:
       sparse-bitfield: 3.0.3
 
+  '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3':
+    optional: true
+
+  '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3':
+    optional: true
+
+  '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3':
+    optional: true
+
+  '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3':
+    optional: true
+
+  '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3':
+    optional: true
+
+  '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3':
+    optional: true
+
   '@napi-rs/wasm-runtime@0.2.7':
     dependencies:
       '@emnapi/core': 1.3.1
@@ -13456,6 +13554,18 @@ snapshots:
       base64-js: 1.5.1
       ieee754: 1.2.1
 
+  bullmq@5.44.0:
+    dependencies:
+      cron-parser: 4.9.0
+      ioredis: 5.6.0
+      msgpackr: 1.11.2
+      node-abort-controller: 3.1.1
+      semver: 7.7.1
+      tslib: 2.8.1
+      uuid: 9.0.1
+    transitivePeerDependencies:
+      - supports-color
+
   bundle-n-require@1.1.2:
     dependencies:
       esbuild: 0.25.1
@@ -13713,6 +13823,8 @@ snapshots:
 
   clsx@2.1.1: {}
 
+  cluster-key-slot@1.1.2: {}
+
   co@4.6.0: {}
 
   collapse-white-space@1.0.6: {}
@@ -14626,7 +14738,7 @@ snapshots:
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.2)
       eslint: 8.56.0
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0)
+      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0)
       eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0)
       eslint-plugin-jsx-a11y: 6.10.2(eslint@8.56.0)
       eslint-plugin-react: 7.37.4(eslint@8.56.0)
@@ -14646,7 +14758,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0):
+  eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0):
     dependencies:
       '@nolyfill/is-core-module': 1.0.39
       debug: 4.4.0
@@ -14661,14 +14773,14 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0))(eslint@8.56.0):
+  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.2)
       eslint: 8.56.0
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0)
+      eslint-import-resolver-typescript: 3.9.0(eslint-plugin-import@2.31.0)(eslint@8.56.0)
     transitivePeerDependencies:
       - supports-color
 
@@ -14683,7 +14795,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 8.56.0
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint@8.56.0))(eslint@8.56.0))(eslint@8.56.0)
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.9.0)(eslint@8.56.0)
       hasown: 2.0.2
       is-core-module: 2.16.1
       is-glob: 4.0.3
@@ -15692,6 +15804,20 @@ snapshots:
 
   intersection-observer@0.12.2: {}
 
+  ioredis@5.6.0:
+    dependencies:
+      '@ioredis/commands': 1.2.0
+      cluster-key-slot: 1.1.2
+      debug: 4.4.0
+      denque: 2.1.0
+      lodash.defaults: 4.2.0
+      lodash.isarguments: 3.1.0
+      redis-errors: 1.2.0
+      redis-parser: 3.0.0
+      standard-as-callback: 2.1.0
+    transitivePeerDependencies:
+      - supports-color
+
   ip-address@9.0.5:
     dependencies:
       jsbn: 1.1.0
@@ -16558,8 +16684,12 @@ snapshots:
 
   lodash.debounce@4.0.8: {}
 
+  lodash.defaults@4.2.0: {}
+
   lodash.includes@4.3.0: {}
 
+  lodash.isarguments@3.1.0: {}
+
   lodash.isboolean@3.0.3: {}
 
   lodash.isinteger@4.0.4: {}
@@ -17481,6 +17611,22 @@ snapshots:
 
   ms@2.1.3: {}
 
+  msgpackr-extract@3.0.3:
+    dependencies:
+      node-gyp-build-optional-packages: 5.2.2
+    optionalDependencies:
+      '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3
+      '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3
+      '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3
+      '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3
+      '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3
+      '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3
+    optional: true
+
+  msgpackr@1.11.2:
+    optionalDependencies:
+      msgpackr-extract: 3.0.3
+
   mssql@11.0.1:
     dependencies:
       '@tediousjs/connection-string': 0.5.0
@@ -17624,6 +17770,11 @@ snapshots:
     optionalDependencies:
       encoding: 0.1.13
 
+  node-gyp-build-optional-packages@5.2.2:
+    dependencies:
+      detect-libc: 2.0.3
+    optional: true
+
   node-gyp@10.3.1:
     dependencies:
       env-paths: 2.2.1
@@ -18499,6 +18650,12 @@ snapshots:
       tiny-invariant: 1.3.3
       victory-vendor: 36.9.2
 
+  redis-errors@1.2.0: {}
+
+  redis-parser@3.0.0:
+    dependencies:
+      redis-errors: 1.2.0
+
   redux@4.2.1:
     dependencies:
       '@babel/runtime': 7.26.10
@@ -19048,6 +19205,8 @@ snapshots:
 
   stackback@0.0.2: {}
 
+  standard-as-callback@2.1.0: {}
+
   state-local@1.0.7: {}
 
   state-toggle@1.0.3: {}

projects/app/.env.template

@@ -20,6 +20,8 @@ AIPROXY_API_TOKEN=xxxxx
 # 强制将图片转成 base64 传递给模型
 MULTIPLE_DATA_TO_BASE64=true
 
+# Redis URL
+REDIS_URL=redis://default:password@127.0.0.1:6379
 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。
 MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin
 
@@ -65,4 +67,4 @@ CHECK_INTERNAL_IP=false
 # # 日志来源ID前缀
 # CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-
 # 自定义跨域，不配置时，默认都允许跨域（逗号分割）
-ALLOWED_ORIGINS=
+ALLOWED_ORIGINS=

projects/app/src/instrumentation.ts

@@ -1,6 +1,6 @@
 import { exit } from 'process';
 
-/* 
+/*
   Init system
 */
 export async function register() {

projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx

@@ -1,19 +1,18 @@
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
-import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react';
+import { Dispatch, ReactNode, SetStateAction, useState } from 'react';
 import { useTranslation } from 'next-i18next';
 import { createContext, useContextSelector } from 'use-context-selector';
-import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
-import { useRequest } from '@fastgpt/web/hooks/useRequest';
+import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { useDisclosure } from '@chakra-ui/react';
 import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
-import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
 import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api';
 import dynamic from 'next/dynamic';
 import { usePagination } from '@fastgpt/web/hooks/usePagination';
 import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
 import { useRouter } from 'next/router';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { WebsiteConfigFormType } from './WebsiteConfig';
 
 const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig'));
 
@@ -66,7 +65,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   const router = useRouter();
   const { parentId = '' } = router.query as { parentId: string };
 
-  const { datasetDetail, datasetId, updateDataset } = useContextSelector(
+  const { datasetDetail, datasetId, updateDataset, loadDatasetDetail } = useContextSelector(
     DatasetPageContext,
     (v) => v
   );
@@ -75,30 +74,31 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({
     content: t('dataset:start_sync_website_tip')
   });
+  const syncWebsite = async () => {
+    await checkTeamWebSyncLimit();
+    await postWebsiteSync({ datasetId: datasetId });
+    await loadDatasetDetail(datasetId);
+  };
   const {
     isOpen: isOpenWebsiteModal,
     onOpen: onOpenWebsiteModal,
     onClose: onCloseWebsiteModal
   } = useDisclosure();
-  const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
+  const { runAsync: onUpdateDatasetWebsiteConfig } = useRequest2(
-    mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
+    async (websiteConfig: WebsiteConfigFormType) => {
-      onCloseWebsiteModal();
-      await checkTeamWebSyncLimit();
       await updateDataset({
         id: datasetId,
-        websiteConfig,
+        websiteConfig: websiteConfig.websiteConfig,
-        status: DatasetStatusEnum.syncing
+        chunkSettings: websiteConfig.chunkSettings
       });
-      const billId = await postCreateTrainingUsage({
+      await syncWebsite();
-        name: t('common:core.dataset.training.Website Sync'),
-        datasetId: datasetId
-      });
-      await postWebsiteSync({ datasetId: datasetId, billId });
-
-      return;
     },
-    errorToast: t('common:common.Update Failed')
+    {
-  });
+      onSuccess() {
+        onCloseWebsiteModal();
+      }
+    }
+  );
 
   // collection list
   const [searchText, setSearchText] = useState('');
@@ -124,7 +124,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   });
 
   const contextValue: CollectionPageContextType = {
-    openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig),
+    openWebSyncConfirm: openWebSyncConfirm(syncWebsite),
     onOpenWebsiteModal,
 
     searchText,
@@ -149,10 +149,6 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
             <WebSiteConfigModal
               onClose={onCloseWebsiteModal}
               onSuccess={onUpdateDatasetWebsiteConfig}
-              defaultValue={{
-                url: datasetDetail?.websiteConfig?.url,
-                selector: datasetDetail?.websiteConfig?.selector
-              }}
             />
           )}
           <ConfirmWebSyncModal />

projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx

@@ -25,6 +25,9 @@ const EmptyCollectionTip = () => {
               {datasetDetail.status === DatasetStatusEnum.syncing && (
                 <>{t('common:core.dataset.status.syncing')}</>
               )}
+              {datasetDetail.status === DatasetStatusEnum.waiting && (
+                <>{t('common:core.dataset.status.waiting')}</>
+              )}
               {datasetDetail.status === DatasetStatusEnum.active && (
                 <>
                   {!datasetDetail?.websiteConfig?.url ? (

projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx

@@ -1,35 +1,23 @@
 import React from 'react';
-import {
+import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react';
-  Box,
-  Flex,
-  MenuButton,
-  Button,
-  Link,
-  useTheme,
-  useDisclosure,
-  HStack
-} from '@chakra-ui/react';
 import {
   getDatasetCollectionPathById,
   postDatasetCollection,
   putDatasetCollectionById
 } from '@/web/core/dataset/api';
-import { useQuery } from '@tanstack/react-query';
 import { useTranslation } from 'next-i18next';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyInput from '@/components/MyInput';
-import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { useRouter } from 'next/router';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyMenu from '@fastgpt/web/components/common/MyMenu';
 import { useEditTitle } from '@/web/common/hooks/useEditTitle';
 import {
   DatasetCollectionTypeEnum,
-  TrainingModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap,
-  DatasetStatusEnum,
+  DatasetStatusEnum
-  DatasetCollectionDataProcessModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import EditFolderModal, { useEditFolder } from '../../EditFolderModal';
 import { TabEnum } from '../../../../pages/dataset/detail/index';
@@ -43,26 +31,35 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import HeaderTagPopOver from './HeaderTagPopOver';
 import MyBox from '@fastgpt/web/components/common/MyBox';
+import Icon from '@fastgpt/web/components/common/Icon';
+import MyTag from '@fastgpt/web/components/common/Tag/index';
 
 const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
 
 const Header = ({}: {}) => {
   const { t } = useTranslation();
-  const theme = useTheme();
-
   const { feConfigs } = useSystemStore();
+  const { isPc } = useSystem();
+
   const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
 
   const router = useRouter();
   const { parentId = '' } = router.query as { parentId: string };
-  const { isPc } = useSystem();
 
-  const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } =
+  const {
-    useContextSelector(CollectionPageContext, (v) => v);
+    searchText,
+    setSearchText,
+    total,
+    getData,
+    pageNum,
+    onOpenWebsiteModal,
+    openWebSyncConfirm
+  } = useContextSelector(CollectionPageContext, (v) => v);
 
-  const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
+  const { data: paths = [] } = useRequest2(() => getDatasetCollectionPathById(parentId), {
-    getDatasetCollectionPathById(parentId)
+    refreshDeps: [parentId],
-  );
+    manual: false
+  });
 
   const { editFolderData, setEditFolderData } = useEditFolder();
   const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
@@ -72,13 +69,14 @@ const Header = ({}: {}) => {
       canEmpty: false
     });
 
+  // Import collection
   const {
     isOpen: isOpenFileSourceSelector,
     onOpen: onOpenFileSourceSelector,
     onClose: onCloseFileSourceSelector
   } = useDisclosure();
 
-  const { runAsync: onCreateCollection, loading: onCreating } = useRequest2(
+  const { runAsync: onCreateCollection } = useRequest2(
     async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => {
       const id = await postDatasetCollection({
         parentId,
@@ -100,7 +98,7 @@ const Header = ({}: {}) => {
   const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset;
 
   return (
-    <MyBox isLoading={onCreating} display={['block', 'flex']} alignItems={'center'} gap={2}>
+    <MyBox display={['block', 'flex']} alignItems={'center'} gap={2}>
       <HStack flex={1}>
         <Box flex={1} fontWeight={'500'} color={'myGray.900'} whiteSpace={'nowrap'}>
           <ParentPath
@@ -121,13 +119,15 @@ const Header = ({}: {}) => {
                   {!isWebSite && <MyIcon name="common/list" mr={2} w={'20px'} color={'black'} />}
                   {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel as any)}({total})
                 </Flex>
+                {/* Website sync */}
                 {datasetDetail?.websiteConfig?.url && (
                   <Flex fontSize={'mini'}>
-                    {t('common:core.dataset.website.Base Url')}:
+                    <Box>{t('common:core.dataset.website.Base Url')}:</Box>
                     <Link
+                      className="textEllipsis"
+                      maxW={'300px'}
                       href={datasetDetail.websiteConfig.url}
                       target="_blank"
-                      mr={2}
                       color={'blue.700'}
                     >
                       {datasetDetail.websiteConfig.url}
@@ -171,12 +171,14 @@ const Header = ({}: {}) => {
         )}
 
         {/* Tag */}
-        {datasetDetail.permission.hasWritePer && feConfigs?.isPlus && <HeaderTagPopOver />}
+        {datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
+          datasetDetail.permission.hasWritePer &&
+          feConfigs?.isPlus && <HeaderTagPopOver />}
       </HStack>
 
       {/* diff collection button */}
       {datasetDetail.permission.hasWritePer && (
-        <Box textAlign={'end'} mt={[3, 0]}>
+        <Box mt={[3, 0]}>
           {datasetDetail?.type === DatasetTypeEnum.dataset && (
             <MyMenu
               offset={[0, 5]}
@@ -233,9 +235,8 @@ const Header = ({}: {}) => {
                       onClick: () => {
                         onOpenCreateVirtualFileModal({
                           defaultVal: '',
-                          onSuccess: (name) => {
+                          onSuccess: (name) =>
-                            onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
+                            onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual })
-                          }
                         });
                       }
                     },
@@ -272,35 +273,60 @@ const Header = ({}: {}) => {
           {datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
             <>
               {datasetDetail?.websiteConfig?.url ? (
-                <Flex alignItems={'center'}>
+                <>
                   {datasetDetail.status === DatasetStatusEnum.active && (
-                    <Button onClick={onOpenWebsiteModal}>{t('common:common.Config')}</Button>
+                    <HStack gap={2}>
+                      <Button
+                        onClick={onOpenWebsiteModal}
+                        leftIcon={<Icon name="change" w={'1rem'} />}
+                      >
+                        {t('dataset:params_config')}
+                      </Button>
+                      <Button
+                        variant={'whitePrimary'}
+                        onClick={openWebSyncConfirm}
+                        leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
+                      >
+                        {t('dataset:immediate_sync')}
+                      </Button>
+                    </HStack>
                   )}
                   {datasetDetail.status === DatasetStatusEnum.syncing && (
-                    <Flex
+                    <MyTag
-                      ml={3}
+                      colorSchema="purple"
-                      alignItems={'center'}
+                      showDot
                       px={3}
-                      py={1}
+                      h={'36px'}
-                      borderRadius="md"
+                      DotStyles={{
-                      border={theme.borders.base}
+                        w: '8px',
+                        h: '8px',
+                        animation: 'zoomStopIcon 0.5s infinite alternate'
+                      }}
                     >
-                      <Box
+                      {t('common:core.dataset.status.syncing')}
-                        animation={'zoomStopIcon 0.5s infinite alternate'}
+                    </MyTag>
-                        bg={'myGray.700'}
-                        w="8px"
-                        h="8px"
-                        borderRadius={'50%'}
-                        mt={'1px'}
-                      ></Box>
-                      <Box ml={2} color={'myGray.600'}>
-                        {t('common:core.dataset.status.syncing')}
-                      </Box>
-                    </Flex>
                   )}
-                </Flex>
+                  {datasetDetail.status === DatasetStatusEnum.waiting && (
+                    <MyTag
+                      colorSchema="gray"
+                      showDot
+                      px={3}
+                      h={'36px'}
+                      DotStyles={{
+                        w: '8px',
+                        h: '8px',
+                        animation: 'zoomStopIcon 0.5s infinite alternate'
+                      }}
+                    >
+                      {t('common:core.dataset.status.waiting')}
+                    </MyTag>
+                  )}
+                </>
               ) : (
-                <Button onClick={onOpenWebsiteModal}>
+                <Button
+                  onClick={onOpenWebsiteModal}
+                  leftIcon={<Icon name="common/setting" w={'18px'} />}
+                >
                   {t('common:core.dataset.Set Website Config')}
                 </Button>
               )}

projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx

@@ -1,110 +1,215 @@
-import React from 'react';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import { useTranslation } from 'next-i18next';
-import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react';
 import { strIsLink } from '@fastgpt/global/common/string/tools';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useForm } from 'react-hook-form';
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
 import { getDocPath } from '@/web/common/system/doc';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { useMyStep } from '@fastgpt/web/hooks/useStep';
+import MyDivider from '@fastgpt/web/components/common/MyDivider';
+import React, { useRef } from 'react';
+import {
+  Box,
+  Link,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  Stack
+} from '@chakra-ui/react';
+import {
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum
+} from '@fastgpt/global/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
+import { useContextSelector } from 'use-context-selector';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import CollectionChunkForm, {
+  collectionChunkForm2StoreChunkData,
+  type CollectionChunkFormType
+} from '../Form/CollectionChunkForm';
+import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils';
+import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
 
-type FormType = {
+export type WebsiteConfigFormType = {
-  url?: string | undefined;
+  websiteConfig: {
-  selector?: string | undefined;
+    url: string;
+    selector: string;
+  };
+  chunkSettings: ChunkSettingsType;
 };
 
 const WebsiteConfigModal = ({
   onClose,
-  onSuccess,
+  onSuccess
-  defaultValue = {
-    url: '',
-    selector: ''
-  }
 }: {
   onClose: () => void;
-  onSuccess: (data: FormType) => void;
+  onSuccess: (data: WebsiteConfigFormType) => void;
-  defaultValue?: FormType;
 }) => {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
   const { toast } = useToast();
-  const { register, handleSubmit } = useForm({
+  const steps = [
-    defaultValues: defaultValue
+    {
+      title: t('dataset:website_info')
+    },
+    {
+      title: t('dataset:params_config')
+    }
+  ];
+
+  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
+  const websiteConfig = datasetDetail.websiteConfig;
+  const chunkSettings = datasetDetail.chunkSettings;
+
+  const {
+    register: websiteInfoForm,
+    handleSubmit: websiteInfoHandleSubmit,
+    getValues: websiteInfoGetValues
+  } = useForm({
+    defaultValues: {
+      url: websiteConfig?.url || '',
+      selector: websiteConfig?.selector || ''
+    }
   });
-  const isEdit = !!defaultValue.url;
+
-  const confirmTip = isEdit
+  const isEdit = !!websiteConfig?.url;
-    ? t('common:core.dataset.website.Confirm Update Tips')
-    : t('common:core.dataset.website.Confirm Create Tips');
 
   const { ConfirmModal, openConfirm } = useConfirm({
     type: 'common'
   });
 
+  const { activeStep, goToPrevious, goToNext, MyStep } = useMyStep({
+    defaultStep: 0,
+    steps
+  });
+
+  const form = useForm<CollectionChunkFormType>({
+    defaultValues: {
+      trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk,
+      imageIndex: chunkSettings?.imageIndex || false,
+      autoIndexes: chunkSettings?.autoIndexes || false,
+
+      chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto,
+      chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size,
+      embeddingChunkSize: chunkSettings?.chunkSize || 2000,
+      qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel),
+      indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512,
+
+      chunkSplitter: chunkSettings?.chunkSplitter || '',
+      qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description
+    }
+  });
+
   return (
     <MyModal
       isOpen
       iconSrc="core/dataset/websiteDataset"
       title={t('common:core.dataset.website.Config')}
       onClose={onClose}
-      maxW={'500px'}
+      w={'550px'}
     >
-      <ModalBody>
+      <ModalBody w={'full'}>
-        <Box fontSize={'sm'} color={'myGray.600'}>
+        <Stack w={'75%'} marginX={'auto'}>
-          {t('common:core.dataset.website.Config Description')}
+          <MyStep />
-          {feConfigs?.docUrl && (
+        </Stack>
-            <Link
+        <MyDivider />
-              href={getDocPath('/docs/guide/knowledge_base/websync/')}
+        {activeStep == 0 && (
-              target="_blank"
+          <>
-              textDecoration={'underline'}
+            <Box
-              fontWeight={'bold'}
+              fontSize={'xs'}
+              color={'myGray.900'}
+              bgColor={'blue.50'}
+              padding={'4'}
+              borderRadius={'8px'}
             >
-              {t('common:common.course.Read Course')}
+              {t('common:core.dataset.website.Config Description')}
-            </Link>
+              {feConfigs?.docUrl && (
-          )}
+                <Link
-        </Box>
+                  href={getDocPath('/docs/guide/knowledge_base/websync/')}
-        <Box mt={2}>
+                  target="_blank"
-          <Box>{t('common:core.dataset.website.Base Url')}</Box>
+                  textDecoration={'underline'}
-          <Input
+                  color={'blue.700'}
-            placeholder={t('common:core.dataset.collection.Website Link')}
+                >
-            {...register('url', {
+                  {t('common:common.course.Read Course')}
-              required: true
+                </Link>
-            })}
+              )}
-          />
+            </Box>
-        </Box>
+            <Box mt={2}>
-        <Box mt={3}>
+              <Box>{t('common:core.dataset.website.Base Url')}</Box>
-          <Box>
+              <Input
-            {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
+                placeholder={t('common:core.dataset.collection.Website Link')}
-          </Box>
+                {...websiteInfoForm('url', {
-          <Input {...register('selector')} placeholder="body .content #document" />
+                  required: true
-        </Box>
+                })}
+              />
+            </Box>
+            <Box mt={3}>
+              <Box>
+                {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
+              </Box>
+              <Input {...websiteInfoForm('selector')} placeholder="body .content #document" />
+            </Box>
+          </>
+        )}
+        {activeStep == 1 && <CollectionChunkForm form={form} />}
       </ModalBody>
       <ModalFooter>
-        <Button variant={'whiteBase'} onClick={onClose}>
+        {activeStep == 0 && (
-          {t('common:common.Close')}
+          <>
-        </Button>
+            <Button variant={'whiteBase'} onClick={onClose}>
-        <Button
+              {t('common:common.Close')}
-          ml={2}
+            </Button>
-          onClick={handleSubmit((data) => {
+            <Button
-            if (!data.url) return;
+              ml={2}
-            // check is link
+              onClick={websiteInfoHandleSubmit((data) => {
-            if (!strIsLink(data.url)) {
+                if (!data.url) return;
-              return toast({
+                // check is link
-                status: 'warning',
+                if (!strIsLink(data.url)) {
-                title: t('common:common.link.UnValid')
+                  return toast({
-              });
+                    status: 'warning',
-            }
+                    title: t('common:common.link.UnValid')
-            openConfirm(
+                  });
-              () => {
+                }
-                onSuccess(data);
+                goToNext();
-              },
+              })}
-              undefined,
+            >
-              confirmTip
+              {t('common:common.Next Step')}
-            )();
+            </Button>
-          })}
+          </>
-        >
+        )}
-          {t('common:core.dataset.website.Start Sync')}
+        {activeStep == 1 && (
-        </Button>
+          <>
+            <Button variant={'whiteBase'} onClick={goToPrevious}>
+              {t('common:common.Last Step')}
+            </Button>
+            <Button
+              ml={2}
+              onClick={form.handleSubmit((data) => {
+                openConfirm(
+                  () =>
+                    onSuccess({
+                      websiteConfig: websiteInfoGetValues(),
+                      chunkSettings: collectionChunkForm2StoreChunkData({
+                        ...data,
+                        agentModel: datasetDetail.agentModel,
+                        vectorModel: datasetDetail.vectorModel
+                      })
+                    }),
+                  undefined,
+                  isEdit
+                    ? t('common:core.dataset.website.Confirm Update Tips')
+                    : t('common:core.dataset.website.Confirm Create Tips')
+                )();
+              })}
+            >
+              {t('common:core.dataset.website.Start Sync')}
+            </Button>
+          </>
+        )}
       </ModalFooter>
       <ConfirmModal />
     </MyModal>
@@ -112,3 +217,42 @@ const WebsiteConfigModal = ({
 };
 
 export default WebsiteConfigModal;
+
+const PromptTextarea = ({
+  defaultValue,
+  onChange,
+  onClose
+}: {
+  defaultValue: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('common:core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common:common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};

projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx

@@ -64,16 +64,6 @@ const CollectionCard = () => {
   const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
   const { feConfigs } = useSystemStore();
 
-  const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
-    content: t('common:dataset.Confirm to delete the file'),
-    type: 'delete'
-  });
-
-  const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
-    title: t('common:Rename')
-  });
-
-  const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
   const [trainingStatesCollection, setTrainingStatesCollection] = useState<{
     collectionId: string;
   }>();
@@ -116,6 +106,11 @@ const CollectionCard = () => {
     [collections, t]
   );
 
+  const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
+
+  const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
+    title: t('common:Rename')
+  });
   const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest2(
     putDatasetCollectionById,
     {
@@ -125,7 +120,12 @@ const CollectionCard = () => {
       successToast: t('common:common.Update Success')
     }
   );
-  const { runAsync: onDelCollection, loading: isDeleting } = useRequest2(
+
+  const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
+    content: t('common:dataset.Confirm to delete the file'),
+    type: 'delete'
+  });
+  const { runAsync: onDelCollection } = useRequest2(
     (collectionId: string) => {
       return delDatasetCollectionById({
         id: collectionId
@@ -163,14 +163,14 @@ const CollectionCard = () => {
     ['refreshCollection'],
     () => {
       getData(pageNum);
-      if (datasetDetail.status === DatasetStatusEnum.syncing) {
+      if (datasetDetail.status !== DatasetStatusEnum.active) {
         loadDatasetDetail(datasetDetail._id);
       }
       return null;
     },
     {
       refetchInterval: 6000,
-      enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
+      enabled: hasTrainingData || datasetDetail.status !== DatasetStatusEnum.active
     }
   );
 
@@ -190,7 +190,7 @@ const CollectionCard = () => {
   });
 
   const isLoading =
-    isUpdating || isDeleting || isSyncing || (isGetting && collections.length === 0) || isDropping;
+    isUpdating || isSyncing || (isGetting && collections.length === 0) || isDropping;
 
   return (
     <MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
@@ -406,9 +406,7 @@ const CollectionCard = () => {
                                 type: 'danger',
                                 onClick: () =>
                                   openDeleteConfirm(
-                                    () => {
+                                    () => onDelCollection(collection._id),
-                                      onDelCollection(collection._id);
-                                    },
                                     undefined,
                                     collection.type === DatasetCollectionTypeEnum.folder
                                       ? t('common:dataset.collections.Confirm to delete the folder')

projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx

@@ -0,0 +1,524 @@
+import MyModal from '@fastgpt/web/components/common/MyModal';
+import { useTranslation } from 'next-i18next';
+import { UseFormReturn } from 'react-hook-form';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
+import {
+  Box,
+  Flex,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  useDisclosure,
+  Checkbox,
+  HStack
+} from '@chakra-ui/react';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
+import {
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum,
+  DatasetCollectionDataProcessModeMap
+} from '@fastgpt/global/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
+import { useContextSelector } from 'use-context-selector';
+import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
+import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
+import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import MySelect from '@fastgpt/web/components/common/MySelect';
+import {
+  chunkAutoChunkSize,
+  getAutoIndexSize,
+  getIndexSizeSelectList,
+  getLLMDefaultChunkSize,
+  getLLMMaxChunkSize,
+  getMaxChunkSize,
+  getMaxIndexSize,
+  minChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
+import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
+import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
+import type { LLMModelItemType, EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
+
+const PromptTextarea = ({
+  defaultValue = '',
+  onChange,
+  onClose
+}: {
+  defaultValue?: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('common:core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common:common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export type CollectionChunkFormType = {
+  trainingType: DatasetCollectionDataProcessModeEnum;
+  imageIndex: boolean;
+  autoIndexes: boolean;
+
+  chunkSettingMode: ChunkSettingModeEnum;
+
+  chunkSplitMode: DataChunkSplitModeEnum;
+  embeddingChunkSize: number;
+  qaChunkSize: number;
+  chunkSplitter?: string;
+  indexSize: number;
+
+  qaPrompt?: string;
+};
+const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkFormType> }) => {
+  const { t } = useTranslation();
+  const { feConfigs } = useSystemStore();
+
+  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
+
+  const vectorModel = datasetDetail.vectorModel;
+  const agentModel = datasetDetail.agentModel;
+
+  const { setValue, register, watch, getValues } = form;
+
+  const trainingType = watch('trainingType');
+  const chunkSettingMode = watch('chunkSettingMode');
+  const chunkSplitMode = watch('chunkSplitMode');
+  const autoIndexes = watch('autoIndexes');
+  const indexSize = watch('indexSize');
+
+  const trainingModeList = useMemo(() => {
+    const list = Object.entries(DatasetCollectionDataProcessModeMap);
+    return list
+      .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
+      .map(([key, value]) => ({
+        title: t(value.label as any),
+        value: key as DatasetCollectionDataProcessModeEnum,
+        tooltip: t(value.tooltip as any)
+      }));
+  }, [t]);
+  const {
+    chunkSizeField,
+    maxChunkSize,
+    minChunkSize: minChunkSizeValue,
+    maxIndexSize
+  } = useMemo(() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+      return {
+        chunkSizeField: 'qaChunkSize',
+        maxChunkSize: getLLMMaxChunkSize(agentModel),
+        minChunkSize: 1000,
+        maxIndexSize: 1000
+      };
+    } else if (autoIndexes) {
+      return {
+        chunkSizeField: 'embeddingChunkSize',
+        maxChunkSize: getMaxChunkSize(agentModel),
+        minChunkSize: minChunkSize,
+        maxIndexSize: getMaxIndexSize(vectorModel)
+      };
+    } else {
+      return {
+        chunkSizeField: 'embeddingChunkSize',
+        maxChunkSize: getMaxChunkSize(agentModel),
+        minChunkSize: minChunkSize,
+        maxIndexSize: getMaxIndexSize(vectorModel)
+      };
+    }
+  }, [trainingType, autoIndexes, agentModel, vectorModel]);
+
+  // Custom split list
+  const customSplitList = [
+    { label: t('dataset:split_sign_null'), value: '' },
+    { label: t('dataset:split_sign_break'), value: '\\n' },
+    { label: t('dataset:split_sign_break2'), value: '\\n\\n' },
+    { label: t('dataset:split_sign_period'), value: '.|。' },
+    { label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
+    { label: t('dataset:split_sign_question'), value: '?|？' },
+    { label: t('dataset:split_sign_semicolon'), value: ';|；' },
+    { label: '=====', value: '=====' },
+    { label: t('dataset:split_sign_custom'), value: 'Other' }
+  ];
+  const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
+  useEffect(() => {
+    if (customListSelectValue === 'Other') {
+      setValue('chunkSplitter', '');
+    } else {
+      setValue('chunkSplitter', customListSelectValue);
+    }
+  }, [customListSelectValue, setValue]);
+
+  // Index size
+  const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
+
+  // QA
+  const qaPrompt = watch('qaPrompt');
+  const {
+    isOpen: isOpenCustomPrompt,
+    onOpen: onOpenCustomPrompt,
+    onClose: onCloseCustomPrompt
+  } = useDisclosure();
+
+  const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
+
+  // Adapt 4.9.0- auto training
+  useEffect(() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
+      setValue('autoIndexes', true);
+      setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
+    }
+  }, [trainingType, setValue]);
+
+  return (
+    <>
+      <Box>
+        <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+          {t('dataset:training_mode')}
+        </Box>
+        <LeftRadio<DatasetCollectionDataProcessModeEnum>
+          list={trainingModeList}
+          px={3}
+          py={2.5}
+          value={trainingType}
+          onChange={(e) => {
+            setValue('trainingType', e);
+          }}
+          defaultBg="white"
+          activeBg="white"
+          gridTemplateColumns={'repeat(2, 1fr)'}
+        />
+      </Box>
+      {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
+        <Box mt={6}>
+          <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+            {t('dataset:enhanced_indexes')}
+          </Box>
+          <HStack gap={[3, 7]}>
+            <HStack flex={'1'} spacing={1}>
+              <MyTooltip label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}>
+                <Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
+                  <FormLabel>{t('dataset:auto_indexes')}</FormLabel>
+                </Checkbox>
+              </MyTooltip>
+              <QuestionTip label={t('dataset:auto_indexes_tips')} />
+            </HStack>
+            <HStack flex={'1'} spacing={1}>
+              <MyTooltip
+                label={
+                  !feConfigs?.isPlus
+                    ? t('common:commercial_function_tip')
+                    : !datasetDetail?.vlmModel
+                      ? t('common:error_vlm_not_config')
+                      : ''
+                }
+              >
+                <Checkbox
+                  isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
+                  {...register('imageIndex')}
+                >
+                  <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
+                </Checkbox>
+              </MyTooltip>
+              <QuestionTip label={t('dataset:image_auto_parse_tips')} />
+            </HStack>
+          </HStack>
+        </Box>
+      )}
+      <Box mt={6}>
+        <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+          {t('dataset:params_setting')}
+        </Box>
+        <LeftRadio<ChunkSettingModeEnum>
+          list={[
+            {
+              title: t('dataset:default_params'),
+              desc: t('dataset:default_params_desc'),
+              value: ChunkSettingModeEnum.auto
+            },
+            {
+              title: t('dataset:custom_data_process_params'),
+              desc: t('dataset:custom_data_process_params_desc'),
+              value: ChunkSettingModeEnum.custom,
+              children: chunkSettingMode === ChunkSettingModeEnum.custom && (
+                <Box mt={5}>
+                  <Box>
+                    <RadioGroup<DataChunkSplitModeEnum>
+                      list={[
+                        {
+                          title: t('dataset:split_chunk_size'),
+                          value: DataChunkSplitModeEnum.size
+                        },
+                        {
+                          title: t('dataset:split_chunk_char'),
+                          value: DataChunkSplitModeEnum.char,
+                          tooltip: t('dataset:custom_split_sign_tip')
+                        }
+                      ]}
+                      value={chunkSplitMode}
+                      onChange={(e) => {
+                        setValue('chunkSplitMode', e);
+                      }}
+                    />
+
+                    {chunkSplitMode === DataChunkSplitModeEnum.size && (
+                      <Box
+                        mt={1.5}
+                        css={{
+                          '& > span': {
+                            display: 'block'
+                          }
+                        }}
+                      >
+                        <MyTooltip
+                          label={t('common:core.dataset.import.Chunk Range', {
+                            min: minChunkSizeValue,
+                            max: maxChunkSize
+                          })}
+                        >
+                          <MyNumberInput
+                            register={register}
+                            name={chunkSizeField}
+                            min={minChunkSizeValue}
+                            max={maxChunkSize}
+                            size={'sm'}
+                            step={100}
+                          />
+                        </MyTooltip>
+                      </Box>
+                    )}
+
+                    {chunkSplitMode === DataChunkSplitModeEnum.char && (
+                      <HStack mt={1.5}>
+                        <Box flex={'1 0 0'}>
+                          <MySelect<string>
+                            list={customSplitList}
+                            size={'sm'}
+                            bg={'myGray.50'}
+                            value={customListSelectValue}
+                            h={'32px'}
+                            onChange={(val) => {
+                              setCustomListSelectValue(val);
+                            }}
+                          />
+                        </Box>
+                        {customListSelectValue === 'Other' && (
+                          <Input
+                            flex={'1 0 0'}
+                            h={'32px'}
+                            size={'sm'}
+                            bg={'myGray.50'}
+                            placeholder="\n;======;==SPLIT=="
+                            {...register('chunkSplitter')}
+                          />
+                        )}
+                      </HStack>
+                    )}
+                  </Box>
+
+                  {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
+                    <Box>
+                      <Flex alignItems={'center'} mt={3}>
+                        <Box>{t('dataset:index_size')}</Box>
+                        <QuestionTip label={t('dataset:index_size_tips')} />
+                      </Flex>
+                      <Box mt={1}>
+                        <MySelect<number>
+                          bg={'myGray.50'}
+                          list={indexSizeSeletorList}
+                          value={indexSize}
+                          onChange={(val) => {
+                            setValue('indexSize', val);
+                          }}
+                        />
+                      </Box>
+                    </Box>
+                  )}
+
+                  {showQAPromptInput && (
+                    <Box mt={3}>
+                      <Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
+                      <Box
+                        position={'relative'}
+                        py={2}
+                        px={3}
+                        bg={'myGray.50'}
+                        fontSize={'xs'}
+                        whiteSpace={'pre-wrap'}
+                        border={'1px'}
+                        borderColor={'borderColor.base'}
+                        borderRadius={'md'}
+                        maxH={'140px'}
+                        overflow={'auto'}
+                        _hover={{
+                          '& .mask': {
+                            display: 'block'
+                          }
+                        }}
+                      >
+                        {qaPrompt}
+
+                        <Box
+                          display={'none'}
+                          className="mask"
+                          position={'absolute'}
+                          top={0}
+                          right={0}
+                          bottom={0}
+                          left={0}
+                          background={
+                            'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
+                          }
+                        >
+                          <Button
+                            size="xs"
+                            variant={'whiteBase'}
+                            leftIcon={<MyIcon name={'edit'} w={'13px'} />}
+                            color={'black'}
+                            position={'absolute'}
+                            right={2}
+                            bottom={2}
+                            onClick={onOpenCustomPrompt}
+                          >
+                            {t('common:core.dataset.import.Custom prompt')}
+                          </Button>
+                        </Box>
+                      </Box>
+                    </Box>
+                  )}
+                </Box>
+              )
+            }
+          ]}
+          gridGap={3}
+          px={3}
+          py={3}
+          defaultBg="white"
+          activeBg="white"
+          value={chunkSettingMode}
+          w={'100%'}
+          onChange={(e) => {
+            setValue('chunkSettingMode', e);
+          }}
+        />
+      </Box>
+      {isOpenCustomPrompt && (
+        <PromptTextarea
+          defaultValue={qaPrompt}
+          onChange={(e) => {
+            setValue('qaPrompt', e);
+          }}
+          onClose={onCloseCustomPrompt}
+        />
+      )}
+    </>
+  );
+};
+
+export default CollectionChunkForm;
+
+export const collectionChunkForm2StoreChunkData = ({
+  trainingType,
+  imageIndex,
+  autoIndexes,
+  chunkSettingMode,
+  chunkSplitMode,
+  embeddingChunkSize,
+  qaChunkSize,
+  chunkSplitter,
+  indexSize,
+  qaPrompt,
+
+  agentModel,
+  vectorModel
+}: CollectionChunkFormType & {
+  agentModel: LLMModelItemType;
+  vectorModel: EmbeddingModelItemType;
+}): ChunkSettingsType => {
+  const trainingModeSize: {
+    autoChunkSize: number;
+    autoIndexSize: number;
+    chunkSize: number;
+    indexSize: number;
+  } = (() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+      return {
+        autoChunkSize: getLLMDefaultChunkSize(agentModel),
+        autoIndexSize: 512,
+        chunkSize: qaChunkSize,
+        indexSize: 512
+      };
+    } else if (autoIndexes) {
+      return {
+        autoChunkSize: chunkAutoChunkSize,
+        autoIndexSize: getAutoIndexSize(vectorModel),
+        chunkSize: embeddingChunkSize,
+        indexSize
+      };
+    } else {
+      return {
+        autoChunkSize: chunkAutoChunkSize,
+        autoIndexSize: getAutoIndexSize(vectorModel),
+        chunkSize: embeddingChunkSize,
+        indexSize
+      };
+    }
+  })();
+
+  const { chunkSize: formatChunkIndex, indexSize: formatIndexSize } = (() => {
+    if (chunkSettingMode === ChunkSettingModeEnum.auto) {
+      return {
+        chunkSize: trainingModeSize.autoChunkSize,
+        indexSize: trainingModeSize.autoIndexSize
+      };
+    } else {
+      return {
+        chunkSize: trainingModeSize.chunkSize,
+        indexSize: trainingModeSize.indexSize
+      };
+    }
+  })();
+
+  return {
+    trainingType,
+    imageIndex,
+    autoIndexes,
+
+    chunkSettingMode,
+    chunkSplitMode,
+
+    chunkSize: formatChunkIndex,
+    indexSize: formatIndexSize,
+
+    chunkSplitter,
+    qaPrompt: trainingType === DatasetCollectionDataProcessModeEnum.qa ? qaPrompt : undefined
+  };
+};

projects/app/src/pageComponents/dataset/detail/Import/Context.tsx

@@ -25,6 +25,14 @@ import {
   getAutoIndexSize,
   getMaxIndexSize
 } from '@fastgpt/global/core/dataset/training/utils';
+import { CollectionChunkFormType } from '../Form/CollectionChunkForm';
+
+type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
+export type ImportFormType = {
+  customPdfParse: boolean;
+
+  webSelector: string;
+} & CollectionChunkFormType;
 
 type TrainingFiledType = {
   chunkOverlapRatio: number;
@@ -51,26 +59,6 @@ type DatasetImportContextType = {
   setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
 } & TrainingFiledType;
 
-type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
-export type ImportFormType = {
-  customPdfParse: boolean;
-
-  trainingType: DatasetCollectionDataProcessModeEnum;
-  imageIndex: boolean;
-  autoIndexes: boolean;
-
-  chunkSettingMode: ChunkSettingModeEnum;
-
-  chunkSplitMode: DataChunkSplitModeEnum;
-  embeddingChunkSize: number;
-  qaChunkSize: number;
-  chunkSplitter: string;
-  indexSize: number;
-
-  qaPrompt: string;
-  webSelector: string;
-};
-
 export const DatasetImportContext = createContext<DatasetImportContextType>({
   importSource: ImportDataSourceEnum.fileLocal,
   goToNext: function (): void {
@@ -314,14 +302,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
         chunkSplitter
       };
     }
-  }, [
+  }, [chunkSettingMode, TrainingModeMap, chunkSplitter]);
-    chunkSettingMode,
-    TrainingModeMap.autoChunkSize,
-    TrainingModeMap.autoIndexSize,
-    TrainingModeMap.chunkSize,
-    TrainingModeMap.indexSize,
-    chunkSplitter
-  ]);
 
   const contextValue = {
     ...TrainingModeMap,

projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx

@@ -1,13 +1,8 @@
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import React, { useCallback } from 'react';
 import {
   Box,
   Flex,
-  Input,
   Button,
-  ModalBody,
-  ModalFooter,
-  Textarea,
-  useDisclosure,
   Checkbox,
   Accordion,
   AccordionItem,
@@ -16,93 +11,26 @@ import {
   AccordionIcon,
   HStack
 } from '@chakra-ui/react';
-import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useTranslation } from 'next-i18next';
-import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
-import {
-  DataChunkSplitModeEnum,
-  DatasetCollectionDataProcessModeEnum,
-  DatasetCollectionDataProcessModeMap
-} from '@fastgpt/global/core/dataset/constants';
-import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
-import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import MyModal from '@fastgpt/web/components/common/MyModal';
-import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
 import MyTag from '@fastgpt/web/components/common/Tag/index';
 import { useContextSelector } from 'use-context-selector';
 import { DatasetImportContext } from '../Context';
 import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
-import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { shadowLight } from '@fastgpt/web/styles/theme';
-import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import CollectionChunkForm from '../../Form/CollectionChunkForm';
-import MySelect from '@fastgpt/web/components/common/MySelect';
+import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
-import { getIndexSizeSelectList } from '@fastgpt/global/core/dataset/training/utils';
-import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
 
 function DataProcess() {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
 
-  const {
+  const { goToNext, processParamsForm, chunkSize } = useContextSelector(
-    goToNext,
+    DatasetImportContext,
-    processParamsForm,
+    (v) => v
-    chunkSizeField,
+  );
-    minChunkSize,
+  const { register } = processParamsForm;
-    maxChunkSize,
-    maxIndexSize,
-    indexSize
-  } = useContextSelector(DatasetImportContext, (v) => v);
-  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
-  const { setValue, register, watch, getValues } = processParamsForm;
-
-  const trainingType = watch('trainingType');
-  const trainingModeList = useMemo(() => {
-    const list = Object.entries(DatasetCollectionDataProcessModeMap);
-    return list
-      .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
-      .map(([key, value]) => ({
-        title: t(value.label as any),
-        value: key as DatasetCollectionDataProcessModeEnum,
-        tooltip: t(value.tooltip as any)
-      }));
-  }, [t]);
-
-  const chunkSettingMode = watch('chunkSettingMode');
-  const chunkSplitMode = watch('chunkSplitMode');
-
-  const customSplitList = [
-    { label: t('dataset:split_sign_null'), value: '' },
-    { label: t('dataset:split_sign_break'), value: '\\n' },
-    { label: t('dataset:split_sign_break2'), value: '\\n\\n' },
-    { label: t('dataset:split_sign_period'), value: '.|。' },
-    { label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
-    { label: t('dataset:split_sign_question'), value: '?|？' },
-    { label: t('dataset:split_sign_semicolon'), value: ';|；' },
-    { label: '=====', value: '=====' },
-    { label: t('dataset:split_sign_custom'), value: 'Other' }
-  ];
-
-  const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
-  useEffect(() => {
-    if (customListSelectValue === 'Other') {
-      setValue('chunkSplitter', '');
-    } else {
-      setValue('chunkSplitter', customListSelectValue);
-    }
-  }, [customListSelectValue, setValue]);
-
-  // Index size
-  const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
-
-  // QA
-  const qaPrompt = watch('qaPrompt');
-  const {
-    isOpen: isOpenCustomPrompt,
-    onOpen: onOpenCustomPrompt,
-    onClose: onCloseCustomPrompt
-  } = useDisclosure();
 
   const Title = useCallback(({ title }: { title: string }) => {
     return (
@@ -116,16 +44,7 @@ function DataProcess() {
     );
   }, []);
 
-  // Adapt auto training
-  useEffect(() => {
-    if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
-      setValue('autoIndexes', true);
-      setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
-    }
-  }, [trainingType, setValue]);
-
   const showFileParseSetting = feConfigs?.showCustomPdfParse;
-  const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
 
   return (
     <>
@@ -179,238 +98,8 @@ function DataProcess() {
             <Title title={t('dataset:import_data_process_setting')} />
 
             <AccordionPanel p={2}>
-              <Box mt={2}>
+              {/* @ts-ignore */}
-                <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+              <CollectionChunkForm form={processParamsForm} />
-                  {t('dataset:training_mode')}
-                </Box>
-                <LeftRadio<DatasetCollectionDataProcessModeEnum>
-                  list={trainingModeList}
-                  px={3}
-                  py={2.5}
-                  value={trainingType}
-                  onChange={(e) => {
-                    setValue('trainingType', e);
-                  }}
-                  defaultBg="white"
-                  activeBg="white"
-                  gridTemplateColumns={'repeat(2, 1fr)'}
-                />
-              </Box>
-              {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
-                <Box mt={6}>
-                  <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
-                    {t('dataset:enhanced_indexes')}
-                  </Box>
-                  <HStack gap={[3, 7]}>
-                    <HStack flex={'1'} spacing={1}>
-                      <MyTooltip
-                        label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}
-                      >
-                        <Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
-                          <FormLabel>{t('dataset:auto_indexes')}</FormLabel>
-                        </Checkbox>
-                      </MyTooltip>
-                      <QuestionTip label={t('dataset:auto_indexes_tips')} />
-                    </HStack>
-                    <HStack flex={'1'} spacing={1}>
-                      <MyTooltip
-                        label={
-                          !feConfigs?.isPlus
-                            ? t('common:commercial_function_tip')
-                            : !datasetDetail?.vlmModel
-                              ? t('common:error_vlm_not_config')
-                              : ''
-                        }
-                      >
-                        <Checkbox
-                          isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
-                          {...register('imageIndex')}
-                        >
-                          <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
-                        </Checkbox>
-                      </MyTooltip>
-                      <QuestionTip label={t('dataset:image_auto_parse_tips')} />
-                    </HStack>
-                  </HStack>
-                </Box>
-              )}
-              <Box mt={6}>
-                <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
-                  {t('dataset:params_setting')}
-                </Box>
-                <LeftRadio<ChunkSettingModeEnum>
-                  list={[
-                    {
-                      title: t('dataset:default_params'),
-                      desc: t('dataset:default_params_desc'),
-                      value: ChunkSettingModeEnum.auto
-                    },
-                    {
-                      title: t('dataset:custom_data_process_params'),
-                      desc: t('dataset:custom_data_process_params_desc'),
-                      value: ChunkSettingModeEnum.custom,
-                      children: chunkSettingMode === ChunkSettingModeEnum.custom && (
-                        <Box mt={5}>
-                          <Box>
-                            <RadioGroup<DataChunkSplitModeEnum>
-                              list={[
-                                {
-                                  title: t('dataset:split_chunk_size'),
-                                  value: DataChunkSplitModeEnum.size
-                                },
-                                {
-                                  title: t('dataset:split_chunk_char'),
-                                  value: DataChunkSplitModeEnum.char,
-                                  tooltip: t('dataset:custom_split_sign_tip')
-                                }
-                              ]}
-                              value={chunkSplitMode}
-                              onChange={(e) => {
-                                setValue('chunkSplitMode', e);
-                              }}
-                            />
-
-                            {chunkSplitMode === DataChunkSplitModeEnum.size && (
-                              <Box
-                                mt={1.5}
-                                css={{
-                                  '& > span': {
-                                    display: 'block'
-                                  }
-                                }}
-                              >
-                                <MyTooltip
-                                  label={t('common:core.dataset.import.Chunk Range', {
-                                    min: minChunkSize,
-                                    max: maxChunkSize
-                                  })}
-                                >
-                                  <MyNumberInput
-                                    register={register}
-                                    name={chunkSizeField}
-                                    min={minChunkSize}
-                                    max={maxChunkSize}
-                                    size={'sm'}
-                                    step={100}
-                                  />
-                                </MyTooltip>
-                              </Box>
-                            )}
-
-                            {chunkSplitMode === DataChunkSplitModeEnum.char && (
-                              <HStack mt={1.5}>
-                                <Box flex={'1 0 0'}>
-                                  <MySelect<string>
-                                    list={customSplitList}
-                                    size={'sm'}
-                                    bg={'myGray.50'}
-                                    value={customListSelectValue}
-                                    h={'32px'}
-                                    onChange={(val) => {
-                                      setCustomListSelectValue(val);
-                                    }}
-                                  />
-                                </Box>
-                                {customListSelectValue === 'Other' && (
-                                  <Input
-                                    flex={'1 0 0'}
-                                    h={'32px'}
-                                    size={'sm'}
-                                    bg={'myGray.50'}
-                                    placeholder="\n;======;==SPLIT=="
-                                    {...register('chunkSplitter')}
-                                  />
-                                )}
-                              </HStack>
-                            )}
-                          </Box>
-
-                          {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
-                            <Box>
-                              <Flex alignItems={'center'} mt={3}>
-                                <Box>{t('dataset:index_size')}</Box>
-                                <QuestionTip label={t('dataset:index_size_tips')} />
-                              </Flex>
-                              <Box mt={1}>
-                                <MySelect<number>
-                                  bg={'myGray.50'}
-                                  list={indexSizeSeletorList}
-                                  value={indexSize}
-                                  onChange={(val) => {
-                                    setValue('indexSize', val);
-                                  }}
-                                />
-                              </Box>
-                            </Box>
-                          )}
-
-                          {showQAPromptInput && (
-                            <Box mt={3}>
-                              <Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
-                              <Box
-                                position={'relative'}
-                                py={2}
-                                px={3}
-                                bg={'myGray.50'}
-                                fontSize={'xs'}
-                                whiteSpace={'pre-wrap'}
-                                border={'1px'}
-                                borderColor={'borderColor.base'}
-                                borderRadius={'md'}
-                                maxH={'140px'}
-                                overflow={'auto'}
-                                _hover={{
-                                  '& .mask': {
-                                    display: 'block'
-                                  }
-                                }}
-                              >
-                                {qaPrompt}
-
-                                <Box
-                                  display={'none'}
-                                  className="mask"
-                                  position={'absolute'}
-                                  top={0}
-                                  right={0}
-                                  bottom={0}
-                                  left={0}
-                                  background={
-                                    'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
-                                  }
-                                >
-                                  <Button
-                                    size="xs"
-                                    variant={'whiteBase'}
-                                    leftIcon={<MyIcon name={'edit'} w={'13px'} />}
-                                    color={'black'}
-                                    position={'absolute'}
-                                    right={2}
-                                    bottom={2}
-                                    onClick={onOpenCustomPrompt}
-                                  >
-                                    {t('common:core.dataset.import.Custom prompt')}
-                                  </Button>
-                                </Box>
-                              </Box>
-                            </Box>
-                          )}
-                        </Box>
-                      )
-                    }
-                  ]}
-                  gridGap={3}
-                  px={3}
-                  py={3}
-                  defaultBg="white"
-                  activeBg="white"
-                  value={chunkSettingMode}
-                  w={'100%'}
-                  onChange={(e) => {
-                    setValue('chunkSettingMode', e);
-                  }}
-                />
-              </Box>
             </AccordionPanel>
           </AccordionItem>
 
@@ -425,57 +114,8 @@ function DataProcess() {
           </Flex>
         </Accordion>
       </Box>
-
-      {isOpenCustomPrompt && (
-        <PromptTextarea
-          defaultValue={qaPrompt}
-          onChange={(e) => {
-            setValue('qaPrompt', e);
-          }}
-          onClose={onCloseCustomPrompt}
-        />
-      )}
     </>
   );
 }
 
 export default React.memo(DataProcess);
-
-const PromptTextarea = ({
-  defaultValue,
-  onChange,
-  onClose
-}: {
-  defaultValue: string;
-  onChange: (e: string) => void;
-  onClose: () => void;
-}) => {
-  const ref = useRef<HTMLTextAreaElement>(null);
-  const { t } = useTranslation();
-
-  return (
-    <MyModal
-      isOpen
-      title={t('common:core.dataset.import.Custom prompt')}
-      iconSrc="modal/edit"
-      w={'600px'}
-      onClose={onClose}
-    >
-      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
-        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
-        <Box>{Prompt_AgentQA.fixedText}</Box>
-      </ModalBody>
-      <ModalFooter>
-        <Button
-          onClick={() => {
-            const val = ref.current?.value || Prompt_AgentQA.description;
-            onChange(val);
-            onClose();
-          }}
-        >
-          {t('common:common.Confirm')}
-        </Button>
-      </ModalFooter>
-    </MyModal>
-  );
-};

projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx

@@ -85,9 +85,13 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
         value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
       },
       {
-        label: t('common:core.dataset.collection.metadata.Chunk Size'),
+        label: t('dataset:chunk_size'),
         value: collection.chunkSize || '-'
       },
+      {
+        label: t('dataset:index_size'),
+        value: collection.indexSize || '-'
+      },
       ...(webSelector
         ? [
             {

projects/app/src/pages/api/admin/initv494.ts

@@ -0,0 +1,53 @@
+import { NextAPI } from '@/service/middleware/entry';
+import { retryFn } from '@fastgpt/global/common/system/utils';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { upsertWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addHours } from 'date-fns';
+import { NextApiRequest, NextApiResponse } from 'next';
+
+const initWebsiteSyncData = async () => {
+  // find out all website dataset
+  const datasets = await MongoDataset.find({ type: DatasetTypeEnum.websiteDataset }).lean();
+
+  console.log('更新站点同步的定时器');
+  // Add scheduler for all website dataset
+  await Promise.all(
+    datasets.map((dataset) => {
+      if (dataset.autoSync) {
+        // 随机生成一个往后 1～24 小时的时间
+        const time = addHours(new Date(), Math.floor(Math.random() * 23) + 1);
+        return retryFn(() =>
+          upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) }, time.getTime())
+        );
+      }
+    })
+  );
+
+  console.log('移除站点同步集合的定时器');
+  // Remove all nextSyncTime
+  await retryFn(() =>
+    MongoDatasetCollection.updateMany(
+      {
+        teamId: datasets.map((dataset) => dataset.teamId),
+        datasetId: datasets.map((dataset) => dataset._id)
+      },
+      {
+        $unset: {
+          nextSyncTime: 1
+        }
+      }
+    )
+  );
+};
+async function handler(req: NextApiRequest, _res: NextApiResponse) {
+  await authCert({ req, authRoot: true });
+
+  await initWebsiteSyncData();
+
+  return { success: true };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/delete.ts

@@ -9,6 +9,8 @@ import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema';
 import { removeImageByPath } from '@fastgpt/service/common/file/image/controller';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { removeWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
 
 async function handler(req: NextApiRequest) {
   const { id: datasetId } = req.query as {
@@ -40,6 +42,13 @@ async function handler(req: NextApiRequest) {
     datasetId: { $in: datasetIds }
   });
 
+  await Promise.all(
+    datasets.map((dataset) => {
+      if (dataset.type === DatasetTypeEnum.websiteDataset)
+        return removeWebsiteSyncJobScheduler(String(dataset._id));
+    })
+  );
+
   // delete all dataset.data and pg data
   await mongoSessionRun(async (session) => {
     // delete dataset data

projects/app/src/pages/api/core/dataset/detail.ts

@@ -5,6 +5,8 @@ import { NextAPI } from '@/service/middleware/entry';
 import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { getWebsiteSyncDatasetStatus } from '@fastgpt/service/core/dataset/websiteSync';
+import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
 
 type Query = {
   id: string;
@@ -28,8 +30,17 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
     per: ReadPermissionVal
   });
 
+  const status = await (async () => {
+    if (dataset.type === DatasetTypeEnum.websiteDataset) {
+      return await getWebsiteSyncDatasetStatus(datasetId);
+    }
+
+    return DatasetStatusEnum.active;
+  })();
+
   return {
     ...dataset,
+    status,
     apiServer: dataset.apiServer
       ? {
           baseUrl: dataset.apiServer.baseUrl,

projects/app/src/pages/api/core/dataset/update.ts

@@ -30,6 +30,13 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
 import { addDays } from 'date-fns';
 import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller';
 import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import {
+  removeWebsiteSyncJobScheduler,
+  upsertWebsiteSyncJobScheduler
+} from '@fastgpt/service/core/dataset/websiteSync';
+import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
+import { isEqual } from 'lodash';
 
 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -62,8 +69,8 @@ async function handler(
     apiServer,
     yuqueServer,
     feishuServer,
-    status,
+    autoSync,
-    autoSync
+    chunkSettings
   } = req.body;
 
   if (!id) {
@@ -114,6 +121,39 @@ async function handler(
   });
 
   const onUpdate = async (session: ClientSession) => {
+    // Website dataset update chunkSettings, need to clean up dataset
+    if (
+      dataset.type === DatasetTypeEnum.websiteDataset &&
+      chunkSettings &&
+      dataset.chunkSettings &&
+      !isEqual(
+        {
+          imageIndex: dataset.chunkSettings.imageIndex,
+          autoIndexes: dataset.chunkSettings.autoIndexes,
+          trainingType: dataset.chunkSettings.trainingType,
+          chunkSettingMode: dataset.chunkSettings.chunkSettingMode,
+          chunkSplitMode: dataset.chunkSettings.chunkSplitMode,
+          chunkSize: dataset.chunkSettings.chunkSize,
+          chunkSplitter: dataset.chunkSettings.chunkSplitter,
+          indexSize: dataset.chunkSettings.indexSize,
+          qaPrompt: dataset.chunkSettings.qaPrompt
+        },
+        {
+          imageIndex: chunkSettings.imageIndex,
+          autoIndexes: chunkSettings.autoIndexes,
+          trainingType: chunkSettings.trainingType,
+          chunkSettingMode: chunkSettings.chunkSettingMode,
+          chunkSplitMode: chunkSettings.chunkSplitMode,
+          chunkSize: chunkSettings.chunkSize,
+          chunkSplitter: chunkSettings.chunkSplitter,
+          indexSize: chunkSettings.indexSize,
+          qaPrompt: chunkSettings.qaPrompt
+        }
+      )
+    ) {
+      await delDatasetRelevantData({ datasets: [dataset], session });
+    }
+
     await MongoDataset.findByIdAndUpdate(
       id,
       {
@@ -123,7 +163,7 @@ async function handler(
         ...(agentModel && { agentModel }),
         ...(vlmModel && { vlmModel }),
         ...(websiteConfig && { websiteConfig }),
-        ...(status && { status }),
+        ...(chunkSettings && { chunkSettings }),
         ...(intro !== undefined && { intro }),
         ...(externalReadUrl !== undefined && { externalReadUrl }),
         ...(!!apiServer?.baseUrl && { 'apiServer.baseUrl': apiServer.baseUrl }),
@@ -143,8 +183,7 @@ async function handler(
       { session }
     );
     await updateSyncSchedule({
-      teamId: dataset.teamId,
+      dataset,
-      datasetId: dataset._id,
       autoSync,
       session
     });
@@ -221,45 +260,54 @@ const updateTraining = async ({
 };
 
 const updateSyncSchedule = async ({
-  teamId,
+  dataset,
-  datasetId,
   autoSync,
   session
 }: {
-  teamId: string;
+  dataset: DatasetSchemaType;
-  datasetId: string;
   autoSync?: boolean;
   session: ClientSession;
 }) => {
   if (typeof autoSync !== 'boolean') return;
 
   // Update all collection nextSyncTime
-  if (autoSync) {
+  if (dataset.type === DatasetTypeEnum.websiteDataset) {
-    await MongoDatasetCollection.updateMany(
+    if (autoSync) {
-      {
+      // upsert Job Scheduler
-        teamId,
+      upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) });
-        datasetId,
+    } else {
-        type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
+      // remove Job Scheduler
-      },
+      removeWebsiteSyncJobScheduler(String(dataset._id));
-      {
+    }
-        $set: {
-          nextSyncTime: addDays(new Date(), 1)
-        }
-      },
-      { session }
-    );
   } else {
-    await MongoDatasetCollection.updateMany(
+    // Other dataset, update the collection sync
-      {
+    if (autoSync) {
-        teamId,
+      await MongoDatasetCollection.updateMany(
-        datasetId
+        {
-      },
+          teamId: dataset.teamId,
-      {
+          datasetId: dataset._id,
-        $unset: {
+          type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
-          nextSyncTime: 1
+        },
-        }
+        {
-      },
+          $set: {
-      { session }
+            nextSyncTime: addDays(new Date(), 1)
-    );
+          }
+        },
+        { session }
+      );
+    } else {
+      await MongoDatasetCollection.updateMany(
+        {
+          teamId: dataset.teamId,
+          datasetId: dataset._id
+        },
+        {
+          $unset: {
+            nextSyncTime: 1
+          }
+        },
+        { session }
+      );
+    }
   }
 };

projects/app/src/web/core/dataset/constants.ts

@@ -47,7 +47,6 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
     avatar: '/icon/logo.svg',
     name: '',
     intro: '',
-    status: 'active',
     vectorModel: defaultVectorModels[0].model,
     agentModel: defaultQAModels[0].model,
     inheritPermission: true

test/cases/pages/api/core/dataset/paths.test.ts

@@ -6,6 +6,7 @@ vi.mock('@fastgpt/service/core/dataset/schema', () => ({
   MongoDataset: {
     findById: vi.fn()
   },
+  ChunkSettings: {},
   DatasetCollectionName: 'datasets'
 }));