Collection tag (#2266)

* feat: collection metadata filter (#2211) * feat: add dataset collection tags (#2231) * dataset page * workflow page * move * fix * add plus filter * fix * fix * fix * perf: collection tag code * fix: collection tags (#2249) * fix * fix * fix tags of dataset page * fix tags of workflow page * doc * add comments * fix: collection tags (#2264) * fix: metadata filter * feat: search filter --------- Co-authored-by: heheer <1239331448@qq.com> Co-authored-by: heheer <heheer@sealos.io>
2025-07-23 13:03:50 +00:00 · 2024-08-05 12:08:46 +08:00
parent 56f6e69bc7
commit fe71efbbd2
46 changed files with 1914 additions and 112 deletions
--- a/packages/global/core/dataset/api.d.ts
+++ b/packages/global/core/dataset/api.d.ts
@@ -74,6 +74,23 @@ export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollecti
  filename?: string;
 };

+/* ================= tag ===================== */
+export type CreateDatasetCollectionTagParams = {
+  datasetId: string;
+  tag: string;
+};
+export type AddTagsToCollectionsParams = {
+  originCollectionIds: string[];
+  collectionIds: string[];
+  datasetId: string;
+  tag: string;
+};
+export type UpdateDatasetCollectionTagParams = {
+  datasetId: string;
+  tagId: string;
+  tag: string;
+};
+
 /* ================= data ===================== */
 export type PgSearchRawType = {
  id: string;
--- a/packages/global/core/dataset/type.d.ts
+++ b/packages/global/core/dataset/type.d.ts
@@ -69,6 +69,13 @@ export type DatasetCollectionSchemaType = {
  };
 };

+export type DatasetCollectionTagsSchemaType = {
+  _id: string;
+  teamId: string;
+  datasetId: string;
+  tag: string;
+};
+
 export type DatasetDataIndexItemType = {
  defaultIndex: boolean;
  dataId: string; // pg data id
@@ -144,6 +151,17 @@ export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentMode
  permission: DatasetPermission;
 };

+/* ================= tag ===================== */
+export type DatasetTagType = {
+  _id: string;
+  tag: string;
+};
+
+export type TagUsageType = {
+  tagId: string;
+  collections: string[];
+};
+
 /* ================= collection ===================== */
 export type DatasetCollectionItemType = CollectionWithDatasetType & {
  sourceName: string;
--- a/packages/global/core/workflow/constants.ts
+++ b/packages/global/core/workflow/constants.ts
@@ -85,6 +85,7 @@ export enum NodeInputKeyEnum {
  datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
  datasetSearchExtensionModel = 'datasetSearchExtensionModel',
  datasetSearchExtensionBg = 'datasetSearchExtensionBg',
+  collectionFilterMatch = 'collectionFilterMatch',

  // concat dataset
  datasetQuoteList = 'system_datasetQuoteList',
--- a/packages/global/core/workflow/template/system/datasetSearch.ts
+++ b/packages/global/core/workflow/template/system/datasetSearch.ts
@@ -90,6 +90,25 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
    {
      ...Input_Template_UserChatInput,
      toolDescription: '需要检索的内容'
+    },
+    {
+      key: NodeInputKeyEnum.collectionFilterMatch,
+      renderTypeList: [FlowNodeInputTypeEnum.JSONEditor, FlowNodeInputTypeEnum.reference],
+      label: '集合元数据过滤',
+      valueType: WorkflowIOValueTypeEnum.object,
+      isPro: true,
+      description: `目前支持标签和创建时间过滤，需按照以下格式填写：
+{
+  "tags": {
+    "$and": ["标签 1","标签 2"],
+    "$or": ["有 $and 标签时，and 生效，or 不生效"]
+  },
+  "createTime": {
+      "$gte": "YYYY-MM-DD HH:mm 格式即可，集合的创建时间大于该时间",
+      "$lte": "YYYY-MM-DD HH:mm 格式即可，集合的创建时间小于该时间,可和 $gte 共同使用"
+  }
+}
+      `
    }
  ],
  outputs: [
--- a/packages/global/core/workflow/type/io.d.ts
+++ b/packages/global/core/workflow/type/io.d.ts
@@ -52,6 +52,7 @@ export type FlowNodeInputItemType = InputComponentPropsType & {

  // render components params
  canEdit?: boolean; // dynamic inputs
+  isPro?: boolean; // Pro version field
 };

 export type FlowNodeOutputItemType = {
--- a/packages/service/common/vectorStore/controller.d.ts
+++ b/packages/service/common/vectorStore/controller.d.ts
@@ -26,9 +26,7 @@ export type EmbeddingRecallProps = {
  datasetIds: string[];

  forbidCollectionIdList: string[];
-  // forbidEmbIndexIdList: string[];
-  // similarity?: number;
-  // efSearch?: number;
+  filterCollectionIdList?: string[];
 };
 export type EmbeddingRecallCtrlProps = EmbeddingRecallProps & {
  vector: number[];
--- a/packages/service/common/vectorStore/milvus/class.ts
+++ b/packages/service/common/vectorStore/milvus/class.ts
@@ -213,19 +213,50 @@ export class MilvusCtrl {
  };
  embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
    const client = await this.getClient();
-    const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
+    const {
+      teamId,
+      datasetIds,
+      vector,
+      limit,
+      forbidCollectionIdList,
+      filterCollectionIdList,
+      retry = 2
+    } = props;

+    // Forbid collection
+    const formatForbidCollectionIdList = (() => {
+      if (!filterCollectionIdList) return forbidCollectionIdList;
+      const list = forbidCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !filterCollectionIdList.includes(id));
+      return list;
+    })();
    const forbidColQuery =
-      forbidCollectionIdList.length > 0
-        ? `and (collectionId not in [${forbidCollectionIdList.map((id) => `"${String(id)}"`).join(',')}])`
+      formatForbidCollectionIdList.length > 0
+        ? `and (collectionId not in [${formatForbidCollectionIdList.map((id) => `"${id}"`).join(',')}])`
        : '';

+    // filter collection id
+    const formatFilterCollectionId = (() => {
+      if (!filterCollectionIdList) return;
+      return filterCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !forbidCollectionIdList.includes(id));
+    })();
+    const collectionIdQuery = formatFilterCollectionId
+      ? `and (collectionId in [${formatFilterCollectionId.map((id) => `"${id}"`)}])`
+      : ``;
+    // Empty data
+    if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
+      return { results: [] };
+    }
+
    try {
      const { results } = await client.search({
        collection_name: DatasetVectorTableName,
        data: vector,
        limit,
-        filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${String(id)}"`).join(',')}]) ${forbidColQuery}`,
+        filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${id}"`).join(',')}]) ${collectionIdQuery} ${forbidColQuery}`,
        output_fields: ['collectionId']
      });

--- a/packages/service/common/vectorStore/pg/class.ts
+++ b/packages/service/common/vectorStore/pg/class.ts
@@ -119,14 +119,44 @@ export class PgVectorCtrl {
    }
  };
  embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
-    const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
+    const {
+      teamId,
+      datasetIds,
+      vector,
+      limit,
+      forbidCollectionIdList,
+      filterCollectionIdList,
+      retry = 2
+    } = props;

+    // Get forbid collection
+    const formatForbidCollectionIdList = (() => {
+      if (!filterCollectionIdList) return forbidCollectionIdList;
+      const list = forbidCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !filterCollectionIdList.includes(id));
+      return list;
+    })();
    const forbidCollectionSql =
-      forbidCollectionIdList.length > 0
-        ? `AND collection_id NOT IN (${forbidCollectionIdList.map((id) => `'${String(id)}'`).join(',')})`
-        : 'AND collection_id IS NOT NULL';
-    // const forbidDataSql =
-    //   forbidEmbIndexIdList.length > 0 ? `AND id NOT IN (${forbidEmbIndexIdList.join(',')})` : '';
+      formatForbidCollectionIdList.length > 0
+        ? `AND collection_id NOT IN (${formatForbidCollectionIdList.map((id) => `'${id}'`).join(',')})`
+        : '';
+
+    // Filter by collectionId
+    const formatFilterCollectionId = (() => {
+      if (!filterCollectionIdList) return;
+
+      return filterCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !forbidCollectionIdList.includes(id));
+    })();
+    const filterCollectionIdSql = formatFilterCollectionId
+      ? `AND collection_id IN (${formatFilterCollectionId.map((id) => `'${id}'`).join(',')})`
+      : '';
+    // Empty data
+    if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
+      return { results: [] };
+    }

    try {
      // const explan: any = await PgClient.query(
@@ -150,6 +180,7 @@ export class PgVectorCtrl {
            from ${DatasetVectorTableName} 
            where team_id='${teamId}'
              AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
+              ${filterCollectionIdSql}
              ${forbidCollectionSql}
            order by score limit ${limit};
        COMMIT;`
--- a/packages/service/core/dataset/collection/schema.ts
+++ b/packages/service/core/dataset/collection/schema.ts
@@ -106,8 +106,10 @@ try {
    updateTime: -1
  });

-  // get forbid
-  // DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
+  // Tag filter
+  DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, tags: 1 });
+  // create time filter
+  DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, createTime: 1 });
 } catch (error) {
  console.log(error);
 }
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -20,6 +20,9 @@ import { hashStr } from '@fastgpt/global/common/string/tools';
 import { jiebaSplit } from '../../../common/string/jieba';
 import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
 import { Types } from '../../../common/mongo';
+import json5 from 'json5';
+import { MongoDatasetCollectionTags } from '../tag/schema';
+import { readFromSecondary } from '../../../common/mongo/utils';

 type SearchDatasetDataProps = {
  teamId: string;
@@ -31,6 +34,20 @@ type SearchDatasetDataProps = {
  usingReRank?: boolean;
  reRankQuery: string;
  queries: string[];
+
+  /* 
+    {
+      tags: {
+        $and: ["str1","str2"],
+        $or: ["str1","str2",null] null means no tags
+      },
+      createTime: {
+        $gte: 'xx',
+        $lte: 'xxx'
+      }
+    }
+  */
+  collectionFilterMatch?: string;
 };

 export async function searchDatasetData(props: SearchDatasetDataProps) {
@@ -43,7 +60,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
    limit: maxTokens,
    searchMode = DatasetSearchModeEnum.embedding,
    usingReRank = false,
-    datasetIds = []
+    datasetIds = [],
+    collectionFilterMatch
  } = props;

  /* init params */
@@ -87,14 +105,148 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
      forbidCollectionIdList: collections.map((item) => String(item._id))
    };
  };
+  /* 
+    Collection metadata filter
+    标签过滤：
+    1. and 先生效
+    2. and 标签和 null 不能共存，否则返回空数组
+  */
+  const filterCollectionByMetadata = async (): Promise<string[] | undefined> => {
+    if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
+
+    let tagCollectionIdList: string[] | undefined = undefined;
+    let createTimeCollectionIdList: string[] | undefined = undefined;
+
+    try {
+      const jsonMatch = json5.parse(collectionFilterMatch);
+
+      // Tag
+      let andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined;
+      let orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined;
+
+      // get andTagIds
+      if (andTags && andTags.length > 0) {
+        // tag 去重
+        andTags = Array.from(new Set(andTags));
+
+        if (andTags.includes(null) && andTags.some((tag) => typeof tag === 'string')) {
+          return [];
+        }
+
+        if (andTags.every((tag) => typeof tag === 'string')) {
+          // Get tagId by tag string
+          const andTagIdList = await MongoDatasetCollectionTags.find(
+            {
+              teamId,
+              datasetId: { $in: datasetIds },
+              tag: { $in: andTags }
+            },
+            '_id',
+            {
+              ...readFromSecondary
+            }
+          ).lean();
+
+          // If you enter a tag that does not exist, none will be found
+          if (andTagIdList.length !== andTags.length) return [];
+
+          // Get collectionId by tagId
+          const collections = await MongoDatasetCollection.find(
+            {
+              teamId,
+              datasetId: { $in: datasetIds },
+              tags: { $all: andTagIdList.map((item) => String(item._id)) }
+            },
+            '_id',
+            {
+              ...readFromSecondary
+            }
+          ).lean();
+          tagCollectionIdList = collections.map((item) => String(item._id));
+        } else if (andTags.every((tag) => tag === null)) {
+          const collections = await MongoDatasetCollection.find(
+            {
+              teamId,
+              datasetId: { $in: datasetIds },
+              $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }]
+            },
+            '_id',
+            {
+              ...readFromSecondary
+            }
+          ).lean();
+          tagCollectionIdList = collections.map((item) => String(item._id));
+        }
+      } else if (orTags && orTags.length > 0) {
+        // Get tagId by tag string
+        const orTagArray = await MongoDatasetCollectionTags.find(
+          {
+            teamId,
+            datasetId: { $in: datasetIds },
+            tag: { $in: orTags.filter((tag) => tag !== null) }
+          },
+          '_id',
+          { ...readFromSecondary }
+        ).lean();
+        const orTagIds = orTagArray.map((item) => String(item._id));
+
+        // Get collections by tagId
+        const collections = await MongoDatasetCollection.find(
+          {
+            teamId,
+            datasetId: { $in: datasetIds },
+            $or: [
+              { tags: { $in: orTagIds } },
+              ...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : [])
+            ]
+          },
+          '_id',
+          { ...readFromSecondary }
+        ).lean();
+
+        tagCollectionIdList = collections.map((item) => String(item._id));
+      }
+
+      // time
+      const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined;
+      const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined;
+      if (getCreateTime || lteCreateTime) {
+        const collections = await MongoDatasetCollection.find(
+          {
+            teamId,
+            datasetId: { $in: datasetIds },
+            createTime: {
+              ...(getCreateTime && { $gte: new Date(getCreateTime) }),
+              ...(lteCreateTime && {
+                $lte: new Date(lteCreateTime)
+              })
+            }
+          },
+          '_id'
+        );
+        createTimeCollectionIdList = collections.map((item) => String(item._id));
+      }
+
+      // Concat tag and time
+      if (tagCollectionIdList && createTimeCollectionIdList) {
+        return tagCollectionIdList.filter((id) => createTimeCollectionIdList!.includes(id));
+      } else if (tagCollectionIdList) {
+        return tagCollectionIdList;
+      } else if (createTimeCollectionIdList) {
+        return createTimeCollectionIdList;
+      }
+    } catch (error) {}
+  };
  const embeddingRecall = async ({
    query,
    limit,
-    forbidCollectionIdList
+    forbidCollectionIdList,
+    filterCollectionIdList
  }: {
    query: string;
    limit: number;
    forbidCollectionIdList: string[];
+    filterCollectionIdList?: string[];
  }) => {
    const { vectors, tokens } = await getVectorsByText({
      model: getVectorModel(model),
@@ -107,7 +259,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
      datasetIds,
      vector: vectors[0],
      limit,
-      forbidCollectionIdList
+      forbidCollectionIdList,
+      filterCollectionIdList
    });

    // get q and a
@@ -165,10 +318,12 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
  };
  const fullTextRecall = async ({
    query,
-    limit
+    limit,
+    filterCollectionIdList
  }: {
    query: string;
    limit: number;
+    filterCollectionIdList?: string[];
  }): Promise<{
    fullTextRecallResults: SearchDataResponseItemType[];
    tokenLen: number;
@@ -188,7 +343,14 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
              $match: {
                teamId: new Types.ObjectId(teamId),
                datasetId: new Types.ObjectId(id),
-                $text: { $search: jiebaSplit({ text: query }) }
+                $text: { $search: jiebaSplit({ text: query }) },
+                ...(filterCollectionIdList && filterCollectionIdList.length > 0
+                  ? {
+                      collectionId: {
+                        $in: filterCollectionIdList.map((id) => new Types.ObjectId(id))
+                      }
+                    }
+                  : {})
              }
            },
            {
@@ -327,19 +489,24 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
    const fullTextRecallResList: SearchDataResponseItemType[][] = [];
    let totalTokens = 0;

-    const { forbidCollectionIdList } = await getForbidData();
-
+    const [{ forbidCollectionIdList }, filterCollectionIdList] = await Promise.all([
+      getForbidData(),
+      filterCollectionByMetadata()
+    ]);
+    console.log(filterCollectionIdList, '===');
    await Promise.all(
      queries.map(async (query) => {
        const [{ tokens, embeddingRecallResults }, { fullTextRecallResults }] = await Promise.all([
          embeddingRecall({
            query,
            limit: embeddingLimit,
-            forbidCollectionIdList
+            forbidCollectionIdList,
+            filterCollectionIdList
          }),
          fullTextRecall({
            query,
-            limit: fullTextLimit
+            limit: fullTextLimit,
+            filterCollectionIdList
          })
        ]);
        totalTokens += tokens;
--- a/packages/service/core/dataset/tag/schema.ts
+++ b/packages/service/core/dataset/tag/schema.ts
@@ -0,0 +1,35 @@
+import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
+import { DatasetCollectionName } from '../schema';
+import { DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type';
+const { Schema } = connectionMongo;
+
+export const DatasetCollectionTagsName = 'dataset_collection_tags';
+
+const DatasetCollectionTagsSchema = new Schema({
+  teamId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamCollectionName,
+    required: true
+  },
+  datasetId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetCollectionName,
+    required: true
+  },
+  tag: {
+    type: String,
+    required: true
+  }
+});
+
+try {
+  DatasetCollectionTagsSchema.index({ teamId: 1, datasetId: 1, tag: 1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDatasetCollectionTags = getMongoModel<DatasetCollectionTagsSchemaType>(
+  DatasetCollectionTagsName,
+  DatasetCollectionTagsSchema
+);
--- a/packages/service/core/workflow/dispatch/dataset/search.ts
+++ b/packages/service/core/workflow/dispatch/dataset/search.ts
@@ -27,6 +27,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
  [NodeInputKeyEnum.datasetSearchUsingExtensionQuery]: boolean;
  [NodeInputKeyEnum.datasetSearchExtensionModel]: string;
  [NodeInputKeyEnum.datasetSearchExtensionBg]: string;
+  [NodeInputKeyEnum.collectionFilterMatch]: string;
 }>;
 export type DatasetSearchResponse = DispatchNodeResultType<{
  [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -49,7 +50,8 @@ export async function dispatchDatasetSearch(

      datasetSearchUsingExtensionQuery,
      datasetSearchExtensionModel,
-      datasetSearchExtensionBg
+      datasetSearchExtensionBg,
+      collectionFilterMatch
    }
  } = props as DatasetSearchProps;

@@ -99,7 +101,8 @@ export async function dispatchDatasetSearch(
    limit,
    datasetIds: datasets.map((item) => item.datasetId),
    searchMode,
-    usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
+    usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
+    collectionFilterMatch
  });

  // count bill results
--- a/packages/web/components/common/Icon/constants.ts
+++ b/packages/web/components/common/Icon/constants.ts
@@ -131,6 +131,7 @@ export const iconPaths = {
  'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'),
  'core/dataset/splitLight': () => import('./icons/core/dataset/splitLight.svg'),
  'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'),
+  'core/dataset/tag': () => import('./icons/core/dataset/tag.svg'),
  'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'),
  'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'),
  'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'),
--- a/packages/web/components/common/Icon/icons/core/dataset/tag.svg
+++ b/packages/web/components/common/Icon/icons/core/dataset/tag.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+  <path d="M6.99509 8.73514C7.91556 8.73514 8.66175 7.98895 8.66175 7.06848C8.66175 6.148 7.91556 5.40181 6.99509 5.40181C6.07461 5.40181 5.32842 6.148 5.32842 7.06848C5.32842 7.98895 6.07461 8.73514 6.99509 8.73514Z" fill="#3370FF"/>
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M3.99565 1.36786C2.54416 1.36786 1.36749 2.54452 1.36749 3.99602V8.73989C1.36749 8.78421 1.36927 8.82834 1.37282 8.87218C1.40208 9.61777 1.70199 10.357 2.27163 10.9267L9.07586 17.7309C10.2775 18.9325 12.2256 18.9325 13.4272 17.7309L17.7314 13.4268C18.933 12.2252 18.933 10.277 17.7314 9.07542L10.9271 2.27119C10.3517 1.69578 9.60329 1.39563 8.85009 1.37157C8.8134 1.3691 8.77653 1.36786 8.73952 1.36786H3.99565ZM3.03416 3.99602C3.03416 3.465 3.46463 3.03453 3.99565 3.03453H8.73865L8.7619 3.03649L8.78665 3.0371C9.13683 3.04575 9.48192 3.183 9.74862 3.4497L16.5528 10.2539C17.1036 10.8046 17.1036 11.6975 16.5528 12.2483L12.2487 16.5524C11.698 17.1031 10.8051 17.1031 10.2544 16.5524L3.45015 9.74818C3.1856 9.48364 3.04846 9.14202 3.03778 8.79481L3.03689 8.76573L3.03416 8.73871V3.99602Z" fill="#3370FF"/>
+</svg>
--- a/packages/web/components/common/MyModal/index.tsx
+++ b/packages/web/components/common/MyModal/index.tsx
@@ -21,6 +21,7 @@ export interface MyModalProps extends ModalContentProps {
  isLoading?: boolean;
  isOpen: boolean;
  onClose?: () => void;
+  closeOnOverlayClick?: boolean;
 }

 const MyModal = ({
@@ -33,6 +34,7 @@ const MyModal = ({
  isLoading,
  w = 'auto',
  maxW = ['90vw', '600px'],
+  closeOnOverlayClick = true,
  ...props
 }: MyModalProps) => {
  const isPc = useSystem();
@@ -44,6 +46,7 @@ const MyModal = ({
      autoFocus={false}
      isCentered={isPc ? isCentered : true}
      blockScrollOnMount={false}
+      closeOnOverlayClick={closeOnOverlayClick}
    >
      <ModalOverlay />
      <ModalContent
--- a/packages/web/components/common/MyPopover/index.tsx
+++ b/packages/web/components/common/MyPopover/index.tsx
@@ -5,22 +5,32 @@ import {
  PopoverContent,
  useDisclosure,
  PlacementWithLogical,
-  PopoverArrow
+  PopoverArrow,
+  PopoverContentProps
 } from '@chakra-ui/react';

+interface Props extends PopoverContentProps {
+  Trigger: React.ReactNode;
+  placement?: PlacementWithLogical;
+  offset?: [number, number];
+  trigger?: 'hover' | 'click';
+  hasArrow?: boolean;
+  children: (e: { onClose: () => void }) => React.ReactNode;
+  onCloseFunc?: () => void;
+  closeOnBlur?: boolean;
+}
+
 const MyPopover = ({
  Trigger,
  placement,
  offset,
  trigger,
-  children
-}: {
-  Trigger: React.ReactNode;
-  placement?: PlacementWithLogical;
-  offset?: [number, number];
-  trigger?: 'hover' | 'click';
-  children: (e: { onClose: () => void }) => React.ReactNode;
-}) => {
+  hasArrow = true,
+  children,
+  onCloseFunc,
+  closeOnBlur = false,
+  ...props
+}: Props) => {
  const firstFieldRef = React.useRef(null);

  const { onOpen, onClose, isOpen } = useDisclosure();
@@ -30,10 +40,13 @@ const MyPopover = ({
      isOpen={isOpen}
      initialFocusRef={firstFieldRef}
      onOpen={onOpen}
-      onClose={onClose}
+      onClose={() => {
+        onClose();
+        onCloseFunc && onCloseFunc();
+      }}
      placement={placement}
      offset={offset}
-      closeOnBlur={false}
+      closeOnBlur={closeOnBlur}
      trigger={trigger}
      openDelay={100}
      closeDelay={100}
@@ -41,8 +54,8 @@ const MyPopover = ({
      lazyBehavior="keepMounted"
    >
      <PopoverTrigger>{Trigger}</PopoverTrigger>
-      <PopoverContent p={4}>
-        <PopoverArrow />
+      <PopoverContent {...props}>
+        {hasArrow && <PopoverArrow />}
        {children({ onClose })}
      </PopoverContent>
    </Popover>
--- a/packages/web/hooks/useScrollPagination.tsx
+++ b/packages/web/hooks/useScrollPagination.tsx
@@ -7,7 +7,6 @@ import {
  useBoolean,
  useLockFn,
  useMemoizedFn,
-  useMount,
  useScroll,
  useVirtualList,
  useRequest
@@ -50,6 +49,7 @@ export function useScrollPagination<
  const { toast } = useToast();
  const [current, setCurrent] = useState(1);
  const [data, setData] = useState<TData['list']>([]);
+  const [total, setTotal] = useState(0);
  const [isLoading, { setTrue, setFalse }] = useBoolean(false);

  const [list] = useVirtualList<TData['list'][0]>(data, {
@@ -71,6 +71,7 @@ export function useScrollPagination<
        ...defaultParams
      } as TParams);

+      setTotal(res.total);
      setCurrent(num);

      if (num === 1) {
@@ -146,6 +147,7 @@ export function useScrollPagination<
  return {
    containerRef,
    list,
+    total,
    data,
    setData,
    isLoading,
--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
@@ -1,7 +1,5 @@
 {
  "App": "App",
-  "click_to_resume": "Resume",
-  "code_editor": "Code edit",
  "Export": "Export",
  "Folder": "Folder",
  "Login": "Login",
@@ -13,6 +11,8 @@
  "UnKnow": "Unknown",
  "Warning": "Warning",
  "add_new": "Add new",
+  "click_to_resume": "Resume",
+  "code_editor": "Code edit",
  "common": {
    "Action": "Action",
    "Add": "Add",
@@ -91,6 +91,8 @@
    "Root folder": "Root folder",
    "Run": "Run",
    "Save": "Save",
+    "Save Failed": "Saved failed",
+    "Save Success": "Saved success",
    "Search": "Search",
    "Select File Failed": "Select File Failed",
    "Select template": "Select template",
@@ -482,7 +484,8 @@
            "success": "Start syncing"
          }
        },
-        "training": {}
+        "training": {
+        }
      },
      "data": {
        "Auxiliary Data": "Auxiliary data",
@@ -505,13 +508,13 @@
        "Data not found": "Data does not exist or has been deleted",
        "Start Sync Failed": "Failed to start syncing",
        "Template does not exist": "Template does not exist",
+        "invalidVectorModelOrQAModel": "Invalid vector model or QA model",
        "unAuthDataset": "Unauthorized to operate this dataset",
        "unAuthDatasetCollection": "Unauthorized to operate this collection",
        "unAuthDatasetData": "Unauthorized to operate this data",
        "unAuthDatasetFile": "Unauthorized to operate this file",
        "unCreateCollection": "Unauthorized to operate this data",
-        "unLinkCollection": "Not a network link collection",
-        "invalidVectorModelOrQAModel": "Invalid vector model or QA model"
+        "unLinkCollection": "Not a network link collection"
      },
      "externalFile": "external file repository",
      "file": "File",
--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -1,26 +1,41 @@
 {
+  "Disabled": "Disabled",
+  "Enable": "Enable",
+  "Enabled": "Enabled",
+  "collection": {
+    "Create update time": "Create/Update time",
+    "Training type": "Training type"
+  },
  "collection_tags": "Tags",
  "common_dataset": "Common dataset",
  "common_dataset_desc": "Can be built by importing files, web links, or manual entry",
  "confirm_to_rebuild_embedding_tip": "Are you sure to switch the knowledge base index?\nSwitching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.\n\nIn addition, you need to be careful to modify the applications that select this knowledge base to avoid mixing them with other index model knowledge bases.",
-  "Disabled": "Disabled",
-  "Enable": "Enable",
-  "Enabled": "Enabled",
+  "dataset": {
+    "no_collections": "no collections",
+    "no_tags": "no tags"
+  },
  "external_file": "External file",
  "external_file_dataset_desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice",
  "external_id": "File id",
  "external_read_url": "External read url",
  "external_read_url_tip": "You can configure the reading address of your file library. This allows users to read and authenticate. You can currently use the {{fileId}} variable to refer to the external file ID.",
  "external_url": "File read url",
+  "filename": "filename",
  "folder_dataset": "Folder",
  "rebuild_embedding_start_tip": "The task of switching index models has begun",
  "rebuilding_index_count": "Rebuilding count: {{count}}",
+  "tag": {
+    "Add New": "Add new",
+    "Add_new_tag": "Add new tag",
+    "Edit_tag": "Edit tag",
+    "add": "Add",
+    "cancel": "Cancel",
+    "delete_tag_confirm": "Confirm to delete tag",
+    "manage": "Manage",
+    "searchOrAddTag": "Search or add tags",
+    "tags": "Tags"
+  },
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The knowledge base has indexes that are being trained or being rebuilt",
  "website_dataset": "Web site",
-  "website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset",
-  "collection": {
-    "Create update time": "Create/Update time",
-    "Training type": "Training type"
-  },
-  "filename": "filename"
-}
+  "website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset"
+}
--- a/packages/web/i18n/zh/common.json
+++ b/packages/web/i18n/zh/common.json
@@ -1,7 +1,5 @@
 {
  "App": "应用",
-  "click_to_resume": "点击恢复",
-  "code_editor": "代码编辑",
  "Export": "导出",
  "Folder": "文件夹",
  "Login": "登录",
@@ -13,6 +11,8 @@
  "UnKnow": "未知",
  "Warning": "提示",
  "add_new": "新增",
+  "click_to_resume": "点击恢复",
+  "code_editor": "代码编辑",
  "common": {
    "Action": "操作",
    "Add": "添加",
@@ -91,6 +91,8 @@
    "Root folder": "根目录",
    "Run": "运行",
    "Save": "保存",
+    "Save Failed": "保存失败",
+    "Save Success": "保存成功",
    "Search": "搜索",
    "Select File Failed": "选择文件异常",
    "Select template": "选择模板",
@@ -482,7 +484,8 @@
            "success": "开始同步"
          }
        },
-        "training": {}
+        "training": {
+        }
      },
      "data": {
        "Auxiliary Data": "辅助数据",
@@ -505,13 +508,13 @@
        "Data not found": "数据不存在或已被删除",
        "Start Sync Failed": "开始同步失败",
        "Template does not exist": "模板不存在",
+        "invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误",
        "unAuthDataset": "无权操作该知识库",
        "unAuthDatasetCollection": "无权操作该数据集",
        "unAuthDatasetData": "无权操作该数据",
        "unAuthDatasetFile": "无权操作该文件",
        "unCreateCollection": "无权操作该数据",
-        "unLinkCollection": "不是网络链接集合",
-        "invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误"
+        "unLinkCollection": "不是网络链接集合"
      },
      "externalFile": "外部文件库",
      "file": "文件",
--- a/packages/web/i18n/zh/dataset.json
+++ b/packages/web/i18n/zh/dataset.json
@@ -1,26 +1,41 @@
 {
+  "Disabled": "已禁用",
+  "Enable": "启用",
+  "Enabled": "已启用",
+  "collection": {
+    "Create update time": "创建/更新时间",
+    "Training type": "训练模式"
+  },
  "collection_tags": "集合标签",
  "common_dataset": "通用知识库",
  "common_dataset_desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
  "confirm_to_rebuild_embedding_tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。\n\n此外，你还需要注意修改选择该知识库的应用，避免它们与其他索引模型知识库混用。",
-  "Disabled": "已禁用",
-  "Enable": "启用",
-  "Enabled": "已启用",
+  "dataset": {
+    "no_collections": "暂无数据集",
+    "no_tags": "暂无标签"
+  },
  "external_file": "外部文件库",
  "external_file_dataset_desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储",
  "external_id": "文件阅读 ID",
  "external_read_url": "外部预览地址",
  "external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。",
  "external_url": "文件访问 URL",
+  "filename": "文件名",
  "folder_dataset": "文件夹",
  "rebuild_embedding_start_tip": "切换索引模型任务已开始",
  "rebuilding_index_count": "重建中索引数量：{{count}}",
+  "tag": {
+    "Add New": "新建",
+    "Add_new_tag": "新建标签",
+    "Edit_tag": "编辑标签",
+    "add": "创建",
+    "cancel": "取消选择",
+    "delete_tag_confirm": "确定删除标签？",
+    "manage": "标签管理",
+    "searchOrAddTag": "搜索或添加标签",
+    "tags": "标签"
+  },
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",
  "website_dataset": "Web 站点同步",
-  "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
-  "collection": {
-    "Create update time": "创建/更新时间",
-    "Training type": "训练模式"
-  },
-  "filename": "文件名"
-}
+  "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库"
+}