Collection tag (#2266)

* feat: collection metadata filter (#2211)

* feat: add dataset collection tags (#2231)

* dataset page

* workflow page

* move

* fix

* add plus filter

* fix

* fix

* fix

* perf: collection tag code

* fix: collection tags (#2249)

* fix

* fix

* fix tags of dataset page

* fix tags of workflow page

* doc

* add comments

* fix: collection tags (#2264)

* fix: metadata filter

* feat: search filter

---------

Co-authored-by: heheer <1239331448@qq.com>
Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
Archer
2024-08-05 12:08:46 +08:00
committed by GitHub
parent 56f6e69bc7
commit fe71efbbd2
46 changed files with 1914 additions and 112 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 294 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 83 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 126 KiB

View File

@@ -4,7 +4,7 @@ description: "FastGPT 对话问题引导"
icon: "code"
draft: false
toc: true
weight: 350
weight: 108
---
![](/imgs/questionGuide.png)

View File

@@ -0,0 +1,50 @@
---
title: "知识库集合标签"
description: "FastGPT 知识库集合标签使用说明"
icon: "developer_guide"
draft: false
toc: true
weight: 108
---
知识库集合标签是 FastGPT 商业版特有功能。它允许你对知识库中的数据集合添加标签进行分类,更高效地管理知识库数据。
而进一步可以在问答中,搜索知识库时添加集合过滤,实现更精确的搜索。
| | | |
| --------------------- | --------------------- | --------------------- |
| ![](/imgs/collection-tags-1.png) | ![](/imgs/collection-tags-2.png) | ![](/imgs/collection-tags-3.png) |
## 标签基础操作说明
在知识库详情页面,可以对标签进行管理,可执行的操作有
- 创建标签
- 修改标签名
- 删除标签
- 将一个标签赋给多个数据集合
- 给一个数据集合添加多个标签
也可以利用标签对数据集合进行筛选
## 知识库搜索-集合过滤说明
利用标签可以在知识库搜索时,通过填写「集合过滤」这一栏来实现更精确的搜索,具体的填写示例如下
```json
{
"tags": {
"$and": ["标签 1","标签 2"],
"$or": ["有 $and 标签时and 生效or 不生效"]
},
"createTime": {
"$gte": "YYYY-MM-DD HH:mm 格式即可,集合的创建时间大于该时间",
"$lte": "YYYY-MM-DD HH:mm 格式即可,集合的创建时间小于该时间,可和 $gte 共同使用"
}
}
```
在填写时有两个注意的点,
- 标签值可以为 `string` 类型的标签名,也可以为 `null`,而 `null` 代表着未设置标签的数据集合
- 标签过滤有 `$and``$or` 两种条件类型,在同时设置了 `$and``$or` 的情况下,只有 `$and` 会生效

View File

@@ -74,6 +74,23 @@ export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollecti
filename?: string;
};
/* ================= tag ===================== */
export type CreateDatasetCollectionTagParams = {
datasetId: string;
tag: string;
};
export type AddTagsToCollectionsParams = {
originCollectionIds: string[];
collectionIds: string[];
datasetId: string;
tag: string;
};
export type UpdateDatasetCollectionTagParams = {
datasetId: string;
tagId: string;
tag: string;
};
/* ================= data ===================== */
export type PgSearchRawType = {
id: string;

View File

@@ -69,6 +69,13 @@ export type DatasetCollectionSchemaType = {
};
};
export type DatasetCollectionTagsSchemaType = {
_id: string;
teamId: string;
datasetId: string;
tag: string;
};
export type DatasetDataIndexItemType = {
defaultIndex: boolean;
dataId: string; // pg data id
@@ -144,6 +151,17 @@ export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentMode
permission: DatasetPermission;
};
/* ================= tag ===================== */
export type DatasetTagType = {
_id: string;
tag: string;
};
export type TagUsageType = {
tagId: string;
collections: string[];
};
/* ================= collection ===================== */
export type DatasetCollectionItemType = CollectionWithDatasetType & {
sourceName: string;

View File

@@ -85,6 +85,7 @@ export enum NodeInputKeyEnum {
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
collectionFilterMatch = 'collectionFilterMatch',
// concat dataset
datasetQuoteList = 'system_datasetQuoteList',

View File

@@ -90,6 +90,25 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
{
...Input_Template_UserChatInput,
toolDescription: '需要检索的内容'
},
{
key: NodeInputKeyEnum.collectionFilterMatch,
renderTypeList: [FlowNodeInputTypeEnum.JSONEditor, FlowNodeInputTypeEnum.reference],
label: '集合元数据过滤',
valueType: WorkflowIOValueTypeEnum.object,
isPro: true,
description: `目前支持标签和创建时间过滤,需按照以下格式填写:
{
"tags": {
"$and": ["标签 1","标签 2"],
"$or": ["有 $and 标签时and 生效or 不生效"]
},
"createTime": {
"$gte": "YYYY-MM-DD HH:mm 格式即可,集合的创建时间大于该时间",
"$lte": "YYYY-MM-DD HH:mm 格式即可,集合的创建时间小于该时间,可和 $gte 共同使用"
}
}
`
}
],
outputs: [

View File

@@ -52,6 +52,7 @@ export type FlowNodeInputItemType = InputComponentPropsType & {
// render components params
canEdit?: boolean; // dynamic inputs
isPro?: boolean; // Pro version field
};
export type FlowNodeOutputItemType = {

View File

@@ -26,9 +26,7 @@ export type EmbeddingRecallProps = {
datasetIds: string[];
forbidCollectionIdList: string[];
// forbidEmbIndexIdList: string[];
// similarity?: number;
// efSearch?: number;
filterCollectionIdList?: string[];
};
export type EmbeddingRecallCtrlProps = EmbeddingRecallProps & {
vector: number[];

View File

@@ -213,19 +213,50 @@ export class MilvusCtrl {
};
embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
const client = await this.getClient();
const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
const {
teamId,
datasetIds,
vector,
limit,
forbidCollectionIdList,
filterCollectionIdList,
retry = 2
} = props;
// Forbid collection
const formatForbidCollectionIdList = (() => {
if (!filterCollectionIdList) return forbidCollectionIdList;
const list = forbidCollectionIdList
.map((id) => String(id))
.filter((id) => !filterCollectionIdList.includes(id));
return list;
})();
const forbidColQuery =
forbidCollectionIdList.length > 0
? `and (collectionId not in [${forbidCollectionIdList.map((id) => `"${String(id)}"`).join(',')}])`
formatForbidCollectionIdList.length > 0
? `and (collectionId not in [${formatForbidCollectionIdList.map((id) => `"${id}"`).join(',')}])`
: '';
// filter collection id
const formatFilterCollectionId = (() => {
if (!filterCollectionIdList) return;
return filterCollectionIdList
.map((id) => String(id))
.filter((id) => !forbidCollectionIdList.includes(id));
})();
const collectionIdQuery = formatFilterCollectionId
? `and (collectionId in [${formatFilterCollectionId.map((id) => `"${id}"`)}])`
: ``;
// Empty data
if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
return { results: [] };
}
try {
const { results } = await client.search({
collection_name: DatasetVectorTableName,
data: vector,
limit,
filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${String(id)}"`).join(',')}]) ${forbidColQuery}`,
filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${id}"`).join(',')}]) ${collectionIdQuery} ${forbidColQuery}`,
output_fields: ['collectionId']
});

View File

@@ -119,14 +119,44 @@ export class PgVectorCtrl {
}
};
embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
const {
teamId,
datasetIds,
vector,
limit,
forbidCollectionIdList,
filterCollectionIdList,
retry = 2
} = props;
// Get forbid collection
const formatForbidCollectionIdList = (() => {
if (!filterCollectionIdList) return forbidCollectionIdList;
const list = forbidCollectionIdList
.map((id) => String(id))
.filter((id) => !filterCollectionIdList.includes(id));
return list;
})();
const forbidCollectionSql =
forbidCollectionIdList.length > 0
? `AND collection_id NOT IN (${forbidCollectionIdList.map((id) => `'${String(id)}'`).join(',')})`
: 'AND collection_id IS NOT NULL';
// const forbidDataSql =
// forbidEmbIndexIdList.length > 0 ? `AND id NOT IN (${forbidEmbIndexIdList.join(',')})` : '';
formatForbidCollectionIdList.length > 0
? `AND collection_id NOT IN (${formatForbidCollectionIdList.map((id) => `'${id}'`).join(',')})`
: '';
// Filter by collectionId
const formatFilterCollectionId = (() => {
if (!filterCollectionIdList) return;
return filterCollectionIdList
.map((id) => String(id))
.filter((id) => !forbidCollectionIdList.includes(id));
})();
const filterCollectionIdSql = formatFilterCollectionId
? `AND collection_id IN (${formatFilterCollectionId.map((id) => `'${id}'`).join(',')})`
: '';
// Empty data
if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
return { results: [] };
}
try {
// const explan: any = await PgClient.query(
@@ -150,6 +180,7 @@ export class PgVectorCtrl {
from ${DatasetVectorTableName}
where team_id='${teamId}'
AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
${filterCollectionIdSql}
${forbidCollectionSql}
order by score limit ${limit};
COMMIT;`

View File

@@ -106,8 +106,10 @@ try {
updateTime: -1
});
// get forbid
// DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
// Tag filter
DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, tags: 1 });
// create time filter
DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, createTime: 1 });
} catch (error) {
console.log(error);
}

View File

@@ -20,6 +20,9 @@ import { hashStr } from '@fastgpt/global/common/string/tools';
import { jiebaSplit } from '../../../common/string/jieba';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { Types } from '../../../common/mongo';
import json5 from 'json5';
import { MongoDatasetCollectionTags } from '../tag/schema';
import { readFromSecondary } from '../../../common/mongo/utils';
type SearchDatasetDataProps = {
teamId: string;
@@ -31,6 +34,20 @@ type SearchDatasetDataProps = {
usingReRank?: boolean;
reRankQuery: string;
queries: string[];
/*
{
tags: {
$and: ["str1","str2"],
$or: ["str1","str2",null] null means no tags
},
createTime: {
$gte: 'xx',
$lte: 'xxx'
}
}
*/
collectionFilterMatch?: string;
};
export async function searchDatasetData(props: SearchDatasetDataProps) {
@@ -43,7 +60,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
limit: maxTokens,
searchMode = DatasetSearchModeEnum.embedding,
usingReRank = false,
datasetIds = []
datasetIds = [],
collectionFilterMatch
} = props;
/* init params */
@@ -87,14 +105,148 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
forbidCollectionIdList: collections.map((item) => String(item._id))
};
};
/*
Collection metadata filter
标签过滤:
1. and 先生效
2. and 标签和 null 不能共存,否则返回空数组
*/
const filterCollectionByMetadata = async (): Promise<string[] | undefined> => {
if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
let tagCollectionIdList: string[] | undefined = undefined;
let createTimeCollectionIdList: string[] | undefined = undefined;
try {
const jsonMatch = json5.parse(collectionFilterMatch);
// Tag
let andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined;
let orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined;
// get andTagIds
if (andTags && andTags.length > 0) {
// tag 去重
andTags = Array.from(new Set(andTags));
if (andTags.includes(null) && andTags.some((tag) => typeof tag === 'string')) {
return [];
}
if (andTags.every((tag) => typeof tag === 'string')) {
// Get tagId by tag string
const andTagIdList = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: andTags }
},
'_id',
{
...readFromSecondary
}
).lean();
// If you enter a tag that does not exist, none will be found
if (andTagIdList.length !== andTags.length) return [];
// Get collectionId by tagId
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
tags: { $all: andTagIdList.map((item) => String(item._id)) }
},
'_id',
{
...readFromSecondary
}
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
} else if (andTags.every((tag) => tag === null)) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }]
},
'_id',
{
...readFromSecondary
}
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
} else if (orTags && orTags.length > 0) {
// Get tagId by tag string
const orTagArray = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: orTags.filter((tag) => tag !== null) }
},
'_id',
{ ...readFromSecondary }
).lean();
const orTagIds = orTagArray.map((item) => String(item._id));
// Get collections by tagId
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [
{ tags: { $in: orTagIds } },
...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : [])
]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
// time
const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined;
const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined;
if (getCreateTime || lteCreateTime) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
createTime: {
...(getCreateTime && { $gte: new Date(getCreateTime) }),
...(lteCreateTime && {
$lte: new Date(lteCreateTime)
})
}
},
'_id'
);
createTimeCollectionIdList = collections.map((item) => String(item._id));
}
// Concat tag and time
if (tagCollectionIdList && createTimeCollectionIdList) {
return tagCollectionIdList.filter((id) => createTimeCollectionIdList!.includes(id));
} else if (tagCollectionIdList) {
return tagCollectionIdList;
} else if (createTimeCollectionIdList) {
return createTimeCollectionIdList;
}
} catch (error) {}
};
const embeddingRecall = async ({
query,
limit,
forbidCollectionIdList
forbidCollectionIdList,
filterCollectionIdList
}: {
query: string;
limit: number;
forbidCollectionIdList: string[];
filterCollectionIdList?: string[];
}) => {
const { vectors, tokens } = await getVectorsByText({
model: getVectorModel(model),
@@ -107,7 +259,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
datasetIds,
vector: vectors[0],
limit,
forbidCollectionIdList
forbidCollectionIdList,
filterCollectionIdList
});
// get q and a
@@ -165,10 +318,12 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
};
const fullTextRecall = async ({
query,
limit
limit,
filterCollectionIdList
}: {
query: string;
limit: number;
filterCollectionIdList?: string[];
}): Promise<{
fullTextRecallResults: SearchDataResponseItemType[];
tokenLen: number;
@@ -188,7 +343,14 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
$match: {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(id),
$text: { $search: jiebaSplit({ text: query }) }
$text: { $search: jiebaSplit({ text: query }) },
...(filterCollectionIdList && filterCollectionIdList.length > 0
? {
collectionId: {
$in: filterCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {})
}
},
{
@@ -327,19 +489,24 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
const fullTextRecallResList: SearchDataResponseItemType[][] = [];
let totalTokens = 0;
const { forbidCollectionIdList } = await getForbidData();
const [{ forbidCollectionIdList }, filterCollectionIdList] = await Promise.all([
getForbidData(),
filterCollectionByMetadata()
]);
console.log(filterCollectionIdList, '===');
await Promise.all(
queries.map(async (query) => {
const [{ tokens, embeddingRecallResults }, { fullTextRecallResults }] = await Promise.all([
embeddingRecall({
query,
limit: embeddingLimit,
forbidCollectionIdList
forbidCollectionIdList,
filterCollectionIdList
}),
fullTextRecall({
query,
limit: fullTextLimit
limit: fullTextLimit,
filterCollectionIdList
})
]);
totalTokens += tokens;

View File

@@ -0,0 +1,35 @@
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
import { DatasetCollectionName } from '../schema';
import { DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type';
const { Schema } = connectionMongo;
export const DatasetCollectionTagsName = 'dataset_collection_tags';
const DatasetCollectionTagsSchema = new Schema({
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
datasetId: {
type: Schema.Types.ObjectId,
ref: DatasetCollectionName,
required: true
},
tag: {
type: String,
required: true
}
});
try {
DatasetCollectionTagsSchema.index({ teamId: 1, datasetId: 1, tag: 1 });
} catch (error) {
console.log(error);
}
export const MongoDatasetCollectionTags = getMongoModel<DatasetCollectionTagsSchemaType>(
DatasetCollectionTagsName,
DatasetCollectionTagsSchema
);

View File

@@ -27,6 +27,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]: string;
[NodeInputKeyEnum.collectionFilterMatch]: string;
}>;
export type DatasetSearchResponse = DispatchNodeResultType<{
[NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -49,7 +50,8 @@ export async function dispatchDatasetSearch(
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg
datasetSearchExtensionBg,
collectionFilterMatch
}
} = props as DatasetSearchProps;
@@ -99,7 +101,8 @@ export async function dispatchDatasetSearch(
limit,
datasetIds: datasets.map((item) => item.datasetId),
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
collectionFilterMatch
});
// count bill results

View File

@@ -131,6 +131,7 @@ export const iconPaths = {
'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'),
'core/dataset/splitLight': () => import('./icons/core/dataset/splitLight.svg'),
'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'),
'core/dataset/tag': () => import('./icons/core/dataset/tag.svg'),
'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'),
'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'),
'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'),

View File

@@ -0,0 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
<path d="M6.99509 8.73514C7.91556 8.73514 8.66175 7.98895 8.66175 7.06848C8.66175 6.148 7.91556 5.40181 6.99509 5.40181C6.07461 5.40181 5.32842 6.148 5.32842 7.06848C5.32842 7.98895 6.07461 8.73514 6.99509 8.73514Z" fill="#3370FF"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M3.99565 1.36786C2.54416 1.36786 1.36749 2.54452 1.36749 3.99602V8.73989C1.36749 8.78421 1.36927 8.82834 1.37282 8.87218C1.40208 9.61777 1.70199 10.357 2.27163 10.9267L9.07586 17.7309C10.2775 18.9325 12.2256 18.9325 13.4272 17.7309L17.7314 13.4268C18.933 12.2252 18.933 10.277 17.7314 9.07542L10.9271 2.27119C10.3517 1.69578 9.60329 1.39563 8.85009 1.37157C8.8134 1.3691 8.77653 1.36786 8.73952 1.36786H3.99565ZM3.03416 3.99602C3.03416 3.465 3.46463 3.03453 3.99565 3.03453H8.73865L8.7619 3.03649L8.78665 3.0371C9.13683 3.04575 9.48192 3.183 9.74862 3.4497L16.5528 10.2539C17.1036 10.8046 17.1036 11.6975 16.5528 12.2483L12.2487 16.5524C11.698 17.1031 10.8051 17.1031 10.2544 16.5524L3.45015 9.74818C3.1856 9.48364 3.04846 9.14202 3.03778 8.79481L3.03689 8.76573L3.03416 8.73871V3.99602Z" fill="#3370FF"/>
</svg>

After

Width:  |  Height:  |  Size: 1.2 KiB

View File

@@ -21,6 +21,7 @@ export interface MyModalProps extends ModalContentProps {
isLoading?: boolean;
isOpen: boolean;
onClose?: () => void;
closeOnOverlayClick?: boolean;
}
const MyModal = ({
@@ -33,6 +34,7 @@ const MyModal = ({
isLoading,
w = 'auto',
maxW = ['90vw', '600px'],
closeOnOverlayClick = true,
...props
}: MyModalProps) => {
const isPc = useSystem();
@@ -44,6 +46,7 @@ const MyModal = ({
autoFocus={false}
isCentered={isPc ? isCentered : true}
blockScrollOnMount={false}
closeOnOverlayClick={closeOnOverlayClick}
>
<ModalOverlay />
<ModalContent

View File

@@ -5,22 +5,32 @@ import {
PopoverContent,
useDisclosure,
PlacementWithLogical,
PopoverArrow
PopoverArrow,
PopoverContentProps
} from '@chakra-ui/react';
interface Props extends PopoverContentProps {
Trigger: React.ReactNode;
placement?: PlacementWithLogical;
offset?: [number, number];
trigger?: 'hover' | 'click';
hasArrow?: boolean;
children: (e: { onClose: () => void }) => React.ReactNode;
onCloseFunc?: () => void;
closeOnBlur?: boolean;
}
const MyPopover = ({
Trigger,
placement,
offset,
trigger,
children
}: {
Trigger: React.ReactNode;
placement?: PlacementWithLogical;
offset?: [number, number];
trigger?: 'hover' | 'click';
children: (e: { onClose: () => void }) => React.ReactNode;
}) => {
hasArrow = true,
children,
onCloseFunc,
closeOnBlur = false,
...props
}: Props) => {
const firstFieldRef = React.useRef(null);
const { onOpen, onClose, isOpen } = useDisclosure();
@@ -30,10 +40,13 @@ const MyPopover = ({
isOpen={isOpen}
initialFocusRef={firstFieldRef}
onOpen={onOpen}
onClose={onClose}
onClose={() => {
onClose();
onCloseFunc && onCloseFunc();
}}
placement={placement}
offset={offset}
closeOnBlur={false}
closeOnBlur={closeOnBlur}
trigger={trigger}
openDelay={100}
closeDelay={100}
@@ -41,8 +54,8 @@ const MyPopover = ({
lazyBehavior="keepMounted"
>
<PopoverTrigger>{Trigger}</PopoverTrigger>
<PopoverContent p={4}>
<PopoverArrow />
<PopoverContent {...props}>
{hasArrow && <PopoverArrow />}
{children({ onClose })}
</PopoverContent>
</Popover>

View File

@@ -7,7 +7,6 @@ import {
useBoolean,
useLockFn,
useMemoizedFn,
useMount,
useScroll,
useVirtualList,
useRequest
@@ -50,6 +49,7 @@ export function useScrollPagination<
const { toast } = useToast();
const [current, setCurrent] = useState(1);
const [data, setData] = useState<TData['list']>([]);
const [total, setTotal] = useState(0);
const [isLoading, { setTrue, setFalse }] = useBoolean(false);
const [list] = useVirtualList<TData['list'][0]>(data, {
@@ -71,6 +71,7 @@ export function useScrollPagination<
...defaultParams
} as TParams);
setTotal(res.total);
setCurrent(num);
if (num === 1) {
@@ -146,6 +147,7 @@ export function useScrollPagination<
return {
containerRef,
list,
total,
data,
setData,
isLoading,

View File

@@ -1,7 +1,5 @@
{
"App": "App",
"click_to_resume": "Resume",
"code_editor": "Code edit",
"Export": "Export",
"Folder": "Folder",
"Login": "Login",
@@ -13,6 +11,8 @@
"UnKnow": "Unknown",
"Warning": "Warning",
"add_new": "Add new",
"click_to_resume": "Resume",
"code_editor": "Code edit",
"common": {
"Action": "Action",
"Add": "Add",
@@ -91,6 +91,8 @@
"Root folder": "Root folder",
"Run": "Run",
"Save": "Save",
"Save Failed": "Saved failed",
"Save Success": "Saved success",
"Search": "Search",
"Select File Failed": "Select File Failed",
"Select template": "Select template",
@@ -482,7 +484,8 @@
"success": "Start syncing"
}
},
"training": {}
"training": {
}
},
"data": {
"Auxiliary Data": "Auxiliary data",
@@ -505,13 +508,13 @@
"Data not found": "Data does not exist or has been deleted",
"Start Sync Failed": "Failed to start syncing",
"Template does not exist": "Template does not exist",
"invalidVectorModelOrQAModel": "Invalid vector model or QA model",
"unAuthDataset": "Unauthorized to operate this dataset",
"unAuthDatasetCollection": "Unauthorized to operate this collection",
"unAuthDatasetData": "Unauthorized to operate this data",
"unAuthDatasetFile": "Unauthorized to operate this file",
"unCreateCollection": "Unauthorized to operate this data",
"unLinkCollection": "Not a network link collection",
"invalidVectorModelOrQAModel": "Invalid vector model or QA model"
"unLinkCollection": "Not a network link collection"
},
"externalFile": "external file repository",
"file": "File",

View File

@@ -1,26 +1,41 @@
{
"Disabled": "Disabled",
"Enable": "Enable",
"Enabled": "Enabled",
"collection": {
"Create update time": "Create/Update time",
"Training type": "Training type"
},
"collection_tags": "Tags",
"common_dataset": "Common dataset",
"common_dataset_desc": "Can be built by importing files, web links, or manual entry",
"confirm_to_rebuild_embedding_tip": "Are you sure to switch the knowledge base index?\nSwitching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.\n\nIn addition, you need to be careful to modify the applications that select this knowledge base to avoid mixing them with other index model knowledge bases.",
"Disabled": "Disabled",
"Enable": "Enable",
"Enabled": "Enabled",
"dataset": {
"no_collections": "no collections",
"no_tags": "no tags"
},
"external_file": "External file",
"external_file_dataset_desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice",
"external_id": "File id",
"external_read_url": "External read url",
"external_read_url_tip": "You can configure the reading address of your file library. This allows users to read and authenticate. You can currently use the {{fileId}} variable to refer to the external file ID.",
"external_url": "File read url",
"filename": "filename",
"folder_dataset": "Folder",
"rebuild_embedding_start_tip": "The task of switching index models has begun",
"rebuilding_index_count": "Rebuilding count: {{count}}",
"tag": {
"Add New": "Add new",
"Add_new_tag": "Add new tag",
"Edit_tag": "Edit tag",
"add": "Add",
"cancel": "Cancel",
"delete_tag_confirm": "Confirm to delete tag",
"manage": "Manage",
"searchOrAddTag": "Search or add tags",
"tags": "Tags"
},
"the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The knowledge base has indexes that are being trained or being rebuilt",
"website_dataset": "Web site",
"website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset",
"collection": {
"Create update time": "Create/Update time",
"Training type": "Training type"
},
"filename": "filename"
}
"website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset"
}

View File

@@ -1,7 +1,5 @@
{
"App": "应用",
"click_to_resume": "点击恢复",
"code_editor": "代码编辑",
"Export": "导出",
"Folder": "文件夹",
"Login": "登录",
@@ -13,6 +11,8 @@
"UnKnow": "未知",
"Warning": "提示",
"add_new": "新增",
"click_to_resume": "点击恢复",
"code_editor": "代码编辑",
"common": {
"Action": "操作",
"Add": "添加",
@@ -91,6 +91,8 @@
"Root folder": "根目录",
"Run": "运行",
"Save": "保存",
"Save Failed": "保存失败",
"Save Success": "保存成功",
"Search": "搜索",
"Select File Failed": "选择文件异常",
"Select template": "选择模板",
@@ -482,7 +484,8 @@
"success": "开始同步"
}
},
"training": {}
"training": {
}
},
"data": {
"Auxiliary Data": "辅助数据",
@@ -505,13 +508,13 @@
"Data not found": "数据不存在或已被删除",
"Start Sync Failed": "开始同步失败",
"Template does not exist": "模板不存在",
"invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误",
"unAuthDataset": "无权操作该知识库",
"unAuthDatasetCollection": "无权操作该数据集",
"unAuthDatasetData": "无权操作该数据",
"unAuthDatasetFile": "无权操作该文件",
"unCreateCollection": "无权操作该数据",
"unLinkCollection": "不是网络链接集合",
"invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误"
"unLinkCollection": "不是网络链接集合"
},
"externalFile": "外部文件库",
"file": "文件",

View File

@@ -1,26 +1,41 @@
{
"Disabled": "已禁用",
"Enable": "启用",
"Enabled": "已启用",
"collection": {
"Create update time": "创建/更新时间",
"Training type": "训练模式"
},
"collection_tags": "集合标签",
"common_dataset": "通用知识库",
"common_dataset_desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
"confirm_to_rebuild_embedding_tip": "确认为知识库切换索引?\n切换索引是一个非常重量的操作需要对您知识库内所有数据进行重新索引时间可能较长请确保账号内剩余积分充足。\n\n此外你还需要注意修改选择该知识库的应用避免它们与其他索引模型知识库混用。",
"Disabled": "已禁用",
"Enable": "启用",
"Enabled": "已启用",
"dataset": {
"no_collections": "暂无数据集",
"no_tags": "暂无标签"
},
"external_file": "外部文件库",
"external_file_dataset_desc": "可以从外部文件库导入文件构建知识库,文件不会进行二次存储",
"external_id": "文件阅读 ID",
"external_read_url": "外部预览地址",
"external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。",
"external_url": "文件访问 URL",
"filename": "文件名",
"folder_dataset": "文件夹",
"rebuild_embedding_start_tip": "切换索引模型任务已开始",
"rebuilding_index_count": "重建中索引数量:{{count}}",
"tag": {
"Add New": "新建",
"Add_new_tag": "新建标签",
"Edit_tag": "编辑标签",
"add": "创建",
"cancel": "取消选择",
"delete_tag_confirm": "确定删除标签?",
"manage": "标签管理",
"searchOrAddTag": "搜索或添加标签",
"tags": "标签"
},
"the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",
"website_dataset": "Web 站点同步",
"website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
"collection": {
"Create update time": "创建/更新时间",
"Training type": "训练模式"
},
"filename": "文件名"
}
"website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库"
}

View File

@@ -10,6 +10,7 @@ import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
import { PaginationProps } from '@fastgpt/web/common/fetch/type';
/* ===== dataset ===== */
@@ -18,6 +19,7 @@ export type GetDatasetCollectionsProps = RequestPaging & {
datasetId: string;
parentId?: string;
searchText?: string;
filterTags?: string[];
simple?: boolean;
selectFolder?: boolean;
};

View File

@@ -1,7 +1,8 @@
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import {
DatasetCollectionSchemaType,
DatasetDataSchemaType
DatasetDataSchemaType,
DatasetTagType
} from '@fastgpt/global/core/dataset/type.d';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
@@ -18,6 +19,7 @@ export type DatasetCollectionsListItemType = {
updateTime: DatasetCollectionSchemaType['updateTime'];
forbid?: DatasetCollectionSchemaType['forbid'];
trainingType?: DatasetCollectionSchemaType['trainingType'];
tags?: string[];
fileId?: string;
rawLink?: string;

View File

@@ -20,6 +20,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
parentId = null,
searchText = '',
selectFolder = false,
filterTags = [],
simple = false
} = req.body as GetDatasetCollectionsProps;
searchText = searchText?.replace(/'/g, '');
@@ -43,7 +44,8 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
? {
name: new RegExp(searchText, 'i')
}
: {})
: {}),
...(filterTags.length ? { tags: { $in: filterTags } } : {})
};
const selectField = {
@@ -57,7 +59,8 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
updateTime: 1,
trainingType: 1,
fileId: 1,
rawLink: 1
rawLink: 1,
tags: 1
};
// not count data amount
@@ -68,6 +71,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
updateTime: -1
})
.lean();
return {
pageNum,
pageSize,

View File

@@ -0,0 +1,191 @@
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
import { Types } from '@fastgpt/service/common/mongo';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
import { startTrainingQueue } from '@/service/core/dataset/training/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
export type GetScrollCollectionsProps = PaginationProps<{
datasetId: string;
parentId?: string | null;
searchText?: string;
selectFolder?: boolean;
filterTags?: string[];
simple?: boolean;
}>;
async function handler(
req: ApiRequestProps<{}, GetScrollCollectionsProps>
): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
let {
datasetId,
pageSize = 10,
current = 1,
parentId = null,
searchText = '',
selectFolder = false,
filterTags = [],
simple = false
} = req.query;
if (!datasetId) {
return Promise.reject(CommonErrEnum.missingParams);
}
searchText = searchText?.replace(/'/g, '');
pageSize = Math.min(pageSize, 30);
// auth dataset and get my role
const { teamId, permission } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const match = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId),
parentId: parentId ? new Types.ObjectId(parentId) : null,
...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
...(searchText
? {
name: new RegExp(searchText, 'i')
}
: {}),
...(filterTags.length ? { tags: { $all: filterTags } } : {})
};
const selectField = {
_id: 1,
parentId: 1,
tmbId: 1,
name: 1,
type: 1,
forbid: 1,
createTime: 1,
updateTime: 1,
trainingType: 1,
fileId: 1,
rawLink: 1,
tags: 1
};
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match)
.select(selectField)
.sort({
updateTime: -1
})
.skip(pageSize * (current - 1))
.limit(pageSize)
.lean();
return {
list: await Promise.all(
collections.map(async (item) => ({
...item,
dataAmount: 0,
trainingAmount: 0,
permission
}))
),
total: await MongoDatasetCollection.countDocuments(match)
};
}
const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([
MongoDatasetCollection.aggregate([
{
$match: match
},
{
$sort: { updateTime: -1 }
},
{
$skip: (current - 1) * pageSize
},
{
$limit: pageSize
},
// count training data
{
$lookup: {
from: DatasetTrainingCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }]
}
}
},
{ $count: 'count' }
],
as: 'trainingCount'
}
},
// count collection total data
{
$lookup: {
from: DatasetDataCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$teamId', '$$team_id'] },
{ $eq: ['$datasetId', '$$dataset_id'] },
{ $eq: ['$collectionId', '$$id'] }
]
}
}
},
{ $count: 'count' }
],
as: 'dataCount'
}
},
{
$project: {
...selectField,
dataAmount: {
$ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0]
},
trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
}
}
}
]),
MongoDatasetCollection.countDocuments(match)
]);
const data = await Promise.all(
collections.map(async (item) => ({
...item,
permission
}))
);
if (data.find((item) => item.trainingAmount > 0)) {
startTrainingQueue();
}
// count collections
return {
list: data,
total
};
}
export default NextAPI(handler);

View File

@@ -10,11 +10,12 @@ export type UpdateDatasetCollectionParams = {
id: string;
parentId?: string;
name?: string;
tags?: string[];
forbid?: boolean;
};
async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
const { id, parentId, name, forbid } = req.body;
const { id, parentId, name, tags, forbid } = req.body;
if (!id) {
return Promise.reject(CommonErrEnum.missingParams);
@@ -32,6 +33,7 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
const updateFields: Record<string, any> = {
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name, updateTime: getCollectionUpdateTime({ name }) }),
...(tags && { tags }),
...(forbid !== undefined && { forbid })
};

View File

@@ -6,6 +6,7 @@ import dynamic from 'next/dynamic';
import InputLabel from './Label';
import type { RenderInputProps } from './type';
import { useSystemStore } from '@/web/common/system/useSystemStore';
const RenderList: {
types: FlowNodeInputTypeEnum[];
@@ -74,7 +75,18 @@ type Props = {
mb?: number;
};
const RenderInput = ({ flowInputList, nodeId, CustomComponent, mb = 5 }: Props) => {
const copyInputs = useMemo(() => JSON.stringify(flowInputList), [flowInputList]);
const { feConfigs } = useSystemStore();
const copyInputs = useMemo(
() =>
JSON.stringify(
flowInputList.filter((input) => {
if (input.isPro && !feConfigs?.isPlus) return false;
return true;
})
),
[feConfigs?.isPlus, flowInputList]
);
const filterInputs = useMemo(() => {
return JSON.parse(copyInputs) as FlowNodeInputItemType[];
}, [copyInputs]);

View File

@@ -90,6 +90,7 @@ const SliderApps = ({ apps, activeAppId }: { apps: AppListItemType[]; activeAppI
<MyPopover
placement="bottom-end"
offset={[20, 10]}
p={4}
trigger="hover"
Trigger={
<HStack

View File

@@ -29,6 +29,8 @@ type CollectionPageContextType = {
pageSize: number;
searchText: string;
setSearchText: Dispatch<SetStateAction<string>>;
filterTags: string[];
setFilterTags: Dispatch<SetStateAction<string[]>>;
};
export const CollectionPageContext = createContext<CollectionPageContextType>({
@@ -52,6 +54,10 @@ export const CollectionPageContext = createContext<CollectionPageContextType>({
searchText: '',
setSearchText: function (value: SetStateAction<string>): void {
throw new Error('Function not implemented.');
},
filterTags: [],
setFilterTags: function (value: SetStateAction<string[]>): void {
throw new Error('Function not implemented.');
}
});
@@ -96,6 +102,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
// collection list
const [searchText, setSearchText] = useState('');
const [filterTags, setFilterTags] = useState<string[]>([]);
const {
data: collections,
Pagination,
@@ -110,7 +117,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
params: {
datasetId,
parentId,
searchText
searchText,
filterTags
},
defaultRequest: false
});
@@ -124,6 +132,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
searchText,
setSearchText,
filterTags,
setFilterTags,
collections,
Pagination,
total,

View File

@@ -32,13 +32,14 @@ import { useContextSelector } from 'use-context-selector';
import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import HeaderTagPopOver from './HeaderTagPopOver';
const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
const Header = ({}: {}) => {
const { t } = useTranslation();
const theme = useTheme();
const { setLoading } = useSystemStore();
const { setLoading, feConfigs } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const router = useRouter();
@@ -154,7 +155,6 @@ const Header = ({}: {}) => {
{isPc && (
<Flex alignItems={'center'} mr={4}>
<MyInput
bg={'myGray.50'}
w={['100%', '250px']}
size={'sm'}
h={'36px'}
@@ -188,7 +188,9 @@ const Header = ({}: {}) => {
{/* diff collection button */}
{datasetDetail.permission.hasWritePer && (
<>
<Flex gap={3}>
{feConfigs?.isPlus && <HeaderTagPopOver />}
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<MyMenu
offset={[0, 5]}
@@ -369,7 +371,7 @@ const Header = ({}: {}) => {
]}
/>
)}
</>
</Flex>
)}
{/* modal */}

View File

@@ -0,0 +1,229 @@
import { Box, Button, Checkbox, Flex, Input, useDisclosure } from '@chakra-ui/react';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { postCreateDatasetCollectionTag } from '@/web/core/dataset/api';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useTranslation } from 'react-i18next';
import { useCallback, useEffect, useState } from 'react';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { CollectionPageContext } from './Context';
import { debounce, isEqual } from 'lodash';
import TagManageModal from './TagManageModal';
import { DatasetTagType } from '@fastgpt/global/core/dataset/type';
const HeaderTagPopOver = () => {
const { t } = useTranslation();
const [searchTag, setSearchTag] = useState('');
const [checkedTags, setCheckedTags] = useState<string[]>([]);
const { datasetDetail, datasetTags, loadDatasetTags, checkedDatasetTag, setCheckedDatasetTag } =
useContextSelector(DatasetPageContext, (v) => v);
const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({
mutationFn: async (tag: string) => {
const id = await postCreateDatasetCollectionTag({
datasetId: datasetDetail._id,
tag
});
return id;
},
onSuccess() {
setSearchTag('');
},
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed')
});
const { filterTags, setFilterTags, getData } = useContextSelector(
CollectionPageContext,
(v) => v
);
const debounceRefetch = useCallback(
debounce(() => {
getData(1);
}, 300),
[]
);
useEffect(() => {
loadDatasetTags({ id: datasetDetail._id, searchKey: searchTag });
}, [searchTag]);
const {
isOpen: isTagManageModalOpen,
onOpen: onOpenTagManageModal,
onClose: onCloseTagManageModal
} = useDisclosure();
const checkTags = (tag: DatasetTagType) => {
let currentCheckedTags = [];
if (checkedTags.includes(tag._id)) {
currentCheckedTags = checkedTags.filter((t) => t !== tag._id);
setCheckedTags(currentCheckedTags);
setCheckedDatasetTag(checkedDatasetTag.filter((t) => t._id !== tag._id));
} else {
currentCheckedTags = [...checkedTags, tag._id];
setCheckedTags([...checkedTags, tag._id]);
setCheckedDatasetTag([...checkedDatasetTag, tag]);
}
if (isEqual(currentCheckedTags, filterTags)) return;
setFilterTags(currentCheckedTags);
debounceRefetch();
};
return (
<>
<MyPopover
placement="bottom"
hasArrow={false}
offset={[2, 2]}
w={'180px'}
closeOnBlur={true}
trigger={'click'}
Trigger={
<Flex
alignItems={'center'}
px={3}
py={2}
w={['140px', '180px']}
borderRadius={'md'}
border={'1px solid'}
borderColor={'myGray.250'}
cursor={'pointer'}
overflow={'hidden'}
h={['28px', '36px']}
fontSize={'sm'}
>
<Flex flex={'1 0 0'}>
{t('dataset:tag.tags')}
<Box as={'span'}>
{checkedTags.length > 0 && (
<Box ml={1} fontSize={'xs'} color={'myGray.600'}>
{`(${checkedTags.length})`}
</Box>
)}
</Box>
</Flex>
<MyIcon name={'core/chat/chevronDown'} w={'14px'} />
</Flex>
}
>
{({ onClose }) => (
<MyBox isLoading={isCreateCollectionTagLoading} onClick={(e) => e.stopPropagation()}>
<Box px={1.5} pt={1.5}>
<Input
pl={2}
h={8}
borderRadius={'4px'}
value={searchTag}
placeholder={t('dataset:tag.searchOrAddTag')}
onChange={(e) => setSearchTag(e.target.value)}
/>
</Box>
<Box my={1} px={1.5} maxH={'240px'} overflow={'auto'}>
{searchTag && !datasetTags.map((item) => item.tag).includes(searchTag) && (
<Flex
alignItems={'center'}
fontSize={'sm'}
px={1}
cursor={'pointer'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'xs'}
onClick={() => {
onCreateCollectionTag(searchTag);
}}
>
<MyIcon name={'common/addLight'} w={'16px'} />
<Box ml={2} py={2}>
{t('dataset:tag.add') + ` "${searchTag}"`}
</Box>
</Flex>
)}
{[
...new Map(
[...checkedDatasetTag, ...datasetTags].map((item) => [item._id, item])
).values()
].map((item) => {
const checked = checkedTags.includes(item._id);
return (
<Flex
alignItems={'center'}
fontSize={'sm'}
px={1}
py={1}
my={1}
cursor={'pointer'}
bg={checked ? '#1118240D' : 'transparent'}
color={checked ? 'primary.700' : 'myGray.600'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'xs'}
key={item._id}
onClick={(e) => {
e.preventDefault();
checkTags(item);
}}
>
<Checkbox
isChecked={checkedTags.includes(item._id)}
onChange={(e) => {
checkTags(item);
}}
size={'md'}
/>
<Box ml={2}>{item.tag}</Box>
</Flex>
);
})}
</Box>
<Flex borderTop={'1px solid #E8EBF0'} color={'myGray.600'}>
<Button
w={'full'}
fontSize={'sm'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'none'}
variant={'unstyled'}
onClick={() => {
setCheckedTags([]);
setFilterTags([]);
debounceRefetch();
onClose();
}}
>
{t('dataset:tag.cancel')}
</Button>
<Box w={'1px'} bg={'myGray.200'}></Box>
<Button
w={'full'}
fontSize={'sm'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'none'}
variant={'unstyled'}
onClick={() => {
onOpenTagManageModal();
setCheckedTags([]);
}}
>
{t('dataset:tag.manage')}
</Button>
</Flex>
</MyBox>
)}
</MyPopover>
{isTagManageModalOpen && (
<TagManageModal
onClose={() => {
onCloseTagManageModal();
debounceRefetch();
}}
/>
)}
</>
);
};
export default HeaderTagPopOver;

View File

@@ -0,0 +1,530 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Input, Button, Flex, Box, Checkbox } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { CollectionPageContext } from './Context';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import {
delDatasetCollectionTag,
getDatasetCollectionTags,
getScrollCollectionList,
getTagUsage,
postAddTagsToCollections,
postCreateDatasetCollectionTag,
updateDatasetCollectionTag
} from '@/web/core/dataset/api';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import MyInput from '@/components/MyInput';
import { DatasetTagType } from '@fastgpt/global/core/dataset/type';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { useQuery } from '@tanstack/react-query';
import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
import MyBox from '@fastgpt/web/components/common/MyBox';
const TagManageModal = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const loadDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadDatasetTags);
const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags);
const { getData } = useContextSelector(CollectionPageContext, (v) => v);
const tagInputRef = useRef<HTMLInputElement>(null);
const editInputRef = useRef<HTMLInputElement>(null);
const [currentAddTag, setCurrentAddTag] = useState<
(DatasetTagType & { collections: string[] }) | undefined
>(undefined);
const [newTag, setNewTag] = useState<string | undefined>(undefined);
const [currentEditTagContent, setCurrentEditTagContent] = useState<string | undefined>(undefined);
const [currentEditTag, setCurrentEditTag] = useState<DatasetTagType | undefined>(undefined);
useEffect(() => {
if (newTag !== undefined && tagInputRef.current) {
tagInputRef.current?.focus();
}
}, [newTag]);
useEffect(() => {
if (currentEditTag !== undefined && editInputRef.current) {
editInputRef.current?.focus();
}
}, [currentEditTag]);
const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({
mutationFn: async (tag: string) => {
const id = await postCreateDatasetCollectionTag({
datasetId: datasetDetail._id,
tag
});
return id;
},
onSuccess() {
fetchData(1);
loadDatasetTags({ id: datasetDetail._id, searchKey: '' });
loadAllDatasetTags({ id: datasetDetail._id });
},
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed')
});
const { mutate: onDeleteCollectionTag, isLoading: isDeleteCollectionTagLoading } = useRequest({
mutationFn: async (tag: string) => {
const id = await delDatasetCollectionTag({
datasetId: datasetDetail._id,
id: tag
});
return id;
},
onSuccess() {
fetchData(1);
loadDatasetTags({ id: datasetDetail._id, searchKey: '' });
loadAllDatasetTags({ id: datasetDetail._id });
},
successToast: t('common:common.Delete Success'),
errorToast: t('common:common.Delete Failed')
});
const { mutate: onUpdateCollectionTag, isLoading: isUpdateCollectionTagLoading } = useRequest({
mutationFn: async (tag: DatasetTagType) => {
const id = await updateDatasetCollectionTag({
datasetId: datasetDetail._id,
tagId: tag._id,
tag: tag.tag
});
return id;
},
onSuccess() {
fetchData(1);
loadDatasetTags({ id: datasetDetail._id, searchKey: '' });
loadAllDatasetTags({ id: datasetDetail._id });
}
});
const { mutate: onSaveCollectionTag, isLoading: isSaveCollectionTagLoading } = useRequest({
mutationFn: async ({
tag,
originCollectionIds,
collectionIds
}: {
tag: string;
originCollectionIds: string[];
collectionIds: string[];
}) => {
try {
await postAddTagsToCollections({
tag,
originCollectionIds,
collectionIds,
datasetId: datasetDetail._id
});
} catch (error) {}
},
onSuccess() {
getData(1);
},
successToast: t('common:common.Save Success'),
errorToast: t('common:common.Save Failed')
});
const {
list,
ScrollList,
isLoading: isRequesting,
fetchData,
total: tagsTotal
} = useScrollPagination(getDatasetCollectionTags, {
refreshDeps: [''],
debounceWait: 300,
itemHeight: 56,
overscan: 10,
pageSize: 10,
defaultParams: {
datasetId: datasetDetail._id,
searchText: ''
}
});
const { data: tagUsages } = useRequest2(() => getTagUsage(datasetDetail._id), {
manual: false
});
return (
<MyModal
isOpen
onClose={onClose}
iconSrc="core/dataset/tag"
title={t('dataset:tag.manage')}
w={'580px'}
h={'600px'}
isLoading={
isRequesting ||
isCreateCollectionTagLoading ||
isDeleteCollectionTagLoading ||
isUpdateCollectionTagLoading ||
isSaveCollectionTagLoading
}
closeOnOverlayClick={false}
>
{currentAddTag === undefined ? (
<>
<Flex
alignItems={'center'}
color={'myGray.900'}
pb={2}
borderBottom={'1px solid #E8EBF0'}
mx={8}
pt={6}
>
<MyIcon name="menu" w={5} />
<Box ml={2} fontWeight={'semibold'} flex={'1 0 0'}>{`${tagsTotal}个标签`}</Box>
<Button
size={'sm'}
leftIcon={<MyIcon name="common/addLight" w={4} />}
variant={'outline'}
fontSize={'xs'}
onClick={() => {
setNewTag('');
}}
>
{t('dataset:tag.Add New')}
</Button>
</Flex>
<ScrollList
px={8}
flex={'1 0 0'}
fontSize={'sm'}
EmptyChildren={<EmptyTip text={t('dataset:dataset.no_tags')} />}
>
{newTag !== undefined && (
<Flex p={2} borderBottom={'1px solid #E8EBF0'}>
<Input
placeholder={t('dataset:tag.Add_new_tag')}
value={newTag}
onChange={(e) => setNewTag(e.target.value)}
ref={tagInputRef}
w={'200px'}
onBlur={() => {
if (newTag && !list.map((item) => item.data.tag).includes(newTag)) {
onCreateCollectionTag(newTag);
}
setNewTag(undefined);
}}
/>
</Flex>
)}
{list.map((listItem) => {
const item = listItem.data;
const tagUsage = tagUsages?.find((tagUsage) => tagUsage.tagId === item._id);
const collections = tagUsage?.collections || [];
const usage = collections.length;
return (
<Flex
py={2}
borderBottom={'1px solid #E8EBF0'}
sx={{
'&:hover .icon-box': {
display: 'flex'
}
}}
key={item._id}
>
<Flex
px={2}
py={1}
flex={'1'}
_hover={{ bg: 'myGray.100' }}
alignItems={'center'}
borderRadius={'4px'}
>
<Flex
flex={'1 0 0'}
alignItems={'center'}
onClick={() => {
setCurrentAddTag({ ...item, collections });
}}
cursor={'pointer'}
>
{currentEditTag?._id !== item._id ? (
<Box
px={3}
py={1.5}
bg={'#DBF3FF'}
color={'#0884DD'}
fontSize={'xs'}
borderRadius={'6px'}
>
{item.tag}
</Box>
) : (
<Input
placeholder={t('dataset:tag.Edit_tag')}
value={
currentEditTagContent !== undefined ? currentEditTagContent : item.tag
}
onChange={(e) => setCurrentEditTagContent(e.target.value)}
ref={editInputRef}
w={'200px'}
onBlur={() => {
if (
currentEditTagContent &&
!list.map((item) => item.data.tag).includes(currentEditTagContent)
) {
onUpdateCollectionTag({
tag: currentEditTagContent,
_id: item._id
});
}
setCurrentEditTag(undefined);
setCurrentEditTagContent(undefined);
}}
/>
)}
<Box as={'span'} color={'myGray.500'} ml={2}>{`(${usage})`}</Box>
</Flex>
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
mr={2}
p={1}
borderRadius={'6px'}
onClick={() => {
setCurrentAddTag({ ...item, collections });
}}
cursor={'pointer'}
>
<MyIcon name="common/addLight" w={4} />
</Box>
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
mr={2}
p={1}
borderRadius={'6px'}
cursor={'pointer'}
onClick={(e) => {
setCurrentEditTag(item);
editInputRef.current?.focus();
}}
>
<MyIcon name="edit" w={4} />
</Box>
<PopoverConfirm
showCancel
content={t('dataset:tag.delete_tag_confirm')}
type="delete"
Trigger={
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
p={1}
borderRadius={'6px'}
cursor={'pointer'}
>
<MyIcon name="delete" w={4} />
</Box>
}
onConfirm={() => onDeleteCollectionTag(item._id)}
/>
</Flex>
</Flex>
);
})}
</ScrollList>
</>
) : (
<AddTagToCollections
currentAddTag={currentAddTag}
setCurrentAddTag={setCurrentAddTag}
onSaveCollectionTag={onSaveCollectionTag}
/>
)}
</MyModal>
);
};
export default TagManageModal;
const AddTagToCollections = ({
currentAddTag,
setCurrentAddTag,
onSaveCollectionTag
}: {
currentAddTag: DatasetTagType & { collections: string[] };
setCurrentAddTag: (tag: (DatasetTagType & { collections: string[] }) | undefined) => void;
onSaveCollectionTag: ({
tag,
originCollectionIds,
collectionIds
}: {
tag: string;
originCollectionIds: string[];
collectionIds: string[];
}) => void;
}) => {
const { t } = useTranslation();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const [selectedCollections, setSelectedCollections] = useState<string[]>([]);
useEffect(() => {
setSelectedCollections(currentAddTag.collections);
}, []);
const [searchText, setSearchText] = useState('');
const {
list: collectionsList,
ScrollList: ScrollListCollections,
isLoading: isCollectionLoading
} = useScrollPagination(getScrollCollectionList, {
refreshDeps: [searchText],
debounceWait: 300,
itemHeight: 29,
overscan: 10,
pageSize: 30,
defaultParams: {
datasetId: datasetDetail._id,
searchText
}
});
const formatCollections = useMemo(
() =>
collectionsList.map((item) => {
const collection = item.data;
const icon = getCollectionIcon(collection.type, collection.name);
return {
id: collection._id,
tags: collection.tags,
name: collection.name,
icon
};
}),
[collectionsList]
);
return (
<MyBox flex={'1 0 0'} isLoading={isCollectionLoading}>
<Flex alignItems={'center'} pb={2} mx={8} pt={6} borderBottom={'1px solid #E8EBF0'}>
<MyIcon
name="common/backFill"
w={4}
cursor={'pointer'}
onClick={() => {
setCurrentAddTag(undefined);
setSearchText('');
}}
/>
{
<Flex alignItems={'center'}>
<Box
ml={2}
px={3}
py={1.5}
bg={'#DBF3FF'}
color={'#0884DD'}
fontSize={'sm'}
borderRadius={'6px'}
>
{currentAddTag.tag}
</Box>
<Box
as={'span'}
fontSize={'sm'}
color={'myGray.500'}
ml={2}
>{`(${selectedCollections.length})`}</Box>
</Flex>
}
<Box flex={'1 0 0'}></Box>
<MyInput
placeholder={t('common:common.Search')}
w={'200px'}
mr={2}
onChange={(e) => {
setSearchText(e.target.value);
}}
/>
<Button
leftIcon={<MyIcon name="save" w={4} />}
onClick={() => {
onSaveCollectionTag({
tag: currentAddTag._id,
originCollectionIds: currentAddTag.collections,
collectionIds: selectedCollections
});
}}
>
{t('common:common.Save')}
</Button>
</Flex>
<ScrollListCollections
px={8}
mt={2}
flex={'1 0 0'}
fontSize={'sm'}
EmptyChildren={<EmptyTip text={t('dataset:dataset.no_collections')} />}
>
{formatCollections.map((collection) => {
return (
<Flex
px={2}
py={1}
flex={'1'}
_hover={{ bg: 'myGray.100' }}
alignItems={'center'}
borderRadius={'4px'}
key={collection.id}
cursor={'pointer'}
onClick={() => {
setSelectedCollections((prev) => {
if (prev.includes(collection.id)) {
return prev.filter((id) => id !== collection.id);
} else {
return [...prev, collection.id];
}
});
}}
>
<Checkbox
size={'md'}
mr={2}
onChange={() => {
setSelectedCollections((prev) => {
if (prev.includes(collection.id)) {
return prev.filter((id) => id !== collection.id);
} else {
return [...prev, collection.id];
}
});
}}
isChecked={selectedCollections.includes(collection.id)}
/>
<MyIcon name={collection.icon as any} w={'16px'} mr={2} />
<Box fontSize={'14px'} borderRadius={'6px'}>
{collection.name}
</Box>
</Flex>
);
})}
</ScrollListCollections>
</MyBox>
);
};

View File

@@ -0,0 +1,287 @@
import { Box, Checkbox, Flex, Input } from '@chakra-ui/react';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { postCreateDatasetCollectionTag, putDatasetCollectionById } from '@/web/core/dataset/api';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useTranslation } from 'react-i18next';
import { useEffect, useMemo, useRef, useState } from 'react';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useDeepCompareEffect } from 'ahooks';
import { DatasetCollectionItemType, DatasetTagType } from '@fastgpt/global/core/dataset/type';
import { isEqual } from 'lodash';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
const TagsPopOver = ({
currentCollection
}: {
currentCollection: DatasetCollectionItemType | DatasetCollectionsListItemType;
}) => {
const { t } = useTranslation();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const datasetTags = useContextSelector(DatasetPageContext, (v) => v.datasetTags);
const loadDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadDatasetTags);
const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags);
const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags);
const [collectionTags, setCollectionTags] = useState<string[]>([]);
const [searchTag, setSearchTag] = useState('');
const [checkedTags, setCheckedTags] = useState<DatasetTagType[]>([]);
const [showTagManage, setShowTagManage] = useState(false);
const [isFocusInput, setIsFocusInput] = useState(false);
const [isUpdateLoading, setIsUpdateLoading] = useState(false);
useEffect(() => {
if (!currentCollection.tags) return;
setCollectionTags(currentCollection.tags);
}, [currentCollection]);
const tagList = useMemo(
() =>
(collectionTags
?.map((tagId) => {
const tagObject = allDatasetTags.find((tag) => tag._id === tagId);
return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null;
})
.filter((tag) => tag !== null) as {
_id: string;
tag: string;
}[]) || [],
[collectionTags, allDatasetTags]
);
useEffect(() => {
if (!isFocusInput) return;
loadDatasetTags({ id: datasetDetail._id, searchKey: searchTag });
}, [datasetDetail._id, isFocusInput, loadDatasetTags, searchTag]);
const [visibleTags, setVisibleTags] = useState<DatasetTagType[]>(tagList);
const [overflowTags, setOverflowTags] = useState<DatasetTagType[]>([]);
const containerRef = useRef<HTMLDivElement>(null);
useDeepCompareEffect(() => {
const calculateTags = () => {
if (!containerRef.current || !tagList) return;
const containerWidth = containerRef.current.offsetWidth;
const tagWidth = 11;
let totalWidth = 30;
let visibleCount = 0;
for (let i = 0; i < tagList.length; i++) {
const tag = tagList[i];
const estimatedWidth = tag.tag.length * tagWidth + 16; // 加上左右 padding 的宽度
if (totalWidth + estimatedWidth <= containerWidth) {
totalWidth += estimatedWidth;
visibleCount++;
} else {
break;
}
}
setVisibleTags(tagList.slice(0, visibleCount));
setOverflowTags(tagList.slice(visibleCount));
};
setTimeout(calculateTags, 100);
setCheckedTags(tagList);
window.addEventListener('resize', calculateTags);
return () => {
window.removeEventListener('resize', calculateTags);
};
}, [tagList]);
const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({
mutationFn: async (tag: string) => {
const id = await postCreateDatasetCollectionTag({
datasetId: datasetDetail._id,
tag
});
return id;
},
onSuccess() {
setSearchTag('');
loadDatasetTags({ id: datasetDetail._id, searchKey: '' });
loadAllDatasetTags({ id: datasetDetail._id });
},
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed')
});
return (
<MyPopover
placement={showTagManage ? 'bottom' : 'bottom-end'}
hasArrow={false}
offset={[2, 2]}
w={'180px'}
trigger={'hover'}
Trigger={
<MyBox
ref={containerRef}
display={'flex'}
isLoading={isUpdateLoading}
size={'xs'}
mt={1}
py={0.5}
px={0.25}
_hover={{
bg: 'myGray.50',
borderRadius: '3px'
}}
onMouseEnter={(e) => {
e.stopPropagation();
if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement)
return;
e.currentTarget.parentElement.parentElement.style.backgroundColor = 'white';
}}
onMouseLeave={(e) => {
if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement)
return;
e.currentTarget.parentElement.parentElement.style.backgroundColor = '';
}}
onClick={(e) => {
e.stopPropagation();
setShowTagManage(true);
}}
cursor={'pointer'}
>
<Flex>
{visibleTags.map((item, index) => (
<Box
key={index}
h={5}
mr={1}
px={2}
fontSize={'11px'}
bg={'#F0FBFF'}
color={'#0884DD'}
borderRadius={'4px'}
>
{item.tag}
</Box>
))}
</Flex>
{overflowTags.length > 0 && (
<Box h={5} px={2} bg={'#1118240D'} borderRadius={'33px'} fontSize={'11px'}>
{`+${overflowTags.length}`}
</Box>
)}
</MyBox>
}
onCloseFunc={async () => {
setShowTagManage(false);
if (isEqual(checkedTags, tagList) || !showTagManage) return;
setIsUpdateLoading(true);
await putDatasetCollectionById({
id: currentCollection._id,
tags: checkedTags.map((tag) => tag._id)
});
setCollectionTags(checkedTags.map((tag) => tag._id));
setIsUpdateLoading(false);
}}
display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'}
>
{({}) => (
<>
{showTagManage ? (
<MyBox isLoading={isCreateCollectionTagLoading} onClick={(e) => e.stopPropagation()}>
<Box px={1.5} pt={1.5}>
<Input
onFocus={() => setIsFocusInput(true)}
onBlur={() => setIsFocusInput(false)}
pl={2}
h={7}
borderRadius={'4px'}
value={searchTag}
placeholder={t('dataset:tag.searchOrAddTag')}
onChange={(e) => setSearchTag(e.target.value)}
/>
</Box>
<Box my={1} px={1.5} maxH={'200px'} overflow={'auto'}>
{searchTag && !datasetTags.map((item) => item.tag).includes(searchTag) && (
<Flex
alignItems={'center'}
fontSize={'xs'}
px={1}
cursor={'pointer'}
_hover={{ bg: '#1118240D', color: '#2B5FD9' }}
borderRadius={'xs'}
onClick={() => {
onCreateCollectionTag(searchTag);
}}
>
<MyIcon name={'common/addLight'} w={'14px'} />
<Box ml={1} py={1}>
{t('dataset:tag.add') + ` "${searchTag}"`}
</Box>
</Flex>
)}
{datasetTags?.map((item) => {
const tagsList = checkedTags.map((tag) => tag.tag);
return (
<Flex
alignItems={'center'}
fontSize={'xs'}
px={1}
py={0.5}
my={0.5}
key={item._id}
cursor={'pointer'}
bg={tagsList.includes(item.tag) ? '#1118240D' : 'transparent'}
color={tagsList.includes(item.tag) ? '#2B5FD9' : 'myGray.600'}
_hover={{ bg: '#1118240D', color: '#2B5FD9' }}
borderRadius={'xs'}
onClick={(e) => {
e.preventDefault();
if (tagsList.includes(item.tag)) {
setCheckedTags(checkedTags.filter((t) => t.tag !== item.tag));
} else {
setCheckedTags([...checkedTags, item]);
}
}}
>
<Checkbox
isChecked={tagsList.includes(item.tag)}
onChange={(e) => {
if (e.target.checked) {
setCheckedTags([...checkedTags, item]);
} else {
setCheckedTags(checkedTags.filter((t) => t._id !== item._id));
}
}}
/>
<Box ml={1}>{item.tag}</Box>
</Flex>
);
})}
</Box>
</MyBox>
) : (
<Flex gap={1} p={3} flexWrap={'wrap'}>
{overflowTags.map((tag, index) => (
<Box
key={index}
h={5}
px={2}
fontSize={'11px'}
bg={'#F0FBFF'}
color={'#0884DD'}
borderRadius={'4px'}
>
{tag.tag}
</Box>
))}
</Flex>
)}
</>
)}
</MyPopover>
);
};
export default TagsPopOver;

View File

@@ -49,6 +49,8 @@ import {
getTrainingTypeLabel
} from '@fastgpt/global/core/dataset/collection/utils';
import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
const Header = dynamic(() => import('./Header'));
const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
@@ -60,6 +62,7 @@ const CollectionCard = () => {
const { t } = useTranslation();
const { datasetT } = useI18n();
const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
const { feConfigs } = useSystemStore();
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
content: t('common:dataset.Confirm to delete the file'),
@@ -244,6 +247,9 @@ const CollectionCard = () => {
</Box>
</MyTooltip>
</Flex>
{feConfigs?.isPlus && !!collection.tags?.length && (
<TagsPopOver currentCollection={collection} />
)}
</Td>
<Td py={2}>
{!checkCollectionIsFolder(collection.type) ? (

View File

@@ -1,4 +1,4 @@
import React, { useCallback, useState, useRef, useMemo } from 'react';
import React, { useState, useRef, useMemo } from 'react';
import {
Box,
Card,
@@ -14,8 +14,7 @@ import {
DrawerOverlay,
DrawerContent,
useDisclosure,
HStack,
Switch
HStack
} from '@chakra-ui/react';
import {
getDatasetDataList,
@@ -26,19 +25,16 @@ import {
import { DeleteIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { debounce } from 'lodash';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput';
import { useLoading } from '@fastgpt/web/hooks/useLoading';
import InputDataModal from '../components/InputDataModal';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import type { DatasetDataListItemType } from '@/global/core/dataset/type.d';
import { TabEnum } from '..';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { DatasetCollectionTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
@@ -54,6 +50,8 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import TagsPopOver from './CollectionCard/TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
const DataCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
@@ -66,6 +64,7 @@ const DataCard = () => {
datasetId: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { feConfigs } = useSystemStore();
const { t } = useTranslation();
const { datasetT } = useI18n();
@@ -224,28 +223,34 @@ const DataCard = () => {
}
/>
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Box lineHeight={1.2}>
{collection?._id && (
<RawSourceBox
collectionId={collection._id}
{...getCollectionSourceData(collection)}
fontSize={['sm', 'md']}
color={'black'}
textDecoration={'none'}
/>
)}
<Box fontSize={'sm'} color={'myGray.500'}>
{t('common:core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
{collection?._id}
<Box>
<Box alignItems={'center'} gap={2} display={isPc ? 'flex' : ''}>
{collection?._id && (
<RawSourceBox
collectionId={collection._id}
{...getCollectionSourceData(collection)}
fontSize={['sm', 'md']}
color={'black'}
textDecoration={'none'}
/>
)}
<Box fontSize={'sm'} color={'myGray.500'}>
{t('common:core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
{collection?._id}
</Box>
</Box>
</Box>
{feConfigs?.isPlus && !!collection?.tags?.length && (
<TagsPopOver currentCollection={collection} />
)}
</Box>
</Flex>
{canWrite && (
<Box>
<Button
mx={2}
ml={2}
mr={isPc ? 2 : 0}
variant={'whitePrimary'}
size={['sm', 'md']}
onClick={() => {

View File

@@ -150,7 +150,7 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
<Box mb={3}>
<LightRowTabs<TabEnum>
m={'auto'}
w={'260px'}
w={'full'}
size={isPc ? 'md' : 'sm'}
list={tabList}
value={currentTab}

View File

@@ -18,6 +18,7 @@ import {
import CollectionPageContextProvider from './components/CollectionCard/Context';
import { useContextSelector } from 'use-context-selector';
import NextHead from '@/components/common/NextHead';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
const CollectionCard = dynamic(() => import('./components/CollectionCard/index'));
const DataCard = dynamic(() => import('./components/DataCard'));
@@ -40,21 +41,26 @@ const Detail = ({ datasetId, currentTab }: Props) => {
const router = useRouter();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const loadDatasetDetail = useContextSelector(DatasetPageContext, (v) => v.loadDatasetDetail);
const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags);
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
useRequest2(() => loadDatasetDetail(datasetId), {
onSuccess: () => {
loadAllDatasetTags({ id: datasetId });
},
onError(err: any) {
router.replace(`/dataset/list`);
toast({
title: t(getErrText(err, t('common:common.Load Failed')) as any),
status: 'error'
});
}
},
manual: false
});
return (
<>
<NextHead title={datasetDetail?.name} icon={datasetDetail?.avatar} />
<PageContainer>
<PageContainer insertProps={{ bg: 'white' }}>
<MyBox display={'flex'} flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
<Slider currentTab={currentTab} />

View File

@@ -6,18 +6,23 @@ import type {
import type {
DatasetItemType,
DatasetListItemType,
DatasetSimpleItemType
DatasetSimpleItemType,
DatasetTagType,
TagUsageType
} from '@fastgpt/global/core/dataset/type.d';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq.d';
import type {
AddTagsToCollectionsParams,
CreateDatasetCollectionParams,
CreateDatasetCollectionTagParams,
CsvTableCreateDatasetCollectionParams,
DatasetUpdateBody,
ExternalFileCreateDatasetCollectionParams,
FileIdCreateDatasetCollectionParams,
LinkCreateDatasetCollectionParams,
PostWebsiteSyncParams,
TextCreateDatasetCollectionParams
TextCreateDatasetCollectionParams,
UpdateDatasetCollectionTagParams
} from '@fastgpt/global/core/dataset/api.d';
import type {
GetTrainingQueueProps,
@@ -43,6 +48,8 @@ import type { UpdateDatasetCollectionParams } from '@/pages/api/core/dataset/col
import type { GetDatasetDataListProps } from '@/pages/api/core/dataset/data/list';
import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller';
import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { GetScrollCollectionsProps } from '@/pages/api/core/dataset/collection/scrollList';
/* ======================== dataset ======================= */
export const getDatasets = (data: GetDatasetListBody) =>
@@ -117,6 +124,32 @@ export const postLinkCollectionSync = (collectionId: string) =>
collectionId
});
/* =============================== tag ==================================== */
export const postCreateDatasetCollectionTag = (data: CreateDatasetCollectionTagParams) =>
POST(`/proApi/core/dataset/tag/create`, data);
export const postAddTagsToCollections = (data: AddTagsToCollectionsParams) =>
POST(`/proApi/core/dataset/tag/addToCollections`, data);
export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) =>
DELETE(`/proApi/core/dataset/tag/delete`, data);
export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) =>
POST(`/proApi/core/dataset/tag/update`, data);
export const getDatasetCollectionTags = (
data: PaginationProps<{
datasetId: string;
searchText?: string;
}>
) => GET<PaginationResponse<DatasetTagType>>(`/proApi/core/dataset/tag/list`, data);
export const getTagUsage = (datasetId: string) =>
GET<TagUsageType[]>(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`);
export const getAllTags = (datasetId: string) =>
GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`);
export const getScrollCollectionList = (data: GetScrollCollectionsProps) =>
GET<PaginationResponse<DatasetCollectionsListItemType>>(
`/core/dataset/collection/scrollList`,
data
);
/* =============================== data ==================================== */
/* get dataset list */
export const getDatasetDataList = (data: GetDatasetDataListProps) =>

View File

@@ -3,7 +3,6 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
import ParentPaths from '@/components/common/ParentPaths';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { getDatasetCollectionPathById, getDatasetCollections } from '@/web/core/dataset/api';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { Box, Flex, ModalFooter, Button, useTheme, Grid, Card, ModalBody } from '@chakra-ui/react';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';

View File

@@ -51,6 +51,7 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
defaultPermission: DatasetDefaultPermissionVal,
inheritPermission: true
},
tags: [],
parentId: '',
name: '',
type: DatasetCollectionTypeEnum.file,

View File

@@ -1,22 +1,30 @@
import { useQuery } from '@tanstack/react-query';
import { ReactNode, useMemo, useState } from 'react';
import { ReactNode, SetStateAction, useMemo, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext } from 'use-context-selector';
import {
getAllTags,
getDatasetById,
getDatasetCollectionTags,
getDatasetTrainingQueue,
getTrainingQueueLen,
putDatasetById
} from '../api';
import { defaultDatasetDetail } from '../constants';
import { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetItemType, DatasetTagType } from '@fastgpt/global/core/dataset/type';
type DatasetPageContextType = {
datasetId: string;
datasetDetail: DatasetItemType;
loadDatasetDetail: (id: string) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateBody) => Promise<void>;
datasetTags: DatasetTagType[];
loadDatasetTags: (data: { id: string; searchKey: string }) => Promise<void>;
allDatasetTags: DatasetTagType[];
loadAllDatasetTags: (data: { id: string }) => Promise<void>;
checkedDatasetTag: DatasetTagType[];
setCheckedDatasetTag: React.Dispatch<SetStateAction<DatasetTagType[]>>;
vectorTrainingMap: {
colorSchema: string;
@@ -52,6 +60,18 @@ export const DatasetPageContext = createContext<DatasetPageContextType>({
},
updateDataset: function (data: DatasetUpdateBody): Promise<void> {
throw new Error('Function not implemented.');
},
datasetTags: [],
loadDatasetTags: function (data: { id: string; searchKey: string }): Promise<void> {
throw new Error('Function not implemented.');
},
allDatasetTags: [],
loadAllDatasetTags: function (data: { id: string }): Promise<void> {
throw new Error('Function not implemented.');
},
checkedDatasetTag: [],
setCheckedDatasetTag: function (): void {
throw new Error('Function not implemented.');
}
});
@@ -85,6 +105,28 @@ export const DatasetPageContextProvider = ({
}
};
// dataset tags
const [datasetTags, setDatasetTags] = useState<DatasetTagType[]>([]);
const loadDatasetTags = async ({ id, searchKey }: { id: string; searchKey: string }) => {
const { list } = await getDatasetCollectionTags({
datasetId: id,
searchText: searchKey,
current: 1,
pageSize: 15
});
setDatasetTags(list);
};
const [checkedDatasetTag, setCheckedDatasetTag] = useState<DatasetTagType[]>([]);
const [allDatasetTags, setAllDatasetTags] = useState<DatasetTagType[]>([]);
const loadAllDatasetTags = async ({ id }: { id: string }) => {
const { list } = await getAllTags(id);
setAllDatasetTags(list);
};
// global queue
const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
['getTrainingQueueLen'],
@@ -152,7 +194,13 @@ export const DatasetPageContextProvider = ({
agentTrainingMap,
rebuildingCount,
trainingCount,
refetchDatasetTraining
refetchDatasetTraining,
datasetTags,
loadDatasetTags,
checkedDatasetTag,
setCheckedDatasetTag,
allDatasetTags,
loadAllDatasetTags
};
return <DatasetPageContext.Provider value={contextValue}>{children}</DatasetPageContext.Provider>;