feat: sync api collection will refresh title;perf: invite link ux (#4237)

* update queue

* feat: sync api collection will refresh title

* sync collection

* remove lock

* perf: invite link ux
This commit is contained in:
Archer
2025-03-19 21:03:21 +08:00
committed by archer
parent 73451dbc64
commit 87e90c37bd
44 changed files with 368 additions and 327 deletions

View File

@@ -64,8 +64,7 @@
"request-ip": "^3.3.0",
"sass": "^1.58.3",
"use-context-selector": "^1.4.4",
"@node-rs/jieba": "2.0.1",
"zustand": "^4.3.5"
"@node-rs/jieba": "2.0.1"
},
"devDependencies": {
"@svgr/webpack": "^6.5.1",

View File

@@ -22,6 +22,9 @@ const NotSufficientModal = dynamic(() => import('@/components/support/wallet/Not
const SystemMsgModal = dynamic(() => import('@/components/support/user/inform/SystemMsgModal'));
const ImportantInform = dynamic(() => import('@/components/support/user/inform/ImportantInform'));
const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal'));
const ManualCopyModal = dynamic(() =>
import('@fastgpt/web/hooks/useCopyData').then((mod) => mod.ManualCopyModal)
);
const pcUnShowLayoutRoute: Record<string, boolean> = {
'/': true,
@@ -162,6 +165,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
</>
)}
<ManualCopyModal />
<Loading loading={loading} zIndex={999999} />
</>
);

View File

@@ -48,15 +48,6 @@ export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & {
collectionId: string;
};
export type GetTrainingQueueProps = {
vectorModel: string;
agentModel: string;
};
export type GetTrainingQueueResponse = {
vectorTrainingCount: number;
agentTrainingCount: number;
};
/* -------------- search ---------------- */
export type SearchTestProps = {
datasetId: string;

View File

@@ -22,7 +22,13 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
function CreateInvitationModal({ onClose }: { onClose: (linkId?: string) => void }) {
function CreateInvitationModal({
onSuccess,
onClose
}: {
onSuccess: (linkId: string) => void;
onClose: () => void;
}) {
const { t } = useTranslation();
const expiresOptions: Array<{ label: string; value: InvitationLinkExpiresType }> = [
{ label: t('account_team:30mins'), value: '30m' }, // 30 mins
@@ -43,12 +49,11 @@ function CreateInvitationModal({ onClose }: { onClose: (linkId?: string) => void
const { runAsync: createInvitationLink, loading } = useRequest2(postCreateInvitationLink, {
manual: true,
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed'),
onSuccess: (data) => {
onClose(data);
},
onFinally: () => onClose()
onSuccess(data);
onClose();
}
});
return (

View File

@@ -35,15 +35,7 @@ import { useCallback } from 'react';
const CreateInvitationModal = dynamic(() => import('./CreateInvitationModal'));
const InviteModal = ({
teamId,
onClose,
onSuccess
}: {
teamId: string;
onClose: () => void;
onSuccess: () => void;
}) => {
const InviteModal = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const {
@@ -57,10 +49,10 @@ const InviteModal = ({
const { isOpen: isOpenCreate, onOpen: onOpenCreate, onClose: onCloseCreate } = useDisclosure();
const isLoading = isLoadingLink;
const { copyData } = useCopyData();
const { userInfo } = useUserStore();
const { feConfigs } = useSystemStore();
const onCopy = useCallback(
(linkId: string) => {
const url = location.origin + `/account/team?invitelinkid=${linkId}`;
@@ -76,7 +68,7 @@ const InviteModal = ({
})
);
},
[copyData]
[copyData, feConfigs?.systemTitle, t, userInfo?.team.memberName, userInfo?.team.teamName]
);
const { runAsync: onForbid, loading: forbiding } = useRequest2(putForbidInvitationLink, {
@@ -131,13 +123,7 @@ const InviteModal = ({
<Td maxW="200px" minW="100px">
{item.description}
</Td>
<Td>
{isForbidden ? (
<Tag colorSchema="gray">{t('account_team:has_forbidden')}</Tag>
) : (
format(new Date(item.expires), 'yyyy-MM-dd HH:mm')
)}
</Td>
<Td>{format(new Date(item.expires), 'yyyy-MM-dd HH:mm')}</Td>
<Td>
{item.usedTimesLimit === -1
? t('account_team:unlimited')
@@ -153,7 +139,6 @@ const InviteModal = ({
cursor="pointer"
_hover={{ bg: 'myGray.100' }}
p="1.5"
w="fit-content"
>
<AvatarGroup max={3} avatars={item.members.map((i) => i.avatar)} />
</Box>
@@ -162,7 +147,7 @@ const InviteModal = ({
closeOnBlur={true}
>
{() => (
<Box py="4" maxH="200px" w="fit-content">
<Box py="4" maxH="200px">
<Flex mx="4" justifyContent="center" alignItems={'center'}>
<Box>{t('account_team:has_invited')}</Box>
<Box
@@ -175,15 +160,16 @@ const InviteModal = ({
{item.members.length}
</Box>
</Flex>
<Divider my="2" mx="4" />
<Divider my="2" />
<Grid
w="fit-content"
mt="2"
gridRowGap="4"
mt="4"
gap={4}
gridTemplateColumns="1fr 1fr"
overflow="auto"
alignItems="center"
mx="4"
maxH={'250px'}
>
{item.members.map((member) => (
<Box key={member.tmbId} justifySelf="start">
@@ -197,7 +183,9 @@ const InviteModal = ({
)}
</Td>
<Td>
{!isForbidden && (
{isForbidden ? (
<Tag colorSchema="red">{t('account_team:has_forbidden')}</Tag>
) : (
<>
<Button
size="sm"
@@ -261,17 +249,11 @@ const InviteModal = ({
</ModalFooter>
{isOpenCreate && (
<CreateInvitationModal
onClose={(linkId?: string) =>
Promise.all([
onCloseCreate(),
refetchInvitationLinkList(),
(() => {
if (linkId) {
onCopy(linkId);
}
})()
])
}
onSuccess={(linkId) => {
refetchInvitationLinkList();
onCopy(linkId);
}}
onClose={onCloseCreate}
/>
)}
</MyModal>

View File

@@ -29,7 +29,6 @@ import {
DatasetCollectionTypeEnum,
DatasetStatusEnum,
DatasetCollectionSyncResultMap,
DatasetTypeEnum,
DatasetCollectionDataProcessModeMap
} from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
@@ -45,7 +44,10 @@ import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import { checkCollectionIsFolder } from '@fastgpt/global/core/dataset/collection/utils';
import {
checkCollectionIsFolder,
collectionCanSync
} from '@fastgpt/global/core/dataset/collection/utils';
import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
@@ -315,8 +317,7 @@ const CollectionCard = () => {
menuList={[
{
children: [
...(collection.type === DatasetCollectionTypeEnum.link ||
datasetDetail.type === DatasetTypeEnum.apiDataset
...(collectionCanSync(collection.type)
? [
{
label: (

View File

@@ -1,4 +1,4 @@
import React, { useCallback } from 'react';
import React, { useCallback, useMemo } from 'react';
import { useTranslation } from 'next-i18next';
import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -9,6 +9,8 @@ import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import ParentPaths from '@/components/common/ParentPaths';
import { getTrainingQueueLen } from '@/web/core/dataset/api';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
export enum TabEnum {
dataCard = 'dataCard',
@@ -24,8 +26,68 @@ const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
const router = useRouter();
const query = router.query;
const { isPc } = useSystem();
const { datasetDetail, vectorTrainingMap, agentTrainingMap, rebuildingCount, paths } =
useContextSelector(DatasetPageContext, (v) => v);
const { datasetDetail, rebuildingCount, paths } = useContextSelector(
DatasetPageContext,
(v) => v
);
// global queue
const {
data: {
vectorTrainingCount = 0,
qaTrainingCount = 0,
autoTrainingCount = 0,
imageTrainingCount = 0
} = {}
} = useRequest2(getTrainingQueueLen, {
manual: false,
retryInterval: 10000
});
const { vectorTrainingMap, qaTrainingMap, autoTrainingMap, imageTrainingMap } = useMemo(() => {
const vectorTrainingMap = (() => {
if (vectorTrainingCount < 1000)
return {
colorSchema: 'green',
tip: t('common:core.dataset.training.Leisure')
};
if (vectorTrainingCount < 20000)
return {
colorSchema: 'yellow',
tip: t('common:core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('common:core.dataset.training.Full')
};
})();
const countLLMMap = (count: number) => {
if (count < 100)
return {
colorSchema: 'green',
tip: t('common:core.dataset.training.Leisure')
};
if (count < 1000)
return {
colorSchema: 'yellow',
tip: t('common:core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('common:core.dataset.training.Full')
};
};
const qaTrainingMap = countLLMMap(qaTrainingCount);
const autoTrainingMap = countLLMMap(autoTrainingCount);
const imageTrainingMap = countLLMMap(imageTrainingCount);
return {
vectorTrainingMap,
qaTrainingMap,
autoTrainingMap,
imageTrainingMap
};
}, [qaTrainingCount, autoTrainingCount, imageTrainingCount, vectorTrainingCount, t]);
const tabList = [
{
@@ -172,12 +234,38 @@ const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
)}
<Box mb={3}>
<Box fontSize={'sm'} pb={1}>
{t('common:core.dataset.training.Agent queue')}({agentTrainingMap.tip})
{t('common:core.dataset.training.Agent queue')}({qaTrainingMap.tip})
</Box>
<Progress
value={100}
size={'xs'}
colorScheme={agentTrainingMap.colorSchema}
colorScheme={qaTrainingMap.colorSchema}
borderRadius={'md'}
isAnimated
hasStripe
/>
</Box>
<Box mb={3}>
<Box fontSize={'sm'} pb={1}>
{t('dataset:auto_training_queue')}({autoTrainingMap.tip})
</Box>
<Progress
value={100}
size={'xs'}
colorScheme={autoTrainingMap.colorSchema}
borderRadius={'md'}
isAnimated
hasStripe
/>
</Box>
<Box mb={3}>
<Box fontSize={'sm'} pb={1}>
{t('dataset:image_training_queue')}({imageTrainingMap.tip})
</Box>
<Progress
value={100}
size={'xs'}
colorScheme={imageTrainingMap.colorSchema}
borderRadius={'md'}
isAnimated
hasStripe

View File

@@ -41,7 +41,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
return Promise.reject(DatasetErrEnum.sameApiCollection);
}
const content = await readApiServerFileContent({
const { title, rawText } = await readApiServerFileContent({
apiServer,
feishuServer,
yuqueServer,
@@ -53,14 +53,14 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
const { collectionId, insertResults } = await createCollectionAndInsertData({
dataset,
rawText: content,
rawText,
relatedId: apiFileId,
createCollectionParams: {
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.apiFile,
name,
name: title || name,
apiFileId,
metadata: {
relatedImgId: apiFileId

View File

@@ -77,7 +77,7 @@ async function handler(
return Promise.reject(i18nT('dataset:collection_not_support_retraining'));
})();
const rawText = await readDatasetSourceRawText({
const { title, rawText } = await readDatasetSourceRawText({
teamId,
tmbId,
customPdfParse,
@@ -100,7 +100,7 @@ async function handler(
teamId: collection.teamId,
tmbId: collection.tmbId,
datasetId: collection.dataset._id,
name: collection.name,
name: title || collection.name,
type: collection.type,
customPdfParse,

View File

@@ -82,7 +82,7 @@ async function handler(
};
})();
const rawText = await readDatasetSourceRawText({
const { rawText } = await readDatasetSourceRawText({
teamId,
tmbId,
type,

View File

@@ -1,27 +1,31 @@
import type { NextApiRequest } from 'next';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { GetTrainingQueueProps } from '@/global/core/dataset/api';
import { NextAPI } from '@/service/middleware/entry';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
export type GetQueueLenResponse = {
vectorTrainingCount: number;
qaTrainingCount: number;
autoTrainingCount: number;
imageTrainingCount: number;
};
async function handler(req: NextApiRequest) {
await authCert({ req, authToken: true });
const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate(
[
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
lockTime: { $lt: new Date('2040/1/1') }
}
},
{
$group: {
_id: '$model',
_id: '$mode',
count: { $sum: 1 }
}
}
@@ -31,12 +35,16 @@ async function handler(req: NextApiRequest) {
}
);
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
const vectorTrainingCount = data.find((item) => item._id === TrainingModeEnum.chunk)?.count || 0;
const qaTrainingCount = data.find((item) => item._id === TrainingModeEnum.qa)?.count || 0;
const autoTrainingCount = data.find((item) => item._id === TrainingModeEnum.auto)?.count || 0;
const imageTrainingCount = data.find((item) => item._id === TrainingModeEnum.image)?.count || 0;
return {
vectorTrainingCount,
agentTrainingCount
qaTrainingCount,
autoTrainingCount,
imageTrainingCount
};
}

View File

@@ -1,6 +1,4 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import axios from 'axios';
import { OAuthEnum } from '@fastgpt/global/support/user/constant';
import type {

View File

@@ -1,6 +1,4 @@
import { create } from 'zustand';
import { createJSONStorage, devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, createJSONStorage, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';

View File

@@ -1,6 +1,4 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
type State = {
localUId?: string;

View File

@@ -27,12 +27,7 @@ import type {
TextCreateDatasetCollectionParams,
UpdateDatasetCollectionTagParams
} from '@fastgpt/global/core/dataset/api.d';
import type {
GetTrainingQueueProps,
GetTrainingQueueResponse,
SearchTestProps,
SearchTestResponse
} from '@/global/core/dataset/api.d';
import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
import type { CreateDatasetParams, InsertOneDatasetDataProps } from '@/global/core/dataset/api.d';
import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants';
@@ -67,6 +62,7 @@ import type {
} from '@/pages/api/core/dataset/apiDataset/listExistId';
import type { GetQuoteDataResponse } from '@/pages/api/core/dataset/data/getQuoteData';
import type { GetQuotePermissionResponse } from '@/pages/api/core/dataset/data/getPermission';
import type { GetQueueLenResponse } from '@/pages/api/core/dataset/training/getQueueLen';
/* ======================== dataset ======================= */
export const getDatasets = (data: GetDatasetListBody) =>
@@ -215,8 +211,8 @@ export const postRebuildEmbedding = (data: rebuildEmbeddingBody) =>
POST(`/core/dataset/training/rebuildEmbedding`, data);
/* get length of system training queue */
export const getTrainingQueueLen = (data: GetTrainingQueueProps) =>
GET<GetTrainingQueueResponse>(`/core/dataset/training/getQueueLen`, data);
export const getTrainingQueueLen = () =>
GET<GetQueueLenResponse>(`/core/dataset/training/getQueueLen`);
export const getDatasetTrainingQueue = (datasetId: string) =>
GET<getDatasetTrainingQueueResponse>(`/core/dataset/training/getDatasetTrainingQueue`, {
datasetId

View File

@@ -1,5 +1,5 @@
import { useQuery } from '@tanstack/react-query';
import { Dispatch, ReactNode, SetStateAction, useMemo, useState } from 'react';
import { Dispatch, ReactNode, SetStateAction, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext } from 'use-context-selector';
import {
@@ -8,7 +8,6 @@ import {
getDatasetCollectionTags,
getDatasetPaths,
getDatasetTrainingQueue,
getTrainingQueueLen,
postCreateDatasetCollectionTag,
putDatasetById
} from '../api';
@@ -37,28 +36,13 @@ type DatasetPageContextType = {
setSearchTagKey: Dispatch<SetStateAction<string>>;
paths: ParentTreePathItemType[];
refetchPaths: () => void;
vectorTrainingMap: {
colorSchema: string;
tip: string;
};
agentTrainingMap: {
colorSchema: string;
tip: string;
};
rebuildingCount: number;
trainingCount: number;
refetchDatasetTraining: () => void;
};
export const DatasetPageContext = createContext<DatasetPageContextType>({
vectorTrainingMap: {
colorSchema: '',
tip: ''
},
agentTrainingMap: {
colorSchema: '',
tip: ''
},
rebuildingCount: 0,
trainingCount: 0,
refetchDatasetTraining: function (): void {
@@ -191,57 +175,6 @@ export const DatasetPageContextProvider = ({
}
);
// global queue
const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
['getTrainingQueueLen'],
() =>
getTrainingQueueLen({
vectorModel: datasetDetail.vectorModel.model,
agentModel: datasetDetail.agentModel.model
}),
{
refetchInterval: 10000
}
);
const { vectorTrainingMap, agentTrainingMap } = useMemo(() => {
const vectorTrainingMap = (() => {
if (vectorTrainingCount < 1000)
return {
colorSchema: 'green',
tip: t('common:core.dataset.training.Leisure')
};
if (vectorTrainingCount < 10000)
return {
colorSchema: 'yellow',
tip: t('common:core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('common:core.dataset.training.Full')
};
})();
const agentTrainingMap = (() => {
if (agentTrainingCount < 100)
return {
colorSchema: 'green',
tip: t('common:core.dataset.training.Leisure')
};
if (agentTrainingCount < 1000)
return {
colorSchema: 'yellow',
tip: t('common:core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('common:core.dataset.training.Full')
};
})();
return {
vectorTrainingMap,
agentTrainingMap
};
}, [agentTrainingCount, t, vectorTrainingCount]);
// training and rebuild queue
const { data: { rebuildingCount = 0, trainingCount = 0 } = {}, refetch: refetchDatasetTraining } =
useQuery(['getDatasetTrainingQueue'], () => getDatasetTrainingQueue(datasetId), {
@@ -273,8 +206,7 @@ export const DatasetPageContextProvider = ({
updateDataset,
paths,
refetchPaths,
vectorTrainingMap,
agentTrainingMap,
rebuildingCount,
trainingCount,
refetchDatasetTraining,

View File

@@ -1,6 +1,4 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { getDatasets } from '@/web/core/dataset/api';

View File

@@ -1,6 +1,4 @@
import { create } from 'zustand';
import { devtools } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, devtools, immer } from '@fastgpt/web/common/zustand';
export type MarkDataStore = {
dataId: string;

View File

@@ -1,6 +1,5 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';

View File

@@ -1,3 +1,5 @@
import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import type { UserUpdateParams } from '@/types/user';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { getTokenLogin, putUserInfo } from '@/web/support/user/api';
@@ -5,9 +7,6 @@ import type { MemberGroupListType } from '@fastgpt/global/support/permission/mem
import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
import type { UserType } from '@fastgpt/global/support/user/type.d';
import type { FeTeamPlanStatusType } from '@fastgpt/global/support/wallet/sub/type';
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { getTeamPlanStatus } from './team/api';
import { getGroupList } from './team/group/api';
import { getOrgList } from './team/org/api';