feat: sync api collection will refresh title;perf: invite link ux (#4237)

* update queue

* feat: sync api collection will refresh title

* sync collection

* remove lock

* perf: invite link ux
This commit is contained in:
Archer
2025-03-19 21:03:21 +08:00
committed by archer
parent 73451dbc64
commit 87e90c37bd
44 changed files with 368 additions and 327 deletions

View File

@@ -1,6 +1,6 @@
import type {
APIFileContentResponse,
APIFileListResponse,
ApiFileReadContentResponse,
APIFileReadResponse,
APIFileServer
} from '@fastgpt/global/core/dataset/apiDataset';
@@ -8,6 +8,7 @@ import axios, { Method } from 'axios';
import { addLog } from '../../../common/system/log';
import { readFileRawTextByUrl } from '../read';
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
type ResponseDataType = {
success: boolean;
@@ -118,17 +119,24 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
tmbId: string;
apiFileId: string;
customPdfParse?: boolean;
}) => {
const data = await request<APIFileContentResponse>(
`/v1/file/content`,
{ id: apiFileId },
'GET'
);
}): Promise<ApiFileReadContentResponse> => {
const data = await request<
{
title?: string;
} & RequireOnlyOne<{
content: string;
previewUrl: string;
}>
>(`/v1/file/content`, { id: apiFileId }, 'GET');
const title = data.title;
const content = data.content;
const previewUrl = data.previewUrl;
if (content) {
return content;
return {
title,
rawText: content
};
}
if (previewUrl) {
const rawText = await readFileRawTextByUrl({
@@ -138,7 +146,10 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
relatedId: apiFileId,
customPdfParse
});
return rawText;
return {
title,
rawText
};
}
return Promise.reject('Invalid content type: content or previewUrl is required');
};

View File

@@ -11,7 +11,6 @@ import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum,
DatasetSourceReadTypeEnum,
DatasetTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
@@ -19,6 +18,7 @@ import { readDatasetSourceRawText } from '../read';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { mongoSessionRun } from '../../../common/mongo/sessionRun';
import { createCollectionAndInsertData, delCollection } from './controller';
import { collectionCanSync } from '@fastgpt/global/core/dataset/collection/utils';
/**
* get all collection by top collectionId
@@ -137,10 +137,7 @@ export const collectionTagsToTagLabel = async ({
export const syncCollection = async (collection: CollectionWithDatasetType) => {
const dataset = collection.dataset;
if (
collection.type !== DatasetCollectionTypeEnum.link &&
dataset.type !== DatasetTypeEnum.apiDataset
) {
if (!collectionCanSync(collection.type)) {
return Promise.reject(DatasetErrEnum.notSupportSync);
}
@@ -155,15 +152,20 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
};
}
if (!collection.apiFileId) return Promise.reject('apiFileId is missing');
if (!dataset.apiServer) return Promise.reject('apiServer not found');
const sourceId = collection.apiFileId;
if (!sourceId) return Promise.reject('apiFileId is missing');
return {
type: DatasetSourceReadTypeEnum.apiFile,
sourceId: collection.apiFileId,
apiServer: dataset.apiServer
sourceId,
apiServer: dataset.apiServer,
feishuServer: dataset.feishuServer,
yuqueServer: dataset.yuqueServer
};
})();
const rawText = await readDatasetSourceRawText({
const { title, rawText } = await readDatasetSourceRawText({
teamId: collection.teamId,
tmbId: collection.tmbId,
...sourceReadType
@@ -196,7 +198,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
createCollectionParams: {
teamId: collection.teamId,
tmbId: collection.tmbId,
name: collection.name,
name: title || collection.name,
datasetId: collection.datasetId,
parentId: collection.parentId,
type: collection.type,

View File

@@ -79,9 +79,12 @@ export const readDatasetSourceRawText = async ({
apiServer?: APIFileServer; // api dataset
feishuServer?: FeishuServer; // feishu dataset
yuqueServer?: YuqueServer; // yuque dataset
}): Promise<string> => {
}): Promise<{
title?: string;
rawText: string;
}> => {
if (type === DatasetSourceReadTypeEnum.fileLocal) {
const { rawText } = await readFileContentFromMongo({
const { filename, rawText } = await readFileContentFromMongo({
teamId,
tmbId,
bucketName: BucketNameEnum.dataset,
@@ -89,14 +92,20 @@ export const readDatasetSourceRawText = async ({
isQAImport,
customPdfParse
});
return rawText;
return {
title: filename,
rawText
};
} else if (type === DatasetSourceReadTypeEnum.link) {
const result = await urlsFetch({
urlList: [sourceId],
selector
});
return result[0]?.content || '';
return {
title: result[0]?.title,
rawText: result[0]?.content || ''
};
} else if (type === DatasetSourceReadTypeEnum.externalFile) {
if (!externalFileId) return Promise.reject('FileId not found');
const rawText = await readFileRawTextByUrl({
@@ -106,9 +115,11 @@ export const readDatasetSourceRawText = async ({
relatedId: externalFileId,
customPdfParse
});
return rawText;
return {
rawText
};
} else if (type === DatasetSourceReadTypeEnum.apiFile) {
const rawText = await readApiServerFileContent({
const { title, rawText } = await readApiServerFileContent({
apiServer,
feishuServer,
yuqueServer,
@@ -116,9 +127,15 @@ export const readDatasetSourceRawText = async ({
teamId,
tmbId
});
return rawText;
return {
title,
rawText
};
}
return '';
return {
title: '',
rawText: ''
};
};
export const readApiServerFileContent = async ({
@@ -137,7 +154,10 @@ export const readApiServerFileContent = async ({
teamId: string;
tmbId: string;
customPdfParse?: boolean;
}) => {
}): Promise<{
title?: string;
rawText: string;
}> => {
if (apiServer) {
return useApiDatasetRequest({ apiServer }).getFileContent({
teamId,
@@ -148,7 +168,10 @@ export const readApiServerFileContent = async ({
}
if (feishuServer || yuqueServer) {
return POST<string>(`/core/dataset/systemApiDataset`, {
return POST<{
title?: string;
rawText: string;
}>(`/core/dataset/systemApiDataset`, {
type: 'content',
feishuServer,
yuqueServer,