mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 21:13:50 +00:00
feat: sync api collection will refresh title;perf: invite link ux (#4237)
* update queue * feat: sync api collection will refresh title * sync collection * remove lock * perf: invite link ux
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import type {
|
||||
APIFileContentResponse,
|
||||
APIFileListResponse,
|
||||
ApiFileReadContentResponse,
|
||||
APIFileReadResponse,
|
||||
APIFileServer
|
||||
} from '@fastgpt/global/core/dataset/apiDataset';
|
||||
@@ -8,6 +8,7 @@ import axios, { Method } from 'axios';
|
||||
import { addLog } from '../../../common/system/log';
|
||||
import { readFileRawTextByUrl } from '../read';
|
||||
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
|
||||
import { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
|
||||
|
||||
type ResponseDataType = {
|
||||
success: boolean;
|
||||
@@ -118,17 +119,24 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
|
||||
tmbId: string;
|
||||
apiFileId: string;
|
||||
customPdfParse?: boolean;
|
||||
}) => {
|
||||
const data = await request<APIFileContentResponse>(
|
||||
`/v1/file/content`,
|
||||
{ id: apiFileId },
|
||||
'GET'
|
||||
);
|
||||
}): Promise<ApiFileReadContentResponse> => {
|
||||
const data = await request<
|
||||
{
|
||||
title?: string;
|
||||
} & RequireOnlyOne<{
|
||||
content: string;
|
||||
previewUrl: string;
|
||||
}>
|
||||
>(`/v1/file/content`, { id: apiFileId }, 'GET');
|
||||
const title = data.title;
|
||||
const content = data.content;
|
||||
const previewUrl = data.previewUrl;
|
||||
|
||||
if (content) {
|
||||
return content;
|
||||
return {
|
||||
title,
|
||||
rawText: content
|
||||
};
|
||||
}
|
||||
if (previewUrl) {
|
||||
const rawText = await readFileRawTextByUrl({
|
||||
@@ -138,7 +146,10 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
|
||||
relatedId: apiFileId,
|
||||
customPdfParse
|
||||
});
|
||||
return rawText;
|
||||
return {
|
||||
title,
|
||||
rawText
|
||||
};
|
||||
}
|
||||
return Promise.reject('Invalid content type: content or previewUrl is required');
|
||||
};
|
||||
|
@@ -11,7 +11,6 @@ import {
|
||||
DatasetCollectionSyncResultEnum,
|
||||
DatasetCollectionTypeEnum,
|
||||
DatasetSourceReadTypeEnum,
|
||||
DatasetTypeEnum,
|
||||
TrainingModeEnum
|
||||
} from '@fastgpt/global/core/dataset/constants';
|
||||
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
|
||||
@@ -19,6 +18,7 @@ import { readDatasetSourceRawText } from '../read';
|
||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||
import { mongoSessionRun } from '../../../common/mongo/sessionRun';
|
||||
import { createCollectionAndInsertData, delCollection } from './controller';
|
||||
import { collectionCanSync } from '@fastgpt/global/core/dataset/collection/utils';
|
||||
|
||||
/**
|
||||
* get all collection by top collectionId
|
||||
@@ -137,10 +137,7 @@ export const collectionTagsToTagLabel = async ({
|
||||
export const syncCollection = async (collection: CollectionWithDatasetType) => {
|
||||
const dataset = collection.dataset;
|
||||
|
||||
if (
|
||||
collection.type !== DatasetCollectionTypeEnum.link &&
|
||||
dataset.type !== DatasetTypeEnum.apiDataset
|
||||
) {
|
||||
if (!collectionCanSync(collection.type)) {
|
||||
return Promise.reject(DatasetErrEnum.notSupportSync);
|
||||
}
|
||||
|
||||
@@ -155,15 +152,20 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
|
||||
};
|
||||
}
|
||||
|
||||
if (!collection.apiFileId) return Promise.reject('apiFileId is missing');
|
||||
if (!dataset.apiServer) return Promise.reject('apiServer not found');
|
||||
const sourceId = collection.apiFileId;
|
||||
|
||||
if (!sourceId) return Promise.reject('apiFileId is missing');
|
||||
|
||||
return {
|
||||
type: DatasetSourceReadTypeEnum.apiFile,
|
||||
sourceId: collection.apiFileId,
|
||||
apiServer: dataset.apiServer
|
||||
sourceId,
|
||||
apiServer: dataset.apiServer,
|
||||
feishuServer: dataset.feishuServer,
|
||||
yuqueServer: dataset.yuqueServer
|
||||
};
|
||||
})();
|
||||
const rawText = await readDatasetSourceRawText({
|
||||
|
||||
const { title, rawText } = await readDatasetSourceRawText({
|
||||
teamId: collection.teamId,
|
||||
tmbId: collection.tmbId,
|
||||
...sourceReadType
|
||||
@@ -196,7 +198,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
|
||||
createCollectionParams: {
|
||||
teamId: collection.teamId,
|
||||
tmbId: collection.tmbId,
|
||||
name: collection.name,
|
||||
name: title || collection.name,
|
||||
datasetId: collection.datasetId,
|
||||
parentId: collection.parentId,
|
||||
type: collection.type,
|
||||
|
@@ -79,9 +79,12 @@ export const readDatasetSourceRawText = async ({
|
||||
apiServer?: APIFileServer; // api dataset
|
||||
feishuServer?: FeishuServer; // feishu dataset
|
||||
yuqueServer?: YuqueServer; // yuque dataset
|
||||
}): Promise<string> => {
|
||||
}): Promise<{
|
||||
title?: string;
|
||||
rawText: string;
|
||||
}> => {
|
||||
if (type === DatasetSourceReadTypeEnum.fileLocal) {
|
||||
const { rawText } = await readFileContentFromMongo({
|
||||
const { filename, rawText } = await readFileContentFromMongo({
|
||||
teamId,
|
||||
tmbId,
|
||||
bucketName: BucketNameEnum.dataset,
|
||||
@@ -89,14 +92,20 @@ export const readDatasetSourceRawText = async ({
|
||||
isQAImport,
|
||||
customPdfParse
|
||||
});
|
||||
return rawText;
|
||||
return {
|
||||
title: filename,
|
||||
rawText
|
||||
};
|
||||
} else if (type === DatasetSourceReadTypeEnum.link) {
|
||||
const result = await urlsFetch({
|
||||
urlList: [sourceId],
|
||||
selector
|
||||
});
|
||||
|
||||
return result[0]?.content || '';
|
||||
return {
|
||||
title: result[0]?.title,
|
||||
rawText: result[0]?.content || ''
|
||||
};
|
||||
} else if (type === DatasetSourceReadTypeEnum.externalFile) {
|
||||
if (!externalFileId) return Promise.reject('FileId not found');
|
||||
const rawText = await readFileRawTextByUrl({
|
||||
@@ -106,9 +115,11 @@ export const readDatasetSourceRawText = async ({
|
||||
relatedId: externalFileId,
|
||||
customPdfParse
|
||||
});
|
||||
return rawText;
|
||||
return {
|
||||
rawText
|
||||
};
|
||||
} else if (type === DatasetSourceReadTypeEnum.apiFile) {
|
||||
const rawText = await readApiServerFileContent({
|
||||
const { title, rawText } = await readApiServerFileContent({
|
||||
apiServer,
|
||||
feishuServer,
|
||||
yuqueServer,
|
||||
@@ -116,9 +127,15 @@ export const readDatasetSourceRawText = async ({
|
||||
teamId,
|
||||
tmbId
|
||||
});
|
||||
return rawText;
|
||||
return {
|
||||
title,
|
||||
rawText
|
||||
};
|
||||
}
|
||||
return '';
|
||||
return {
|
||||
title: '',
|
||||
rawText: ''
|
||||
};
|
||||
};
|
||||
|
||||
export const readApiServerFileContent = async ({
|
||||
@@ -137,7 +154,10 @@ export const readApiServerFileContent = async ({
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
customPdfParse?: boolean;
|
||||
}) => {
|
||||
}): Promise<{
|
||||
title?: string;
|
||||
rawText: string;
|
||||
}> => {
|
||||
if (apiServer) {
|
||||
return useApiDatasetRequest({ apiServer }).getFileContent({
|
||||
teamId,
|
||||
@@ -148,7 +168,10 @@ export const readApiServerFileContent = async ({
|
||||
}
|
||||
|
||||
if (feishuServer || yuqueServer) {
|
||||
return POST<string>(`/core/dataset/systemApiDataset`, {
|
||||
return POST<{
|
||||
title?: string;
|
||||
rawText: string;
|
||||
}>(`/core/dataset/systemApiDataset`, {
|
||||
type: 'content',
|
||||
feishuServer,
|
||||
yuqueServer,
|
||||
|
Reference in New Issue
Block a user