feat: sync api collection will refresh title;perf: invite link ux (#4237)

* update queue

* feat: sync api collection will refresh title

* sync collection

* remove lock

* perf: invite link ux

packages/service/core/dataset/apiDataset/api.ts

@@ -1,6 +1,6 @@
 import type {
-  APIFileContentResponse,
   APIFileListResponse,
+  ApiFileReadContentResponse,
   APIFileReadResponse,
   APIFileServer
 } from '@fastgpt/global/core/dataset/apiDataset';
@@ -8,6 +8,7 @@ import axios, { Method } from 'axios';
 import { addLog } from '../../../common/system/log';
 import { readFileRawTextByUrl } from '../read';
 import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
+import { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
 
 type ResponseDataType = {
   success: boolean;
@@ -118,17 +119,24 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
     tmbId: string;
     apiFileId: string;
     customPdfParse?: boolean;
-  }) => {
-    const data = await request<APIFileContentResponse>(
-      `/v1/file/content`,
-      { id: apiFileId },
-      'GET'
-    );
+  }): Promise<ApiFileReadContentResponse> => {
+    const data = await request<
+      {
+        title?: string;
+      } & RequireOnlyOne<{
+        content: string;
+        previewUrl: string;
+      }>
+    >(`/v1/file/content`, { id: apiFileId }, 'GET');
+    const title = data.title;
     const content = data.content;
     const previewUrl = data.previewUrl;
 
     if (content) {
-      return content;
+      return {
+        title,
+        rawText: content
+      };
     }
     if (previewUrl) {
       const rawText = await readFileRawTextByUrl({
@@ -138,7 +146,10 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
         relatedId: apiFileId,
         customPdfParse
       });
-      return rawText;
+      return {
+        title,
+        rawText
+      };
     }
     return Promise.reject('Invalid content type: content or previewUrl is required');
   };

packages/service/core/dataset/collection/utils.ts

@@ -11,7 +11,6 @@ import {
   DatasetCollectionSyncResultEnum,
   DatasetCollectionTypeEnum,
   DatasetSourceReadTypeEnum,
-  DatasetTypeEnum,
   TrainingModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
@@ -19,6 +18,7 @@ import { readDatasetSourceRawText } from '../read';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { mongoSessionRun } from '../../../common/mongo/sessionRun';
 import { createCollectionAndInsertData, delCollection } from './controller';
+import { collectionCanSync } from '@fastgpt/global/core/dataset/collection/utils';
 
 /**
  * get all collection by top collectionId
@@ -137,10 +137,7 @@ export const collectionTagsToTagLabel = async ({
 export const syncCollection = async (collection: CollectionWithDatasetType) => {
   const dataset = collection.dataset;
 
-  if (
-    collection.type !== DatasetCollectionTypeEnum.link &&
-    dataset.type !== DatasetTypeEnum.apiDataset
-  ) {
+  if (!collectionCanSync(collection.type)) {
     return Promise.reject(DatasetErrEnum.notSupportSync);
   }
 
@@ -155,15 +152,20 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
       };
     }
 
-    if (!collection.apiFileId) return Promise.reject('apiFileId is missing');
-    if (!dataset.apiServer) return Promise.reject('apiServer not found');
+    const sourceId = collection.apiFileId;
+
+    if (!sourceId) return Promise.reject('apiFileId is missing');
+
     return {
       type: DatasetSourceReadTypeEnum.apiFile,
-      sourceId: collection.apiFileId,
-      apiServer: dataset.apiServer
+      sourceId,
+      apiServer: dataset.apiServer,
+      feishuServer: dataset.feishuServer,
+      yuqueServer: dataset.yuqueServer
     };
   })();
-  const rawText = await readDatasetSourceRawText({
+
+  const { title, rawText } = await readDatasetSourceRawText({
     teamId: collection.teamId,
     tmbId: collection.tmbId,
     ...sourceReadType
@@ -196,7 +198,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
       createCollectionParams: {
         teamId: collection.teamId,
         tmbId: collection.tmbId,
-        name: collection.name,
+        name: title || collection.name,
         datasetId: collection.datasetId,
         parentId: collection.parentId,
         type: collection.type,

packages/service/core/dataset/read.ts

@@ -79,9 +79,12 @@ export const readDatasetSourceRawText = async ({
   apiServer?: APIFileServer; // api dataset
   feishuServer?: FeishuServer; // feishu dataset
   yuqueServer?: YuqueServer; // yuque dataset
-}): Promise<string> => {
+}): Promise<{
+  title?: string;
+  rawText: string;
+}> => {
   if (type === DatasetSourceReadTypeEnum.fileLocal) {
-    const { rawText } = await readFileContentFromMongo({
+    const { filename, rawText } = await readFileContentFromMongo({
       teamId,
       tmbId,
       bucketName: BucketNameEnum.dataset,
@@ -89,14 +92,20 @@ export const readDatasetSourceRawText = async ({
       isQAImport,
       customPdfParse
     });
-    return rawText;
+    return {
+      title: filename,
+      rawText
+    };
   } else if (type === DatasetSourceReadTypeEnum.link) {
     const result = await urlsFetch({
       urlList: [sourceId],
       selector
     });
 
-    return result[0]?.content || '';
+    return {
+      title: result[0]?.title,
+      rawText: result[0]?.content || ''
+    };
   } else if (type === DatasetSourceReadTypeEnum.externalFile) {
     if (!externalFileId) return Promise.reject('FileId not found');
     const rawText = await readFileRawTextByUrl({
@@ -106,9 +115,11 @@ export const readDatasetSourceRawText = async ({
       relatedId: externalFileId,
       customPdfParse
     });
-    return rawText;
+    return {
+      rawText
+    };
   } else if (type === DatasetSourceReadTypeEnum.apiFile) {
-    const rawText = await readApiServerFileContent({
+    const { title, rawText } = await readApiServerFileContent({
       apiServer,
       feishuServer,
       yuqueServer,
@@ -116,9 +127,15 @@ export const readDatasetSourceRawText = async ({
       teamId,
       tmbId
     });
-    return rawText;
+    return {
+      title,
+      rawText
+    };
   }
-  return '';
+  return {
+    title: '',
+    rawText: ''
+  };
 };
 
 export const readApiServerFileContent = async ({
@@ -137,7 +154,10 @@ export const readApiServerFileContent = async ({
   teamId: string;
   tmbId: string;
   customPdfParse?: boolean;
-}) => {
+}): Promise<{
+  title?: string;
+  rawText: string;
+}> => {
   if (apiServer) {
     return useApiDatasetRequest({ apiServer }).getFileContent({
       teamId,
@@ -148,7 +168,10 @@ export const readApiServerFileContent = async ({
   }
 
   if (feishuServer || yuqueServer) {
-    return POST<string>(`/core/dataset/systemApiDataset`, {
+    return POST<{
+      title?: string;
+      rawText: string;
+    }>(`/core/dataset/systemApiDataset`, {
       type: 'content',
       feishuServer,
       yuqueServer,