External dataset (#1519)

* perf: local file create collection

* rename middleware

* perf: remove code

* feat: next14

* feat: external file dataset

* collection tags field

* external file dataset doc

* fix: ts

projects/app/src/web/core/dataset/api.ts

@@ -14,6 +14,7 @@ import type {
   CreateDatasetCollectionParams,
   CsvTableCreateDatasetCollectionParams,
   DatasetUpdateBody,
+  ExternalFileCreateDatasetCollectionParams,
   FileIdCreateDatasetCollectionParams,
   LinkCreateDatasetCollectionParams,
   PostWebsiteSyncParams,
@@ -44,6 +45,7 @@ import type {
   PostPreviewFilesChunksProps,
   PreviewChunksResponse
 } from '@/pages/api/core/dataset/file/getPreviewChunks';
+import type { readCollectionSourceResponse } from '@/pages/api/core/dataset/collection/read';
 
 /* ======================== dataset ======================= */
 export const getDatasets = (data: { parentId?: string; type?: DatasetTypeEnum }) =>
@@ -85,7 +87,9 @@ export const getDatasetCollectionById = (id: string) =>
 export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
   POST<string>(`/core/dataset/collection/create`, data);
 export const postCreateDatasetFileCollection = (data: FileIdCreateDatasetCollectionParams) =>
-  POST<{ collectionId: string }>(`/core/dataset/collection/create/file`, data, { timeout: 120000 });
+  POST<{ collectionId: string }>(`/core/dataset/collection/create/fileId`, data, {
+    timeout: 120000
+  });
 export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectionParams) =>
   POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data);
 export const postCreateDatasetTextCollection = (data: TextCreateDatasetCollectionParams) =>
@@ -94,6 +98,12 @@ export const postCreateDatasetCsvTableCollection = (data: CsvTableCreateDatasetC
   POST<{ collectionId: string }>(`/core/dataset/collection/create/csvTable`, data, {
     timeout: 120000
   });
+export const postCreateDatasetExternalFileCollection = (
+  data: ExternalFileCreateDatasetCollectionParams
+) =>
+  POST<{ collectionId: string }>(`/proApi/core/dataset/collection/create/externalFileUrl`, data, {
+    timeout: 120000
+  });
 
 export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
   POST(`/core/dataset/collection/update`, data);
@@ -144,6 +154,6 @@ export const getDatasetTrainingQueue = (datasetId: string) =>
 export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
   POST<PreviewChunksResponse>('/core/dataset/file/getPreviewChunks', data);
 
-/* ================== file ======================== */
-export const getFileViewUrl = (fileId: string) =>
-  GET<string>('/core/dataset/file/getPreviewUrl', { fileId });
+/* ================== read source ======================== */
+export const getCollectionSource = (collectionId: string) =>
+  GET<readCollectionSourceResponse>('/core/dataset/collection/read', { collectionId });

projects/app/src/web/core/dataset/constants.ts

@@ -1,5 +1,9 @@
 import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/model';
-import { DatasetTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import {
+  DatasetCollectionTypeEnum,
+  DatasetTypeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constants';
 import type {
   DatasetCollectionItemType,
   DatasetItemType
@@ -46,7 +50,7 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
   },
   parentId: '',
   name: '',
-  type: 'file',
+  type: DatasetCollectionTypeEnum.file,
   updateTime: new Date(),
   canWrite: false,
   sourceName: '',

projects/app/src/web/core/dataset/hooks/readCollectionSource.ts

@@ -0,0 +1,34 @@
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { getCollectionSource } from '@/web/core/dataset/api';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { useToast } from '@fastgpt/web/hooks/useToast';
+import { useTranslation } from 'next-i18next';
+
+export function getCollectionSourceAndOpen(collectionId: string) {
+  const { toast } = useToast();
+  const { t } = useTranslation();
+  const { setLoading } = useSystemStore();
+
+  return async () => {
+    try {
+      setLoading(true);
+      const { value: url } = await getCollectionSource(collectionId);
+
+      if (!url) {
+        throw new Error('No file found');
+      }
+
+      if (url.startsWith('/')) {
+        window.open(`${location.origin}${url}`, '_blank');
+      } else {
+        window.open(url, '_blank');
+      }
+    } catch (error) {
+      toast({
+        title: getErrText(error, t('error.fileNotFound')),
+        status: 'error'
+      });
+    }
+    setLoading(false);
+  };
+}

projects/app/src/web/core/dataset/type.d.ts

@@ -28,8 +28,8 @@ export type ImportSourceItemType = {
   rawText?: string;
 
   // external file
-  sourceUrl?: string;
-  externalId?: string;
+  externalFileUrl?: string;
+  externalFileId?: string;
 };
 
 export type ImportSourceParamsType = UseFormReturn<

projects/app/src/web/core/dataset/utils.ts

@@ -1,11 +0,0 @@
-import { getFileViewUrl } from '@/web/core/dataset/api';
-import { strIsLink } from '@fastgpt/global/common/string/tools';
-
-export async function getFileAndOpen(fileId: string) {
-  if (strIsLink(fileId)) {
-    return window.open(fileId, '_blank');
-  }
-  const url = await getFileViewUrl(fileId);
-  const asPath = `${location.origin}${url}`;
-  window.open(asPath, '_blank');
-}