Add externalfile api params (#2745)

* feat: external dataset api

* perf: doc

projects/app/src/global/core/dataset/type.d.ts

@@ -21,6 +21,8 @@ export type DatasetCollectionsListItemType = {
   trainingType?: DatasetCollectionSchemaType['trainingType'];
   tags?: string[];
 
+  externalFileId?: string;
+
   fileId?: string;
   rawLink?: string;
   permission: DatasetPermission;

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -10,6 +10,7 @@ import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
 
 async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
   const { id } = req.query as { id: string };
@@ -35,6 +36,10 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
   return {
     ...collection,
     ...getCollectionSourceData(collection),
+    tags: await collectionTagsToTagLabel({
+      datasetId: collection.datasetId._id,
+      tags: collection.tags
+    }),
     permission,
     file
   };

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -11,6 +11,8 @@ import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { PagingData } from '@/types';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
 
 async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectionsListItemType>> {
   let {
@@ -60,12 +62,15 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
     trainingType: 1,
     fileId: 1,
     rawLink: 1,
-    tags: 1
+    tags: 1,
+    externalFileId: 1
   };
 
   // not count data amount
   if (simple) {
-    const collections = await MongoDatasetCollection.find(match)
+    const collections = await MongoDatasetCollection.find(match, undefined, {
+      ...readFromSecondary
+    })
       .select(selectField)
       .sort({
         updateTime: -1
@@ -78,6 +83,10 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
       data: await Promise.all(
         collections.map(async (item) => ({
           ...item,
+          tags: await collectionTagsToTagLabel({
+            datasetId,
+            tags: item.tags
+          }),
           dataAmount: 0,
           trainingAmount: 0,
           permission
@@ -153,12 +162,18 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
         }
       }
     ]),
-    MongoDatasetCollection.countDocuments(match)
+    MongoDatasetCollection.countDocuments(match, {
+      ...readFromSecondary
+    })
   ]);
 
   const data = await Promise.all(
     collections.map(async (item) => ({
       ...item,
+      tags: await collectionTagsToTagLabel({
+        datasetId,
+        tags: item.tags
+      }),
       permission
     }))
   );

projects/app/src/pages/api/core/dataset/collection/update.ts

@@ -1,5 +1,8 @@
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+import {
+  createOrGetCollectionTags,
+  getCollectionUpdateTime
+} from '@fastgpt/service/core/dataset/collection/utils';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
@@ -11,11 +14,16 @@ import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 
 export type UpdateDatasetCollectionParams = {
-  id: string;
+  id?: string;
   parentId?: string;
   name?: string;
-  tags?: string[];
+  tags?: string[]; // Not tag id, is tag label
   forbid?: boolean;
+  createTime?: Date;
+
+  // External file id
+  datasetId?: string;
+  externalFileId?: string;
 };
 
 // Set folder collection children forbid status
@@ -65,14 +73,22 @@ const updateFolderChildrenForbid = async ({
 };
 
 async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
-  const { id, parentId, name, tags, forbid } = req.body;
+  let { datasetId, externalFileId, id, parentId, name, tags, forbid, createTime } = req.body;
+
+  if (datasetId && externalFileId) {
+    const collection = await MongoDatasetCollection.findOne({ datasetId, externalFileId }, '_id');
+    if (!collection) {
+      return Promise.reject(CommonErrEnum.fileNotFound);
+    }
+    id = collection._id;
+  }
 
   if (!id) {
     return Promise.reject(CommonErrEnum.missingParams);
   }
 
   // 凭证校验
-  const { collection } = await authDatasetCollection({
+  const { collection, teamId } = await authDatasetCollection({
     req,
     authToken: true,
     authApiKey: true,
@@ -81,6 +97,13 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
   });
 
   await mongoSessionRun(async (session) => {
+    const collectionTags = await createOrGetCollectionTags({
+      tags,
+      teamId,
+      datasetId: collection.datasetId._id,
+      session
+    });
+
     await MongoDatasetCollection.updateOne(
       {
         _id: id
@@ -89,8 +112,9 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
         $set: {
           ...(parentId !== undefined && { parentId: parentId || null }),
           ...(name && { name, updateTime: getCollectionUpdateTime({ name }) }),
-          ...(tags && { tags }),
-          ...(forbid !== undefined && { forbid })
+          ...(collectionTags !== undefined && { tags: collectionTags }),
+          ...(forbid !== undefined && { forbid }),
+          ...(createTime !== undefined && { createTime })
         }
       },
       {

projects/app/src/pages/dataset/detail/components/CollectionCard/TagsPopOver.tsx

@@ -35,8 +35,8 @@ const TagsPopOver = ({
   const tagList = useMemo(
     () =>
       (collectionTags
-        ?.map((tagId) => {
-          const tagObject = allDatasetTags.find((tag) => tag._id === tagId);
+        ?.map((item) => {
+          const tagObject = allDatasetTags.find((tag) => tag.tag === item);
           return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null;
         })
         .filter((tag) => tag !== null) as {
@@ -153,9 +153,9 @@ const TagsPopOver = ({
         setIsUpdateLoading(true);
         await putDatasetCollectionById({
           id: currentCollection._id,
-          tags: checkedTags.map((tag) => tag._id)
+          tags: checkedTags.map((tag) => tag.tag)
         });
-        setCollectionTags(checkedTags.map((tag) => tag._id));
+        setCollectionTags(checkedTags.map((tag) => tag.tag));
         setIsUpdateLoading(false);
       }}
       display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'}

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -1,10 +1,9 @@
-import React, { useState, useRef, useMemo } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Box, Card, IconButton, Flex, Button, useTheme } from '@chakra-ui/react';
 import {
   getDatasetDataList,
   delOneDatasetDataById,
-  getDatasetCollectionById,
-  putDatasetDataById
+  getDatasetCollectionById
 } from '@/web/core/dataset/api';
 import { useQuery } from '@tanstack/react-query';
 import { useToast } from '@fastgpt/web/hooks/useToast';
@@ -21,7 +20,6 @@ import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
 import { useContextSelector } from 'use-context-selector';
-import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import MyTag from '@fastgpt/web/components/common/Tag/index';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
@@ -96,24 +94,27 @@ const DataCard = () => {
       <Flex flexDirection={'column'} h={'100%'}>
         {/* Header */}
         <Flex alignItems={'center'} px={6}>
-          <Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
-            <Box>
-              <Box alignItems={'center'} gap={2} display={isPc ? 'flex' : ''}>
-                {collection?._id && (
-                  <RawSourceBox
-                    collectionId={collection._id}
-                    {...getCollectionSourceData(collection)}
-                    fontSize={['sm', 'md']}
-                    color={'black'}
-                    textDecoration={'none'}
-                  />
-                )}
-              </Box>
-              {feConfigs?.isPlus && !!collection?.tags?.length && (
-                <TagsPopOver currentCollection={collection} />
+          <Box flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
+            <Box
+              className="textEllipsis"
+              alignItems={'center'}
+              gap={2}
+              display={isPc ? 'flex' : ''}
+            >
+              {collection?._id && (
+                <RawSourceBox
+                  collectionId={collection._id}
+                  {...getCollectionSourceData(collection)}
+                  fontSize={['sm', 'md']}
+                  color={'black'}
+                  textDecoration={'none'}
+                />
               )}
             </Box>
-          </Flex>
+            {feConfigs?.isPlus && !!collection?.tags?.length && (
+              <TagsPopOver currentCollection={collection} />
+            )}
+          </Box>
           {canWrite && (
             <Box>
               <Button

projects/app/src/pages/dataset/detail/components/Info.tsx

@@ -243,8 +243,9 @@ const Info = ({ datasetId }: { datasetId: string }) => {
                 const vectorModel = vectorModelList.find((item) => item.model === e);
                 if (!vectorModel) return;
                 return onOpenConfirmRebuild(() => {
-                  setValue('vectorModel', vectorModel);
-                  return onRebuilding(vectorModel);
+                  return onRebuilding(vectorModel).then(() => {
+                    setValue('vectorModel', vectorModel);
+                  });
                 })();
               }}
             />