4.6.7 fix (#752)

projects/app/src/pages/api/core/dataset/collection/create/text.ts

@@ -16,11 +16,15 @@ import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dat
 import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
 import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
+import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
+import { getQAModel, getVectorModel } from '@/service/core/ai/model';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
     const {
+      name,
       text,
       trainingType = TrainingModeEnum.chunk,
       chunkSize = 512,
@@ -29,7 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       ...body
     } = req.body as TextCreateDatasetCollectionParams;
 
-    const { teamId, tmbId } = await authDataset({
+    const { teamId, tmbId, dataset } = await authDataset({
       req,
       authToken: true,
       authApiKey: true,
@@ -52,21 +56,32 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       insertLen: predictDataLimitLength(trainingType, chunks)
     });
 
-    // 3. create collection
-    const collectionId = await createOneCollection({
-      ...body,
-      teamId,
-      tmbId,
-      type: DatasetCollectionTypeEnum.virtual,
+    // 3. create collection and training bill
+    const [collectionId, { billId }] = await Promise.all([
+      createOneCollection({
+        ...body,
+        teamId,
+        tmbId,
+        type: DatasetCollectionTypeEnum.virtual,
 
-      trainingType,
-      chunkSize,
-      chunkSplitter,
-      qaPrompt,
+        name,
+        trainingType,
+        chunkSize,
+        chunkSplitter,
+        qaPrompt,
 
-      hashRawText: hashStr(text),
-      rawTextLength: text.length
-    });
+        hashRawText: hashStr(text),
+        rawTextLength: text.length
+      }),
+      createTrainingBill({
+        teamId,
+        tmbId,
+        appName: name,
+        billSource: BillSourceEnum.training,
+        vectorModel: getVectorModel(dataset.vectorModel)?.name,
+        agentModel: getQAModel(dataset.agentModel)?.name
+      })
+    ]);
 
     // 4. push chunks to training queue
     const insertResults = await pushDataToTrainingQueue({
@@ -74,6 +89,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       tmbId,
       collectionId,
       trainingMode: trainingType,
+      prompt: qaPrompt,
+      billId,
       data: chunks.map((text, index) => ({
         q: text,
         chunkIndex: index
@@ -90,3 +107,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
   }
 }
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '10mb'
+    }
+  }
+};

projects/app/src/pages/api/core/dataset/data/pushData.ts

@@ -3,8 +3,10 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
-import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
-import type { PushDatasetDataProps } from '@/global/core/dataset/api.d';
+import type {
+  PushDatasetDataProps,
+  PushDatasetDataResponse
+} from '@fastgpt/global/core/dataset/api.d';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
 import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
@@ -39,7 +41,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       insertLen: predictDataLimitLength(collection.trainingType, data)
     });
 
-    jsonRes<PushDataResponse>(res, {
+    jsonRes<PushDatasetDataResponse>(res, {
       data: await pushDataToTrainingQueue({
         ...req.body,
         teamId,

projects/app/src/pages/api/plusApi/[...path].ts

@@ -12,16 +12,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const method = (req.method || 'POST') as Method;
     const { path = [], ...query } = req.query as any;
 
-    const url = `/${path?.join('/')}`;
+    const url = `/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
 
     if (!url) {
       throw new Error('url is empty');
     }
 
-    const data = {
-      ...req.body,
-      ...query
-    };
+    const data = req.body || query;
 
     const repose = await request(
       url,
@@ -56,3 +53,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
   }
 }
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '10mb'
+    },
+    responseLimit: '10mb'
+  }
+};

projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx

@@ -27,7 +27,7 @@ const Upload = dynamic(() => import('../commonProgress/Upload'));
 const PreviewRawText = dynamic(() => import('../components/PreviewRawText'));
 
 type FileItemType = ImportSourceItemType & { file: File };
-const fileType = '.txt, .docx, .pdf, .md, .html';
+const fileType = '.txt, .docx, .csv, .pdf, .md, .html';
 const maxSelectFileCount = 1000;
 
 const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {

projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx

@@ -14,7 +14,8 @@ import { useImportStore } from '../Provider';
 import { feConfigs } from '@/web/common/system/staticData';
 
 import dynamic from 'next/dynamic';
-import { fileDownload, readCsvContent } from '@/web/common/file/utils';
+import { fileDownload } from '@/web/common/file/utils';
+import { readCsvContent } from '@fastgpt/web/common/file/read/csv';
 
 const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
 const Upload = dynamic(() => import('../commonProgress/Upload'));
@@ -56,7 +57,7 @@ const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () =
       {
         for await (const selectFile of files) {
           const { file, folderPath } = selectFile;
-          const { header, data } = await readCsvContent(file);
+          const { header, data } = await readCsvContent({ file });
 
           const filterData: FileItemType['chunks'] = data
             .filter((item) => item[0])

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -193,7 +193,10 @@ const InputDataModal = ({
       // not exactly same
       await putDatasetDataById({
         id: dataId,
-        ...e
+        ...e,
+        indexes: e.indexes.map((index) =>
+          index.defaultIndex ? getDefaultIndex({ q: e.q, a: e.a }) : index
+        )
       });
 
       return {

projects/app/src/pages/dataset/detail/components/Test.tsx

@@ -35,7 +35,8 @@ import dynamic from 'next/dynamic';
 import { useForm } from 'react-hook-form';
 import MySelect from '@/components/Select';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { fileDownload, readCsvContent } from '@/web/common/file/utils';
+import { fileDownload } from '@/web/common/file/utils';
+import { readCsvContent } from '@fastgpt/web/common/file/read/csv';
 import { delay } from '@fastgpt/global/common/system/utils';
 import QuoteItem from '@/components/core/dataset/QuoteItem';
 
@@ -125,7 +126,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
   const { mutate: onFileTest, isLoading: fileTestIsLoading } = useRequest({
     mutationFn: async ({ searchParams }: FormType) => {
       if (!selectFile) return Promise.reject('File is not selected');
-      const { data } = await readCsvContent(selectFile);
+      const { data } = await readCsvContent({ file: selectFile });
       const testList = data.slice(0, 100);
       const results: SearchTestResponse[] = [];