dataset save raw file

client/package.json

@@ -31,6 +31,7 @@
     "i18next": "^22.5.1",
     "immer": "^9.0.19",
     "js-cookie": "^3.0.5",
+    "jschardet": "^3.0.0",
     "jsdom": "^22.1.0",
     "jsonwebtoken": "^9.0.0",
     "lodash": "^4.17.21",

client/pnpm-lock.yaml

@@ -71,6 +71,9 @@ dependencies:
   js-cookie:
     specifier: ^3.0.5
     version: registry.npmmirror.com/js-cookie@3.0.5
+  jschardet:
+    specifier: ^3.0.0
+    version: registry.npmmirror.com/jschardet@3.0.0
   jsdom:
     specifier: ^22.1.0
     version: registry.npmmirror.com/jsdom@22.1.0
@@ -8918,6 +8921,13 @@ packages:
       argparse: registry.npmmirror.com/argparse@2.0.1
     dev: true
 
+  registry.npmmirror.com/jschardet@3.0.0:
+    resolution: {integrity: sha512-lJH6tJ77V8Nzd5QWRkFYCLc13a3vADkh3r/Fi8HupZGWk2OVVDfnZP8V/VgQgZ+lzW0kG2UGb5hFgt3V3ndotQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jschardet/-/jschardet-3.0.0.tgz}
+    name: jschardet
+    version: 3.0.0
+    engines: {node: '>=0.1.90'}
+    dev: false
+
   registry.npmmirror.com/jsdom@22.1.0:
     resolution: {integrity: sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsdom/-/jsdom-22.1.0.tgz}
     name: jsdom

client/public/locales/en/common.json

@@ -88,13 +88,16 @@
   },
   "file": {
     "Click to download CSV template": "Click to download CSV template",
+    "Click to view file": "Click to view file",
     "Create File": "Create File",
     "Create file": "Create file",
     "Drag and drop": "Drag and drop files here",
     "Fetch Url": "Fetch Url",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
+    "Parse": "{{name}} Parsing...",
     "Release the mouse to upload the file": "Release the mouse to upload the file",
     "Select a maximum of 10 files": "Select a maximum of 10 files",
+    "Uploading": "Uploading: {{name}}, Progress: {{percent}}%",
     "max 10": "Max 10 files",
     "select a document": "select a document",
     "support": "support {{fileExtension}} file",

client/public/locales/zh/common.json

@@ -88,13 +88,16 @@
   },
   "file": {
     "Click to download CSV template": "点击下载 CSV 模板",
+    "Click to view file": "点击查看原始文件",
     "Create File": "创建新文件",
     "Create file": "创建文件",
     "Drag and drop": "拖拽文件至此",
     "Fetch Url": "链接读取",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
+    "Parse": "{{name}} 解析中...",
     "Release the mouse to upload the file": "松开鼠标上传文件",
     "Select a maximum of 10 files": "最多选择10个文件",
+    "Uploading": "正在上传 {{name}}，进度: {{percent}}%",
     "max 10": "最多选择 10 个文件",
     "select a document": "选择文件",
     "support": "支持 {{fileExtension}} 文件",

client/src/api/plugins/kb.ts

@@ -1,5 +1,5 @@
 import { GET, POST, PUT, DELETE } from '../request';
-import type { KbItemType, KbListItemType } from '@/types/plugin';
+import type { DatasetItemType, KbItemType, KbListItemType } from '@/types/plugin';
 import { RequestPaging } from '@/types/index';
 import { TrainingModeEnum } from '@/constants/plugin';
 import {
@@ -13,6 +13,7 @@ import {
 import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById';
 import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
 import type { KbUpdateParams, CreateKbParams } from '../request/kb';
+import { QuoteItemType } from '@/types/chat';
 
 /* knowledge base */
 export const getKbList = () => GET<KbListItemType[]>(`/plugins/kb/list`);
@@ -58,7 +59,7 @@ export const getTrainingData = (data: { kbId: string; init: boolean }) =>
 export const getTrainingQueueLen = () => GET<number>(`/plugins/kb/data/getQueueLen`);
 
 export const getKbDataItemById = (dataId: string) =>
-  GET<KbDataItemType>(`/plugins/kb/data/getDataById`, { dataId });
+  GET<QuoteItemType>(`/plugins/kb/data/getDataById`, { dataId });
 
 /**
  * 直接push数据
@@ -69,10 +70,8 @@ export const postKbDataFromList = (data: PushDataProps) =>
 /**
  * insert one data to dataset
  */
-export const insertData2Kb = (data: {
+export const insertData2Kb = (data: { kbId: string; data: DatasetItemType }) =>
-  kbId: string;
+  POST<string>(`/plugins/kb/data/insertData`, data);
-  data: { a: string; q: string; source?: string };
-}) => POST<string>(`/plugins/kb/data/insertData`, data);
 
 /**
  * 更新一条数据

client/src/api/request.ts

@@ -1,4 +1,9 @@
-import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
+import axios, {
+  Method,
+  InternalAxiosRequestConfig,
+  AxiosResponse,
+  AxiosProgressEvent
+} from 'axios';
 import { clearToken, getToken } from '@/utils/user';
 import { TOKEN_ERROR_CODE } from '@/service/errorCode';
 
@@ -6,6 +11,7 @@ interface ConfigType {
   headers?: { [key: string]: string };
   hold?: boolean;
   timeout?: number;
+  onUploadProgress?: (progressEvent: AxiosProgressEvent) => void;
 }
 interface ResponseDataType {
   code: number;

client/src/api/system.ts

@@ -1,6 +1,20 @@
 import { GET, POST, PUT } from './request';
 import type { InitDateResponse } from '@/pages/api/system/getInitData';
+import { AxiosProgressEvent } from 'axios';
 
 export const getInitData = () => GET<InitDateResponse>('/system/getInitData');
 
 export const uploadImg = (base64Img: string) => POST<string>('/system/uploadImage', { base64Img });
+
+export const postUploadFiles = (
+  data: FormData,
+  onUploadProgress: (progressEvent: AxiosProgressEvent) => void
+) =>
+  POST<string[]>('/plugins/file/upload', data, {
+    onUploadProgress,
+    headers: {
+      'Content-Type': 'multipart/form-data; charset=utf-8'
+    }
+  });
+
+export const getFileViewUrl = (fileId: string) => GET<string>('/plugins/file/readUrl', { fileId });

client/src/components/ChatBox/ContextModal.tsx

@@ -21,7 +21,13 @@ const ContextModal = ({
       minW={['90vw', '600px']}
       isCentered
     >
-      <ModalBody pt={0} whiteSpace={'pre-wrap'} textAlign={'justify'} fontSize={'sm'}>
+      <ModalBody
+        pt={0}
+        whiteSpace={'pre-wrap'}
+        textAlign={'justify'}
+        wordBreak={'break-all'}
+        fontSize={'sm'}
+      >
         {context.map((item, i) => (
           <Box
             key={i}

client/src/components/ChatBox/QuoteModal.tsx

@@ -6,15 +6,12 @@ import { useToast } from '@/hooks/useToast';
 import { getErrText } from '@/utils/tools';
 import { QuoteItemType } from '@/types/chat';
 import MyIcon from '@/components/Icon';
-import InputDataModal from '@/pages/kb/detail/components/InputDataModal';
+import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal';
 import MyModal from '../MyModal';
+import { KbDataItemType } from '@/types/plugin';
 
-type SearchType = {
+type SearchType = KbDataItemType & {
   kb_id?: string;
-  id?: string;
-  q: string;
-  a?: string;
-  source?: string | undefined;
 };
 
 const QuoteModal = ({
@@ -29,12 +26,7 @@ const QuoteModal = ({
   const theme = useTheme();
   const { toast } = useToast();
   const { setIsLoading, Loading } = useLoading();
-  const [editDataItem, setEditDataItem] = useState<{
+  const [editDataItem, setEditDataItem] = useState<QuoteItemType>();
-    kbId: string;
-    dataId: string;
-    a: string;
-    q: string;
-  }>();
 
   /**
    * click edit, get new kbDataItem
@@ -44,19 +36,14 @@ const QuoteModal = ({
       if (!item.id) return;
       try {
         setIsLoading(true);
-        const data = (await getKbDataItemById(item.id)) as QuoteItemType;
+        const data = await getKbDataItemById(item.id);
 
         if (!data) {
           onUpdateQuote(item.id, '已删除');
           throw new Error('该数据已被删除');
         }
 
-        setEditDataItem({
+        setEditDataItem(data);
-          kbId: data.kb_id,
-          dataId: data.id,
-          q: data.q,
-          a: data.a
-        });
       } catch (err) {
         toast({
           status: 'warning',
@@ -85,7 +72,13 @@ const QuoteModal = ({
           </>
         }
       >
-        <ModalBody pt={0} whiteSpace={'pre-wrap'} textAlign={'justify'} fontSize={'sm'}>
+        <ModalBody
+          pt={0}
+          whiteSpace={'pre-wrap'}
+          textAlign={'justify'}
+          wordBreak={'break-all'}
+          fontSize={'sm'}
+        >
           {rawSearch.map((item, i) => (
             <Box
               key={i}
@@ -98,7 +91,7 @@ const QuoteModal = ({
               _hover={{ '& .edit': { display: 'flex' } }}
               overflow={'hidden'}
             >
-              {item.source && <Box color={'myGray.600'}>({item.source})</Box>}
+              {item.source && <RawFileText filename={item.source} fileId={item.file_id} />}
               <Box>{item.q}</Box>
               <Box>{item.a}</Box>
               {item.id && (
@@ -136,10 +129,13 @@ const QuoteModal = ({
       {editDataItem && (
         <InputDataModal
           onClose={() => setEditDataItem(undefined)}
-          onSuccess={() => onUpdateQuote(editDataItem.dataId, '手动修改')}
+          onSuccess={() => onUpdateQuote(editDataItem.id, '手动修改')}
-          onDelete={() => onUpdateQuote(editDataItem.dataId, '已删除')}
+          onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
-          kbId={editDataItem.kbId}
+          kbId={editDataItem.kb_id}
-          defaultValues={editDataItem}
+          defaultValues={{
+            ...editDataItem,
+            dataId: editDataItem.id
+          }}
         />
       )}
     </>

client/src/components/Loading/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
-import { Spinner, Flex } from '@chakra-ui/react';
+import { Spinner, Flex, Box } from '@chakra-ui/react';
 
-const Loading = ({ fixed = true }: { fixed?: boolean }) => {
+const Loading = ({ fixed = true, text = '' }: { fixed?: boolean; text?: string }) => {
   return (
     <Flex
       position={fixed ? 'fixed' : 'absolute'}
@@ -13,8 +13,14 @@ const Loading = ({ fixed = true }: { fixed?: boolean }) => {
       bottom={0}
       alignItems={'center'}
       justifyContent={'center'}
+      flexDirection={'column'}
     >
       <Spinner thickness="4px" speed="0.65s" emptyColor="myGray.100" color="myBlue.600" size="xl" />
+      {text && (
+        <Box mt={2} color="myBlue.700" fontWeight={'bold'}>
+          {text}
+        </Box>
+      )}
     </Flex>
   );
 };

client/src/constants/common.ts

@@ -15,7 +15,8 @@ export const fileImgs = [
 
 export enum TrackEventName {
   windowError = 'windowError',
-  pageError = 'pageError'
+  pageError = 'pageError',
+  wordReadError = 'wordReadError'
 }
 
 export const htmlTemplate = `<!DOCTYPE html>

client/src/hooks/useLoading.tsx

@@ -5,8 +5,16 @@ export const useLoading = (props?: { defaultLoading: boolean }) => {
   const [isLoading, setIsLoading] = useState(props?.defaultLoading || false);
 
   const Loading = useCallback(
-    ({ loading, fixed = true }: { loading?: boolean; fixed?: boolean }): JSX.Element | null => {
+    ({
-      return isLoading || loading ? <LoadingComponent fixed={fixed} /> : null;
+      loading,
+      fixed = true,
+      text = ''
+    }: {
+      loading?: boolean;
+      fixed?: boolean;
+      text?: string;
+    }): JSX.Element | null => {
+      return isLoading || loading ? <LoadingComponent fixed={fixed} text={text} /> : null;
     },
     [isLoading]
   );

client/src/pages/api/admin/initv43.ts

@@ -0,0 +1,35 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+import { PgTrainingTableName } from '@/constants/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    await authUser({ req, authRoot: true });
+
+    const { rowCount } = await PgClient.query(`SELECT 1
+    FROM   information_schema.columns 
+    WHERE  table_schema = 'public'
+    AND    table_name   = '${PgTrainingTableName}'
+    AND    column_name  = 'file_id'`);
+
+    if (rowCount > 0) {
+      return jsonRes(res, {
+        data: '已经存在file_id字段'
+      });
+    }
+
+    jsonRes(res, {
+      data: await PgClient.query(
+        `ALTER TABLE ${PgTrainingTableName} ADD COLUMN file_id VARCHAR(100)`
+      )
+    });
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}

client/src/pages/api/openapi/kb/pushData.ts

@@ -9,12 +9,11 @@ import { startQueue } from '@/service/utils/tools';
 import { PgClient } from '@/service/pg';
 import { modelToolMap } from '@/utils/plugin';
 import { getVectorModel } from '@/service/utils/data';
-
+import { DatasetItemType } from '@/types/plugin';
-export type DateItemType = { a: string; q: string; source?: string };
 
 export type Props = {
   kbId: string;
-  data: DateItemType[];
+  data: DatasetItemType[];
   mode: `${TrainingModeEnum}`;
   prompt?: string;
 };
@@ -95,7 +94,7 @@ export async function pushDataToKb({
 
   // 过滤重复的 qa 内容
   const set = new Set();
-  const filterData: DateItemType[] = [];
+  const filterData: DatasetItemType[] = [];
 
   data.forEach((item) => {
     if (!item.q) return;
@@ -120,13 +119,10 @@ export async function pushDataToKb({
   // 数据库去重
   const insertData = (
     await Promise.allSettled(
-      filterData.map(async ({ q, a = '', source }) => {
+      filterData.map(async (data) => {
+        let { q, a } = data;
         if (mode !== TrainingModeEnum.index) {
-          return Promise.resolve({
+          return Promise.resolve(data);
-            q,
-            a,
-            source
-          });
         }
 
         if (!q) {
@@ -152,23 +148,17 @@ export async function pushDataToKb({
           console.log(error);
           error;
         }
-        return Promise.resolve({
+        return Promise.resolve(data);
-          q,
-          a,
-          source
-        });
       })
     )
   )
     .filter((item) => item.status === 'fulfilled')
-    .map<DateItemType>((item: any) => item.value);
+    .map<DatasetItemType>((item: any) => item.value);
 
   // 插入记录
   const insertRes = await TrainingData.insertMany(
     insertData.map((item) => ({
-      q: item.q,
+      ...item,
-      a: item.a,
-      source: item.source,
       userId,
       kbId,
       mode,

client/src/pages/api/openapi/kb/searchTest.ts

@@ -41,7 +41,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     const response: any = await PgClient.query(
       `BEGIN;
         SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
-        select id,q,a,source,(vector <#> '[${
+        select id, q, a, source, file_id, (vector <#> '[${
           vectors[0]
         }]') * -1 AS score from ${PgTrainingTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
         vectors[0]
@@ -49,7 +49,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
         COMMIT;`
     );
 
-    jsonRes<Response>(res, { data: response?.[2]?.rows || [] });
+    jsonRes<Response>(res, {
+      data: response?.[2]?.rows || []
+    });
   } catch (err) {
     console.log(err);
     jsonRes(res, {

client/src/pages/api/plugins/file/read.ts

@@ -3,6 +3,7 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { authFileToken } from './readUrl';
+import jschardet from 'jschardet';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -12,6 +13,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const { fileId, userId } = await authFileToken(token);
 
+    if (!fileId) {
+      throw new Error('fileId is empty');
+    }
+
     const gridFs = new GridFSStorage('dataset', userId);
 
     const [file, buffer] = await Promise.all([
@@ -19,9 +24,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       gridFs.download(fileId)
     ]);
 
-    res.setHeader('encoding', file.encoding);
+    const encoding = jschardet.detect(buffer)?.encoding;
+
+    res.setHeader('encoding', encoding);
     res.setHeader('Content-Type', file.contentType);
     res.setHeader('Cache-Control', 'public, max-age=3600');
+    res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
 
     res.end(buffer);
   } catch (error) {

client/src/pages/api/plugins/file/upload.ts

@@ -28,9 +28,10 @@ class UploadModel {
     limits: {
       fieldSize: maxSize
     },
+    preservePath: true,
     storage: multer.diskStorage({
       filename: (_req, file, cb) => {
-        const { ext } = path.parse(file.originalname);
+        const { ext } = path.parse(decodeURIComponent(file.originalname));
         cb(null, nanoid() + ext);
       }
     })
@@ -44,8 +45,13 @@ class UploadModel {
           return reject(error);
         }
 
-        // @ts-ignore
+        resolve({
-        resolve({ files: req.files });
+          // @ts-ignore
+          files: req.files?.map((file) => ({
+            ...file,
+            originalname: decodeURIComponent(file.originalname)
+          }))
+        });
       });
     });
   }
@@ -56,9 +62,9 @@ const upload = new UploadModel();
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { userId } = await authUser({ req });
+    const { userId } = await authUser({ req, authToken: true });
 
-    const { files } = await upload.doUpload(req, res);
+    const { files = [] } = await upload.doUpload(req, res);
 
     const gridFs = new GridFSStorage('dataset', userId);
 

client/src/pages/api/plugins/kb/data/getDataById.ts

@@ -30,7 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const where: any = [['user_id', userId], 'AND', ['id', dataId]];
 
     const searchRes = await PgClient.select<KbDataItemType>(PgTrainingTableName, {
-      fields: ['kb_id', 'id', 'q', 'a', 'source'],
+      fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'],
       where,
       limit: 1
     });

client/src/pages/api/plugins/kb/data/getDataList.ts

@@ -43,7 +43,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const [searchRes, total] = await Promise.all([
       PgClient.select<KbDataItemType>(PgTrainingTableName, {
-        fields: ['id', 'q', 'a', 'source'],
+        fields: ['id', 'q', 'a', 'source', 'file_id'],
         where,
         order: [{ field: 'id', mode: 'DESC' }],
         limit: pageSize,

client/src/pages/api/plugins/kb/data/insertData.ts

@@ -8,10 +8,11 @@ import { insertKbItem, PgClient } from '@/service/pg';
 import { modelToolMap } from '@/utils/plugin';
 import { getVectorModel } from '@/service/utils/data';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
+import { DatasetItemType } from '@/types/plugin';
 
 export type Props = {
   kbId: string;
-  data: { a: string; q: string; source?: string };
+  data: DatasetItemType;
 };
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

client/src/pages/kb/detail/components/DataCard.tsx

@@ -198,8 +198,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
             onClick={() =>
               setEditInputData({
                 dataId: item.id,
-                q: item.q,
+                ...item
-                a: item.a
               })
             }
           >

client/src/pages/kb/detail/components/Import/Chunk.tsx

@@ -109,10 +109,9 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
           return {
             ...file,
             tokens: splitRes.tokens,
-            chunks: splitRes.chunks.map((chunk) => ({
+            chunks: file.chunks.map((chunk, i) => ({
-              q: chunk,
+              ...chunk,
-              a: '',
+              q: splitRes.chunks[i]
-              source: file.filename
             }))
           };
         })

client/src/pages/kb/detail/components/Import/Csv.tsx

@@ -1,11 +1,10 @@
-import React, { useState, useCallback, useMemo } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Box, Flex, Button, useTheme, Image } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
 import { getErrText } from '@/utils/tools';
-import { vectorModelList } from '@/store/static';
 import MyIcon from '@/components/Icon';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
 import { TrainingModeEnum } from '@/constants/plugin';

client/src/pages/kb/detail/components/Import/FileSelect.tsx

@@ -2,7 +2,13 @@ import MyIcon from '@/components/Icon';
 import { useLoading } from '@/hooks/useLoading';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useToast } from '@/hooks/useToast';
-import { fileDownload, readCsvContent, simpleText, splitText2Chunks } from '@/utils/file';
+import {
+  fileDownload,
+  readCsvContent,
+  simpleText,
+  splitText2Chunks,
+  uploadFiles
+} from '@/utils/file';
 import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
 import { fileImgs } from '@/constants/common';
 import { DragEvent, useCallback, useState } from 'react';
@@ -11,7 +17,8 @@ import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
-import { FetchResultItem } from '@/types/plugin';
+import { FetchResultItem, DatasetItemType } from '@/types/plugin';
+import { getErrText } from '@/utils/tools';
 
 const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -22,7 +29,7 @@ const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云
 export type FileItemType = {
   id: string;
   filename: string;
-  chunks: { q: string; a: string; source?: string }[];
+  chunks: DatasetItemType[];
   text: string;
   icon: string;
   tokens: number;
@@ -58,7 +65,7 @@ const FileSelect = ({
   });
 
   const [isDragging, setIsDragging] = useState(false);
-  const [selecting, setSelecting] = useState(false);
+  const [selectingText, setSelectingText] = useState<string>();
 
   const {
     isOpen: isOpenUrlFetch,
@@ -73,7 +80,6 @@ const FileSelect = ({
 
   const onSelectFile = useCallback(
     async (files: File[]) => {
-      setSelecting(true);
       try {
         // Parse file by file
         const chunkFiles: FileItemType[] = [];
@@ -88,19 +94,31 @@ const FileSelect = ({
             continue;
           }
 
-          let text = await (async () => {
+          // parse and upload files
-            switch (extension) {
+          let [text, filesId] = await Promise.all([
-              case 'txt':
+            (async () => {
-              case 'md':
+              switch (extension) {
-                return readTxtContent(file);
+                case 'txt':
-              case 'pdf':
+                case 'md':
-                return readPdfContent(file);
+                  return readTxtContent(file);
-              case 'doc':
+                case 'pdf':
-              case 'docx':
+                  return readPdfContent(file);
-                return readDocContent(file);
+                case 'doc':
-            }
+                case 'docx':
-            return '';
+                  return readDocContent(file);
-          })();
+              }
+              return '';
+            })(),
+            uploadFiles(files, (percent) => {
+              if (percent < 100) {
+                setSelectingText(
+                  t('file.Uploading', { name: file.name.slice(0, 20), percent }) || ''
+                );
+              } else {
+                setSelectingText(t('file.Parse', { name: file.name.slice(0, 20) }) || '');
+              }
+            })
+          ]);
 
           if (text) {
             text = simpleText(text);
@@ -117,7 +135,8 @@ const FileSelect = ({
               chunks: splitRes.chunks.map((chunk) => ({
                 q: chunk,
                 a: '',
-                source: file.name
+                source: file.name,
+                file_id: filesId[0]
               }))
             };
             chunkFiles.unshift(fileItem);
@@ -139,7 +158,8 @@ const FileSelect = ({
               chunks: data.map((item) => ({
                 q: item[0],
                 a: item[1],
-                source: item[2] || file.name
+                source: item[2] || file.name,
+                file_id: filesId[0]
               }))
             };
 
@@ -150,13 +170,13 @@ const FileSelect = ({
       } catch (error: any) {
         console.log(error);
         toast({
-          title: typeof error === 'string' ? error : '解析文件失败',
+          title: getErrText(error, '解析文件失败'),
           status: 'error'
         });
       }
-      setSelecting(false);
+      setSelectingText(undefined);
     },
-    [chunkLen, onPushFiles, toast]
+    [chunkLen, onPushFiles, t, toast]
   );
   const onUrlFetch = useCallback(
     (e: FetchResultItem[]) => {
@@ -353,7 +373,9 @@ const FileSelect = ({
           {t('file.Click to download CSV template')}
         </Box>
       )}
-      <FileSelectLoading loading={selecting} fixed={false} />
+      {selectingText !== undefined && (
+        <FileSelectLoading loading text={selectingText} fixed={false} />
+      )}
       <File onSelect={onSelectFile} />
       {isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
       {isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}

client/src/pages/kb/detail/components/Import/QA.tsx

@@ -97,10 +97,9 @@ const QAImport = ({ kbId }: { kbId: string }) => {
           return {
             ...file,
             tokens: splitRes.tokens,
-            chunks: splitRes.chunks.map((chunk) => ({
+            chunks: file.chunks.map((chunk, i) => ({
-              q: chunk,
+              ...chunk,
-              a: '',
+              q: splitRes.chunks[i]
-              source: file.filename
             }))
           };
         })

client/src/pages/kb/detail/components/InputDataModal.tsx

@@ -1,7 +1,8 @@
 import React, { useState, useCallback } from 'react';
-import { Box, Flex, Button, Textarea, IconButton } from '@chakra-ui/react';
+import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
 import { insertData2Kb, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb';
+import { getFileViewUrl } from '@/api/system';
 import { useToast } from '@/hooks/useToast';
 import { getErrText } from '@/utils/tools';
 import MyIcon from '@/components/Icon';
@@ -10,8 +11,10 @@ import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useUserStore } from '@/store/user';
 import { useQuery } from '@tanstack/react-query';
+import { DatasetItemType } from '@/types/plugin';
+import { useTranslation } from 'react-i18next';
 
-export type FormData = { dataId?: string; a: string; q: string; source?: string };
+export type FormData = { dataId?: string } & DatasetItemType;
 
 const InputDataModal = ({
   onClose,
@@ -29,12 +32,13 @@ const InputDataModal = ({
   kbId: string;
   defaultValues?: FormData;
 }) => {
+  const { t } = useTranslation();
   const [loading, setLoading] = useState(false);
   const { toast } = useToast();
 
   const { kbDetail, getKbDetail } = useUserStore();
 
-  const { register, handleSubmit, reset } = useForm<FormData>({
+  const { getValues, register, handleSubmit, reset } = useForm<FormData>({
     defaultValues
   });
 
@@ -183,7 +187,16 @@ const InputDataModal = ({
           </Box>
         </Box>
 
-        <Flex px={6} pt={2} pb={4} alignItems={'center'}>
+        <Flex px={6} pt={['34px', 2]} pb={4} alignItems={'center'} position={'relative'}>
+          <RawFileText
+            fileId={getValues('file_id')}
+            filename={getValues('source')}
+            position={'absolute'}
+            left={'50%'}
+            top={['16px', '50%']}
+            transform={'translate(-50%,-50%)'}
+          />
+
           <Box flex={1}>
             {defaultValues.dataId && onDelete && (
               <IconButton
@@ -217,15 +230,17 @@ const InputDataModal = ({
               />
             )}
           </Box>
-          <Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
+          <Box>
-            取消
+            <Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
-          </Button>
+              取消
-          <Button
+            </Button>
-            isLoading={loading}
+            <Button
-            onClick={handleSubmit(defaultValues.dataId ? updateData : sureImportData)}
+              isLoading={loading}
-          >
+              onClick={handleSubmit(defaultValues.dataId ? updateData : sureImportData)}
-            {defaultValues.dataId ? '确认变更' : '确认导入'}
+            >
-          </Button>
+              {defaultValues.dataId ? '确认变更' : '确认导入'}
+            </Button>
+          </Box>
         </Flex>
       </Flex>
     </MyModal>
@@ -233,3 +248,44 @@ const InputDataModal = ({
 };
 
 export default InputDataModal;
+
+interface RawFileTextProps extends BoxProps {
+  filename?: string;
+  fileId?: string;
+}
+export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProps) {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  return (
+    <MyTooltip label={fileId ? t('file.Click to view file') || '' : ''} shouldWrapChildren={false}>
+      <Box
+        color={'myGray.600'}
+        display={'inline-block'}
+        {...(!!fileId
+          ? {
+              cursor: 'pointer',
+              textDecoration: ['underline', 'none'],
+              _hover: {
+                textDecoration: 'underline'
+              },
+              onClick: async () => {
+                try {
+                  const url = await getFileViewUrl(fileId);
+                  const asPath = `${location.origin}${url}`;
+                  window.open(asPath, '_blank');
+                } catch (error) {
+                  toast({
+                    title: getErrText(error, '获取文件地址失败'),
+                    status: 'error'
+                  });
+                }
+              }
+            }
+          : {})}
+        {...props}
+      >
+        {filename}
+      </Box>
+    </MyTooltip>
+  );
+}

client/src/pages/kb/detail/components/Test.tsx

@@ -207,8 +207,7 @@ const Test = ({ kbId }: { kbId: string }) => {
 
                       setEditData({
                         dataId: data.id,
-                        q: data.q,
+                        ...data
-                        a: data.a
                       });
                     } catch (err) {
                       toast({

client/src/service/events/generateQA.ts

@@ -38,7 +38,7 @@ export async function generateQA(): Promise<any> {
       prompt: 1,
       q: 1,
       source: 1,
-      model: 1
+      file_id: 1
     });
 
     // task preemption
@@ -136,7 +136,8 @@ A2:
       kbId,
       data: responseList.map((item) => ({
         ...item,
-        source: data.source
+        source: data.source,
+        file_id: data.file_id
       })),
       userId,
       mode: TrainingModeEnum.index

client/src/service/events/generateVector.ts

@@ -38,6 +38,7 @@ export async function generateVector(): Promise<any> {
       q: 1,
       a: 1,
       source: 1,
+      file_id: 1,
       vectorModel: 1
     });
 
@@ -74,6 +75,7 @@ export async function generateVector(): Promise<any> {
         q: dataItems[i].q,
         a: dataItems[i].a,
         source: data.source,
+        file_id: data.file_id,
         vector
       }))
     });

client/src/service/models/trainingData.ts

@@ -49,6 +49,10 @@ const TrainingDataSchema = new Schema({
   source: {
     type: String,
     default: ''
+  },
+  file_id: {
+    type: String,
+    default: ''
   }
 });
 

client/src/service/moduleDispatch/kb/search.ts

@@ -42,7 +42,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
   const res: any = await PgClient.query(
     `BEGIN;
     SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
-    select kb_id,id,q,a,source from ${PgTrainingTableName} where kb_id IN (${kbList
+    select kb_id,id,q,a,source,file_id from ${PgTrainingTableName} where kb_id IN (${kbList
       .map((item) => `'${item.kbId}'`)
       .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
       vectors[0]

client/src/service/pg.ts

@@ -1,8 +1,8 @@
 import { Pool } from 'pg';
 import type { QueryResultRow } from 'pg';
 import { PgTrainingTableName } from '@/constants/plugin';
-import { exit } from 'process';
 import { addLog } from './utils/tools';
+import { DatasetItemType } from '@/types/plugin';
 
 export const connectPg = async (): Promise<Pool> => {
   if (global.pgClient) {
@@ -45,7 +45,7 @@ type DeleteProps = {
   where: WhereProps;
 };
 
-type ValuesProps = { key: string; value: string | number }[];
+type ValuesProps = { key: string; value?: string | number }[];
 type UpdateProps = {
   values: ValuesProps;
   where: WhereProps;
@@ -168,18 +168,16 @@ export const insertKbItem = ({
 }: {
   userId: string;
   kbId: string;
-  data: {
+  data: (DatasetItemType & {
     vector: number[];
-    q: string;
+  })[];
-    a: string;
-    source?: string;
-  }[];
 }) => {
   return PgClient.insert(PgTrainingTableName, {
     values: data.map((item) => [
       { key: 'user_id', value: userId },
       { key: 'kb_id', value: kbId },
       { key: 'source', value: item.source?.slice(0, 30)?.trim() || '' },
+      { key: 'file_id', value: item.file_id },
       { key: 'q', value: item.q.replace(/'/g, '"') },
       { key: 'a', value: item.a.replace(/'/g, '"') },
       { key: 'vector', value: `[${item.vector}]` }
@@ -196,10 +194,11 @@ export async function initPg() {
           id BIGSERIAL PRIMARY KEY,
           vector VECTOR(1536) NOT NULL,
           user_id VARCHAR(50) NOT NULL,
-          kb_id VARCHAR(50) NOT NULL,
+          kb_id VARCHAR(50),
           source VARCHAR(100),
+          file_id VARCHAR(100),
           q TEXT NOT NULL,
-          a TEXT NOT NULL
+          a TEXT
       );
       CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id);
       CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id);

client/src/types/chat.d.ts

@@ -3,6 +3,7 @@ import type { InitChatResponse, InitShareChatResponse } from '@/api/response/cha
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { ClassifyQuestionAgentItemType } from './app';
 import { ChatItemSchema } from './mongoSchema';
+import { KbDataItemType } from './plugin';
 
 export type ExportChatType = 'md' | 'pdf' | 'html';
 
@@ -41,12 +42,8 @@ export type ShareChatType = InitShareChatResponse & {
   history: ShareChatHistoryItemType;
 };
 
-export type QuoteItemType = {
+export type QuoteItemType = KbDataItemType & {
   kb_id: string;
-  id: string;
-  q: string;
-  a: string;
-  source?: string;
 };
 
 export type ChatHistoryItemResType = {

client/src/types/mongoSchema.d.ts

@@ -78,6 +78,7 @@ export interface TrainingDataSchema {
   q: string;
   a: string;
   source: string;
+  file_id: string;
 }
 
 export interface ChatSchema {

client/src/types/plugin.d.ts

@@ -20,12 +20,15 @@ export interface KbItemType {
   tags: string;
 }
 
-export interface KbDataItemType {
+export type DatasetItemType = {
-  id: string;
   q: string; // 提问词
   a: string; // 原文
-  source: string;
+  source?: string;
-}
+  file_id?: string;
+};
+export type KbDataItemType = DatasetItemType & {
+  id: string;
+};
 
 export type KbTestItemType = {
   id: string;

client/src/utils/file.ts

@@ -2,7 +2,23 @@ import mammoth from 'mammoth';
 import Papa from 'papaparse';
 import { getOpenAiEncMap } from './plugin/openai';
 import { getErrText } from './tools';
-import { uploadImg } from '@/api/system';
+import { uploadImg, postUploadFiles } from '@/api/system';
+
+/**
+ * upload file to mongo gridfs
+ */
+export const uploadFiles = (files: File[], percentListen?: (percent: number) => void) => {
+  const form = new FormData();
+  files.forEach((file) => {
+    form.append('file', file, encodeURIComponent(file.name));
+  });
+  return postUploadFiles(form, (e) => {
+    if (!e.total) return;
+
+    const percent = Math.round((e.loaded / e.total) * 100);
+    percentListen && percentListen(percent);
+  });
+};
 
 /**
  * 读取 txt 文件内容
@@ -37,7 +53,11 @@ export const readPdfContent = (file: File) =>
       const readPDFPage = async (doc: any, pageNo: number) => {
         const page = await doc.getPage(pageNo);
         const tokenizedText = await page.getTextContent();
-        const pageText = tokenizedText.items.map((token: any) => token.str).join(' ');
+
+        const pageText = tokenizedText.items
+          .map((token: any) => token.str)
+          .filter((item: string) => item)
+          .join('');
         return pageText;
       };
 
@@ -54,12 +74,12 @@ export const readPdfContent = (file: File) =>
           const pageTexts = await Promise.all(pageTextPromises);
           resolve(pageTexts.join('\n'));
         } catch (err) {
-          console.log(err, 'pdfjs error');
+          console.log(err, 'pdf load error');
           reject('解析 PDF 失败');
         }
       };
       reader.onerror = (err) => {
-        console.log(err, 'reader error');
+        console.log(err, 'pdf load error');
         reject('解析 PDF 失败');
       };
     } catch (error) {
@@ -83,10 +103,18 @@ export const readDocContent = (file: File) =>
           });
           resolve(res?.value);
         } catch (error) {
+          window.umami?.track('wordReadError', {
+            err: error?.toString()
+          });
+          console.log('error doc read:', error);
+
           reject('读取 doc 文件失败, 请转换成 PDF');
         }
       };
       reader.onerror = (err) => {
+        window.umami?.track('wordReadError', {
+          err: err?.toString()
+        });
         console.log('error doc read:', err);
 
         reject('读取 doc 文件失败');