4.6.7 first pr (#726)

2025-08-01 20:27:45 +00:00 · 2024-01-10 23:35:04 +08:00
parent 414b693303
commit 006ad17c6a
186 changed files with 2996 additions and 1838 deletions
--- a/projects/app/src/web/common/file/controller.ts
+++ b/projects/app/src/web/common/file/controller.ts
@@ -1,10 +1,8 @@
 import { postUploadImg, postUploadFiles } from '@/web/common/file/api';
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
-import {
-  compressBase64ImgAndUpload as compressBase64ImgAndUploadControl,
-  type CompressImgProps
-} from '@fastgpt/web/common/file/img';
+import { preUploadImgProps } from '@fastgpt/global/common/file/api';
+import { compressBase64Img, type CompressImgProps } from '@fastgpt/web/common/file/img';

 /**
 * upload file to mongo gridfs
@@ -34,57 +32,45 @@ export const uploadFiles = ({
  });
 };

-export const getUploadMdImgController = ({
-  base64Img,
-  metadata
-}: {
-  base64Img: string;
-  metadata: Record<string, any>;
-}) =>
-  compressBase64ImgAndUpload({
-    base64Img,
+export const getUploadBase64ImgController = (props: CompressImgProps & UploadImgProps) =>
+  compressBase64Img({
    maxW: 4000,
    maxH: 4000,
    maxSize: 1024 * 1024 * 5,
-    metadata
+    ...props
  });

 /**
 * compress image. response base64
 * @param maxSize The max size of the compressed image
 */
-export const compressBase64ImgAndUpload = ({
-  expiredTime,
-  metadata,
-  shareId,
-  ...props
-}: UploadImgProps & CompressImgProps) => {
-  return compressBase64ImgAndUploadControl({
-    ...props,
-    uploadController: (base64Img) =>
-      postUploadImg({
-        shareId,
-        base64Img,
-        expiredTime,
-        metadata
-      })
-  });
-};
-export const compressImgFileAndUpload = async ({
-  file,
+export const compressBase64ImgAndUpload = async ({
+  base64Img,
  maxW,
  maxH,
  maxSize,
-  expiredTime,
-  shareId
-}: {
-  file: File;
-  maxW?: number;
-  maxH?: number;
-  maxSize?: number;
-  expiredTime?: Date;
-  shareId?: string;
-}) => {
+  ...props
+}: UploadImgProps & CompressImgProps) => {
+  const compressUrl = await compressBase64Img({
+    base64Img,
+    maxW,
+    maxH,
+    maxSize
+  });
+
+  return postUploadImg({
+    ...props,
+    base64Img: compressUrl
+  });
+};
+
+export const compressImgFileAndUpload = async ({
+  file,
+  ...props
+}: preUploadImgProps &
+  CompressImgProps & {
+    file: File;
+  }) => {
  const reader = new FileReader();
  reader.readAsDataURL(file);

@@ -94,16 +80,12 @@ export const compressImgFileAndUpload = async ({
    };
    reader.onerror = (err) => {
      console.log(err);
-      reject('压缩图片异常');
+      reject('Load image error');
    };
  });

  return compressBase64ImgAndUpload({
    base64Img,
-    maxW,
-    maxH,
-    maxSize,
-    expiredTime,
-    shareId
+    ...props
  });
 };
--- a/projects/app/src/web/common/file/utils.ts
+++ b/projects/app/src/web/common/file/utils.ts
@@ -1,80 +1,5 @@
-import mammoth from 'mammoth';
 import Papa from 'papaparse';
-import { compressBase64ImgAndUpload } from './controller';
-import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
-import { htmlStr2Md } from '@fastgpt/web/common/string/markdown';
-import { readPdfFile } from '@fastgpt/global/common/file/read/index';
-import { readFileRawText } from '@fastgpt/web/common/file/read';
-
-/**
- * read pdf to raw text
- */
-export const readPdfContent = (file: File) =>
-  new Promise<string>((resolve, reject) => {
-    try {
-      let reader = new FileReader();
-      reader.readAsArrayBuffer(file);
-      reader.onload = async (event) => {
-        if (!event?.target?.result) return reject('解析 PDF 失败');
-        try {
-          const content = await readPdfFile({ pdf: event.target.result });
-
-          resolve(content);
-        } catch (err) {
-          console.log(err, 'pdf load error');
-          reject('解析 PDF 失败');
-        }
-      };
-      reader.onerror = (err) => {
-        console.log(err, 'pdf load error');
-        reject('解析 PDF 失败');
-      };
-    } catch (error) {
-      reject('浏览器不支持文件内容读取');
-    }
-  });
-
-/**
- * read docx to markdown
- */
-export const readDocContent = (file: File, metadata: Record<string, any>) =>
-  new Promise<string>((resolve, reject) => {
-    try {
-      const reader = new FileReader();
-      reader.readAsArrayBuffer(file);
-      reader.onload = async ({ target }) => {
-        if (!target?.result) return reject('读取 doc 文件失败');
-        try {
-          const buffer = target.result as ArrayBuffer;
-          const { value: html } = await mammoth.convertToHtml({
-            arrayBuffer: buffer
-          });
-          const md = htmlStr2Md(html);
-
-          const rawText = await uploadMarkdownBase64(md, metadata);
-
-          resolve(rawText);
-        } catch (error) {
-          window.umami?.track('wordReadError', {
-            err: error?.toString()
-          });
-          console.log('error doc read:', error);
-
-          reject('读取 doc 文件失败, 请转换成 PDF');
-        }
-      };
-      reader.onerror = (err) => {
-        window.umami?.track('wordReadError', {
-          err: err?.toString()
-        });
-        console.log('error doc read:', err);
-
-        reject('读取 doc 文件失败');
-      };
-    } catch (error) {
-      reject('浏览器不支持文件内容读取');
-    }
-  });
+import { readFileRawText } from '@fastgpt/web/common/file/read/rawText';

 /**
 * read csv to json
@@ -85,7 +10,7 @@ export const readDocContent = (file: File, metadata: Record<string, any>) =>
 */
 export const readCsvContent = async (file: File) => {
  try {
-    const textArr = await readFileRawText(file);
+    const { rawText: textArr } = await readFileRawText(file);
    const csvArr = Papa.parse(textArr).data as string[][];
    if (csvArr.length === 0) {
      throw new Error('csv 解析失败');
@@ -99,44 +24,6 @@ export const readCsvContent = async (file: File) => {
  }
 };

-/**
- * format markdown
- * 1. upload base64
- * 2. replace \
- */
-export const uploadMarkdownBase64 = async (rawText: string = '', metadata: Record<string, any>) => {
-  // match base64, upload and replace it
-  const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
-  const base64Arr = rawText.match(base64Regex) || [];
-  // upload base64 and replace it
-  await Promise.all(
-    base64Arr.map(async (base64Img) => {
-      try {
-        const str = await compressBase64ImgAndUpload({
-          base64Img,
-          maxW: 4329,
-          maxH: 4329,
-          maxSize: 1024 * 1024 * 5,
-          metadata
-        });
-
-        rawText = rawText.replace(base64Img, str);
-      } catch (error) {
-        rawText = rawText.replace(base64Img, '');
-        rawText = rawText.replace(/!\[.*\]\(\)/g, '');
-      }
-    })
-  );
-
-  // Remove white space on both sides of the picture
-  const trimReg = /(!\[.*\]\(.*\))\s*/g;
-  if (trimReg.test(rawText)) {
-    rawText = rawText.replace(trimReg, '$1');
-  }
-
-  return simpleMarkdownText(rawText);
-};
-
 /**
 * file download by text
 */
--- a/projects/app/src/web/common/hooks/useConfirm.tsx
+++ b/projects/app/src/web/common/hooks/useConfirm.tsx
@@ -105,7 +105,7 @@ export const useConfirm = (props?: {
              )}

              <Button
-                {...(bg && { bg: `${bg} !important` })}
+                bg={bg ? bg : map.bg}
                isDisabled={countDownAmount > 0}
                ml={4}
                isLoading={isLoading}
@@ -120,7 +120,7 @@ export const useConfirm = (props?: {
          </MyModal>
        );
      },
-      [customContent, iconSrc, isOpen, onClose, showCancel, t, title]
+      [customContent, iconSrc, isOpen, map.bg, onClose, showCancel, t, title]
    )
  };
 };
--- a/projects/app/src/web/core/dataset/api.ts
+++ b/projects/app/src/web/core/dataset/api.ts
@@ -74,7 +74,7 @@ export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
  POST<string>(`/core/dataset/collection/create`, data);
 export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
  POST(`/core/dataset/collection/update`, data);
-export const delDatasetCollectionById = (params: { collectionId: string }) =>
+export const delDatasetCollectionById = (params: { id: string }) =>
  DELETE(`/core/dataset/collection/delete`, params);
 export const postLinkCollectionSync = (collectionId: string) =>
  POST<`${DatasetCollectionSyncResultEnum}`>(`/core/dataset/collection/sync/link`, {
@@ -86,8 +86,8 @@ export const postLinkCollectionSync = (collectionId: string) =>
 export const getDatasetDataList = (data: GetDatasetDataListProps) =>
  POST(`/core/dataset/data/list`, data);

-export const getDatasetDataItemById = (dataId: string) =>
-  GET<DatasetDataItemType>(`/core/dataset/data/detail`, { dataId });
+export const getDatasetDataItemById = (id: string) =>
+  GET<DatasetDataItemType>(`/core/dataset/data/detail`, { id });

 /**
 * push data to training queue
@@ -109,8 +109,8 @@ export const putDatasetDataById = (data: UpdateDatasetDataProps) =>
 /**
 * 删除一条知识库数据
 */
-export const delOneDatasetDataById = (dataId: string) =>
-  DELETE<string>(`/core/dataset/data/delete`, { dataId });
+export const delOneDatasetDataById = (id: string) =>
+  DELETE<string>(`/core/dataset/data/delete`, { id });

 /* ================ training ==================== */
 /* get length of system training queue */
--- a/projects/app/src/web/core/dataset/store/dataset.ts
+++ b/projects/app/src/web/core/dataset/store/dataset.ts
@@ -13,7 +13,7 @@ import { defaultDatasetDetail } from '@/constants/dataset';
 import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
 import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constant';
 import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
-import { checkTeamWebSyncLimit } from '@/web/support/user/api';
+import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';

 type State = {
  allDatasets: DatasetListItemType[];
--- a/projects/app/src/web/core/dataset/utils.ts
+++ b/projects/app/src/web/core/dataset/utils.ts
@@ -5,17 +5,17 @@ import { strIsLink } from '@fastgpt/global/common/string/tools';
 import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';

 export async function chunksUpload({
-  collectionId,
  billId,
-  mode,
+  collectionId,
+  trainingMode,
  chunks,
  prompt,
  rate = 150,
  onUploading
 }: {
-  collectionId: string;
  billId: string;
-  mode: `${TrainingModeEnum}`;
+  collectionId: string;
+  trainingMode: `${TrainingModeEnum}`;
  chunks: PushDatasetDataChunkProps[];
  prompt?: string;
  rate?: number;
@@ -24,8 +24,8 @@ export async function chunksUpload({
  async function upload(data: PushDatasetDataChunkProps[]) {
    return postChunks2Dataset({
      collectionId,
+      trainingMode,
      data,
-      mode,
      prompt,
      billId
    });
--- a/projects/app/src/web/styles/theme.ts
+++ b/projects/app/src/web/styles/theme.ts
@@ -195,6 +195,7 @@ const Input: ComponentStyleConfig = {
  baseStyle: {
    fontsize: '14px'
  },
+  sizes: {},
  variants: {
    outline: {
      field: {
--- a/projects/app/src/web/support/user/api.ts
+++ b/projects/app/src/web/support/user/api.ts
@@ -71,8 +71,3 @@ export const postLogin = ({ password, ...props }: PostLoginProps) =>
 export const loginOut = () => GET('/support/user/account/loginout');

 export const putUserInfo = (data: UserUpdateParams) => PUT('/support/user/account/update', data);
-
-/* team limit */
-export const checkTeamExportDatasetLimit = (datasetId: string) =>
-  GET(`/support/user/team/limit/exportDatasetLimit`, { datasetId });
-export const checkTeamWebSyncLimit = () => GET(`/support/user/team/limit/webSyncLimit`);
--- a/projects/app/src/web/support/user/team/api.ts
+++ b/projects/app/src/web/support/user/team/api.ts
@@ -39,3 +39,8 @@ export const updateInviteResult = (data: UpdateInviteProps) =>
  PUT('/plusApi/support/user/team/member/updateInvite', data);
 export const delLeaveTeam = (teamId: string) =>
  DELETE('/plusApi/support/user/team/member/leave', { teamId });
+
+/* team limit */
+export const checkTeamExportDatasetLimit = (datasetId: string) =>
+  GET(`/support/user/team/limit/exportDatasetLimit`, { datasetId });
+export const checkTeamWebSyncLimit = () => GET(`/support/user/team/limit/webSyncLimit`);
--- a/projects/app/src/web/support/wallet/sub/api.ts
+++ b/projects/app/src/web/support/wallet/sub/api.ts
@@ -0,0 +1,13 @@
+import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
+import { SubDatasetSizeParams } from '@fastgpt/global/support/wallet/sub/api';
+import { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type';
+
+export const getTeamDatasetValidSub = () =>
+  GET<{
+    sub: TeamSubSchema;
+    maxSize: number;
+    usedSize: number;
+  }>(`/support/wallet/sub/getDatasetSub`);
+
+export const postExpandTeamDatasetSub = (data: SubDatasetSizeParams) =>
+  POST('/plusApi/support/wallet/sub/datasetSize/expand', data);