V4.8.18 feature (#3565)

* feat: org CRUD (#3380) * feat: add org schema * feat: org manage UI * feat: OrgInfoModal * feat: org tree view * feat: org management * fix: init root org * feat: org permission for app * feat: org support for dataset * fix: disable org role control * styles: opt type signatures * fix: remove unused permission * feat: delete org collaborator * perf: Team org ui (#3499) * perf: org ui * perf: org ui * feat: org auth for app & dataset (#3498) * feat: auth org resource permission * feat: org auth support for app & dataset * perf: org permission check (#3500) * i18n (#3501) * name * i18n * feat: support dataset changeOwner (#3483) * feat: support dataset changeOwner * chore: update dataset change owner api * feat: permission manage UI for org (#3503) * perf: password check;perf: image upload check;perf: sso login check (#3509) * perf: password check * perf: image upload check * perf: sso login check * force show update notification modal & fix login page text (#3512) * fix login page English text * update notification modal * perf: notify account (#3515) * perf(plugin): improve searXNG empty result handling and documentation (#3507) * perf(plugin): improve searXNG empty result handling and documentation * 修改了文档和代码部分无搜索的结果的反馈 * refactor: org pathId (#3516) * optimize payment process (#3517) * feat: support wecom sso (#3518) * feat: support wecom sso * chore: remove unused wecom js-sdk dependency * fix qrcode script (#3520) * fix qrcode script * i18n * perf: full text collection and search code;perf: rename function (#3519) * perf: full text collection and search code * perf: rename function * perf: notify modal * remove invalid code * perf: sso login * perf: pay process * 4.8.18 test (#3524) * perf: remove local token * perf: index * perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528) * perf: text encoding * perf: leave team code * perf: full text search code * fix: http status * perf: embedding search and vector avatar * perf: async read file (#3531) * refactor: team permission manager (#3535) * perf: classify org, group and member * refactor: team per manager * fix: missing functions * 4.8.18 test (#3543) * perf: login check * doc * perf: llm model config * perf: team clb config * fix: MemberModal UI (#3553) * fix: adapt MemberModal title and icon * fix: adapt member modal * fix: search input placeholder * fix: add button text * perf: org permission (#3556) * docs:用户答疑的官方文档补充 (#3540) * docs:用户答疑的官方文档补充 * 问题回答的内容修补 * share link random avatar (#3541) * share link random avatar * fix * delete unused code * share page avatar (#3558) * feat: init 4818 * share page avatar * feat: tmp upgrade code (#3559) * feat: tmp upgrade code * fulltext search test * update action * full text tmp code (#3561) * full text tmp code * fix: init * fix: init * remove tmp code * remove tmp code * 4818-alpha * 4.8.18 test (#3562) * full text tmp code * fix: init * upgrade code * account log * account log * perf: dockerfile * upgrade code * chore: update docs app template submission (#3564) --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
2025-07-23 05:12:39 +00:00 · 2025-01-11 15:15:38 +08:00
parent bb669ca3ff
commit 10d8c56e23
205 changed files with 5305 additions and 2428 deletions
--- a/packages/service/common/file/gridfs/controller.ts
+++ b/packages/service/common/file/gridfs/controller.ts
@@ -4,7 +4,7 @@ import fsp from 'fs/promises';
 import fs from 'fs';
 import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
 import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
-import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
+import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
 import { readRawContentByFileBuffer } from '../read/utils';
@@ -36,7 +36,6 @@ export async function uploadFile({
  path,
  filename,
  contentType,
-  encoding,
  metadata = {}
 }: {
  bucketName: `${BucketNameEnum}`;
@@ -45,7 +44,6 @@ export async function uploadFile({
  path: string;
  filename: string;
  contentType?: string;
-  encoding: string;
  metadata?: Record<string, any>;
 }) {
  if (!path) return Promise.reject(`filePath is empty`);
@@ -59,7 +57,7 @@ export async function uploadFile({
  // Add default metadata
  metadata.teamId = teamId;
  metadata.uid = uid;
-  metadata.encoding = encoding;
+  metadata.encoding = await detectFileEncodingByPath(path);

  // create a gridfs bucket
  const bucket = getGridBucket(bucketName);
--- a/packages/service/common/file/image/controller.ts
+++ b/packages/service/common/file/image/controller.ts
@@ -1,43 +1,92 @@
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
 import { MongoImage } from './schema';
-import { ClientSession } from '../../../common/mongo';
+import { ClientSession, Types } from '../../../common/mongo';
 import { guessBase64ImageType } from '../utils';
 import { readFromSecondary } from '../../mongo/utils';
+import { addHours } from 'date-fns';

 export const maxImgSize = 1024 * 1024 * 12;
 const base64MimeRegex = /data:image\/([^\)]+);base64/;
 export async function uploadMongoImg({
-  type,
  base64Img,
  teamId,
-  expiredTime,
  metadata,
-  shareId
+  shareId,
+  forever = false
 }: UploadImgProps & {
  teamId: string;
+  forever?: Boolean;
 }) {
  if (base64Img.length > maxImgSize) {
    return Promise.reject('Image too large');
  }

  const [base64Mime, base64Data] = base64Img.split(',');
+  // Check if mime type is valid
+  if (!base64MimeRegex.test(base64Mime)) {
+    return Promise.reject('Invalid image mime type');
+  }
+
  const mime = `image/${base64Mime.match(base64MimeRegex)?.[1] ?? 'image/jpeg'}`;
  const binary = Buffer.from(base64Data, 'base64');
  const extension = mime.split('/')[1];

  const { _id } = await MongoImage.create({
-    type,
    teamId,
    binary,
-    expiredTime,
    metadata: Object.assign({ mime }, metadata),
-    shareId
+    shareId,
+    expiredTime: forever ? undefined : addHours(new Date(), 1)
  });

  return `${process.env.FE_DOMAIN || ''}${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(_id)}.${extension}`;
 }

+const getIdFromPath = (path?: string) => {
+  if (!path) return;
+
+  const paths = path.split('/');
+  const name = paths[paths.length - 1];
+
+  if (!name) return;
+
+  const id = name.split('.')[0];
+  if (!id || !Types.ObjectId.isValid(id)) return;
+
+  return id;
+};
+// 删除旧的头像，新的头像去除过期时间
+export const refreshSourceAvatar = async (
+  path?: string,
+  oldPath?: string,
+  session?: ClientSession
+) => {
+  const newId = getIdFromPath(path);
+  const oldId = getIdFromPath(oldPath);
+
+  if (!newId) return;
+
+  await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session });
+
+  if (oldId) {
+    await MongoImage.deleteOne({ _id: oldId }, { session });
+  }
+};
+export const removeImageByPath = (path?: string, session?: ClientSession) => {
+  if (!path) return;
+
+  const paths = path.split('/');
+  const name = paths[paths.length - 1];
+
+  if (!name) return;
+
+  const id = name.split('.')[0];
+  if (!id || !Types.ObjectId.isValid(id)) return;
+
+  return MongoImage.deleteOne({ _id: id }, { session });
+};
+
 export async function readMongoImg({ id }: { id: string }) {
  const formatId = id.replace(/\.[^/.]+$/, '');

--- a/packages/service/common/file/image/schema.ts
+++ b/packages/service/common/file/image/schema.ts
@@ -1,8 +1,7 @@
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
-import { connectionMongo, getMongoModel, type Model } from '../../mongo';
+import { connectionMongo, getMongoModel } from '../../mongo';
 import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
-import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
-const { Schema, model, models } = connectionMongo;
+const { Schema } = connectionMongo;

 const ImageSchema = new Schema({
  teamId: {
@@ -14,27 +13,15 @@ const ImageSchema = new Schema({
    type: Date,
    default: () => new Date()
  },
-  expiredTime: {
-    type: Date
-  },
-  binary: {
-    type: Buffer
-  },
-  type: {
-    type: String,
-    enum: Object.keys(mongoImageTypeMap),
-    required: true
-  },
-  metadata: {
-    type: Object
-  }
+  expiredTime: Date,
+  binary: Buffer,
+  metadata: Object
 });

 try {
  // tts expired（60 Minutes）
  ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 * 60 });
  ImageSchema.index({ type: 1 });
-  ImageSchema.index({ createTime: 1 });
  // delete related img
  ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
--- a/packages/service/common/file/read/utils.ts
+++ b/packages/service/common/file/read/utils.ts
@@ -1,5 +1,4 @@
 import { uploadMongoImg } from '../image/controller';
-import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
 import FormData from 'form-data';

 import { WorkerNameEnum, runWorker } from '../../../worker/utils';
@@ -8,7 +7,6 @@ import type { ReadFileResponse } from '../../../worker/readFile/type';
 import axios from 'axios';
 import { addLog } from '../../system/log';
 import { batchRun } from '@fastgpt/global/common/fn/utils';
-import { addHours } from 'date-fns';
 import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';

 export type readRawTextByLocalFileParams = {
@@ -22,7 +20,7 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam

  const extension = path?.split('.')?.pop()?.toLowerCase() || '';

-  const buffer = fs.readFileSync(path);
+  const buffer = await fs.promises.readFile(path);

  const { rawText } = await readRawContentByFileBuffer({
    extension,
@@ -114,10 +112,9 @@ export const readRawContentByFileBuffer = async ({
  if (imageList) {
    await batchRun(imageList, async (item) => {
      const src = await uploadMongoImg({
-        type: MongoImageTypeEnum.collectionImage,
        base64Img: `data:${item.mime};base64,${item.base64}`,
        teamId,
-        expiredTime: addHours(new Date(), 1),
+        // expiredTime: addHours(new Date(), 1),
        metadata: {
          ...metadata,
          mime: item.mime
--- a/packages/service/common/middle/reqFrequencyLimit.ts
+++ b/packages/service/common/middle/reqFrequencyLimit.ts
@@ -9,10 +9,10 @@ import { jsonRes } from '../response';
 // unit: times/s
 // how to use?
 // export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip
-export function useReqFrequencyLimit(seconds: number, limit: number) {
+export function useReqFrequencyLimit(seconds: number, limit: number, force = false) {
  return async (req: ApiRequestProps, res: NextApiResponse) => {
    const ip = requestIp.getClientIp(req);
-    if (!ip || process.env.USE_IP_LIMIT !== 'true') {
+    if (!ip || (process.env.USE_IP_LIMIT !== 'true' && !force)) {
      return;
    }
    try {
@@ -22,10 +22,9 @@ export function useReqFrequencyLimit(seconds: number, limit: number) {
        expiredTime: addSeconds(new Date(), seconds)
      });
    } catch (_) {
-      res.status(429);
      jsonRes(res, {
        code: 429,
-        message: ERROR_ENUM.tooManyRequest
+        error: ERROR_ENUM.tooManyRequest
      });
    }
  };
--- a/packages/service/common/response/index.ts
+++ b/packages/service/common/response/index.ts
@@ -33,7 +33,7 @@ export const jsonRes = <T = any>(

    addLog.error(`Api response error: ${url}`, ERROR_RESPONSE[errResponseKey]);

-    return res.json(ERROR_RESPONSE[errResponseKey]);
+    return res.status(code).json(ERROR_RESPONSE[errResponseKey]);
  }

  // another error