perf: unlogin user fetch data (#3775)

* model config

* feat: normalization embedding

* perf: unlogin user fetch data

docSite/content/zh-cn/docs/development/upgrading/4821.md

@@ -18,12 +18,14 @@ weight: 804
 5. 新增 - Doubao1.5 模型预设。阿里 embedding3 预设。
 6. 新增 - 向量模型支持归一化配置，以便适配未归一化的向量模型，例如 Doubao 的 embedding 模型。
 6. 新增 - AI 对话节点，支持输出思考过程结果，可用于其他节点引用。
-7. 优化 - 模型未配置时错误提示。
+7. 优化 - 网站嵌入式聊天窗口，增加窗口位置适配。
-8. 优化 - 适配非 Stream 模式思考输出。
+8. 优化 - 模型未配置时错误提示。
-9. 优化 - 增加 TTS voice 未配置时的空指针保护。
+9. 优化 - 适配非 Stream 模式思考输出。
-10. 优化 - Markdown 链接解析分割规则，改成严格匹配模式，牺牲兼容多种情况，减少误解析。
+10. 优化 - 增加 TTS voice 未配置时的空指针保护。
-11. 修复 - 简易模式，切换到其他非视觉模型时候，会强制关闭图片识别。
+11. 优化 - Markdown 链接解析分割规则，改成严格匹配模式，牺牲兼容多种情况，减少误解析。
-12. 修复 - o1,o3 模型，在测试时候字段映射未生效导致报错。
+12. 优化 - 减少未登录用户的数据获取范围，提高系统隐私性。
-13. 修复 - 公众号对话空指针异常。
+13. 修复 - 简易模式，切换到其他非视觉模型时候，会强制关闭图片识别。
-14. 修复 - 多个音频/视频文件展示异常。
+14. 修复 - o1,o3 模型，在测试时候字段映射未生效导致报错。
-15. 修复 - 分享链接鉴权报错后无限循环。
+15. 修复 - 公众号对话空指针异常。
+16. 修复 - 多个音频/视频文件展示异常。
+17. 修复 - 分享链接鉴权报错后无限循环。

packages/global/common/file/constants.ts

@@ -16,7 +16,7 @@ export const bucketNameMap = {
   }
 };
 
-export const ReadFileBaseUrl = `${process.env.FE_DOMAIN || ''}${process.env.NEXT_PUBLIC_BASE_URL || ''}/api/common/file/read`;
+export const ReadFileBaseUrl = `${process.env.FILE_DOMAIN || process.env.FE_DOMAIN || ''}${process.env.NEXT_PUBLIC_BASE_URL || ''}/api/common/file/read`;
 
 export const documentFileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
 export const imageFileType =

packages/service/common/file/image/controller.ts

@@ -5,6 +5,7 @@ import { ClientSession, Types } from '../../../common/mongo';
 import { guessBase64ImageType } from '../utils';
 import { readFromSecondary } from '../../mongo/utils';
 import { addHours } from 'date-fns';
+import { imageFileType } from '@fastgpt/global/common/file/constants';
 
 export const maxImgSize = 1024 * 1024 * 12;
 const base64MimeRegex = /data:image\/([^\)]+);base64/;
@@ -32,6 +33,10 @@ export async function uploadMongoImg({
   const binary = Buffer.from(base64Data, 'base64');
   const extension = mime.split('/')[1];
 
+  if (!imageFileType.includes(`.${extension}`)) {
+    return Promise.reject('Invalid image file type');
+  }
+
   const { _id } = await MongoImage.create({
     teamId,
     binary,

projects/app/.env.template

@@ -32,6 +32,8 @@ SANDBOX_URL=http://localhost:3001
 PRO_URL=
 # 页面的地址，用于自动补全相对路径资源的 domain，注意后面不要跟 /
 FE_DOMAIN=http://localhost:3000
+# 文件域名，也是指向 FastGPT 服务，但是如果希望内容足够安全，可以独立分配一个域名，避免高危文件读取到主域名的内容。
+FILE_DOMAIN=http://localhost:3000
 # 二级路由，需要打包时候就确定
 # NEXT_PUBLIC_BASE_URL=/fastai
 

projects/app/src/components/Layout/index.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo } from 'react';
+import React, { useEffect, useMemo } from 'react';
 import { Box, Flex } from '@chakra-ui/react';
 import { useRouter } from 'next/router';
 import { useLoading } from '@fastgpt/web/hooks/useLoading';
@@ -12,6 +12,8 @@ import { useI18nLng } from '@fastgpt/web/hooks/useI18n';
 import Auth from './auth';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import { useMount } from 'ahooks';
+import { useTranslation } from 'next-i18next';
+import { useToast } from '@fastgpt/web/hooks/useToast';
 
 const Navbar = dynamic(() => import('./navbar'));
 const NavbarPhone = dynamic(() => import('./navbarPhone'));
@@ -50,8 +52,11 @@ export const navbarWidth = '64px';
 
 const Layout = ({ children }: { children: JSX.Element }) => {
   const router = useRouter();
+  const { t } = useTranslation();
+  const { toast } = useToast();
   const { Loading } = useLoading();
-  const { loading, feConfigs, notSufficientModalType } = useSystemStore();
+  const { loading, feConfigs, notSufficientModalType, llmModelList, embeddingModelList } =
+    useSystemStore();
   const { isPc } = useSystem();
   const { userInfo, isUpdateNotification, setIsUpdateNotification } = useUserStore();
   const { setUserDefaultLng } = useI18nLng();
@@ -82,6 +87,20 @@ const Layout = ({ children }: { children: JSX.Element }) => {
     setUserDefaultLng();
   });
 
+  // Check model invalid
+  useEffect(() => {
+    if (
+      userInfo?.username === 'root' &&
+      (llmModelList.length === 0 || embeddingModelList.length === 0)
+    ) {
+      toast({
+        status: 'warning',
+        title: t('login:model_not_config')
+      });
+      router.push('/account/model');
+    }
+  }, [embeddingModelList.length, llmModelList.length, router, t, toast, userInfo?.username]);
+
   return (
     <>
       <Box h={'100%'} bg={'myGray.100'}>

projects/app/src/global/common/api/systemRes.d.ts

@@ -15,7 +15,7 @@ export type InitDateResponse = {
 
   feConfigs?: FastGPTFeConfigsType;
   subPlans?: SubPlanType;
-  systemVersion: string;
+  systemVersion?: string;
 
   activeModelList?: SystemModelItemType[];
   defaultModels?: SystemDefaultModelType;

projects/app/src/pages/api/common/file/read.ts

@@ -6,6 +6,20 @@ import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gri
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { stream2Encoding } from '@fastgpt/service/common/file/gridfs/utils';
 
+const previewableExtensions = [
+  'jpg',
+  'jpeg',
+  'png',
+  'gif',
+  'bmp',
+  'webp',
+  'txt',
+  'log',
+  'csv',
+  'md',
+  'json'
+];
+
 // Abandoned, use: file/read/[filename].ts
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -39,7 +53,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     })();
 
     const extension = file.filename.split('.').pop() || '';
-    const disposition = ['html', 'htm'].includes(extension) ? 'attachment' : 'inline';
+    const disposition = previewableExtensions.includes(extension) ? 'inline' : 'attachment';
 
     res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`);
     res.setHeader('Cache-Control', 'public, max-age=31536000');

projects/app/src/pages/api/common/file/read/[filename].ts

@@ -6,6 +6,19 @@ import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gri
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { stream2Encoding } from '@fastgpt/service/common/file/gridfs/utils';
 
+const previewableExtensions = [
+  'jpg',
+  'jpeg',
+  'png',
+  'gif',
+  'bmp',
+  'webp',
+  'txt',
+  'log',
+  'csv',
+  'md',
+  'json'
+];
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
@@ -38,7 +51,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     })();
 
     const extension = file.filename.split('.').pop() || '';
-    const disposition = ['html', 'htm'].includes(extension) ? 'attachment' : 'inline';
+    const disposition = previewableExtensions.includes(extension) ? 'inline' : 'attachment';
 
     res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`);
     res.setHeader('Cache-Control', 'public, max-age=31536000');

projects/app/src/pages/api/common/system/getInitData.ts

@@ -3,6 +3,7 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { NextAPI } from '@/service/middleware/entry';
 import { InitDateResponse } from '@/global/common/api/systemRes';
 import { SystemModelItemType } from '@fastgpt/service/core/ai/type';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
 
 async function handler(
   req: ApiRequestProps<{}, { bufferId?: string }>,
@@ -24,22 +25,36 @@ async function handler(
     requestAuth: undefined
   })) as SystemModelItemType[];
 
-  // If bufferId is the same as the current bufferId, return directly
+  try {
-  if (bufferId && global.systemInitBufferId && global.systemInitBufferId === bufferId) {
+    await authCert({ req, authToken: true });
+    // If bufferId is the same as the current bufferId, return directly
+    if (bufferId && global.systemInitBufferId && global.systemInitBufferId === bufferId) {
+      return {
+        bufferId: global.systemInitBufferId,
+        systemVersion: global.systemVersion
+      };
+    }
+
     return {
       bufferId: global.systemInitBufferId,
-      systemVersion: global.systemVersion || '0.0.0'
+      feConfigs: global.feConfigs,
+      subPlans: global.subPlans,
+      systemVersion: global.systemVersion,
+      activeModelList,
+      defaultModels: global.systemDefaultModel
+    };
+  } catch (error) {
+    const unAuthBufferId = global.systemInitBufferId ? `unAuth_${global.systemInitBufferId}` : '';
+    if (bufferId && unAuthBufferId === bufferId) {
+      return {
+        bufferId: unAuthBufferId
+      };
+    }
+    return {
+      bufferId: unAuthBufferId,
+      feConfigs: global.feConfigs
     };
   }
-
-  return {
-    bufferId: global.systemInitBufferId,
-    feConfigs: global.feConfigs,
-    subPlans: global.subPlans,
-    systemVersion: global.systemVersion || '0.0.0',
-    activeModelList,
-    defaultModels: global.systemDefaultModel
-  };
 }
 
 export default NextAPI(handler);

projects/app/src/pages/login/index.tsx

@@ -42,7 +42,7 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => {
   const router = useRouter();
   const { t } = useTranslation();
   const { lastRoute = '' } = router.query as { lastRoute: string };
-  const { feConfigs, llmModelList } = useSystemStore();
+  const { feConfigs } = useSystemStore();
   const [pageType, setPageType] = useState<`${LoginPageTypeEnum}`>(LoginPageTypeEnum.passwordLogin);
   const { setUserInfo } = useUserStore();
   const { setLastChatAppId } = useChatStore();
@@ -63,16 +63,6 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => {
     (res: ResLogin) => {
       setUserInfo(res.user);
 
-      // Check that the model is available
-      if (res.user.username === 'root' && llmModelList?.length === 0) {
-        toast({
-          status: 'warning',
-          title: t('login:model_not_config')
-        });
-        router.push('/account/model');
-        return;
-      }
-
       const decodeLastRoute = decodeURIComponent(lastRoute);
       // 检查是否是当前的 route
       const navigateTo =
@@ -81,7 +71,7 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => {
         router.push(navigateTo);
       }, 300);
     },
-    [setUserInfo, llmModelList?.length, lastRoute, toast, t, router]
+    [setUserInfo, lastRoute, router]
   );
 
   const DynamicComponent = useMemo(() => {

projects/app/src/web/common/system/useSystemStore.ts

@@ -104,7 +104,7 @@ export const useSystemStore = create<State>()(
           return null;
         },
 
-        gitStar: 15600,
+        gitStar: 20000,
         async loadGitStar() {
           if (!get().feConfigs?.show_git) return;
           try {

projects/app/src/web/context/useInitApp.ts

@@ -6,6 +6,7 @@ import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/i
 import { useMemoizedFn, useMount } from 'ahooks';
 import { TrackEventName } from '../common/system/constants';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import { useUserStore } from '../support/user/useUserStore';
 
 export const useInitApp = () => {
   const router = useRouter();
@@ -16,6 +17,7 @@ export const useInitApp = () => {
     sourceDomain?: string;
   };
   const { loadGitStar, setInitd, feConfigs } = useSystemStore();
+  const { userInfo } = useUserStore();
   const [scripts, setScripts] = useState<FastGPTFeConfigsType['scripts']>([]);
   const [title, setTitle] = useState(process.env.SYSTEM_NAME || 'AI');
 
@@ -62,6 +64,7 @@ export const useInitApp = () => {
   });
 
   useRequest2(initFetch, {
+    refreshDeps: [userInfo?.username],
     manual: false,
     pollingInterval: 300000 // 5 minutes refresh
   });

projects/app/src/web/support/user/useUserStore.ts

@@ -1,10 +1,8 @@
 import type { UserUpdateParams } from '@/types/user';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { getTokenLogin, putUserInfo } from '@/web/support/user/api';
-import { getTeamMembers } from '@/web/support/user/team/api';
 import type { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
-import type { OrgMemberSchemaType, OrgType } from '@fastgpt/global/support/user/team/org/type';
+import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
-import type { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
 import type { UserType } from '@fastgpt/global/support/user/type.d';
 import type { FeTeamPlanStatusType } from '@fastgpt/global/support/wallet/sub/type';
 import { create } from 'zustand';