4.6.2-alpha (#517)

2025-07-29 09:44:47 +00:00 · 2023-11-25 21:58:00 +08:00
parent 9cb4280a16
commit 3acbf1ab17
39 changed files with 617 additions and 183 deletions
--- a/projects/app/src/pages/_app.tsx
+++ b/projects/app/src/pages/_app.tsx
@@ -93,7 +93,7 @@ function App({ Component, pageProps }: AppProps) {
  return (
    <>
      <Head>
-        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || 'GPT'}</title>
+        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || ''}</title>
        <meta
          name="description"
          content="FastGPT 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！"
--- a/projects/app/src/pages/api/admin/initv46.ts
+++ b/projects/app/src/pages/api/admin/initv46.ts
@@ -295,7 +295,7 @@ async function initPgData() {
    ]);
  } catch (error) {
    console.log(error);
-    console.log('column exits');
+    console.log('column exists');
  }

  const { rows } = await PgClient.query<{ user_id: string }>(`
--- a/projects/app/src/pages/api/admin/initv462.ts
+++ b/projects/app/src/pages/api/admin/initv462.ts
@@ -2,18 +2,9 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { delay } from '@/utils/tools';
-import { PgClient } from '@fastgpt/service/common/pg';
-import {
-  DatasetDataIndexTypeEnum,
-  PgDatasetTableName
-} from '@fastgpt/global/core/dataset/constant';
-
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
-import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
-import { defaultQAModels } from '@fastgpt/global/core/ai/model';
-import { MongoApp } from '@fastgpt/service/core/app/schema';
+import { jiebaSplit } from '@/service/core/dataset/utils';

 let success = 0;
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
@@ -22,6 +13,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { limit = 50 } = req.body as { limit: number };
    await authCert({ req, authRoot: true });
    await connectToDatabase();
+    success = 0;
+
+    console.log(
+      'total',
+      await MongoDatasetData.countDocuments({ fullTextToken: { $exists: false } })
+    );

    await initFullTextToken(limit);

@@ -37,4 +34,31 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    });
  }
 }
-export async function initFullTextToken(limit = 50) {}
+export async function initFullTextToken(limit = 50): Promise<any> {
+  try {
+    const dataList = await MongoDatasetData.find({ fullTextToken: { $exists: false } }, '_id q a')
+      .limit(limit)
+      .lean();
+    if (dataList.length === 0) return;
+
+    const result = await Promise.allSettled(
+      dataList.map((item) => {
+        const text = item.q + (item.a || '');
+        const tokens = jiebaSplit({ text });
+
+        return MongoDatasetData.findByIdAndUpdate(item._id, {
+          $set: {
+            fullTextToken: tokens
+          }
+        });
+      })
+    );
+
+    success += result.filter((item) => item.status === 'fulfilled').length;
+    console.log(`success: ${success}`);
+    return initFullTextToken(limit);
+  } catch (error) {
+    await delay(1000);
+    return initFullTextToken(limit);
+  }
+}
--- a/projects/app/src/pages/api/core/app/create.ts
+++ b/projects/app/src/pages/api/core/app/create.ts
@@ -6,6 +6,7 @@ import type { CreateAppParams } from '@fastgpt/global/core/app/api.d';
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
+import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -39,7 +40,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      teamId,
      tmbId,
      modules,
-      type
+      type,
+      simpleTemplateId: SimpleModeTemplate_FastGPT_Universal.id
    });

    jsonRes(res, {
--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
@@ -41,7 +41,7 @@ function simpleChatTemplate({
    {
      moduleId: 'userChatInput',
      name: '用户问题(对话入口)',
-      logo: '/imgs/module/userChatInput.png',
+      avatar: '/imgs/module/userChatInput.png',
      flowType: 'questionInput',
      position: {
        x: 464.32198615344566,
@@ -73,7 +73,7 @@ function simpleChatTemplate({
    {
      moduleId: 'history',
      name: '聊天记录',
-      logo: '/imgs/module/history.png',
+      avatar: '/imgs/module/history.png',
      flowType: 'historyNode',
      position: {
        x: 452.5466249541586,
@@ -114,7 +114,7 @@ function simpleChatTemplate({
    {
      moduleId: 'chatModule',
      name: 'AI 对话',
-      logo: '/imgs/module/AI.png',
+      avatar: '/imgs/module/AI.png',
      flowType: 'chatNode',
      showStatus: true,
      position: {
@@ -284,7 +284,7 @@ function datasetTemplate({
    {
      moduleId: 'userChatInput',
      name: '用户问题(对话入口)',
-      logo: '/imgs/module/userChatInput.png',
+      avatar: '/imgs/module/userChatInput.png',
      flowType: 'questionInput',
      position: {
        x: 464.32198615344566,
@@ -320,7 +320,7 @@ function datasetTemplate({
    {
      moduleId: 'history',
      name: '聊天记录',
-      logo: '/imgs/module/history.png',
+      avatar: '/imgs/module/history.png',
      flowType: 'historyNode',
      position: {
        x: 452.5466249541586,
@@ -361,7 +361,7 @@ function datasetTemplate({
    {
      moduleId: 'datasetSearch',
      name: '知识库搜索',
-      logo: '/imgs/module/db.png',
+      avatar: '/imgs/module/db.png',
      flowType: 'datasetSearchNode',
      showStatus: true,
      position: {
@@ -454,7 +454,7 @@ function datasetTemplate({
    {
      moduleId: 'chatModule',
      name: 'AI 对话',
-      logo: '/imgs/module/AI.png',
+      avatar: '/imgs/module/AI.png',
      flowType: 'chatNode',
      showStatus: true,
      position: {
--- a/projects/app/src/pages/api/core/dataset/data/insertData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts
@@ -15,6 +15,7 @@ import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controll
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
 import { InsertOneDatasetDataProps } from '@/global/core/dataset/api';
+import { simpleText } from '@fastgpt/global/common/string/tools';

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -46,8 +47,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    ] = await Promise.all([getCollectionWithDataset(collectionId), authTeamBalance(teamId)]);

    // format data
-    const formatQ = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-    const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
+    const formatQ = simpleText(q);
+    const formatA = simpleText(a);
+    const formatIndexes = indexes?.map((item) => ({
+      ...item,
+      text: simpleText(item.text)
+    }));

    // token check
    const token = countPromptTokens(formatQ, 'system');
@@ -72,7 +77,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      q: formatQ,
      a: formatA,
      model: vectorModelData.model,
-      indexes
+      indexes: formatIndexes
    });

    pushGenerateVectorBill({
--- a/projects/app/src/pages/api/core/dataset/data/pushData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts
@@ -13,6 +13,7 @@ import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
 import { getQAModel, getVectorModel } from '@/service/core/ai/model';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
+import { simpleText } from '@fastgpt/global/common/string/tools';

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -72,6 +73,21 @@ export async function pushDataToDatasetCollection({
    collectionId
  });

+  // format q and a, remove empty char
+  data.forEach((item) => {
+    item.q = simpleText(item.q);
+    item.a = simpleText(item.a);
+
+    item.indexes = item.indexes
+      ?.map((index) => {
+        return {
+          ...index,
+          text: simpleText(index.text)
+        };
+      })
+      .filter(Boolean);
+  });
+
  // filter repeat or equal content
  const set = new Set();
  const filterResult: Record<string, PushDatasetDataChunkProps[]> = {
--- a/projects/app/src/pages/api/core/dataset/searchTest.ts
+++ b/projects/app/src/pages/api/core/dataset/searchTest.ts
@@ -1,9 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
-import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
+import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
 import { connectToDatabase } from '@/service/mongo';
-import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
@@ -22,6 +21,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      throw new Error('缺少参数');
    }

+    const start = Date.now();
+
    // auth dataset role
    const { dataset, teamId, tmbId, apikey } = await authDataset({
      req,
@@ -61,8 +62,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      });
    }

-    jsonRes<SearchDataResponseItemType[]>(res, {
-      data: searchRes
+    jsonRes<SearchTestResponse>(res, {
+      data: {
+        list: searchRes,
+        duration: `${((Date.now() - start) / 1000).toFixed(3)}s`
+      }
    });
  } catch (err) {
    jsonRes(res, {
--- a/projects/app/src/pages/api/core/plugin/detail.ts
+++ b/projects/app/src/pages/api/core/plugin/detail.ts
@@ -1,17 +1,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
 import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    const { id } = req.query as { id: string };
    await connectToDatabase();
-    await authPluginCrud({ req, authToken: true, id, per: 'r' });
+    const { plugin } = await authPluginCrud({ req, authToken: true, id, per: 'r' });

    jsonRes(res, {
-      data: await MongoPlugin.findOne({ id })
+      data: plugin
    });
  } catch (err) {
    jsonRes(res, {
--- a/projects/app/src/pages/dataset/detail/components/Info.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info.tsx
@@ -230,16 +230,13 @@ const Info = (
          placeholder={'标签,使用空格分割。'}
          maxLength={30}
          onChange={(e) => {
-            setValue(
-              'tags',
-              e.target.value.split(' ').filter((item) => item)
-            );
+            setValue('tags', e.target.value.split(' ').filter(Boolean));
            setRefresh(!refresh);
          }}
        />
        <Flex w={'100%'} pl={['90px', '160px']} mt={2}>
          {getValues('tags')
-            .filter((item) => item)
+            .filter(Boolean)
            .map((item, i) => (
              <Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
                {item}
--- a/projects/app/src/pages/dataset/detail/components/Test.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Test.tsx
@@ -16,6 +16,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { useTranslation } from 'next-i18next';
 import { feConfigs } from '@/web/common/system/staticData';
+import { SearchTestResponse } from '../../../../global/core/dataset/api';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);

 const Test = ({ datasetId }: { datasetId: string }) => {
@@ -37,20 +38,21 @@ const Test = ({ datasetId }: { datasetId: string }) => {
  );

  const { mutate, isLoading } = useRequest({
-    mutationFn: () => postSearchText({ datasetId, text: inputText.trim(), rerank, limit: 20 }),
-    onSuccess(res: SearchDataResponseItemType[]) {
-      if (!res || res.length === 0) {
+    mutationFn: () => postSearchText({ datasetId, text: inputText.trim(), rerank, limit: 30 }),
+    onSuccess(res: SearchTestResponse) {
+      if (!res || res.list.length === 0) {
        return toast({
          status: 'warning',
          title: t('dataset.test.noResult')
        });
      }
-      const testItem = {
+      const testItem: SearchTestStoreItemType = {
        id: nanoid(),
        datasetId,
        text: inputText.trim(),
        time: new Date(),
-        results: res
+        results: res.list,
+        duration: res.duration
      };
      pushDatasetTestItem(testItem);
      setDatasetTestItem(testItem);
@@ -176,7 +178,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
          <>
            <Flex alignItems={'center'}>
              <Box fontSize={'3xl'} color={'myGray.600'}>
-                测试结果
+                {t('core.dataset.test.Test Result')}
              </Box>
              <MyTooltip
                label={
@@ -185,12 +187,13 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                forceShow
              >
                <QuestionOutlineIcon
-                  ml={2}
+                  mx={2}
                  color={'myGray.600'}
                  cursor={'pointer'}
                  fontSize={'lg'}
                />
              </MyTooltip>
+              <Box>({datasetTestItem.duration})</Box>
            </Flex>
            <Grid
              mt={1}
--- a/projects/app/src/pages/dataset/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/dataset/list/component/CreateModal.tsx
@@ -178,7 +178,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
        <Flex mt={2} flexWrap={'wrap'}>
          {getValues('tags')
            .split(' ')
-            .filter((item) => item)
+            .filter(Boolean)
            .map((item, i) => (
              <Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
                {item}
--- a/projects/app/src/pages/dataset/list/index.tsx
+++ b/projects/app/src/pages/dataset/list/index.tsx
@@ -390,13 +390,11 @@ const Kb = () => {
            </Flex>
            <Box flex={'1 0 0'} overflow={'hidden'} pt={2}>
              <Flex>
-                {dataset.tags
-                  .filter((item) => item)
-                  .map((tag, i) => (
-                    <Tag key={i} mr={2} mb={2}>
-                      {tag}
-                    </Tag>
-                  ))}
+                {dataset.tags.filter(Boolean).map((tag, i) => (
+                  <Tag key={i} mr={2} mb={2}>
+                    {tag}
+                  </Tag>
+                ))}
              </Flex>
            </Box>
            <Flex alignItems={'center'} fontSize={'sm'}>
--- a/projects/app/src/pages/plugin/edit/Preview.tsx
+++ b/projects/app/src/pages/plugin/edit/Preview.tsx
@@ -34,7 +34,7 @@ const PreviewPlugin = ({
        item: {
          moduleId: 'plugin',
          flowType: FlowNodeTypeEnum.pluginModule,
-          logo: plugin.avatar,
+          avatar: plugin.avatar,
          name: plugin.name,
          intro: plugin.intro,
          ...formatPluginToPreviewModule(plugin._id, modules)
--- a/projects/app/src/pages/plugin/edit/index.tsx
+++ b/projects/app/src/pages/plugin/edit/index.tsx
@@ -48,27 +48,32 @@ const Render = ({ pluginId }: Props) => {
    return copyTemplates;
  }, [nodes]);

-  const { data } = useQuery(['getOnePlugin', pluginId], () => getOnePlugin(pluginId), {
-    onError: (error) => {
-      toast({
-        status: 'warning',
-        title: getErrText(error, t('plugin.Load Plugin Failed'))
-      });
-      router.replace('/plugin/list');
+  const { data: pluginDetail } = useQuery(
+    ['getOnePlugin', pluginId],
+    () => getOnePlugin(pluginId),
+    {
+      onError: (error) => {
+        toast({
+          status: 'warning',
+          title: getErrText(error, t('plugin.Load Plugin Failed'))
+        });
+        router.replace('/plugin/list');
+      }
    }
-  });
+  );
+  console.log(pluginDetail);

  useQuery(['getPlugTemplates'], () => loadPluginTemplates());
  const filterPlugins = useMemo(() => {
    return pluginModuleTemplates.filter((item) => item.id !== pluginId);
  }, [pluginId, pluginModuleTemplates]);

-  return data ? (
+  return pluginDetail ? (
    <Flow
      systemTemplates={filterTemplates}
      pluginTemplates={filterPlugins}
-      modules={data?.modules || []}
-      Header={<Header plugin={data} onClose={() => router.back()} />}
+      modules={pluginDetail?.modules || []}
+      Header={<Header plugin={pluginDetail} onClose={() => router.back()} />}
    />
  ) : (
    <Loading />
--- a/projects/app/src/pages/plugin/list/component/EditModal.tsx
+++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx
@@ -24,17 +24,81 @@ import MyModal from '@/components/MyModal';
 import { useTranslation } from 'next-i18next';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import MyIcon from '@/components/Icon';
+import { CreateOnePluginParams } from '@fastgpt/global/core/plugin/controller';

-export type FormType = {
+export type FormType = CreateOnePluginParams & {
  id?: string;
-  avatar: string;
-  name: string;
-  intro: string;
 };
-export const defaultForm = {
+export const defaultForm: FormType = {
  avatar: '/icon/logo.svg',
  name: '',
-  intro: ''
+  intro: '',
+  modules: [
+    {
+      moduleId: 'w90mfp',
+      name: '定义插件输入',
+      avatar: '/imgs/module/input.png',
+      flowType: 'pluginInput',
+      showStatus: false,
+      position: {
+        x: 616.4226348688949,
+        y: -165.05298493910115
+      },
+      inputs: [
+        {
+          key: 'question',
+          valueType: 'string',
+          type: 'target',
+          label: '用户问题',
+          required: true,
+          edit: true,
+          connected: false
+        }
+      ],
+      outputs: [
+        {
+          key: 'question',
+          valueType: 'string',
+          label: '用户问题',
+          type: 'source',
+          edit: true,
+          targets: []
+        }
+      ]
+    },
+    {
+      moduleId: 'tze1ju',
+      name: '定义插件输出',
+      avatar: '/imgs/module/output.png',
+      flowType: 'pluginOutput',
+      showStatus: false,
+      position: {
+        x: 1607.7142331269126,
+        y: -151.8669210746189
+      },
+      inputs: [
+        {
+          key: 'answer',
+          type: 'target',
+          valueType: 'string',
+          label: '答案',
+          required: true,
+          edit: true,
+          connected: true
+        }
+      ],
+      outputs: [
+        {
+          key: 'answer',
+          valueType: 'string',
+          label: '答案',
+          type: 'source',
+          edit: true,
+          targets: []
+        }
+      ]
+    }
+  ]
 };

 const CreateModal = ({