ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-01 20:27:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

v4.5.1 (#417)

Browse Source
This commit is contained in:
Archer
2023-10-22 23:54:04 +08:00
committed by GitHub
parent 3091a90df6
commit a3534407bf
365 changed files with 7266 additions and 6055 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
projects/app/src/web/common/api/fetch.ts
View File

@@ -1,9 +1,9 @@
import { sseResponseEventEnum, TaskResponseKeyEnum } from '@/constants/chat';
import { getErrText } from '@/utils/tools';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { parseStreamChunk, SSEParseData } from '@/utils/sse';
import type { ChatHistoryItemResType } from '@/types/chat';
import { StartChatFnProps } from '@/components/ChatBox';
import { getToken } from '@/utils/user';
import { getToken } from '@/web/support/user/auth';
type StreamFetchProps = {
url?: string;

6
projects/app/src/web/common/api/plugin.ts
View File

@@ -1,6 +0,0 @@
import { GET, POST, PUT, DELETE } from './request';
import type { FetchResultItem } from '@/global/common/api/pluginRes.d';
export const postFetchUrls = (urlList: string[]) =>
POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });

4
projects/app/src/web/common/api/request.ts
View File

@@ -4,8 +4,8 @@ import axios, {
AxiosResponse,
AxiosProgressEvent
} from 'axios';
import { clearToken, getToken } from '@/utils/user';
import { TOKEN_ERROR_CODE } from '@fastgpt/common/constant/errorCode';
import { clearToken, getToken } from '@/web/support/user/auth';
import { TOKEN_ERROR_CODE } from '@fastgpt/global/common/error/errorCode';
interface ConfigType {
headers?: { [key: string]: string };

15
projects/app/src/web/common/api/bill.ts → projects/app/src/web/common/bill/api.ts
View File

@@ -1,8 +1,9 @@
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import type { CreateTrainingBillType } from '@/global/common/api/billReq.d';
import type { CreateTrainingBillType } from '@fastgpt/global/common/bill/types/billReq.d';
import type { PaySchema } from '@/types/mongoSchema';
import type { PagingData, RequestPaging } from '@/types';
import { UserBillType } from '@/types/user';
import { delay } from '@/utils/tools';
export const getUserBills = (data: RequestPaging) =>
POST<PagingData<UserBillType>>(`/user/getBill`, data);
@@ -20,8 +21,14 @@ export const getPayCode = (amount: number) =>
export const checkPayResult = (payId: string) =>
GET<number>(`/plusApi/support/user/pay/checkPayResult`, { payId }).then(() => {
try {
GET('/user/account/paySuccess');
} catch (error) {}
async function startQueue() {
try {
await GET('/user/account/paySuccess');
} catch (error) {
await delay(1000);
startQueue();
}
}
startQueue();
return 'success';
});

2
projects/app/src/web/common/hooks/useSelectFile.tsx → projects/app/src/web/common/file/hooks/useSelectFile.tsx
View File

@@ -1,6 +1,6 @@
import React, { useRef, useCallback } from 'react';
import { Box } from '@chakra-ui/react';
import { useToast } from './useToast';
import { useToast } from '@/web/common/hooks/useToast';
import { useTranslation } from 'react-i18next';
export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {

2
projects/app/src/web/common/utils/file.ts → projects/app/src/web/common/file/utils.ts
View File

@@ -1,6 +1,6 @@
import mammoth from 'mammoth';
import Papa from 'papaparse';
import { postUploadImg, postUploadFiles, getFileViewUrl } from '@/web/common/api/system';
import { postUploadImg, postUploadFiles, getFileViewUrl } from '@/web/common/system/api';
/**
* upload file to mongo gridfs

16
projects/app/src/web/common/hooks/useDrag.tsx Normal file
View File

@@ -0,0 +1,16 @@
import { useState } from 'react';
export const useDrag = () => {
const [moveDataId, setMoveDataId] = useState<string>();
const [dragStartId, setDragStartId] = useState<string>();
const [dragTargetId, setDragTargetId] = useState<string>();
return {
moveDataId,
setMoveDataId,
dragStartId,
setDragStartId,
dragTargetId,
setDragTargetId
};
};

12
projects/app/src/web/common/hooks/useEditTitle.tsx
View File

@@ -1,12 +1,14 @@
import React, { useCallback, useRef } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button, Box } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';
export const useEditTitle = ({
title,
tip,
placeholder = ''
}: {
title: string;
tip?: string;
placeholder?: string;
}) => {
const { isOpen, onOpen, onClose } = useDisclosure();
@@ -51,6 +53,12 @@ export const useEditTitle = ({
() => (
<MyModal isOpen={isOpen} onClose={onClose} title={title}>
<ModalBody>
{!!tip && (
<Box mb={2} color={'myGray.500'} fontSize={'sm'}>
{tip}
</Box>
)}
<Input
ref={inputRef}
defaultValue={defaultValue.current}
@@ -67,7 +75,7 @@ export const useEditTitle = ({
</ModalFooter>
</MyModal>
),
[isOpen, onClose, onclickConfirm, placeholder, title]
[isOpen, onClose, onclickConfirm, placeholder, tip, title]
);
return {

2
projects/app/src/web/common/hooks/useRequest.tsx
View File

@@ -1,7 +1,7 @@
import { useToast } from '@/web/common/hooks/useToast';
import { useMutation } from '@tanstack/react-query';
import type { UseMutationOptions } from '@tanstack/react-query';
import { getErrText } from '@/utils/tools';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useTranslation } from 'react-i18next';
interface Props extends UseMutationOptions<any, any, any, any> {

6
projects/app/src/web/common/plugin/api.ts Normal file
View File

@@ -0,0 +1,6 @@
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
export const postFetchUrls = (urlList: string[]) =>
POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });

15
projects/app/src/web/common/plusApi/censor.ts Normal file
View File

@@ -0,0 +1,15 @@
import { POST } from '@fastgpt/service/common/api/plusRequest';
export const postTextCensor = (data: { text: string }) =>
POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
.then((res) => {
if (res?.code === 5000) {
return Promise.reject(res);
}
})
.catch((err) => {
if (err?.code === 5000) {
return Promise.reject(err.message);
}
return Promise.resolve('');
});

2
projects/app/src/web/common/api/system.ts → projects/app/src/web/common/system/api.ts
View File

@@ -1,4 +1,4 @@
import { GET, POST, PUT } from './request';
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import type { InitDateResponse } from '@/global/common/api/systemRes';
import { AxiosProgressEvent } from 'axios';

4
projects/app/src/web/common/store/static.ts → projects/app/src/web/common/system/staticData.ts
View File

@@ -1,7 +1,7 @@
import type { InitDateResponse } from '@/global/common/api/systemRes';
import { getSystemInitData } from '@/web/common/api/system';
import { getSystemInitData } from '@/web/common/system/api';
import { delay } from '@/utils/tools';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
import {
defaultChatModels,
defaultQAModels,

2
projects/app/src/web/common/store/global.ts → projects/app/src/web/common/system/useSystemStore.ts
View File

@@ -21,7 +21,7 @@ type State = {
loadGitStar: () => Promise<void>;
};
export const useGlobalStore = create<State>()(
export const useSystemStore = create<State>()(
devtools(
persist(
immer((set, get) => ({

3
projects/app/src/web/common/utils/eventbus.ts
View File

@@ -1,5 +1,6 @@
export enum EventNameEnum {
guideClick = 'guideClick'
guideClick = 'guideClick',
updaterNode = 'updaterNode'
}
type EventNameType = `${EventNameEnum}`;

2
projects/app/src/web/common/utils/voice.ts
View File

@@ -1,6 +1,6 @@
import { useState, useCallback, useEffect, useMemo } from 'react';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@/utils/tools';
import { getErrText } from '@fastgpt/global/common/error/utils';
export const useAudioPlay = (props?: { ttsUrl?: string }) => {
const { ttsUrl } = props || {};

0
projects/app/src/web/core/api/ai.ts → projects/app/src/web/core/ai/api.ts
View File

0
projects/app/src/web/core/api/app.ts → projects/app/src/web/core/app/api.ts
View File

573
projects/app/src/web/core/app/basicSettings.ts Normal file
View File

@@ -0,0 +1,573 @@
import type { AppModuleItemType, VariableItemType } from '@/types/app';
import { chatModelList } from '@/web/common/system/staticData';
import {
FlowInputItemTypeEnum,
FlowModuleTypeEnum,
FlowValueTypeEnum,
SpecialInputKeyEnum
} from '@/constants/flow';
import { SystemInputEnum } from '@/constants/app';
import type { SelectedDatasetType } from '@/types/core/dataset';
import type { FlowInputItemType } from '@/types/core/app/flow';
import type { AIChatProps } from '@/types/core/aiChat';
import { getGuideModule, splitGuideModule } from '@/global/core/app/modules/utils';
export type EditFormType = {
chatModel: AIChatProps;
kb: {
list: SelectedDatasetType;
searchSimilarity: number;
searchLimit: number;
searchEmptyText: string;
};
guide: {
welcome: {
text: string;
};
};
variables: VariableItemType[];
questionGuide: boolean;
};
export const getDefaultAppForm = (): EditFormType => {
const defaultChatModel = chatModelList[0];
return {
chatModel: {
model: defaultChatModel?.model,
systemPrompt: '',
temperature: 0,
[SystemInputEnum.isResponseAnswerText]: true,
quotePrompt: '',
quoteTemplate: '',
maxToken: defaultChatModel ? defaultChatModel.maxToken / 2 : 4000,
frequency: 0.5,
presence: -0.5
},
kb: {
list: [],
searchSimilarity: 0.4,
searchLimit: 5,
searchEmptyText: ''
},
guide: {
welcome: {
text: ''
}
},
variables: [],
questionGuide: false
};
};
export const appModules2Form = (modules: AppModuleItemType[]) => {
const defaultAppForm = getDefaultAppForm();
const updateVal = ({
formKey,
inputs,
key
}: {
formKey: string;
inputs: FlowInputItemType[];
key: string;
}) => {
const propertyPath = formKey.split('.');
let currentObj: any = defaultAppForm;
for (let i = 0; i < propertyPath.length - 1; i++) {
currentObj = currentObj[propertyPath[i]];
}
const val =
inputs.find((item) => item.key === key)?.value ||
currentObj[propertyPath[propertyPath.length - 1]];
currentObj[propertyPath[propertyPath.length - 1]] = val;
};
modules.forEach((module) => {
if (module.flowType === FlowModuleTypeEnum.chatNode) {
updateVal({
formKey: 'chatModel.model',
inputs: module.inputs,
key: 'model'
});
updateVal({
formKey: 'chatModel.temperature',
inputs: module.inputs,
key: 'temperature'
});
updateVal({
formKey: 'chatModel.maxToken',
inputs: module.inputs,
key: 'maxToken'
});
updateVal({
formKey: 'chatModel.systemPrompt',
inputs: module.inputs,
key: 'systemPrompt'
});
updateVal({
formKey: 'chatModel.quoteTemplate',
inputs: module.inputs,
key: 'quoteTemplate'
});
updateVal({
formKey: 'chatModel.quotePrompt',
inputs: module.inputs,
key: 'quotePrompt'
});
} else if (module.flowType === FlowModuleTypeEnum.datasetSearchNode) {
updateVal({
formKey: 'kb.list',
inputs: module.inputs,
key: 'datasets'
});
updateVal({
formKey: 'kb.searchSimilarity',
inputs: module.inputs,
key: 'similarity'
});
updateVal({
formKey: 'kb.searchLimit',
inputs: module.inputs,
key: 'limit'
});
// empty text
const emptyOutputs = module.outputs.find((item) => item.key === 'isEmpty')?.targets || [];
const emptyOutput = emptyOutputs[0];
if (emptyOutput) {
const target = modules.find((item) => item.moduleId === emptyOutput.moduleId);
defaultAppForm.kb.searchEmptyText =
target?.inputs?.find((item) => item.key === SpecialInputKeyEnum.answerText)?.value || '';
}
} else if (module.flowType === FlowModuleTypeEnum.userGuide) {
const { welcomeText, variableModules, questionGuide } = splitGuideModule(
getGuideModule(modules)
);
if (welcomeText) {
defaultAppForm.guide.welcome = {
text: welcomeText
};
}
defaultAppForm.variables = variableModules;
defaultAppForm.questionGuide = !!questionGuide;
}
});
return defaultAppForm;
};
const chatModelInput = (formData: EditFormType): FlowInputItemType[] => [
{
key: 'model',
value: formData.chatModel.model,
type: 'custom',
label: '对话模型',
connected: true
},
{
key: 'temperature',
value: formData.chatModel.temperature,
type: 'slider',
label: '温度',
connected: true
},
{
key: 'maxToken',
value: formData.chatModel.maxToken,
type: 'custom',
label: '回复上限',
connected: true
},
{
key: 'systemPrompt',
value: formData.chatModel.systemPrompt || '',
type: 'textarea',
label: '系统提示词',
connected: true
},
{
key: SystemInputEnum.isResponseAnswerText,
value: true,
type: 'hidden',
label: '返回AI内容',
connected: true
},
{
key: 'quoteTemplate',
value: formData.chatModel.quoteTemplate || '',
type: 'hidden',
label: '引用内容模板',
connected: true
},
{
key: 'quotePrompt',
value: formData.chatModel.quotePrompt || '',
type: 'hidden',
label: '引用内容提示词',
connected: true
},
{
key: 'switch',
type: 'target',
label: '触发器',
connected: formData.kb.list.length > 0 && !!formData.kb.searchEmptyText
},
{
key: 'quoteQA',
type: 'target',
label: '引用内容',
connected: formData.kb.list.length > 0
},
{
key: 'history',
type: 'target',
label: '聊天记录',
connected: true
},
{
key: 'userChatInput',
type: 'target',
label: '用户问题',
connected: true
}
];
const userGuideTemplate = (formData: EditFormType): AppModuleItemType[] => [
{
name: '用户引导',
flowType: FlowModuleTypeEnum.userGuide,
inputs: [
{
key: SystemInputEnum.welcomeText,
type: FlowInputItemTypeEnum.hidden,
label: '开场白',
value: formData.guide.welcome.text
},
{
key: SystemInputEnum.variables,
type: FlowInputItemTypeEnum.hidden,
label: '对话框变量',
value: formData.variables
},
{
key: SystemInputEnum.questionGuide,
type: FlowInputItemTypeEnum.hidden,
label: '问题引导',
value: formData.questionGuide
}
],
outputs: [],
position: {
x: 447.98520778293346,
y: 721.4016845336229
},
moduleId: 'userGuide'
}
];
const simpleChatTemplate = (formData: EditFormType): AppModuleItemType[] => [
{
name: '用户问题(对话入口)',
flowType: FlowModuleTypeEnum.questionInput,
inputs: [
{
key: 'userChatInput',
connected: true,
label: '用户问题',
type: 'target'
}
],
outputs: [
{
key: 'userChatInput',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
],
position: {
x: 464.32198615344566,
y: 1602.2698463081606
},
moduleId: 'userChatInput'
},
{
name: '聊天记录',
flowType: FlowModuleTypeEnum.historyNode,
inputs: [
{
key: 'maxContext',
value: 6,
connected: true,
type: 'numberInput',
label: '最长记录数'
},
{
key: 'history',
type: 'hidden',
label: '聊天记录',
connected: true
}
],
outputs: [
{
key: 'history',
targets: [
{
moduleId: 'chatModule',
key: 'history'
}
]
}
],
position: {
x: 452.5466249541586,
y: 1276.3930310334215
},
moduleId: 'history'
},
{
name: 'AI 对话',
flowType: FlowModuleTypeEnum.chatNode,
inputs: chatModelInput(formData),
showStatus: true,
outputs: [
{
key: 'answerText',
label: 'AI回复',
description: '直接响应，无需配置',
type: 'hidden',
targets: []
},
{
key: 'finish',
label: '回复结束',
description: 'AI 回复完成后触发',
valueType: 'boolean',
type: 'source',
targets: []
}
],
position: {
x: 981.9682828103937,
y: 890.014595014464
},
moduleId: 'chatModule'
}
];
const kbTemplate = (formData: EditFormType): AppModuleItemType[] => [
{
name: '用户问题(对话入口)',
flowType: FlowModuleTypeEnum.questionInput,
inputs: [
{
key: 'userChatInput',
label: '用户问题',
type: 'target',
connected: true
}
],
outputs: [
{
key: 'userChatInput',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
},
{
moduleId: 'kbSearch',
key: 'userChatInput'
}
]
}
],
position: {
x: 464.32198615344566,
y: 1602.2698463081606
},
moduleId: 'userChatInput'
},
{
name: '聊天记录',
flowType: FlowModuleTypeEnum.historyNode,
inputs: [
{
key: 'maxContext',
value: 6,
connected: true,
type: 'numberInput',
label: '最长记录数'
},
{
key: 'history',
type: 'hidden',
label: '聊天记录',
connected: true
}
],
outputs: [
{
key: 'history',
targets: [
{
moduleId: 'chatModule',
key: 'history'
}
]
}
],
position: {
x: 452.5466249541586,
y: 1276.3930310334215
},
moduleId: 'history'
},
{
name: '知识库搜索',
flowType: FlowModuleTypeEnum.datasetSearchNode,
showStatus: true,
inputs: [
{
key: 'datasets',
value: formData.kb.list,
type: FlowInputItemTypeEnum.custom,
label: '关联的知识库',
connected: true
},
{
key: 'similarity',
value: formData.kb.searchSimilarity,
type: FlowInputItemTypeEnum.slider,
label: '相似度',
connected: true
},
{
key: 'limit',
value: formData.kb.searchLimit,
type: FlowInputItemTypeEnum.slider,
label: '单次搜索上限',
connected: true
},
{
key: 'switch',
type: FlowInputItemTypeEnum.target,
label: '触发器',
connected: false
},
{
key: 'userChatInput',
type: FlowInputItemTypeEnum.target,
label: '用户问题',
connected: true
}
],
outputs: [
{
key: 'isEmpty',
targets: formData.kb.searchEmptyText
? [
{
moduleId: 'emptyText',
key: 'switch'
}
]
: []
},
{
key: 'unEmpty',
targets: formData.kb.searchEmptyText
? [
{
moduleId: 'chatModule',
key: 'switch'
}
]
: []
},
{
key: 'quoteQA',
targets: [
{
moduleId: 'chatModule',
key: 'quoteQA'
}
]
}
],
position: {
x: 956.0838440206068,
y: 887.462827870246
},
moduleId: 'kbSearch'
},
...(formData.kb.searchEmptyText
? [
{
name: '指定回复',
flowType: FlowModuleTypeEnum.answerNode,
inputs: [
{
key: 'switch',
type: FlowInputItemTypeEnum.target,
label: '触发器',
connected: true
},
{
key: SpecialInputKeyEnum.answerText,
value: formData.kb.searchEmptyText,
type: FlowInputItemTypeEnum.textarea,
valueType: FlowValueTypeEnum.string,
label: '回复的内容',
connected: true
}
],
outputs: [],
position: {
x: 1553.5815811529146,
y: 637.8753731306779
},
moduleId: 'emptyText'
}
]
: []),
{
name: 'AI 对话',
flowType: FlowModuleTypeEnum.chatNode,
inputs: chatModelInput(formData),
showStatus: true,
outputs: [
{
key: 'answerText',
label: 'AI回复',
description: '直接响应，无需配置',
type: 'hidden',
targets: []
},
{
key: 'finish',
label: '回复结束',
description: 'AI 回复完成后触发',
valueType: 'boolean',
type: 'source',
targets: []
}
],
position: {
x: 1551.71405495818,
y: 977.4911578918461
},
moduleId: 'chatModule'
}
];
export const appForm2Modules = (formData: EditFormType) => {
const modules = [
...userGuideTemplate(formData),
...(formData.kb.list.length > 0 ? kbTemplate(formData) : simpleChatTemplate(formData))
];
return modules as AppModuleItemType[];
};

0
projects/app/src/web/core/api/chat.ts → projects/app/src/web/core/chat/api.ts
View File

2
projects/app/src/web/core/store/chat.ts → projects/app/src/web/core/chat/storeChat.ts
View File

@@ -4,7 +4,7 @@ import { immer } from 'zustand/middleware/immer';
import { ChatHistoryItemType } from '@/types/chat';
import type { InitChatResponse } from '@/global/core/api/chatRes.d';
import { delChatHistoryById, getChatHistory, clearChatHistoryByAppId } from '@/web/core/api/chat';
import { delChatHistoryById, getChatHistory, clearChatHistoryByAppId } from '@/web/core/chat/api';
type State = {
history: ChatHistoryItemType[];

4
projects/app/src/web/core/store/shareChat.ts → projects/app/src/web/core/chat/storeShareChat.ts
View File

@@ -70,7 +70,7 @@ export const useShareChatStore = create<State>()(
...item,
title: prompts[prompts.length - 2]?.value,
updateTime: new Date(),
chats: chatHistory.chats.concat(prompts).slice(-50),
chats: chatHistory.chats.concat(prompts).slice(-30),
variables
}
: item
@@ -90,7 +90,7 @@ export const useShareChatStore = create<State>()(
historyList.sort((a, b) => new Date(b.updateTime) - new Date(a.updateTime));
set((state) => {
state.shareChatHistory = historyList.slice(0, 100);
state.shareChatHistory = historyList.slice(0, 50);
});
},
delOneShareHistoryByChatId(chatId: string) {

84
projects/app/src/web/core/api/dataset.ts → projects/app/src/web/core/dataset/api.ts
View File

@@ -4,21 +4,26 @@ import type {
DatasetUpdateParams,
CreateDatasetParams,
SearchTestProps,
GetFileListProps,
UpdateFileProps,
MarkFileUsedProps,
GetDatasetCollectionsProps,
PushDataProps,
UpdateDatasetDataPrams,
GetDatasetDataListProps
GetDatasetDataListProps,
CreateDatasetCollectionParams,
UpdateDatasetCollectionParams,
SetOneDatasetDataProps
} from '@/global/core/api/datasetReq.d';
import type { SearchTestResponseType, PushDataResponse } from '@/global/core/api/datasetRes.d';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import type { DatasetFileItemType } from '@/types/core/dataset/file';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import type { GSFileInfoType } from '@/types/common/file';
import type { QuoteItemType } from '@/types/chat';
import { getToken } from '@/utils/user';
import { getToken } from '@/web/support/user/auth';
import download from 'downloadjs';
import type { DatasetDataItemType } from '@/types/core/dataset/data';
import type {
DatasetCollectionSchemaType,
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
import { PagingData } from '@/types';
/* ======================== dataset ======================= */
export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
@@ -41,35 +46,35 @@ export const putDatasetById = (data: DatasetUpdateParams) => PUT(`/core/dataset/
export const delDatasetById = (id: string) => DELETE(`/core/dataset/delete?id=${id}`);
/* =========== search test ============ */
export const postSearchText = (data: SearchTestProps) =>
POST<SearchTestResponseType>(`/core/dataset/searchTest`, data);
POST<SearchDataResponseItemType[]>(`/core/dataset/searchTest`, data);
/* ============================= file ==================================== */
export const getDatasetFiles = (data: GetFileListProps) =>
POST<DatasetFileItemType[]>(`/core/dataset/file/list`, data);
export const delDatasetFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/core/dataset/file/delById`, params);
export const getFileInfoById = (fileId: string) =>
GET<GSFileInfoType>(`/core/dataset/file/detail`, { fileId });
export const delDatasetEmptyFiles = (kbId: string) =>
DELETE(`/core/dataset/file/delEmptyFiles`, { kbId });
export const updateDatasetFile = (data: UpdateFileProps) => PUT(`/core/dataset/file/update`, data);
export const putMarkFilesUsed = (data: MarkFileUsedProps) =>
PUT(`/core/dataset/file/markUsed`, data);
/* ============================= collections ==================================== */
export const getDatasetCollections = (data: GetDatasetCollectionsProps) =>
POST<PagingData<DatasetCollectionsListItemType>>(`/core/dataset/collection/list`, data);
export const getDatasetCollectionPathById = (parentId: string) =>
GET<ParentTreePathItemType[]>(`/core/dataset/collection/paths`, { parentId });
export const getDatasetCollectionById = (id: string) =>
GET<DatasetCollectionSchemaType>(`/core/dataset/collection/detail`, { id });
export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
POST<string>(`/core/dataset/collection/create`, data);
export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
POST(`/core/dataset/collection/update`, data);
export const delDatasetCollectionById = (params: { collectionId: string }) =>
DELETE(`/core/dataset/collection/delById`, params);
/* =============================== data ==================================== */
/* kb data */
/* get dataset list */
export const getDatasetDataList = (data: GetDatasetDataListProps) =>
POST(`/core/dataset/data/getDataList`, data);
/**
* export and download data
*/
export const exportDatasetData = (data: { kbId: string }) =>
fetch(`/api/core/dataset/data/exportAll?kbId=${data.kbId}`, {
export const exportDatasetData = (data: { datasetId: string }) =>
fetch(`/api/core/dataset/data/exportAll?datasetId=${data.datasetId}`, {
method: 'GET',
headers: {
token: getToken()
@@ -84,20 +89,11 @@ export const exportDatasetData = (data: { kbId: string }) =>
})
.then((blob) => download(blob, 'dataset.csv', 'text/csv'));
/**
* 获取模型正在拆分数据的数量
*/
export const getTrainingData = (data: { kbId: string; init: boolean }) =>
POST<{
qaListLen: number;
vectorListLen: number;
}>(`/core/dataset/data/getTrainingData`, data);
/* get length of system training queue */
export const getTrainingQueueLen = () => GET<number>(`/core/dataset/data/getQueueLen`);
export const getDatasetDataItemById = (dataId: string) =>
GET<QuoteItemType>(`/core/dataset/data/getDataById`, { dataId });
GET<DatasetDataItemType>(`/core/dataset/data/getDataById`, { dataId });
/**
* push data to training queue
@@ -108,16 +104,22 @@ export const postChunks2Dataset = (data: PushDataProps) =>
/**
* insert one data to dataset (immediately insert)
*/
export const postData2Dataset = (data: { kbId: string; data: DatasetDataItemType }) =>
export const postData2Dataset = (data: SetOneDatasetDataProps) =>
POST<string>(`/core/dataset/data/insertData`, data);
/**
* 更新一条数据
*/
export const putDatasetDataById = (data: UpdateDatasetDataPrams) =>
export const putDatasetDataById = (data: SetOneDatasetDataProps) =>
PUT('/core/dataset/data/updateData', data);
/**
* 删除一条知识库数据
*/
export const delOneDatasetDataById = (dataId: string) =>
DELETE(`/core/dataset/data/delDataById?dataId=${dataId}`);
/* ================== file ======================== */
export const getFileInfoById = (fileId: string) =>
GET<GSFileInfoType>(`/core/dataset/file/detail`, { fileId });
export const delDatasetEmptyFiles = (datasetId: string) =>
DELETE(`/core/dataset/file/delEmptyFiles`, { datasetId });

211
projects/app/src/web/core/dataset/components/SelectCollections.tsx Normal file
View File

@@ -0,0 +1,211 @@
import MyIcon from '@/components/Icon';
import MyModal from '@/components/MyModal';
import ParentPaths from '@/components/common/ParentPaths';
import { useLoading } from '@/web/common/hooks/useLoading';
import { useRequest } from '@/web/common/hooks/useRequest';
import { getDatasetCollectionPathById, getDatasetCollections } from '@/web/core/dataset/api';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { Box, Flex, ModalFooter, Button, useTheme, Grid, Card, Image } from '@chakra-ui/react';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import { useQuery } from '@tanstack/react-query';
import React, { useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
const SelectCollections = ({
datasetId,
type,
defaultSelectedId = [],
onClose,
onChange,
onSuccess,
title,
tip,
max = 1,
CustomFooter
}: {
datasetId: string;
type: 'folder' | 'collection';
onClose: () => void;
onChange?: (e: { parentId: string; collectionIds: string[] }) => void | Promise<void>;
onSuccess?: (e: { parentId: string; collectionIds: string[] }) => void | Promise<void>;
defaultSelectedId?: string[];
title?: string;
tip?: string;
max?: number;
CustomFooter?: React.ReactNode;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { Loading } = useLoading();
const [selectedDatasetCollectionIds, setSelectedDatasetCollectionIds] =
useState<string[]>(defaultSelectedId);
const [parentId, setParentId] = useState('');
useQuery(['loadDatasetDetail', datasetId], () => loadDatasetDetail(datasetId));
const { data, isLoading } = useQuery(['getDatasetCollections', parentId], () =>
getDatasetCollections({
datasetId,
parentId,
selectFolder: type === 'folder',
simple: true,
pageNum: 1,
pageSize: 50
})
);
const formatCollections = useMemo(
() =>
data?.data.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
return {
...collection,
icon
};
}) || [],
[data]
);
const collections = useMemo(
() =>
type === 'folder'
? formatCollections.filter((item) => item._id !== defaultSelectedId[0])
: formatCollections,
[defaultSelectedId, formatCollections, type]
);
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
const { mutate, isLoading: isResponding } = useRequest({
mutationFn: async () => {
if (type === 'folder') {
await onSuccess?.({ parentId: paths[paths.length - 1]?.parentId || '', collectionIds: [] });
} else {
await onSuccess?.({
parentId: paths[paths.length - 1]?.parentId || '',
collectionIds: selectedDatasetCollectionIds
});
}
return null;
},
errorToast: t('common.Request Error')
});
return (
<MyModal isOpen onClose={onClose} maxW={['90vw', '900px']} h={['90vh', '80vh']} isCentered>
<Flex flexDirection={'column'} flex={'1 0 0'}>
<Box flex={'1 0 0'} px={4} py={2}>
<Flex flexDirection={'column'} h={'100%'} position={'relative'}>
<Box>
<ParentPaths
paths={paths.map((path, i) => ({
parentId: path.parentId,
parentName: path.parentName
}))}
FirstPathDom={
<>
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{title || type === 'folder'
? t('common.Select One Folder')
: t('dataset.collections.Select Collection')}
</Box>
{!!tip && (
<Box fontSize={'sm'} color={'myGray.500'}>
{tip}
</Box>
)}
</>
}
onClick={(e) => {
setParentId(e);
}}
/>
</Box>
<Box flex={'1 0 0'} overflowY={'auto'} mt={2}>
<Grid
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={3}
userSelect={'none'}
>
{collections.map((item) =>
(() => {
const selected = selectedDatasetCollectionIds.includes(item._id);
return (
<Card
key={item._id}
p={3}
border={theme.borders.base}
boxShadow={'sm'}
cursor={'pointer'}
_hover={{
boxShadow: 'md'
}}
{...(selected
? {
bg: 'myBlue.300'
}
: {})}
onClick={() => {
if (item.type === DatasetCollectionTypeEnum.folder) {
setParentId(item._id);
} else {
let result: string[] = [];
if (max === 1) {
result = [item._id];
} else if (selected) {
result = selectedDatasetCollectionIds.filter((id) => id !== item._id);
} else if (selectedDatasetCollectionIds.length < max) {
result = [...selectedDatasetCollectionIds, item._id];
}
setSelectedDatasetCollectionIds(result);
onChange && onChange({ parentId, collectionIds: result });
}
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Image src={item.icon} w={'18px'} alt={''} />
<Box ml={3} fontSize={'sm'}>
{item.name}
</Box>
</Flex>
</Card>
);
})()
)}
</Grid>
{collections.length === 0 && (
<Flex mt={'10vh'} flexDirection={'column'} alignItems={'center'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
{t('common.folder.No Folder')}
</Box>
</Flex>
)}
</Box>
<Loading loading={isLoading} fixed={false} />
</Flex>
</Box>
{CustomFooter ? (
<>{CustomFooter}</>
) : (
<ModalFooter>
<Button
isLoading={isResponding}
isDisabled={type === 'collection' && selectedDatasetCollectionIds.length === 0}
onClick={mutate}
>
{type === 'folder' ? t('common.Confirm Move') : t('Confirm')}
</Button>
</ModalFooter>
)}
</Flex>
</MyModal>
);
};
export default SelectCollections;

87
projects/app/src/web/core/dataset/store/dataset.ts Normal file
View File

@@ -0,0 +1,87 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { DatasetItemType, DatasetsItemType } from '@/types/core/dataset';
import { getAllDataset, getDatasets, getDatasetById, putDatasetById } from '@/web/core/dataset/api';
import { defaultKbDetail } from '@/constants/dataset';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
type State = {
allDatasets: DatasetsItemType[];
loadAllDatasets: () => Promise<DatasetsItemType[]>;
myDatasets: DatasetsItemType[];
loadDatasets: (parentId?: string) => Promise<any>;
setDatasets(val: DatasetsItemType[]): void;
datasetDetail: DatasetItemType;
loadDatasetDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateParams) => Promise<any>;
};
export const useDatasetStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
allDatasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.allDatasets = res;
});
return res;
},
myDatasets: [],
async loadDatasets(parentId = '') {
const res = await getDatasets({ parentId });
set((state) => {
state.myDatasets = res;
});
return res;
},
setDatasets(val) {
set((state) => {
state.myDatasets = val;
});
},
datasetDetail: defaultKbDetail,
async loadDatasetDetail(id: string, init = false) {
if (!id || (id === get().datasetDetail._id && !init)) return get().datasetDetail;
const data = await getDatasetById(id);
set((state) => {
state.datasetDetail = data;
});
return data;
},
async updateDataset(data) {
if (get().datasetDetail._id === data.id) {
set((state) => {
state.datasetDetail = {
...state.datasetDetail,
...data
};
});
}
set((state) => {
state.myDatasets = state.myDatasets = state.myDatasets.map((item) =>
item._id === data.id
? {
...item,
...data,
tags: data.tags?.split(' ') || []
}
: item
);
});
await putDatasetById(data);
}
})),
{
name: 'datasetStore',
partialize: (state) => ({})
}
)
)
);

40
projects/app/src/web/core/dataset/store/markdata.ts Normal file
View File

@@ -0,0 +1,40 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
export type MarkDataStore = {
chatItemId: string;
datasetId?: string;
collectionId?: string;
q: string;
a: string;
};
export type MarkDataCallback = (data: MarkDataStore) => void;
type State = {
markData?: MarkDataStore;
markDataCallback?: MarkDataCallback;
startMarkData: (data: MarkDataStore, cb: MarkDataCallback) => void;
updateMarkData: (data: MarkDataStore) => void;
};
export const useSearchTestStore = create<State>()(
devtools(
immer((set, get) => ({
markData: undefined,
markDataCallback: undefined,
startMarkData(data, cb) {
set((state) => {
state.markData = data;
state.markDataCallback = cb;
});
},
updateMarkData(data: MarkDataStore) {
set((state) => {
state.markData = data;
});
}
}))
)
);

52
projects/app/src/web/core/dataset/store/searchTest.ts Normal file
View File

@@ -0,0 +1,52 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
export type SearchTestStoreItemType = {
id: string;
datasetId: string;
text: string;
time: Date;
results: SearchDataResponseItemType[];
};
type State = {
datasetTestList: SearchTestStoreItemType[];
pushDatasetTestItem: (data: SearchTestStoreItemType) => void;
delDatasetTestItemById: (id: string) => void;
updateDatasetItemById: (data: SearchTestStoreItemType) => void;
};
export const useSearchTestStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
datasetTestList: [],
pushDatasetTestItem(data) {
set((state) => {
state.datasetTestList = [data, ...state.datasetTestList].slice(0, 100);
});
},
delDatasetTestItemById(id) {
set((state) => {
state.datasetTestList = state.datasetTestList.filter((item) => item.id !== id);
});
},
updateDatasetItemById(data: SearchTestStoreItemType) {
set((state) => {
state.datasetTestList = state.datasetTestList.map((item) =>
item.id === data.id ? data : item
);
});
}
})),
{
name: 'searchTestStore',
partialize: (state) => ({
datasetTestList: state.datasetTestList
})
}
)
)
);

24
projects/app/src/web/core/utils/dataset.ts → projects/app/src/web/core/dataset/utils.ts
View File

@@ -1,30 +1,28 @@
import { postCreateTrainingBill } from '@/web/common/api/bill';
import { postChunks2Dataset } from '@/web/core/api/dataset';
import { TrainingModeEnum } from '@/constants/plugin';
import type { DatasetDataItemType } from '@/types/core/dataset/data';
import { postChunks2Dataset } from '@/web/core/dataset/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetChunkItemType } from '@fastgpt/global/core/dataset/type';
import { delay } from '@/utils/tools';
export async function chunksUpload({
kbId,
collectionId,
billId,
mode,
chunks,
prompt,
rate = 150,
onUploading
}: {
kbId: string;
collectionId: string;
billId: string;
mode: `${TrainingModeEnum}`;
chunks: DatasetDataItemType[];
chunks: DatasetChunkItemType[];
prompt?: string;
rate?: number;
onUploading?: (insertLen: number, total: number) => void;
}) {
// create training bill
const billId = await postCreateTrainingBill({ name: 'dataset.Training Name' });
async function upload(data: DatasetDataItemType[]) {
async function upload(data: DatasetChunkItemType[]) {
return postChunks2Dataset({
kbId,
collectionId,
data,
mode,
prompt,
@@ -37,7 +35,7 @@ export async function chunksUpload({
for (let i = 0; i < chunks.length; i += rate) {
try {
const { insertLen } = await upload(chunks.slice(i, i + rate));
onUploading && onUploading(i + rate, chunks.length);
onUploading && onUploading(insertLen, chunks.length);
successInsert += insertLen;
} catch (error) {
if (retryTimes === 0) {

110
projects/app/src/web/core/store/dataset.ts
View File

@@ -1,110 +0,0 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { SearchTestItemType } from '@/types/core/dataset';
import type { DatasetItemType, DatasetsItemType } from '@/types/core/dataset';
import { getAllDataset, getDatasets, getDatasetById, putDatasetById } from '@/web/core/api/dataset';
import { defaultKbDetail } from '@/constants/dataset';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
type State = {
allDatasets: DatasetsItemType[];
loadAllDatasets: () => Promise<DatasetsItemType[]>;
myKbList: DatasetsItemType[];
loadKbList: (parentId?: string) => Promise<any>;
setKbList(val: DatasetsItemType[]): void;
kbDetail: DatasetItemType;
getKbDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateParams) => Promise<any>;
kbTestList: SearchTestItemType[];
pushKbTestItem: (data: SearchTestItemType) => void;
delKbTestItemById: (id: string) => void;
updateKbItemById: (data: SearchTestItemType) => void;
};
export const useDatasetStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
allDatasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.allDatasets = res;
});
return res;
},
myKbList: [],
async loadKbList(parentId = '') {
const res = await getDatasets({ parentId });
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getDatasetById(id);
set((state) => {
state.kbDetail = data;
});
return data;
},
async updateDataset(data) {
if (get().kbDetail._id === data.id) {
set((state) => {
state.kbDetail = {
...state.kbDetail,
...data
};
});
}
set((state) => {
state.myKbList = state.myKbList = state.myKbList.map((item) =>
item._id === data.id
? {
...item,
...data,
tags: data.tags?.split(' ') || []
}
: item
);
});
await putDatasetById(data);
},
kbTestList: [],
pushKbTestItem(data) {
set((state) => {
state.kbTestList = [data, ...state.kbTestList].slice(0, 500);
});
},
delKbTestItemById(id) {
set((state) => {
state.kbTestList = state.kbTestList.filter((item) => item.id !== id);
});
},
updateKbItemById(data: SearchTestItemType) {
set((state) => {
state.kbTestList = state.kbTestList.map((item) => (item.id === data.id ? data : item));
});
}
})),
{
name: 'kbStore',
partialize: (state) => ({
kbTestList: state.kbTestList
})
}
)
)
);

2
projects/app/src/web/support/api/openapi.ts → projects/app/src/web/support/openapi/api.ts
View File

@@ -1,6 +1,6 @@
import { GET, POST, DELETE } from '@/web/common/api/request';
import type { EditApiKeyProps, GetApiKeyProps } from '@/global/support/api/openapiReq.d';
import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
/**
* crete a api key

2
projects/app/src/web/support/api/outLink.ts → projects/app/src/web/support/outLink/api.ts
View File

@@ -1,6 +1,6 @@
import { GET, POST, DELETE } from '@/web/common/api/request';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import type { OutLinkEditType, OutLinkSchema } from '@fastgpt/support/outLink/type.d';
import type { OutLinkEditType, OutLinkSchema } from '@fastgpt/global/support/outLink/type.d';
/**
* 初始化分享聊天

2
projects/app/src/web/support/api/user.ts → projects/app/src/web/support/user/api.ts
View File

@@ -1,5 +1,5 @@
import { GET, POST, PUT } from '@/web/common/api/request';
import { hashStr } from '@fastgpt/common/tools/str';
import { hashStr } from '@fastgpt/global/common/string/tools';
import type { ResLogin, PromotionRecordType } from '@/global/support/api/userRes.d';
import { UserAuthTypeEnum } from '@/constants/common';
import { UserType, UserUpdateParams } from '@/types/user';

18
projects/app/src/web/support/user/auth.ts Normal file
View File

@@ -0,0 +1,18 @@
import { loginOut } from '@/web/support/user/api';
const tokenKey = 'token';
export const clearToken = () => {
try {
loginOut();
localStorage.removeItem(tokenKey);
} catch (error) {
error;
}
};
export const setToken = (token: string) => {
localStorage.setItem(tokenKey, token);
};
export const getToken = () => {
return localStorage.getItem(tokenKey) || '';
};

6
projects/app/src/web/support/hooks/useSendCode.ts → projects/app/src/web/support/user/hooks/useSendCode.ts
View File

@@ -1,9 +1,9 @@
import { useState, useMemo, useCallback } from 'react';
import { sendAuthCode } from '@/web/support/api/user';
import { sendAuthCode } from '@/web/support/user/api';
import { UserAuthTypeEnum } from '@/constants/common';
import { useToast } from '@/web/common/hooks/useToast';
import { feConfigs } from '@/web/common/store/static';
import { getErrText } from '@/utils/tools';
import { feConfigs } from '@/web/common/system/staticData';
import { getErrText } from '@fastgpt/global/common/error/utils';
let timer: any;

6
projects/app/src/web/support/store/user.ts → projects/app/src/web/support/user/useUserStore.ts
View File

@@ -2,9 +2,9 @@ import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { UserType, UserUpdateParams } from '@/types/user';
import { getMyModels, getModelById, putAppById } from '@/web/core/api/app';
import { formatPrice } from '@fastgpt/common/bill/index';
import { getTokenLogin, putUserInfo } from '@/web/support/api/user';
import { getMyModels, getModelById, putAppById } from '@/web/core/app/api';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { getTokenLogin, putUserInfo } from '@/web/support/user/api';
import { defaultApp } from '@/constants/model';
import { AppListItemType, AppUpdateParams } from '@/types/app';