Dataset folder manager (#274)

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit
This commit is contained in:
Archer
2023-09-10 16:37:32 +08:00
committed by GitHub
parent a1a63260dd
commit 7917766024
83 changed files with 1996 additions and 702 deletions

View File

@@ -1,6 +1,12 @@
import { GET, POST, PUT, DELETE } from '../request';
import type { DatasetItemType, KbItemType, KbListItemType } from '@/types/plugin';
import { RequestPaging } from '@/types/index';
import type {
DatasetItemType,
FileInfo,
KbFileItemType,
KbItemType,
KbListItemType,
KbPathItemType
} from '@/types/plugin';
import { TrainingModeEnum } from '@/constants/plugin';
import {
Props as PushDataProps,
@@ -10,13 +16,17 @@ import {
Props as SearchTestProps,
Response as SearchTestResponse
} from '@/pages/api/openapi/kb/searchTest';
import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById';
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type { KbUpdateParams, CreateKbParams } from '../request/kb';
import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
import { QuoteItemType } from '@/types/chat';
/* knowledge base */
export const getKbList = () => GET<KbListItemType[]>(`/plugins/kb/list`);
export const getKbList = (parentId?: string) =>
GET<KbListItemType[]>(`/plugins/kb/list`, { parentId });
export const getAllDataset = () => GET<KbListItemType[]>(`/plugins/kb/allDataset`);
export const getKbPaths = (parentId?: string) =>
GET<KbPathItemType[]>('/plugins/kb/paths', { parentId });
export const getKbById = (id: string) => GET<KbItemType>(`/plugins/kb/detail?id=${id}`);
@@ -26,25 +36,27 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat
export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
/* kb file */
export const getKbFiles = (data: { kbId: string; searchText: string }) =>
GET<KbFileItemType[]>(`/plugins/kb/file/list`, data);
export const deleteKbFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/plugins/kb/file/delFileByFileId`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/plugins/kb/file/getFileInfo`, { fileId });
export const delEmptyFiles = (kbId: string) =>
DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId });
/* kb data */
type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data);
/**
* 获取导出数据(不分页)
*/
export const getExportDataList = (kbId: string) =>
GET<[string, string, string][]>(
`/plugins/kb/data/exportModelData`,
{ kbId },
{
timeout: 600000
}
);
export const getExportDataList = (data: { kbId: string; fileId: string }) =>
GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
timeout: 600000
});
/**
* 获取模型正在拆分数据的数量

View File

@@ -1,12 +1,23 @@
import { KbTypeEnum } from '@/constants/kb';
import type { RequestPaging } from '@/types';
export type KbUpdateParams = {
id: string;
name: string;
tags: string;
avatar: string;
tags?: string;
name?: string;
avatar?: string;
};
export type CreateKbParams = {
parentId?: string;
name: string;
tags: string[];
avatar: string;
vectorModel: string;
vectorModel?: string;
type: `${KbTypeEnum}`;
};
export type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
fileId: string;
};

View File

@@ -19,7 +19,7 @@ const QuoteModal = ({
rawSearch = [],
onClose
}: {
onUpdateQuote: (quoteId: string, sourceText: string) => Promise<void>;
onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
rawSearch: SearchType[];
onClose: () => void;
}) => {
@@ -129,7 +129,7 @@ const QuoteModal = ({
{editDataItem && (
<InputDataModal
onClose={() => setEditDataItem(undefined)}
onSuccess={() => onUpdateQuote(editDataItem.id, '手动修改')}
onSuccess={() => onUpdateQuote(editDataItem.id)}
onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
kbId={editDataItem.kb_id}
defaultValues={{

View File

@@ -44,7 +44,7 @@ const ResponseTags = ({
};
}, [responseData]);
const updateQuote = useCallback(async (quoteId: string, sourceText: string) => {}, []);
const updateQuote = useCallback(async (quoteId: string, sourceText?: string) => {}, []);
const TagStyles: BoxProps = {
mr: 2,

View File

@@ -12,7 +12,7 @@ import {
import MyModal from '../MyModal';
import { useTranslation } from 'next-i18next';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useToast } from '@/hooks/useToast';
import Avatar from '../Avatar';
import MyIcon from '@/components/Icon';
@@ -29,10 +29,10 @@ const SelectDataset = ({
const theme = useTheme();
const { isPc } = useGlobalStore();
const { toast } = useToast();
const { myKbList, loadKbList } = useUserStore();
const { myKbList, loadKbList } = useDatasetStore();
const [selectedId, setSelectedId] = useState<string>();
useQuery(['loadKbList'], loadKbList);
useQuery(['loadKbList'], () => loadKbList());
return (
<MyModal isOpen={true} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered={!isPc}>

View File

@@ -1,5 +1,5 @@
import React, { useMemo } from 'react';
import { Box, ModalBody, useTheme, ModalHeader, Flex } from '@chakra-ui/react';
import { Box, ModalBody, useTheme, Flex } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@/types/chat';
import { useTranslation } from 'react-i18next';

View File

@@ -55,6 +55,7 @@ const SelectDataset = dynamic(() => import('./SelectDataset'));
const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal'));
import styles from './index.module.scss';
import Script from 'next/script';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
@@ -293,7 +294,7 @@ const ChatBox = (
* user confirm send prompt
*/
const sendPrompt = useCallback(
async (variables: Record<string, any> = {}, inputVal = '') => {
async (variables: Record<string, any> = {}, inputVal = '', history = chatHistory) => {
if (!onStartChat) return;
if (isChatting) {
toast({
@@ -314,7 +315,7 @@ const ChatBox = (
}
const newChatList: ChatSiteItemType[] = [
...chatHistory,
...history,
{
dataId: nanoid(),
obj: 'Human',
@@ -407,6 +408,22 @@ const ChatBox = (
]
);
// retry input
const retryInput = useCallback(
async (index: number) => {
if (!onDelMessage) return;
const delHistory = chatHistory.slice(index);
setChatHistory((state) => (index === 0 ? [] : state.slice(0, index)));
await Promise.all(
delHistory.map((item, i) => onDelMessage({ contentId: item.dataId, index: index + i }))
);
sendPrompt(variables, delHistory[0].value, chatHistory.slice(0, index));
},
[chatHistory, onDelMessage, sendPrompt, variables]
);
// output data
useImperativeHandle(ref, () => ({
getChatHistory: () => chatHistory,
@@ -470,7 +487,7 @@ const ChatBox = (
);
const statusBoxData = useMemo(() => {
const colorMap = {
loading: '#67c13b',
loading: 'myGray.700',
running: '#67c13b',
finish: 'myBlue.600'
};
@@ -484,6 +501,7 @@ const ChatBox = (
};
}, [chatHistory, isChatting, t]);
// page change and abort request
useEffect(() => {
return () => {
controller.current?.abort('leave');
@@ -492,16 +510,7 @@ const ChatBox = (
};
}, [router.query]);
useEffect(() => {
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
// page destroy and abort request
useEffect(() => {
const listen = () => {
cancelBroadcast();
@@ -513,8 +522,22 @@ const ChatBox = (
};
}, []);
// add guide text listener
useEffect(() => {
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
return (
<Flex flexDirection={'column'} h={'100%'}>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
<Box ref={ChatBoxRef} flex={'1 0 0'} h={0} w={'100%'} overflow={'overlay'} px={[4, 0]} pb={3}>
<Box id="chat-container" maxW={['100%', '92%']} h={'100%'} mx={'auto'}>
{showEmpty && <Empty />}
@@ -616,7 +639,7 @@ const ChatBox = (
justifyContent={'flex-end'}
mr={3}
>
<MyTooltip label={'复制'}>
<MyTooltip label={t('common.Copy')}>
<MyIcon
{...controlIconStyle}
name={'copy'}
@@ -624,8 +647,18 @@ const ChatBox = (
onClick={() => onclickCopy(item.value)}
/>
</MyTooltip>
{!!onDelMessage && (
<MyTooltip label={t('chat.retry')}>
<MyIcon
{...controlIconStyle}
name={'retryLight'}
_hover={{ color: 'green.500' }}
onClick={() => retryInput(index)}
/>
</MyTooltip>
)}
{onDelMessage && (
<MyTooltip label={'删除'}>
<MyTooltip label={t('common.Delete')}>
<MyIcon
{...controlIconStyle}
mr={0}

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694331723034"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5978"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M512 70.283C267.486 70.283 69.268 268.046 69.268 512S267.486 953.717 512 953.717 954.732 755.954 954.732 512 756.514 70.283 512 70.283m223.045 488.321H558.603v176.442c0 25.738-20.866 46.604-46.604 46.604s-46.604-20.866-46.604-46.604V558.604H288.953c-25.738 0-46.604-20.866-46.604-46.604s20.866-46.604 46.604-46.604h176.442V288.954c0-25.738 20.866-46.604 46.604-46.604s46.604 20.866 46.604 46.604v176.442h176.442c25.738 0 46.604 20.866 46.604 46.604s-20.866 46.604-46.604 46.604z"
p-id="5979"></path>
</svg>

After

Width:  |  Height:  |  Size: 867 B

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694067364830"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5118"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M727.950222 274.773333l-55.296-9.329777a38.741333 38.741333 0 0 0-12.856889 76.344888l193.308445 32.597334c1.991111 0.113778 1.991111 0.113778 2.844444 0 2.275556 0.227556 4.266667 0.113778 7.850667-0.284445l0.682667-0.056889a28.216889 28.216889 0 0 0 5.632-0.967111c1.080889 0 1.080889 0 3.185777-0.568889a15.530667 15.530667 0 0 0 4.039111-2.332444l1.137778-0.796444 0.796445-0.398223a28.444444 28.444444 0 0 0 4.152889-2.730666 37.091556 37.091556 0 0 0 6.542222-6.826667l0.796444-0.967111c1.080889-1.422222 1.080889-1.422222 2.161778-3.128889a37.432889 37.432889 0 0 0 3.697778-9.557333c0.568889-1.194667 0.568889-1.194667 1.137778-3.128889 0.113778-1.763556 0.113778-1.763556 0-2.503111v0.910222a36.579556 36.579556 0 0 0-0.341334-10.24l-0.113778-0.967111a22.755556 22.755556 0 0 0-0.682666-3.982222c0-1.080889 0-1.080889-0.568889-3.128889l-68.494222-183.751111a38.798222 38.798222 0 0 0-49.777778-22.755556 38.798222 38.798222 0 0 0-22.755556 49.777778l16.270223 43.804444A397.880889 397.880889 0 0 0 512 113.777778C292.408889 113.777778 113.777778 292.408889 113.777778 512s178.631111 398.222222 398.222222 398.222222 398.222222-178.631111 398.222222-398.222222a38.684444 38.684444 0 1 0-77.368889 0c0 176.924444-143.928889 320.853333-320.853333 320.853333S191.146667 688.924444 191.146667 512 335.075556 191.146667 512 191.146667c80.099556 0 157.070222 29.980444 215.950222 83.626666z"
p-id="5119"></path>
</svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

View File

@@ -0,0 +1,3 @@
<svg width="24" height="24" viewBox="0 0 24 24">
<path d="M8.3 5.7a1 1 0 011.4-1.4l7.71 7.7-7.7 7.7a1 1 0 11-1.42-1.4l6.3-6.3-6.3-6.3z" fill-rule="nonzero"></path>
</svg>

After

Width:  |  Height:  |  Size: 174 B

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694224177076"
class="icon" viewBox="0 0 1026 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3984"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64.125" height="64">
<path
d="M989.365124 873.455175c21.85764 24.973422 33.760499 46.831061 35.714294 65.567202 1.948078 18.730424-5.271103 37.076377-21.661831 55.026425-18.736141 21.075836-39.416072 31.030616-62.055515 29.858625-22.633727-1.171991-44.491366-10.344968-65.567202-27.513213L679.093265 806.715982c-35.128298 22.633727-72.786383 40.197876-112.989976 52.68673-40.197876 12.488855-82.545355 18.730424-127.036721 18.730424-60.882095 0-117.863745-11.512672-170.940663-34.536586-53.078347-23.029631-99.523509-54.446147-139.331197-94.252406-39.811976-39.811976-71.228492-86.25285-94.252406-139.329768C11.512672 556.93603 0 499.950092 0 439.066568c0-60.883524 11.512672-117.863745 34.542303-170.940663 23.023914-53.078347 54.44043-99.523509 94.252406-139.331197 39.807688-39.811976 86.25285-71.228492 139.331197-94.252406 53.076918-23.029631 110.058568-34.542303 170.940663-34.542303 60.883524 0 117.869462 11.512672 170.94638 34.542303 53.078347 23.023914 99.517792 54.44043 139.329768 94.252406 39.807688 39.807688 71.222775 86.25285 94.252406 139.331197 23.023914 53.076918 34.536586 110.057139 34.536586 170.940663 0 46.054974-6.633185 89.764536-19.903844 131.134403s-32.002511 79.619664-56.198417 114.742246l38.639985 38.639985c18.730424 18.730424 38.439889 38.249797 59.124108 58.543829s39.61188 39.416072 56.784413 57.371837C973.754771 857.448917 984.680017 868.771497 989.365124 873.455175L989.365124 873.455175zM443.751675 731.779995c40.588063 0 78.83786-7.609369 114.742246-22.829535 35.904385-15.224454 67.13081-36.105911 93.66641-62.641511 26.541317-26.541317 47.422774-57.762025 62.641511-93.66641 15.218737-35.910102 22.835252-74.154183 22.835252-114.747963 0-40.589492-7.615086-78.832143-22.835252-114.742246-15.218737-35.905815-36.100194-67.125093-62.641511-93.667839-26.5356-26.5356-57.762025-47.415628-93.66641-62.641511-35.904385-15.218737-74.154183-22.828106-114.742246-22.828106-40.589492 0-78.83929 7.609369-114.743675 22.828106-35.904385 15.225883-67.129381 36.105911-93.66641 62.641511-26.541317 26.542747-47.422774 57.762025-62.641511 93.667839-15.218737 35.910102-22.829535 74.152753-22.829535 114.742246 0 40.59378 7.610798 78.83786 22.829535 114.747963 15.218737 35.904385 36.100194 67.125093 62.641511 93.66641 26.53703 26.5356 57.762025 47.417057 93.66641 62.641511C364.912385 724.170627 403.162183 731.779995 443.751675 731.779995L443.751675 731.779995zM443.751675 731.779995"
p-id="3985"></path>
</svg>

After

Width:  |  Height:  |  Size: 2.7 KiB

View File

@@ -79,7 +79,11 @@ const map = {
promotionLight: require('./icons/light/promotion.svg').default,
logsLight: require('./icons/light/logs.svg').default,
badLight: require('./icons/light/bad.svg').default,
markLight: require('./icons/light/mark.svg').default
markLight: require('./icons/light/mark.svg').default,
retryLight: require('./icons/light/retry.svg').default,
rightArrowLight: require('./icons/light/rightArrow.svg').default,
searchLight: require('./icons/light/search.svg').default,
plusFill: require('./icons/fill/plus.svg').default
};
export type IconName = keyof typeof map;

View File

@@ -0,0 +1,28 @@
import React from 'react';
import { Flex, Input, InputProps } from '@chakra-ui/react';
interface Props extends InputProps {
leftIcon?: React.ReactNode;
}
const MyInput = ({ leftIcon, ...props }: Props) => {
return (
<Flex position={'relative'} alignItems={'center'}>
<Input w={'100%'} pl={leftIcon ? '30px' : 3} {...props} />
{leftIcon && (
<Flex
alignItems={'center'}
position={'absolute'}
left={3}
w={'20px'}
zIndex={10}
transform={'translateY(1.5px)'}
>
{leftIcon}
</Flex>
)}
</Flex>
);
};
export default MyInput;

View File

@@ -0,0 +1,51 @@
import React from 'react';
import { Menu, MenuList, MenuItem } from '@chakra-ui/react';
interface Props {
width: number;
offset?: [number, number];
Button: React.ReactNode;
menuList: {
isActive?: boolean;
child: React.ReactNode;
onClick: () => void;
}[];
}
const MyMenu = ({ width, offset = [0, 10], Button, menuList }: Props) => {
const menuItemStyles = {
borderRadius: 'sm',
py: 3,
display: 'flex',
alignItems: 'center',
_hover: {
backgroundColor: 'myWhite.600',
color: 'hover.blue'
}
};
return (
<Menu offset={offset} autoSelect={false} isLazy>
{Button}
<MenuList
minW={`${width}px !important`}
p={'6px'}
border={'1px solid #fff'}
boxShadow={'0px 2px 4px rgba(161, 167, 179, 0.25), 0px 0px 1px rgba(121, 141, 159, 0.25);'}
>
{menuList.map((item, i) => (
<MenuItem
key={i}
{...menuItemStyles}
onClick={item.onClick}
color={item.isActive ? 'hover.blue' : ''}
>
{item.child}
</MenuItem>
))}
</MenuList>
</Menu>
);
};
export default MyMenu;

View File

@@ -10,7 +10,8 @@ export const fileImgs = [
{ suffix: 'csv', src: '/imgs/files/csv.svg' },
{ suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
{ suffix: 'txt', src: '/imgs/files/txt.svg' },
{ suffix: 'md', src: '/imgs/files/markdown.svg' }
{ suffix: 'md', src: '/imgs/files/markdown.svg' },
{ suffix: '.', src: '/imgs/files/file.svg' }
];
export enum TrackEventName {

View File

@@ -14,3 +14,24 @@ export const defaultKbDetail: KbItemType = {
maxToken: 3000
}
};
export enum KbTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export const KbTypeMap = {
[KbTypeEnum.folder]: {
name: 'folder'
},
[KbTypeEnum.dataset]: {
name: 'dataset'
}
};
export const FolderAvatarSrc = '/imgs/files/folder.svg';
export const OtherFileId = 'other';

View File

@@ -1,4 +1,4 @@
import { useCallback, useRef } from 'react';
import { useCallback, useRef, useState } from 'react';
import {
AlertDialog,
AlertDialogBody,
@@ -11,21 +11,25 @@ import {
} from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
export const useConfirm = (props: { title?: string; content: string }) => {
export const useConfirm = (props: { title?: string | null; content?: string | null }) => {
const { t } = useTranslation();
const { title = t('Warning'), content } = props;
const [customContent, setCustomContent] = useState(content);
const { isOpen, onOpen, onClose } = useDisclosure();
const cancelRef = useRef(null);
const confirmCb = useRef<any>();
const cancelCb = useRef<any>();
return {
openConfirm: useCallback(
(confirm?: any, cancel?: any) => {
(confirm?: any, cancel?: any, customContent?: string) => {
confirmCb.current = confirm;
cancelCb.current = cancel;
customContent && setCustomContent(customContent);
return onOpen;
},
[onOpen]
@@ -44,7 +48,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
{title}
</AlertDialogHeader>
<AlertDialogBody>{content}</AlertDialogBody>
<AlertDialogBody>{customContent}</AlertDialogBody>
<AlertDialogFooter>
<Button
@@ -70,7 +74,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
</AlertDialogOverlay>
</AlertDialog>
),
[content, isOpen, onClose, title]
[customContent, isOpen, onClose, t, title]
)
};
};

View File

@@ -1,4 +1,4 @@
import React, { useCallback, useRef } from 'react';
import React, { useCallback, useRef, useState } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';

View File

@@ -41,16 +41,14 @@ function App({ Component, pageProps }: AppProps) {
const { setLastRoute } = useGlobalStore();
const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
const [googleClientVerKey, setGoogleVerKey] = useState<string>();
useEffect(() => {
// get init data
(async () => {
const {
feConfigs: { scripts, googleClientVerKey }
feConfigs: { scripts }
} = await clientInitData();
setScripts(scripts || []);
setGoogleVerKey(googleClientVerKey);
})();
// add window error track
window.onerror = function (msg, url) {
@@ -94,20 +92,10 @@ function App({ Component, pageProps }: AppProps) {
/>
<link rel="icon" href="/favicon.ico" />
</Head>
<Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
{scripts?.map((item, i) => (
<Script key={i} strategy="lazyOnload" {...item}></Script>
))}
{googleClientVerKey && (
<>
<Script
src={`https://www.recaptcha.net/recaptcha/api.js?render=${googleClientVerKey}`}
strategy="lazyOnload"
></Script>
</>
)}
<QueryClientProvider client={queryClient}>
<ChakraProvider theme={theme}>
<ColorModeScript initialColorMode={theme.config.initialColorMode} />

View File

@@ -25,7 +25,7 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
const { mutate: onSubmit, isLoading } = useRequest({
mutationFn: (data: FormType) => {
if (data.newPsw !== data.confirmPsw) {
return Promise.reject(t('commom.Password inconsistency'));
return Promise.reject(t('common.Password inconsistency'));
}
return updatePasswordByOld(data);
},

View File

@@ -13,6 +13,7 @@ import UserInfo from './components/Info';
import { serviceSideProps } from '@/utils/i18n';
import { feConfigs } from '@/store/static';
import { useTranslation } from 'react-i18next';
import Script from 'next/script';
const Promotion = dynamic(() => import('./components/Promotion'));
const BillTable = dynamic(() => import('./components/BillTable'));
@@ -97,51 +98,54 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
);
return (
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Box>
)}
<>
<Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Box>
)}
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.info && <UserInfo />}
{currentTab === TabEnum.promotion && <Promotion />}
{currentTab === TabEnum.bill && <BillTable />}
{currentTab === TabEnum.pay && <PayRecordTable />}
{currentTab === TabEnum.inform && <InformTable />}
</Box>
</Flex>
<ConfirmModal />
</PageContainer>
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.info && <UserInfo />}
{currentTab === TabEnum.promotion && <Promotion />}
{currentTab === TabEnum.bill && <BillTable />}
{currentTab === TabEnum.pay && <PayRecordTable />}
{currentTab === TabEnum.inform && <InformTable />}
</Box>
</Flex>
<ConfirmModal />
</PageContainer>
</>
);
};

View File

@@ -0,0 +1,32 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum, KbTypeMap } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
await KB.updateMany(
{
type: { $exists: false }
},
{
$set: {
type: KbTypeEnum.dataset,
parentId: null
}
}
);
jsonRes(res, {});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

View File

@@ -88,7 +88,7 @@ export async function pushDataToKb({
]);
const modeMaxToken = {
[TrainingModeEnum.index]: vectorModel.maxToken,
[TrainingModeEnum.index]: vectorModel.maxToken * 1.5,
[TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8
};
@@ -146,7 +146,6 @@ export async function pushDataToKb({
}
} catch (error) {
console.log(error);
error;
}
return Promise.resolve(data);
})

View File

@@ -50,7 +50,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await PgClient.update(PgTrainingTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]],
values: [
{ key: 'source', value: '手动修改' },
{ key: 'a', value: a.replace(/'/g, '"') },
...(q
? [

View File

@@ -69,7 +69,7 @@ export async function getVector({
.then(async (res) => {
if (!res.data?.data?.[0]?.embedding) {
// @ts-ignore
return Promise.reject(res.data?.error?.message || 'Embedding API Error');
return Promise.reject(res.data?.err?.message || 'Embedding API Error');
}
return {
tokenLen: res.data.usage.total_tokens || 0,

View File

@@ -0,0 +1,34 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find({
userId,
type: 'dataset'
});
const data = kbList.map((item) => ({
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
}));
jsonRes<KbListItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -6,11 +6,7 @@ import type { CreateKbParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { name, tags, avatar, vectorModel } = req.body as CreateKbParams;
if (!name || !vectorModel) {
throw new Error('缺少参数');
}
const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateKbParams;
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
@@ -22,7 +18,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId,
tags,
vectorModel,
avatar
avatar,
parentId: parentId || null,
type
});
jsonRes(res, { data: _id });

View File

@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { kbId } = req.query as {
let { kbId, fileId } = req.query as {
kbId: string;
fileId: string;
};
if (!kbId) {
@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000
);
// auth export times
const authTimes = await User.findOne(
@@ -35,21 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
);
if (!authTimes) {
throw new Error('上次导出未到半小时,每半小时仅可导出一次。');
const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`;
throw new Error(`上次导出未到 ${minutes},每 ${minutes}仅可导出一次。`);
}
// 统计数据
const count = await PgClient.count(PgTrainingTableName, {
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
const where: any = [['kb_id', kbId], 'AND', ['user_id', userId]];
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgTrainingTableName,
{
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
where,
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count
limit: 1000000
}
);
@@ -78,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export const config = {
api: {
bodyParser: {
sizeLimit: '100mb'
sizeLimit: '200mb'
}
}
};

View File

@@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -12,12 +13,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
kbId,
pageNum = 1,
pageSize = 10,
searchText = ''
searchText = '',
fileId = ''
} = req.body as {
kbId: string;
pageNum: number;
pageSize: number;
searchText: string;
fileId: string;
};
if (!kbId) {
throw new Error('缺少参数');
@@ -33,6 +36,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
['user_id', userId],
'AND',
['kb_id', kbId],
...(fileId
? fileId === OtherFileId
? ["AND (file_id IS NULL OR file_id = '')"]
: ['AND', ['file_id', fileId]]
: []),
...(searchText
? [
'AND',

View File

@@ -3,12 +3,12 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { Types } from 'mongoose';
import { PgTrainingTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
id: string;
};
@@ -20,26 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const deletedIds = [id, ...(await findAllChildrenIds(id))];
// delete training data
await TrainingData.deleteMany({
userId,
kbId: id
kbId: { $in: deletedIds }
});
// delete all pg data
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', id]]
where: [
['user_id', userId],
'AND',
`kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
]
});
// delete related files
const gridFs = new GridFSStorage('dataset', userId);
await gridFs.deleteFilesByKbId(id);
await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
// delete kb data
await KB.findOneAndDelete({
_id: id,
await KB.deleteMany({
_id: { $in: deletedIds },
userId
});
@@ -51,3 +55,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
async function findAllChildrenIds(id: string) {
// find children
const children = await KB.find({ parentId: id });
let allChildrenIds = children.map((child) => String(child._id));
for (const child of children) {
const grandChildrenIds = await findAllChildrenIds(child._id);
allChildrenIds = allChildrenIds.concat(grandChildrenIds);
}
return allChildrenIds;
}

View File

@@ -0,0 +1,55 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId, kbId } = req.query as { fileId: string; kbId: string };
if (!fileId || !kbId) {
throw new Error('fileId and kbId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
await PgClient.delete(PgTrainingTableName, {
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
});
} else {
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
await gridFs.findAndAuthFile(fileId);
// delete all pg data
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
});
// delete file
await bucket.delete(new Types.ObjectId(fileId));
}
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -0,0 +1,59 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId } = req.query as { kbId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
// 1 hours expired
.find({
uploadDate: { $lte: new Date(Date.now() - 60 * 1000) },
['metadata.kbId']: kbId,
['metadata.userId']: userId
})
.sort({ _id: -1 })
.toArray();
const data = await Promise.all(
files.map(async (file) => {
return {
id: file._id,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
);
await Promise.all(
data
.filter((item) => item.chunkLength === 0)
.map((file) => bucket.delete(new Types.ObjectId(file.id)))
);
jsonRes(res);
} catch (err) {
jsonRes(res);
}
}

View File

@@ -0,0 +1,43 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { OtherFileId } from '@/constants/kb';
import type { FileInfo } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { kbId: string; fileId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
return jsonRes<FileInfo>(res, {
data: {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadDate: new Date(),
encoding: '',
contentType: ''
}
});
}
const gridFs = new GridFSStorage('dataset', userId);
const file = await gridFs.findAndAuthFile(fileId);
jsonRes<FileInfo>(res, {
data: file
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -0,0 +1,84 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { KbFileItemType } from '@/types/plugin';
import { FileStatusEnum, OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { kbId, searchText } = req.query as { kbId: string; searchText: string };
searchText = searchText.replace(/'/g, '');
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
.find({ ['metadata.kbId']: kbId, ...(searchText && { filename: { $regex: searchText } }) })
.sort({ _id: -1 })
.toArray();
async function GetOtherData() {
return {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadTime: new Date(),
status: (await TrainingData.findOne({ userId, kbId, file_id: '' }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
})
};
}
const data = await Promise.all([
GetOtherData(),
...files.map(async (file) => {
return {
id: String(file._id),
size: file.length,
filename: file.filename,
uploadTime: file.uploadDate,
status: (await TrainingData.findOne({ userId, kbId, file_id: file._id }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
]);
jsonRes<KbFileItemType[]>(res, {
data: data.flat().filter((item) => item.chunkLength > 0)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -2,29 +2,25 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { KbListItemType } from '@/types/plugin';
import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { parentId } = req.query as { parentId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find(
{
userId
},
'_id avatar name tags vectorModel'
).sort({ updateTime: -1 });
const kbList = await KB.find({
userId,
parentId: parentId || null
}).sort({ updateTime: -1 });
const data = await Promise.all(
kbList.map(async (item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
tags: item.tags,
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
}))
);

View File

@@ -0,0 +1,36 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbPathItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId } = req.query as { parentId: string };
jsonRes<KbPathItemType[]>(res, {
data: await getParents(parentId)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
async function getParents(parentId?: string): Promise<KbPathItemType[]> {
if (!parentId) {
return [];
}
const parent = await KB.findById(parentId, 'name parentId');
if (!parent) return [];
const paths = await getParents(parent.parentId);
paths.push({ parentId, parentName: parent.name });
return paths;
}

View File

@@ -6,7 +6,7 @@ import type { KbUpdateParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id, name, tags, avatar } = req.body as KbUpdateParams;
const { id, name, avatar, tags = '' } = req.body as KbUpdateParams;
if (!id || !name) {
throw new Error('缺少参数');
@@ -23,8 +23,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId
},
{
avatar,
name,
...(name && { name }),
...(avatar && { avatar }),
tags: tags.split(' ').filter((item) => item)
}
);

View File

@@ -1,4 +1,4 @@
import type { FeConfigsType } from '@/types';
import type { FeConfigsType, SystemEnvType } from '@/types';
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { readFileSync } from 'fs';
@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
const defaultSystemEnv = {
const defaultSystemEnv: SystemEnvType = {
vectorMaxProcess: 15,
qaMaxProcess: 15,
pgIvfflatProbe: 20
};
const defaultFeConfigs = {
const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true,
show_register: false,
show_appStore: false,
@@ -44,7 +44,7 @@ const defaultFeConfigs = {
show_doc: true,
systemTitle: 'FastGPT',
authorText: 'Made by FastGPT Team.',
gitLoginKey: '',
exportLimitMinutes: 0,
scripts: []
};
const defaultChatModels = [
@@ -99,8 +99,10 @@ export async function getInitConfig() {
const res = JSON.parse(readFileSync(filename, 'utf-8'));
console.log(res);
global.systemEnv = res.SystemParams || defaultSystemEnv;
global.feConfigs = res.FeConfig || defaultFeConfigs;
global.systemEnv = res.SystemParams
? { ...defaultSystemEnv, ...res.SystemParams }
: defaultSystemEnv;
global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs;
global.chatModels = res.ChatModels || defaultChatModels;
global.qaModel = res.QAModel || defaultQAModel;
global.vectorModels = res.VectorModels || defaultVectorModels;

View File

@@ -1,8 +1,8 @@
import React, { useMemo, useState } from 'react';
import React, { useMemo } from 'react';
import { NodeProps } from 'reactflow';
import { FlowModuleItemType } from '@/types/flow';
import { Flex, Box, Button, useTheme, useDisclosure, Grid } from '@chakra-ui/react';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useQuery } from '@tanstack/react-query';
import NodeCard from '../modules/NodeCard';
import Divider from '../modules/Divider';
@@ -21,7 +21,7 @@ const KBSelect = ({
onChange: (e: SelectedKbType) => void;
}) => {
const theme = useTheme();
const { myKbList, loadKbList } = useUserStore();
const { datasets, loadAllDatasets } = useDatasetStore();
const {
isOpen: isOpenKbSelect,
onOpen: onOpenKbSelect,
@@ -29,11 +29,11 @@ const KBSelect = ({
} = useDisclosure();
const showKbList = useMemo(
() => myKbList.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)),
[myKbList, activeKbs]
() => datasets.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)),
[datasets, activeKbs]
);
useQuery(['initkb'], loadKbList);
useQuery(['loadAllDatasets'], loadAllDatasets);
return (
<>
@@ -58,12 +58,7 @@ const KBSelect = ({
))}
</Grid>
{isOpenKbSelect && (
<KBSelectModal
kbList={myKbList}
activeKbs={activeKbs}
onChange={onChange}
onClose={onCloseKbSelect}
/>
<KBSelectModal activeKbs={activeKbs} onChange={onChange} onClose={onCloseKbSelect} />
)}
</>
);

View File

@@ -56,18 +56,21 @@ import MyIcon from '@/components/Icon';
import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox';
import { addVariable } from '../VariableEditModal';
import { KBSelectModal, KbParamsModal } from '../KBSelectModal';
import { KbParamsModal } from '../KBSelectModal';
import { AppTypeEnum } from '@/constants/app';
import { useDatasetStore } from '@/store/dataset';
const VariableEditModal = dynamic(() => import('../VariableEditModal'));
const InfoModal = dynamic(() => import('../InfoModal'));
const KBSelectModal = dynamic(() => import('../KBSelectModal'));
const Settings = ({ appId }: { appId: string }) => {
const theme = useTheme();
const router = useRouter();
const { t } = useTranslation();
const { toast } = useToast();
const { appDetail, updateAppDetail, loadKbList, myKbList } = useUserStore();
const { appDetail, updateAppDetail } = useUserStore();
const { loadAllDatasets, datasets } = useDatasetStore();
const { isPc } = useGlobalStore();
const [editVariable, setEditVariable] = useState<VariableItemType>();
@@ -122,8 +125,8 @@ const Settings = ({ appId }: { appId: string }) => {
);
}, [getValues, refresh]);
const selectedKbList = useMemo(
() => myKbList.filter((item) => kbList.find((kb) => kb.kbId === item._id)),
[myKbList, kbList]
() => datasets.filter((item) => kbList.find((kb) => kb.kbId === item._id)),
[datasets, kbList]
);
/* 点击删除 */
@@ -167,7 +170,7 @@ const Settings = ({ appId }: { appId: string }) => {
appModule2Form();
}, [appModule2Form]);
useQuery(['initkb', appId], () => loadKbList());
useQuery(['loadAllDatasets'], loadAllDatasets);
const BoxStyles: BoxProps = {
bg: 'myWhite.200',
@@ -304,6 +307,23 @@ const Settings = ({ appId }: { appId: string }) => {
</Button>
</Flex>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}></Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
{/* variable */}
<Box mt={2} {...BoxStyles}>
<Flex alignItems={'center'}>
@@ -498,24 +518,6 @@ const Settings = ({ appId }: { appId: string }) => {
</Grid>
</Box>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}></Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
<ConfirmSaveModal />
<ConfirmDelModal />
{settingAppInfo && (
@@ -548,7 +550,6 @@ const Settings = ({ appId }: { appId: string }) => {
)}
{isOpenKbSelect && (
<KBSelectModal
kbList={myKbList}
activeKbs={selectedKbList.map((item) => ({
kbId: item._id,
vectorModel: item.vectorModel

View File

@@ -1,4 +1,4 @@
import React, { useState } from 'react';
import React, { useMemo, useState } from 'react';
import {
Card,
Flex,
@@ -8,10 +8,12 @@ import {
ModalHeader,
ModalFooter,
useTheme,
Textarea
Textarea,
Grid,
Divider
} from '@chakra-ui/react';
import { getKbPaths } from '@/api/plugins/kb';
import Avatar from '@/components/Avatar';
import { KbListItemType } from '@/types/plugin';
import { useForm } from 'react-hook-form';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import type { SelectedKbType } from '@/types/plugin';
@@ -21,6 +23,10 @@ import MySlider from '@/components/Slider';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/kb';
import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query';
import { useDatasetStore } from '@/store/dataset';
export type KbParamsType = {
searchSimilarity: number;
@@ -29,20 +35,42 @@ export type KbParamsType = {
};
export const KBSelectModal = ({
kbList,
activeKbs = [],
onChange,
onClose
}: {
kbList: KbListItemType[];
activeKbs: SelectedKbType;
onChange: (e: SelectedKbType) => void;
onClose: () => void;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const [selectedKbList, setSelectedKbList] = useState<SelectedKbType>(activeKbs);
const { isPc } = useGlobalStore();
const { toast } = useToast();
const [parentId, setParentId] = useState<string>();
const { myKbList, loadKbList, datasets, loadAllDatasets } = useDatasetStore();
const { data } = useQuery(['loadKbList', parentId], () => {
return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
});
useQuery(['loadAllDatasets'], loadAllDatasets);
const paths = useMemo(
() => [
{
parentId: '',
parentName: t('kb.My Dataset')
},
...(data?.[1] || [])
],
[data, t]
);
const filterKbList = useMemo(() => {
return {
selected: datasets.filter((item) => selectedKbList.find((kb) => kb.kbId === item._id)),
unSelected: myKbList.filter((item) => !selectedKbList.find((kb) => kb.kbId === item._id))
};
}, [myKbList, datasets, selectedKbList]);
return (
<MyModal
@@ -54,10 +82,43 @@ export const KBSelectModal = ({
>
<Flex flexDirection={'column'} h={['90vh', 'auto']}>
<ModalHeader>
<Box>({selectedKbList.length})</Box>
<Box fontSize={'sm'} color={'myGray.500'} fontWeight={'normal'}>
</Box>
{!!parentId ? (
<Flex flex={1}>
{paths.map((item, i) => (
<Flex key={item.parentId} mr={2} alignItems={'center'}>
<Box
fontSize={'lg'}
borderRadius={'md'}
{...(i === paths.length - 1
? {
cursor: 'default'
}
: {
cursor: 'pointer',
_hover: {
color: 'myBlue.600'
},
onClick: () => {
setParentId(item.parentId);
}
})}
>
{item.parentName}
</Box>
{i !== paths.length - 1 && (
<MyIcon name={'rightArrowLight'} color={'myGray.500'} />
)}
</Flex>
))}
</Flex>
) : (
<Box>({selectedKbList.length})</Box>
)}
{isPc && (
<Box fontSize={'sm'} color={'myGray.500'} fontWeight={'normal'}>
</Box>
)}
</ModalHeader>
<ModalBody
@@ -65,72 +126,132 @@ export const KBSelectModal = ({
maxH={'80vh'}
overflowY={'auto'}
display={'grid'}
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={3}
userSelect={'none'}
>
{kbList.map((item) =>
(() => {
const selected = !!selectedKbList.find((kb) => kb.kbId === item._id);
const active = !!activeKbs.find((kb) => kb.kbId === item._id);
return (
<Card
key={item._id}
p={3}
border={theme.borders.base}
boxShadow={'sm'}
h={'80px'}
cursor={'pointer'}
order={active ? 0 : 1}
_hover={{
boxShadow: 'md'
}}
{...(selected
? {
bg: 'myBlue.300'
}
: {})}
onClick={() => {
if (selected) {
setSelectedKbList((state) => state.filter((kb) => kb.kbId !== item._id));
} else {
const vectorModel = selectedKbList[0]?.vectorModel?.model;
<Grid
h={'auto'}
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={3}
>
{filterKbList.selected.map((item) =>
(() => {
return (
<Card
key={item._id}
p={3}
border={theme.borders.base}
boxShadow={'sm'}
bg={'myBlue.300'}
>
<Flex alignItems={'center'} h={'38px'}>
<Avatar src={item.avatar} w={['24px', '28px']}></Avatar>
<Box flex={'1 0 0'} mx={3}>
{item.name}
</Box>
<MyIcon
name={'delete'}
w={'14px'}
cursor={'pointer'}
_hover={{ color: 'red.500' }}
onClick={() => {
setSelectedKbList((state) => state.filter((kb) => kb.kbId !== item._id));
}}
/>
</Flex>
</Card>
);
})()
)}
</Grid>
if (vectorModel && vectorModel !== item.vectorModel.model) {
return toast({
status: 'warning',
title: '仅能选择同一个索引模型的知识库'
});
}
setSelectedKbList((state) => [
...state,
{ kbId: item._id, vectorModel: item.vectorModel }
]);
{filterKbList.selected.length > 0 && <Divider my={3} />}
<Grid
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={3}
>
{filterKbList.unSelected.map((item) =>
(() => {
return (
<MyTooltip
key={item._id}
label={
item.type === KbTypeEnum.dataset
? t('kb.Select Dataset')
: t('kb.Select Folder')
}
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Avatar src={item.avatar} w={['24px', '28px', '32px']}></Avatar>
<Box ml={3} fontWeight={'bold'} fontSize={['md', 'lg', 'xl']}>
{item.name}
</Box>
</Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{item.vectorModel.name}</Box>
</Flex>
</Card>
);
})()
>
<Card
p={3}
border={theme.borders.base}
boxShadow={'sm'}
h={'80px'}
cursor={'pointer'}
_hover={{
boxShadow: 'md'
}}
onClick={() => {
if (item.type === KbTypeEnum.folder) {
setParentId(item._id);
} else if (item.type === KbTypeEnum.dataset) {
const vectorModel = selectedKbList[0]?.vectorModel?.model;
if (vectorModel && vectorModel !== item.vectorModel.model) {
return toast({
status: 'warning',
title: '仅能选择同一个索引模型的知识库'
});
}
setSelectedKbList((state) => [
...state,
{ kbId: item._id, vectorModel: item.vectorModel }
]);
}
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Avatar src={item.avatar} w={['24px', '28px']}></Avatar>
<Box
className="textEllipsis"
ml={3}
fontWeight={'bold'}
fontSize={['md', 'lg', 'xl']}
>
{item.name}
</Box>
</Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{item.type === KbTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{item.vectorModel.name}</Box>
</>
)}
</Flex>
</Card>
</MyTooltip>
);
})()
)}
</Grid>
{filterKbList.unSelected.length === 0 && (
<Flex mt={5} flexDirection={'column'} alignItems={'center'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
西~
</Box>
</Flex>
)}
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
// filter out the kb that is not in the kbList
// filter out the kb that is not in the kList
const filterKbList = selectedKbList.filter((kb) => {
return kbList.find((item) => item._id === kb.kbId);
return datasets.find((item) => item._id === kb.kbId);
});
onClose();

View File

@@ -1,12 +1,13 @@
import React, { useCallback, useState, useRef } from 'react';
import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react';
import React, { useCallback, useState, useRef, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react';
import type { KbDataItemType } from '@/types/plugin';
import { usePagination } from '@/hooks/usePagination';
import {
getKbDataList,
getExportDataList,
delOneKbDataByDataId,
getTrainingData
getTrainingData,
getFileInfoById
} from '@/api/plugins/kb';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { fileDownload } from '@/utils/file';
@@ -18,12 +19,19 @@ import { debounce } from 'lodash';
import { getErrText } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@/components/Icon';
import MyTooltip from '@/components/MyTooltip';
import MyInput from '@/components/MyInput';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { feConfigs } from '@/store/static';
const DataCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { fileId = '' } = router.query as { fileId: string };
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
@@ -45,7 +53,8 @@ const DataCard = ({ kbId }: { kbId: string }) => {
pageSize: 24,
params: {
kbId,
searchText
searchText,
fileId
},
onChange() {
if (BoxRef.current) {
@@ -72,33 +81,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
[getData, pageNum, refetchTrainingData]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(kbId),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'data.csv'
});
toast({
title: '导出成功,下次导出需要半小时后',
status: 'success'
});
},
onError(err: any) {
toast({
title: getErrText(err, '导出异常'),
status: 'error'
});
console.log(err);
}
});
// get first page data
const getFirstData = useCallback(
debounce(() => {
getData(1);
@@ -113,57 +96,100 @@ const DataCard = ({ kbId }: { kbId: string }) => {
enabled: qaListLen > 0 || vectorListLen > 0
});
// get file info
const { data: fileInfo } = useQuery(['getFileInfo', fileId], () => getFileInfoById(fileId));
const fileIcon = useMemo(
() =>
fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(fileInfo?.filename || ''))?.src,
[fileInfo?.filename]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
mutationFn: () => getExportDataList({ kbId, fileId }),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
const filenameSplit = fileInfo?.filename?.split('.') || [];
const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
fileDownload({
text,
type: 'text/csv',
filename
});
},
successToast: `导出成功,下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`,
errorToast: '导出异常'
});
return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
: {total}
</Box>
<Flex alignItems={'center'}>
<Flex
className="textEllipsis"
flex={'1 0 0'}
mr={[3, 5]}
fontSize={['sm', 'md']}
alignItems={'center'}
>
<Image src={fileIcon} w={'16px'} mr={2} alt={''} />
{t(fileInfo?.filename || 'Filename')}
</Flex>
<Button
mr={2}
size={['sm', 'md']}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={`${feConfigs} 分钟能导出 1 次`}
onClick={onclickExport}
>
{t('dataset.Export')}
</Button>
<Box>
<MyTooltip label={'刷新'}>
<IconButton
icon={<RepeatIcon />}
size={['sm', 'md']}
aria-label={'refresh'}
variant={'base'}
isLoading={isLoading}
mr={[2, 4]}
size={'sm'}
onClick={() => {
getData(pageNum);
getTrainingData({ kbId, init: true });
}}
/>
</MyTooltip>
<Button
mr={2}
size={'sm'}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={'半小时仅能导出1次'}
onClick={() => onclickExport()}
>
</Button>
</Box>
</Flex>
<Flex my={4}>
{qaListLen > 0 || vectorListLen > 0 ? (
<Box fontSize={'xs'}>
{qaListLen > 0 ? `${qaListLen}条数据正在拆分,` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引,` : ''}
...
<Flex my={3} alignItems={'center'}>
<Box>
<Box as={'span'} fontSize={['md', 'lg']}>
{total}
</Box>
) : (
<Box fontSize={'xs'}>~</Box>
)}
<Box as={'span'}>
{(qaListLen > 0 || vectorListLen > 0) && (
<>
({qaListLen > 0 ? `${qaListLen}条数据正在拆分,` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引,` : ''}
... )
</>
)}
</Box>
</Box>
<Box flex={1} mr={1} />
<Input
maxW={['60%', '300px']}
size={'sm'}
value={searchText}
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['200px', '300px']}
placeholder="根据匹配知识,预期答案和来源进行搜索"
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
@@ -217,7 +243,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
</Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
{item.source?.trim()}
ID:{item.id}
</Box>
<IconButton
className="delete"

View File

@@ -0,0 +1,208 @@
import React, { useCallback, useState, useRef, useMemo } from 'react';
import {
Box,
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Image
} from '@chakra-ui/react';
import { getKbFiles, deleteKbFileById } from '@/api/plugins/kb';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { debounce } from 'lodash';
import { formatFileSize } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import MyIcon from '@/components/Icon';
import MyInput from '@/components/MyInput';
import dayjs from 'dayjs';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { useLoading } from '@/hooks/useLoading';
import { FileStatusEnum } from '@/constants/kb';
import { useRouter } from 'next/router';
const FileCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { Loading } = useLoading();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('kb.Confirm to delete the file')
});
const {
data: files = [],
refetch,
isInitialLoading
} = useQuery(['getFiles', kbId], () => getKbFiles({ kbId, searchText }), {
refetchInterval: 6000,
refetchOnWindowFocus: true
});
const debounceRefetch = useCallback(
debounce(() => {
refetch();
lastSearch.current = searchText;
}, 300),
[]
);
const formatFiles = useMemo(
() =>
files.map((file) => ({
...file,
icon: fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.filename))?.src
})),
[files]
);
const totalDataLength = useMemo(
() => files.reduce((sum, item) => sum + item.chunkLength, 0),
[files]
);
const { mutate: onDeleteFile, isLoading } = useRequest({
mutationFn: (fileId: string) =>
deleteKbFileById({
fileId,
kbId
}),
onSuccess() {
refetch();
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const statusMap = {
[FileStatusEnum.embedding]: {
color: 'myGray.500',
text: t('file.Embedding')
},
[FileStatusEnum.ready]: {
color: 'green.500',
text: t('file.Ready')
}
};
return (
<Box ref={BoxRef} position={'relative'} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'} px={5}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
{t('kb.Files', { total: files.length })}
</Box>
<Flex alignItems={'center'}>
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['100%', '200px']}
placeholder={t('common.Search') || ''}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
debounceRefetch();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
refetch();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
refetch();
}
}}
/>
</Flex>
</Flex>
<TableContainer mt={[0, 3]}>
<Table variant={'simple'} fontSize={'sm'}>
<Thead>
<Tr>
<Th>{t('kb.Filename')}</Th>
<Th>
{t('kb.Chunk Length')}({totalDataLength})
</Th>
<Th>{t('kb.Upload Time')}</Th>
<Th>{t('kb.File Size')}</Th>
<Th>{t('common.Status')}</Th>
<Th />
</Tr>
</Thead>
<Tbody>
{formatFiles.map((file) => (
<Tr
key={file.id}
_hover={{ bg: 'myWhite.600' }}
cursor={'pointer'}
title={'点击查看数据详情'}
onClick={() =>
router.push({
query: {
kbId,
fileId: file.id,
currentTab: 'dataCard'
}
})
}
>
<Td>
<Flex alignItems={'center'}>
<Image src={file.icon} w={'16px'} mr={2} alt={''} />
<Box maxW={['300px', '400px']} className="textEllipsis">
{t(file.filename)}
</Box>
</Flex>
</Td>
<Td fontSize={'md'} fontWeight={'bold'}>
{file.chunkLength}
</Td>
<Td>{dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>{formatFileSize(file.size)}</Td>
<Td
display={'flex'}
alignItems={'center'}
_before={{
content: '""',
w: '10px',
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: statusMap[file.status].color
}}
>
{statusMap[file.status].text}
</Td>
<Td onClick={(e) => e.stopPropagation()}>
<MyIcon
name={'delete'}
w={'14px'}
_hover={{ color: 'red.600' }}
onClick={() =>
openConfirm(() => {
onDeleteFile(file.id);
})()
}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<ConfirmModal />
<Loading loading={isInitialLoading || isLoading} />
</Box>
);
};
export default React.memo(FileCard);

View File

@@ -26,12 +26,12 @@ import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
const ChunkImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const vectorModel = kbDetail.vectorModel;
const unitPrice = vectorModel?.price || 0.2;
@@ -86,7 +86,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},
@@ -106,12 +106,15 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
text: file.text,
maxLen: chunkLen
});
return {
...file,
tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({
...chunk,
q: splitRes.chunks[i]
chunks: splitRes.chunks.map((chunk) => ({
a: '',
source: file.filename,
file_id: file.id,
q: chunk
}))
};
})

View File

@@ -20,7 +20,7 @@ const CreateFileModal = ({
});
return (
<MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
<MyModal title={t('file.Create File')} isOpen onClose={() => {}} w={'600px'} top={'15vh'}>
<ModalBody>
<Box mb={1} fontSize={'sm'}>

View File

@@ -10,12 +10,12 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.csv';
const CsvImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const theme = useTheme();
@@ -42,7 +42,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
.flat()
.filter((item) => item?.q);
const filterChunks = chunks.filter((item) => item.q.length < maxToken);
const filterChunks = chunks.filter((item) => item.q.length < maxToken * 1.5);
if (filterChunks.length !== chunks.length) {
toast({
@@ -73,7 +73,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},

View File

@@ -19,13 +19,13 @@ import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import { FetchResultItem, DatasetItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
export type FileItemType = {
id: string;
@@ -55,7 +55,7 @@ const FileSelect = ({
showCreateFile = true,
...props
}: Props) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation();
@@ -129,7 +129,7 @@ const FileSelect = ({
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
id: filesId[0],
filename: file.name,
icon,
text,

View File

@@ -6,14 +6,14 @@ import { useRequest } from '@/hooks/useRequest';
import { getErrText } from '@/utils/tools';
import { postKbDataFromList } from '@/api/plugins/kb';
import { TrainingModeEnum } from '@/constants/plugin';
import { useUserStore } from '@/store/user';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/store/dataset';
type ManualFormType = { q: string; a: string };
const ManualImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const { register, handleSubmit, reset } = useForm({

View File

@@ -74,7 +74,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},
@@ -97,9 +97,11 @@ const QAImport = ({ kbId }: { kbId: string }) => {
return {
...file,
tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({
...chunk,
q: splitRes.chunks[i]
chunks: splitRes.chunks.map((chunk) => ({
a: '',
source: file.filename,
file_id: file.id,
q: chunk
}))
};
})

View File

@@ -12,7 +12,7 @@ import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delKbById, putKbById } from '@/api/plugins/kb';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useToast } from '@/hooks/useToast';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useConfirm } from '@/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form';
import { compressImg } from '@/utils/file';
@@ -47,7 +47,7 @@ const Info = (
multiple: false
});
const { kbDetail, getKbDetail, loadKbList, myKbList } = useUserStore();
const { kbDetail, getKbDetail, loadKbList } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {

View File

@@ -9,10 +9,10 @@ import MyIcon from '@/components/Icon';
import MyModal from '@/components/MyModal';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
import { useQuery } from '@tanstack/react-query';
import { DatasetItemType } from '@/types/plugin';
import { useTranslation } from 'react-i18next';
import { useDatasetStore } from '@/store/dataset';
export type FormData = { dataId?: string } & DatasetItemType;
@@ -36,7 +36,7 @@ const InputDataModal = ({
const [loading, setLoading] = useState(false);
const { toast } = useToast();
const { kbDetail, getKbDetail } = useUserStore();
const { kbDetail, getKbDetail } = useDatasetStore();
const { getValues, register, handleSubmit, reset } = useForm<FormData>({
defaultValues
@@ -261,13 +261,11 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp
<Box
color={'myGray.600'}
display={'inline-block'}
whiteSpace={'nowrap'}
{...(!!fileId
? {
cursor: 'pointer',
textDecoration: ['underline', 'none'],
_hover: {
textDecoration: 'underline'
},
textDecoration: 'underline',
onClick: async () => {
try {
const url = await getFileViewUrl(fileId);

View File

@@ -1,6 +1,6 @@
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
import { useKbStore } from '@/store/kb';
import { useDatasetStore } from '@/store/dataset';
import type { KbTestItemType } from '@/types/plugin';
import { searchText, getKbDataItemById } from '@/api/plugins/kb';
import MyIcon from '@/components/Icon';
@@ -13,16 +13,14 @@ import { useToast } from '@/hooks/useToast';
import { customAlphabet } from 'nanoid';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const Test = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const theme = useTheme();
const { toast } = useToast();
const { setLoading } = useGlobalStore();
const { kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } = useKbStore();
const { kbDetail, kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } =
useDatasetStore();
const [inputText, setInputText] = useState('');
const [kbTestItem, setKbTestItem] = useState<KbTestItemType>();
const [editData, setEditData] = useState<FormData>();

View File

@@ -1,17 +1,15 @@
import React, { useCallback, useRef } from 'react';
import React, { useCallback, useEffect, useRef } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { KbItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools';
import { useGlobalStore } from '@/store/global';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import DataCard from './components/DataCard';
import MyIcon from '@/components/Icon';
import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer';
@@ -19,11 +17,17 @@ import Avatar from '@/components/Avatar';
import Info from './components/Info';
import { serviceSideProps } from '@/utils/i18n';
import { useTranslation } from 'react-i18next';
import { getTrainingQueueLen } from '@/api/plugins/kb';
import { delEmptyFiles, getTrainingQueueLen } from '@/api/plugins/kb';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { feConfigs } from '@/store/static';
import Script from 'next/script';
import FileCard from './components/FileCard';
import { useDatasetStore } from '@/store/dataset';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
});
const ImportData = dynamic(() => import('./components/Import'), {
ssr: false
});
@@ -32,7 +36,8 @@ const Test = dynamic(() => import('./components/Test'), {
});
enum TabEnum {
data = 'data',
dataCard = 'dataCard',
dataset = 'dataset',
import = 'import',
test = 'test',
info = 'info'
@@ -45,10 +50,10 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
const { toast } = useToast();
const router = useRouter();
const { isPc } = useGlobalStore();
const { kbDetail, getKbDetail } = useUserStore();
const { kbDetail, getKbDetail } = useDatasetStore();
const tabList = useRef([
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' },
{ label: '数据集', id: TabEnum.dataset, icon: 'overviewLight' },
{ label: '导入数据', id: TabEnum.import, icon: 'importLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: '配置', id: TabEnum.info, icon: 'settingLight' }
@@ -85,105 +90,117 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
});
const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
refetchInterval: 5000
refetchInterval: 10000
});
return (
<PageContainer>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
</Flex>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
)}
useEffect(() => {
return () => {
try {
delEmptyFiles(kbId);
} catch (error) {}
};
}, [kbId]);
{!!kbDetail._id && (
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.data && <DataCard kbId={kbId} />}
{currentTab === TabEnum.import && <ImportData kbId={kbId} />}
{currentTab === TabEnum.test && <Test kbId={kbId} />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
</Box>
)}
</Box>
</PageContainer>
return (
<>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<PageContainer>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
</Flex>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
)}
{!!kbDetail._id && (
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.dataset && <FileCard kbId={kbId} />}
{currentTab === TabEnum.dataCard && <DataCard kbId={kbId} />}
{currentTab === TabEnum.import && <ImportData kbId={kbId} />}
{currentTab === TabEnum.test && <Test kbId={kbId} />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
</Box>
)}
</Box>
</PageContainer>
</>
);
};
export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.data;
const currentTab = context?.query?.currentTab || TabEnum.dataset;
const kbId = context?.query?.kbId;
return {

View File

@@ -18,7 +18,7 @@ import MySelect from '@/components/Select';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import Tag from '@/components/Tag';
const CreateModal = ({ onClose }: { onClose: () => void }) => {
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const [refresh, setRefresh] = useState(false);
const { toast } = useToast();
const router = useRouter();
@@ -28,7 +28,9 @@ const CreateModal = ({ onClose }: { onClose: () => void }) => {
avatar: '/icon/logo.svg',
name: '',
tags: [],
vectorModel: vectorModelList[0].model
vectorModel: vectorModelList[0].model,
type: 'dataset',
parentId
}
});
const InputRef = useRef<HTMLInputElement>(null);

View File

@@ -0,0 +1,85 @@
import React, { useMemo, useRef } from 'react';
import { ModalFooter, ModalBody, Input, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'react-i18next';
import { useRequest } from '@/hooks/useRequest';
import { postCreateKb, putKbById } from '@/api/plugins/kb';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
const EditFolderModal = ({
onClose,
onSuccess,
id,
parentId,
name
}: {
onClose: () => void;
onSuccess: () => void;
id?: string;
parentId?: string;
name?: string;
}) => {
const { t } = useTranslation();
const inputRef = useRef<HTMLInputElement>(null);
const typeMap = useMemo(
() =>
id
? {
title: t('kb.Edit Folder')
}
: {
title: t('kb.Create Folder')
},
[id, t]
);
const { mutate: onSave, isLoading } = useRequest({
mutationFn: () => {
const val = inputRef.current?.value;
if (!val) return Promise.resolve('');
if (id) {
return putKbById({
id,
name: val
});
}
return postCreateKb({
parentId,
name: val,
type: KbTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: []
});
},
onSuccess: (res) => {
if (!res) return;
onSuccess();
onClose();
}
});
return (
<MyModal isOpen onClose={onClose} title={typeMap.title}>
<ModalBody>
<Input
ref={inputRef}
defaultValue={name}
placeholder={t('kb.Folder Name') || ''}
autoFocus
maxLength={20}
/>
</ModalBody>
<ModalFooter>
<Button mr={3} variant={'base'} onClick={onClose}>
{t('common.Cancel')}
</Button>
<Button isLoading={isLoading} onClick={onSave}>
{t('Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default EditFolderModal;

View File

@@ -1,77 +1,184 @@
import React, { useCallback } from 'react';
import React, { useCallback, useMemo, useRef, useState } from 'react';
import {
Box,
Card,
Flex,
Grid,
useTheme,
Button,
useDisclosure,
Card,
IconButton,
useDisclosure
MenuButton,
Image
} from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { delKbById } from '@/api/plugins/kb';
import { delKbById, getKbPaths } from '@/api/plugins/kb';
import { useTranslation } from 'react-i18next';
import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon';
import Tag from '@/components/Tag';
import { serviceSideProps } from '@/utils/i18n';
import dynamic from 'next/dynamic';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/hooks/useRequest';
import { useGlobalStore } from '@/store/global';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false });
const Kb = () => {
const { t } = useTranslation();
const theme = useTheme();
const router = useRouter();
const { parentId } = router.query as { parentId: string };
const { toast } = useToast();
const { openConfirm, ConfirmModal } = useConfirm({
title: '删除提示',
content: '确认删除该知识库?知识库相关的文件、记录将永久删除,无法恢复!'
const { setLoading } = useGlobalStore();
const DeleteTipsMap = useRef({
[KbTypeEnum.folder]: t('kb.deleteFolderTips'),
[KbTypeEnum.dataset]: t('kb.deleteDatasetTips')
});
const { myKbList, loadKbList, setKbList } = useUserStore();
const { openConfirm, ConfirmModal } = useConfirm({
title: t('common.Delete Warning'),
content: ''
});
const { myKbList, loadKbList, setKbList } = useDatasetStore();
const {
isOpen: isOpenCreateModal,
onOpen: onOpenCreateModal,
onClose: onCloseCreateModal
} = useDisclosure();
const { refetch } = useQuery(['loadKbList'], () => loadKbList());
const [editFolderData, setEditFolderData] = useState<{
id?: string;
name?: string;
}>();
/* 点击删除 */
const onclickDelKb = useCallback(
async (id: string) => {
try {
delKbById(id);
toast({
title: '删除成功',
status: 'success'
});
setKbList(myKbList.filter((item) => item._id !== id));
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
});
}
const { mutate: onclickDelKb } = useRequest({
mutationFn: async (id: string) => {
setLoading(true);
await delKbById(id);
return id;
},
[toast, setKbList, myKbList]
onSuccess(id: string) {
setKbList(myKbList.filter((item) => item._id !== id));
},
onSettled() {
setLoading(false);
},
successToast: t('common.Delete Success'),
errorToast: t('kb.Delete Dataset Error')
});
const { data, refetch } = useQuery(['loadKbList', parentId], () => {
return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
});
const paths = useMemo(
() => [
{
parentId: '',
parentName: t('kb.My Dataset')
},
...(data?.[1] || [])
],
[data, t]
);
return (
<PageContainer>
<Flex pt={3} px={5} alignItems={'center'}>
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
</Box>
<Button leftIcon={<AddIcon />} variant={'base'} onClick={onOpenCreateModal}>
</Button>
{/* url path */}
{!!parentId ? (
<Flex flex={1}>
{paths.map((item, i) => (
<Flex key={item.parentId} mr={2} alignItems={'center'}>
<Box
fontSize={'lg'}
px={2}
py={1}
borderRadius={'md'}
{...(i === paths.length - 1
? {
cursor: 'default'
}
: {
cursor: 'pointer',
_hover: {
bg: 'myGray.100'
},
onClick: () => {
router.push({
query: {
parentId: item.parentId
}
});
}
})}
>
{item.parentName}
</Box>
{i !== paths.length - 1 && <MyIcon name={'rightArrowLight'} color={'myGray.500'} />}
</Flex>
))}
</Flex>
) : (
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
</Box>
)}
<MyMenu
offset={[-30, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
>
<Flex
alignItems={'center'}
border={theme.borders.base}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={1} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/module/db.png'} alt={''} w={'20px'} mr={1} />
{t('Dataset')}
</Flex>
),
onClick: onOpenCreateModal
}
]}
/>
</Flex>
<Grid
p={5}
@@ -86,7 +193,7 @@ const Kb = () => {
py={4}
px={5}
cursor={'pointer'}
h={'140px'}
h={'130px'}
border={theme.borders.md}
boxShadow={'none'}
userSelect={'none'}
@@ -98,18 +205,28 @@ const Kb = () => {
display: 'block'
}
}}
onClick={() =>
router.push({
pathname: '/kb/detail',
query: {
kbId: kb._id
}
})
}
onClick={() => {
if (kb.type === KbTypeEnum.folder) {
router.push({
pathname: '/kb/list',
query: {
parentId: kb._id
}
});
} else if (kb.type === KbTypeEnum.dataset) {
router.push({
pathname: '/kb/detail',
query: {
kbId: kb._id
}
});
}
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Avatar src={kb.avatar} borderRadius={'lg'} w={'28px'} />
<Box ml={3}>{kb.name}</Box>
<IconButton
className="delete"
position={'absolute'}
@@ -126,7 +243,11 @@ const Kb = () => {
}}
onClick={(e) => {
e.stopPropagation();
openConfirm(() => onclickDelKb(kb._id))();
openConfirm(
() => onclickDelKb(kb._id),
undefined,
DeleteTipsMap.current[kb.type]
)();
}}
/>
</Flex>
@@ -140,8 +261,14 @@ const Kb = () => {
</Flex>
</Box>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box>
{kb.type === KbTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box>
</>
)}
</Flex>
</Card>
))}
@@ -155,7 +282,15 @@ const Kb = () => {
</Flex>
)}
<ConfirmModal />
{isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} />}
{isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} parentId={parentId} />}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
onSuccess={refetch}
parentId={parentId}
{...editFolderData}
/>
)}
</PageContainer>
);
};

View File

@@ -13,6 +13,7 @@ import { serviceSideProps } from '@/utils/i18n';
import { setToken } from '@/utils/user';
import { feConfigs } from '@/store/static';
import CommunityModal from '@/components/CommunityModal';
import Script from 'next/script';
const RegisterForm = dynamic(() => import('./components/RegisterForm'));
const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm'));
@@ -53,70 +54,77 @@ const Login = () => {
}
return (
<Flex
alignItems={'center'}
justifyContent={'center'}
className={styles.loginPage}
h={'100%'}
px={[0, '10vw']}
>
<>
{feConfigs.googleClientVerKey && (
<Script
src={`https://www.recaptcha.net/recaptcha/api.js?render=${feConfigs.googleClientVerKey}`}
></Script>
)}
<Flex
height="100%"
w={'100%'}
maxW={'1240px'}
maxH={['auto', 'max(660px,80vh)']}
backgroundColor={'#fff'}
alignItems={'center'}
justifyContent={'center'}
py={[5, 10]}
px={'5vw'}
borderRadius={isPc ? 'md' : 'none'}
gap={5}
className={styles.loginPage}
h={'100%'}
px={[0, '10vw']}
>
{isPc && (
<Image
src={'/icon/loginLeft.svg'}
order={pageType === PageTypeEnum.login ? 0 : 2}
flex={'1 0 0'}
w="0"
maxW={'600px'}
height={'100%'}
maxH={'450px'}
alt=""
/>
)}
<Box
position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
<Flex
height="100%"
w={'100%'}
maxW={'1240px'}
maxH={['auto', 'max(660px,80vh)']}
backgroundColor={'#fff'}
alignItems={'center'}
justifyContent={'center'}
py={[5, 10]}
px={'5vw'}
borderRadius={isPc ? 'md' : 'none'}
gap={5}
>
<DynamicComponent type={pageType} />
{feConfigs?.show_contact && (
<Box
fontSize={'sm'}
color={'myGray.600'}
cursor={'pointer'}
position={'absolute'}
right={5}
bottom={3}
onClick={onOpen}
>
</Box>
{isPc && (
<Image
src={'/icon/loginLeft.svg'}
order={pageType === PageTypeEnum.login ? 0 : 2}
flex={'1 0 0'}
w="0"
maxW={'600px'}
height={'100%'}
maxH={'450px'}
alt=""
/>
)}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />}
</Flex>
<Box
position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
borderRadius={isPc ? 'md' : 'none'}
>
<DynamicComponent type={pageType} />
{feConfigs?.show_contact && (
<Box
fontSize={'sm'}
color={'myGray.600'}
cursor={'pointer'}
position={'absolute'}
right={5}
bottom={3}
onClick={onOpen}
>
</Box>
)}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />}
</Flex>
</>
);
};

View File

@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
import { modelToolMap } from '@/utils/plugin';
import { gptMessage2ChatType } from '@/utils/adapt';
import { addLog } from '../utils/tools';
import { splitText2Chunks } from '@/utils/file';
const reduceQueue = () => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -157,7 +158,7 @@ A2:
console.log('openai error: 生成QA错误');
console.log(err.response?.status, err.response?.statusText, err.response?.data);
} else {
console.log('生成QA错误:', err);
addLog.error('生成 QA 错误', err);
}
// message error or openai account error
@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
}
}
// empty result. direct split chunk
if (result.length === 0) {
const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
splitRes.chunks.forEach((item) => {
result.push({
q: item,
a: ''
});
});
}
return result;
}

View File

@@ -96,9 +96,7 @@ export async function generateVector(): Promise<any> {
data: err.response?.data
});
} else {
addLog.info('openai error: 生成向量错误', {
err
});
addLog.error('openai error: 生成向量错误', err);
}
// message error or openai account error

View File

@@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose';
import fs from 'fs';
import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode';
import type { FileInfo } from '@/types/plugin';
enum BucketNameEnum {
dataset = 'dataset'
}
type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
};
export class GridFSStorage {
readonly type = 'gridfs';
readonly bucket: `${BucketNameEnum}`;
@@ -88,6 +81,7 @@ export class GridFSStorage {
filename: file.filename,
contentType: file.metadata?.contentType,
encoding: file.metadata?.encoding,
uploadDate: file.uploadDate,
size: file.length
};
}

View File

@@ -55,7 +55,6 @@ const BillSchema = new Schema({
try {
BillSchema.index({ userId: 1 });
// BillSchema.index({ time: -1 });
BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 });
} catch (error) {
console.log(error);

View File

@@ -48,24 +48,7 @@ const ChatItemSchema = new Schema({
}
},
[TaskResponseKeyEnum.responseData]: {
type: [
{
moduleName: String,
price: String,
model: String,
tokens: Number,
question: String,
answer: String,
temperature: Number,
maxToken: Number,
quoteList: Array,
completeMessages: Array,
similarity: Number,
limit: Number,
cqList: Array,
cqResult: String
}
],
type: Array,
default: []
}
});

View File

@@ -1,7 +1,13 @@
import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema';
import { KbTypeMap } from '@/constants/kb';
const kbSchema = new Schema({
parentId: {
type: Schema.Types.ObjectId,
ref: 'kb',
default: null
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
@@ -24,6 +30,12 @@ const kbSchema = new Schema({
required: true,
default: 'text-embedding-ada-002'
},
type: {
type: String,
enum: Object.keys(KbTypeMap),
required: true,
default: 'dataset'
},
tags: {
type: [String],
default: []

View File

@@ -13,6 +13,7 @@ export const connectPg = async (): Promise<Pool> => {
connectionString: process.env.PG_URL,
max: Number(process.env.DB_MAX_LINK || 5),
keepAlive: true,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 5000
});
@@ -107,6 +108,7 @@ class Pg {
}
LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
`;
const pg = await connectPg();
return pg.query<T>(sql);
}

View File

@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '接口连接异常';
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) {
@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '接口连接异常';
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) {

View File

@@ -155,7 +155,7 @@ export const authUser = async ({
})();
return {
userId: uid,
userId: String(uid),
appId,
authType,
user

View File

@@ -0,0 +1,86 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { type KbTestItemType } from '@/types/plugin';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById, getAllDataset } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
type State = {
datasets: KbListItemType[];
loadAllDatasets: () => Promise<KbListItemType[]>;
myKbList: KbListItemType[];
loadKbList: (parentId?: string) => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
kbTestList: KbTestItemType[];
pushKbTestItem: (data: KbTestItemType) => void;
delKbTestItemById: (id: string) => void;
updateKbItemById: (data: KbTestItemType) => void;
};
export const useDatasetStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
datasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.datasets = res;
});
return res;
},
myKbList: [],
async loadKbList(parentId) {
const res = await getKbList(parentId);
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
},
kbTestList: [],
pushKbTestItem(data) {
set((state) => {
state.kbTestList = [data, ...state.kbTestList].slice(0, 500);
});
},
delKbTestItemById(id) {
set((state) => {
state.kbTestList = state.kbTestList.filter((item) => item.id !== id);
});
},
updateKbItemById(data: KbTestItemType) {
set((state) => {
state.kbTestList = state.kbTestList.map((item) => (item.id === data.id ? data : item));
});
}
})),
{
name: 'kbStore',
partialize: (state) => ({
kbTestList: state.kbTestList
})
}
)
)
);

View File

@@ -1,42 +0,0 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { type KbTestItemType } from '@/types/plugin';
type State = {
kbTestList: KbTestItemType[];
pushKbTestItem: (data: KbTestItemType) => void;
delKbTestItemById: (id: string) => void;
updateKbItemById: (data: KbTestItemType) => void;
};
export const useKbStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
kbTestList: [],
pushKbTestItem(data) {
set((state) => {
state.kbTestList = [data, ...state.kbTestList].slice(0, 500);
});
},
delKbTestItemById(id) {
set((state) => {
state.kbTestList = state.kbTestList.filter((item) => item.id !== id);
});
},
updateKbItemById(data: KbTestItemType) {
set((state) => {
state.kbTestList = state.kbTestList.map((item) => (item.id === data.id ? data : item));
});
}
})),
{
name: 'kbStore',
partialize: (state) => ({
kbTestList: state.kbTestList
})
}
)
)
);

View File

@@ -7,9 +7,7 @@ import { formatPrice } from '@/utils/user';
import { getTokenLogin, putUserInfo } from '@/api/user';
import { defaultApp } from '@/constants/model';
import { AppListItemType, AppUpdateParams } from '@/types/app';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
import type { AppSchema } from '@/types/mongoSchema';
type State = {
@@ -24,12 +22,6 @@ type State = {
loadAppDetail: (id: string, init?: boolean) => Promise<AppSchema>;
updateAppDetail(appId: string, data: AppUpdateParams): Promise<void>;
clearAppModules(): void;
// kb
myKbList: KbListItemType[];
loadKbList: () => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
};
export const useUserStore = create<State>()(
@@ -106,31 +98,6 @@ export const useUserStore = create<State>()(
modules: []
};
});
},
myKbList: [],
async loadKbList() {
const res = await getKbList();
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val: KbListItemType[]) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
}
})),
{

View File

@@ -28,6 +28,7 @@ export type FeConfigsType = {
beianText?: string;
googleClientVerKey?: string;
gitLoginKey?: string;
exportLimitMinutes?: number;
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
@@ -58,7 +59,6 @@ declare global {
interface Window {
['pdfjs-dist/build/pdf']: any;
particlesJS: any;
grecaptcha: any;
QRCode: any;
umami?: {

View File

@@ -6,6 +6,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
import type { AppModuleItemType } from './app';
import { ChatSourceEnum, OutLinkTypeEnum } from '@/constants/chat';
import { AppTypeEnum } from '@/constants/app';
import { KbTypeEnum } from '@/constants/kb';
export interface UserModelSchema {
_id: string;
@@ -166,15 +167,17 @@ export interface OutLinkSchema {
type: `${OutLinkTypeEnum}`;
}
export interface kbSchema {
export type kbSchema = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
}
type: `${KbTypeEnum}`;
};
export interface informSchema {
_id: string;

View File

@@ -1,15 +1,18 @@
import { FileStatusEnum } from '@/constants/kb';
import { VectorModelItemType } from './model';
import type { kbSchema } from './mongoSchema';
export type SelectedKbType = { kbId: string; vectorModel: VectorModelItemType }[];
export type KbListItemType = {
_id: string;
avatar: string;
name: string;
tags: string[];
export type KbListItemType = Omit<kbSchema, 'vectorModel'> & {
vectorModel: VectorModelItemType;
};
export type KbPathItemType = {
parentId: string;
parentName: string;
};
/* kb type */
export interface KbItemType {
_id: string;
@@ -20,6 +23,15 @@ export interface KbItemType {
tags: string;
}
export type KbFileItemType = {
id: string;
size: number;
filename: string;
uploadTime: Date;
chunkLength: number;
status: `${FileStatusEnum}`;
};
export type DatasetItemType = {
q: string; // 提问词
a: string; // 原文
@@ -42,3 +54,12 @@ export type FetchResultItem = {
url: string;
content: string;
};
export type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
uploadDate: Date;
};

View File

@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
const token = adaptMessages.reduce((sum, item) => {
const text = `${item.role}\n${item.content}`;
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
const tokens = encodeText.length + 3; // 补充估算值
/* use textLen as tokens if encode error */
const tokens = (() => {
try {
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
return encodeText.length + 3; // 补充估算值
} catch (error) {
return text.length;
}
})();
return sum + tokens;
}, 0);
@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
try {
const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
} catch (error) {
return text.slice(0, length);
}
};
export const authOpenAiKey = async (key: string) => {

View File

@@ -44,7 +44,7 @@ export class SSEParseData {
data: parseData
};
} catch (error) {
if (typeof item.data === 'string' && !item.data.startsWith(": ping")) {
if (typeof item.data === 'string' && !item.data.startsWith(': ping')) {
this.storeReadData += item.data;
} else {
this.storeReadData = '';

View File

@@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
return queryParams.toString();
};
/**
* parse string to query object
*/
export const parseQueryString = (str: string) => {
const queryObject: Record<string, any> = {};
@@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => {
return target.format('YYYY/M/D');
};
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};
export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
/**
* voice broadcast