mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-29 01:40:51 +00:00
kb
This commit is contained in:
archer
@@ -11,6 +11,7 @@ import {
|
||||
Response as SearchTestResponse
|
||||
} from '@/pages/api/openapi/kb/searchTest';
|
||||
import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById';
|
||||
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
|
||||
|
||||
export type KbUpdateParams = {
|
||||
id: string;
|
||||
@@ -71,8 +72,7 @@ export const postKbDataFromList = (data: PushDataProps) =>
|
||||
/**
|
||||
* 更新一条数据
|
||||
*/
|
||||
export const putKbDataById = (data: { dataId: string; a: string; q?: string }) =>
|
||||
PUT('/openapi/kb/updateData', data);
|
||||
export const putKbDataById = (data: UpdateDataProps) => PUT('/openapi/kb/updateData', data);
|
||||
/**
|
||||
* 删除一条知识库数据
|
||||
*/
|
||||
|
||||
@@ -63,6 +63,9 @@ function responseError(err: any) {
|
||||
);
|
||||
return Promise.reject({ message: 'token过期，重新登录' });
|
||||
}
|
||||
if (err?.response?.data) {
|
||||
return Promise.reject(err?.response?.data);
|
||||
}
|
||||
return Promise.reject(err);
|
||||
}
|
||||
|
||||
|
||||
22
client/src/components/Icon/close.tsx
Normal file
22
client/src/components/Icon/close.tsx
Normal file
@@ -0,0 +1,22 @@
|
||||
import React from 'react';
|
||||
import { Flex, type FlexProps } from '@chakra-ui/react';
|
||||
import MyIcon from '@/components/Icon';
|
||||
|
||||
const CloseIcon = (props: FlexProps) => {
|
||||
return (
|
||||
<Flex
|
||||
cursor={'pointer'}
|
||||
w={'22px'}
|
||||
h={'22px'}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
borderRadius={'50%'}
|
||||
_hover={{ bg: 'myGray.200' }}
|
||||
{...props}
|
||||
>
|
||||
<MyIcon name={'closeLight'} w={'12px'} color={'myGray.500'} />
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export default CloseIcon;
|
||||
25
client/src/components/Icon/delete.tsx
Normal file
25
client/src/components/Icon/delete.tsx
Normal file
@@ -0,0 +1,25 @@
|
||||
import React from 'react';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import { IconProps } from '@chakra-ui/react';
|
||||
|
||||
const DeleteIcon = (props: IconProps) => {
|
||||
return (
|
||||
<MyIcon
|
||||
className="delete"
|
||||
name={'delete' as any}
|
||||
w={'14px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
display={['block', 'none']}
|
||||
cursor={'pointer'}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default DeleteIcon;
|
||||
|
||||
export const hoverDeleteStyles = {
|
||||
'& .delete': {
|
||||
display: 'block'
|
||||
}
|
||||
};
|
||||
1
client/src/components/Icon/icons/file/csv.svg
Normal file
1
client/src/components/Icon/icons/file/csv.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689489239258" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="23019" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M145.6 0C100.8 0 64 36.8 64 81.6v860.8C64 987.2 100.8 1024 145.6 1024h732.8c44.8 0 81.6-36.8 81.6-81.6V324.8L656 0H145.6z" fill="#45B058" p-id="23020"></path><path d="M388.8 691.2c1.6 1.6 3.2 4.8 3.2 8 0 6.4-4.8 11.2-11.2 11.2-3.2 0-6.4 0-8-3.2-11.2-12.8-30.4-22.4-48-22.4-41.6 0-73.6 32-73.6 78.4 0 44.8 32 78.4 73.6 78.4 17.6 0 35.2-8 48-22.4 1.6-1.6 4.8-3.2 8-3.2 6.4 0 11.2 4.8 11.2 11.2 0 3.2-1.6 6.4-3.2 8-14.4 16-35.2 27.2-64 27.2-56 0-99.2-40-99.2-99.2s43.2-99.2 99.2-99.2c28.8 0 49.6 11.2 64 27.2z m108.8 171.2c-28.8 0-51.2-9.6-67.2-24-3.2-1.6-3.2-4.8-3.2-8 0-6.4 3.2-12.8 11.2-12.8 1.6 0 4.8 1.6 6.4 3.2 12.8 11.2 32 20.8 54.4 20.8 33.6 0 44.8-19.2 44.8-33.6 0-49.6-113.6-22.4-113.6-91.2 0-30.4 27.2-52.8 65.6-52.8 24 0 46.4 8 62.4 20.8 1.6 1.6 3.2 4.8 3.2 8 0 6.4-4.8 11.2-11.2 11.2-1.6 0-4.8 0-6.4-1.6-14.4-11.2-32-17.6-49.6-17.6-24 0-40 12.8-40 30.4 0 43.2 113.6 19.2 113.6 91.2 0 27.2-19.2 56-70.4 56z m272-179.2L702.4 848c-3.2 8-9.6 12.8-17.6 12.8h-1.6c-8 0-16-4.8-19.2-12.8l-65.6-164.8c-1.6-1.6-1.6-3.2-1.6-4.8 0-6.4 4.8-12.8 12.8-12.8 4.8 0 9.6 3.2 11.2 8l62.4 160 62.4-160c1.6-4.8 6.4-8 11.2-8 8 0 14.4 6.4 14.4 12.8 0 1.6-1.6 3.2-1.6 4.8z" fill="#FFFFFF" p-id="23021"></path><path d="M960 326.4v16H755.2s-100.8-20.8-97.6-108.8c0 0 3.2 92.8 96 92.8H960z" fill="#349C42" p-id="23022"></path><path d="M657.6 0v233.6c0 25.6 17.6 92.8 97.6 92.8H960L657.6 0z" fill="#FFFFFF" p-id="23023"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.7 KiB |
1
client/src/components/Icon/icons/file/indexImport.svg
Normal file
1
client/src/components/Icon/icons/file/indexImport.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689488775813" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6900" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M448 768h448V256H448v512z" fill="#7839ee" opacity=".1" p-id="6901"></path><path d="M960 928H384v-64h576v64z m0-768H384v-64h576v64zM128 736H64v-64h64v64z m0-128H64v-64h64v64z m0-128H64v-64h64v64z m0-128H64v-64h64v64z m0-128H64v-64h64v64zM320 800H256v-64h64v64z m0-128H256v-64h64v64z m0-128H256v-64h64v64z m0-128H256v-64h64v64z m0-128H256v-64h64v64zM640 256h64v512h-64z" fill="#7839ee" p-id="6902"></path><path d="M672 160l96 128H576l96-128zM672 864l96-128H576l96 128z" fill="#7839ee" p-id="6903"></path></svg>
|
||||
|
After Width: | Height: | Size: 839 B |
1
client/src/components/Icon/icons/file/manualImport.svg
Normal file
1
client/src/components/Icon/icons/file/manualImport.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689489413405" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2128" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M753 521.1c-23.2-60.8-116.5-56.6-143.1-4.8-15.9-52.8-121.7-70.4-146.8-6.4 0-51.1-0.8-271.7-0.8-271.7-0.1-32.3-26.3-58.3-58.5-58.3s-58.4 26-58.5 58.3l-1.9 433.7s-23.1-53.6-81.8-83.9c-94.5-48.6-137.3 17.9-92.8 62.6C266.4 748.7 322.2 830.3 356 922c28.6 77.5 87.4 94.2 87.4 94.2h366c48.2-31 62.3-134.1 62.3-134.1V582.2c-0.3-111.6-120.1-93.8-118.7-61.1z" p-id="2129" fill="#4e83fd"></path><path d="M286.3 442.9v-72.4c-36.3-32.1-59.6-78.5-59.6-130.8 0-96.7 78.5-175.2 175.2-175.2 96.7 0 175.2 78.5 175.2 175.2 0 51.2-22.4 96.9-57.4 128.9v74.1c69.9-40.6 117.6-115.4 117.6-202.1 0-129.4-105-234.4-234.4-234.4s-234.4 105-234.4 234.4c-0.1 86.9 47.7 161.8 117.8 202.3z" p-id="2130" fill="#4e83fd"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
1
client/src/components/Icon/icons/file/qaImport.svg
Normal file
1
client/src/components/Icon/icons/file/qaImport.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689489324421" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="28738" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M166.32 808.98l-99.28 99.28c-15.31 15.32-39.71 15.32-55.03 0-15.32-15.31-15.32-39.71 0-55.03l98.71-99.28c15.32-15.31 39.71-15.31 55.03 0 15.32 15.32 15.32 39.72 0.57 55.03z m48.22 48.79l-99.28 99.27c-15.31 15.32-15.31 39.71 0 55.03 15.32 15.32 39.71 15.32 55.03 0l99.28-99.28c15.32-15.32 15.32-39.71 0-55.03-15.32-15.31-39.72-15.31-55.03 0.01z m625.14-545.16c0 71.47-57.86 129.34-129.34 129.34S581 384.08 581 312.61c0-71.48 57.86-129.34 129.34-129.34 72.05 0 129.91 57.86 129.34 129.34z m-77.71 0.57c0-28.36-23.26-51.63-51.62-51.63-28.36 0-51.62 23.26-51.62 51.63 0 28.36 23.26 51.62 51.62 51.62 28.93 0 52.19-23.26 51.62-51.62z m236.56 68.07c-27.8 104.39-83.39 203.65-164.51 285.35h-3.97l-5.11 5.67c10.21 38.57 11.35 78.28 2.84 116.86-9.07 46.52-32.33 90.2-66.94 124.8l-2.27 2.27c-38.57 36.87-85.09 59.57-134.45 67.51-51.62 7.94-105.51 0-152.6-23.26-17.58-10.21-25.53-32.9-16.45-51.63 1.7-2.84 3.41-5.67 5.67-8.5 14.18-15.32 24.96-32.9 32.9-51.63l3.4-7.94c-10.21 1.7-20.42 4.53-30.63 5.1-29.5 4.54-60.14 6.24-90.2 5.67-10.21 0-19.86-3.97-26.1-11.35L181.63 671.14c-8.51-7.95-11.35-18.72-10.78-29.5 0-28.37 2.27-58.43 5.67-86.79 1.7-10.78 2.84-20.99 5.1-31.2l-8.51 3.4c-17.58 7.95-35.17 18.72-51.06 33.47-14.75 14.18-39.14 12.48-53.32-2.27-2.27-2.84-3.97-7.37-6.24-10.21-23.26-47.09-30.63-99.28-22.69-150.9 8.22-51.8 32.65-99.66 69.78-136.71 34.61-34.61 78.85-57.86 123.67-66.94 39.14-8.51 79.42-7.94 116.86 2.27l7.95-7.95C439.75 106.69 538.45 51.66 642.83 23.3c108.35-28.36 222.38-28.36 329.59 0 13.62 4.54 23.82 15.32 27.23 27.23l0.57 0.57v0.56c28.37 107.78 28.37 221.81-1.69 329.59zM936.12 92.51c-89.06-19.86-182.67-18.16-270.03 5.1-91.34 24.96-178.7 73.18-250.74 145.22l-10.78 10.78-10.21 11.35c-10.21 11.35-26.66 16.45-42.55 10.78-31.2-12.48-65.81-14.75-98.71-7.38-31.2 6.81-61.27 22.13-86.23 46.52-26.66 27.8-43.12 60.7-48.79 95.3-3.4 20.99-3.4 42.55 0.57 62.97 9.64-5.67 18.72-10.78 28.93-14.18 26.66-11.34 55.6-18.15 84.53-18.72 3.97-0.57 7.94 0 11.91 0.56 19.86 5.67 31.77 26.66 26.1 46.52-7.95 26.1-13.05 52.2-16.45 78.29-1.7 22.12-3.4 43.12-3.97 64.67l146.36 146.93c22.12-1.14 43.11-2.27 64.67-5.11 27.23-3.97 52.76-8.51 78.29-15.89 3.97-0.56 7.94-1.13 11.91-1.13 20.42 0.57 36.31 18.72 36.31 39.14-1.7 29.5-7.38 58.43-19.29 85.66-3.97 9.08-9.07 19.29-14.18 28.36 20.42 3.97 41.98 3.97 62.4 1.13 33.47-5.1 66.94-20.99 93.03-47.09l1.13-1.13c24.39-24.96 39.71-55.6 47.09-86.79 6.81-32.34 3.97-66.94-7.94-99.28-4.54-14.18-1.7-30.64 10.21-41.41l11.91-11.35 8.51-8.51 2.27-1.13c72.04-72.61 119.7-159.41 145.79-250.17 23.27-89.05 25.54-180.95 7.95-270.01z" p-id="28739" fill="#FB7C3C"></path></svg>
|
||||
|
After Width: | Height: | Size: 2.9 KiB |
1
client/src/components/Icon/icons/file/uploadFile.svg
Normal file
1
client/src/components/Icon/icons/file/uploadFile.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689491332665" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5892" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M393.2 701.7c-8.2 0-16.4-3.1-22.6-9.4-12.5-12.5-12.5-32.8 0-45.3l115.3-115.3c14-14 36.9-14 50.9 0L652.1 647c12.5 12.5 12.5 32.8 0 45.3s-32.8 12.5-45.3 0l-95.5-95.5-95.5 95.5c-6.2 6.2-14.4 9.4-22.6 9.4z" fill="" p-id="5893"></path><path d="M511.3 921.1c-17.7 0-32-14.3-32-32v-276c0-17.7 14.3-32 32-32s32 14.3 32 32v276c0 17.6-14.3 32-32 32z" fill="" p-id="5894"></path><path d="M732.7 784.9c-17.7 0-32-14.3-32-32s14.3-32 32-32c90.6 0 164.3-73.7 164.3-164.3 0-82.9-61.9-153-144-163.1l-22.7-2.8-4.7-22.4c-20.8-99.9-110.1-172.4-212.4-172.4-102.2 0-191.5 72.5-212.4 172.4l-4.7 22.4-22.7 2.8c-82.1 10.1-144 80.2-144 163.1 0 90.6 73.7 164.3 164.3 164.3 17.7 0 32 14.3 32 32s-14.3 32-32 32c-61 0-118.3-23.8-161.5-66.9-43.1-43.1-66.9-100.5-66.9-161.5 0-107.6 75.2-199.7 178.2-222.8 15.8-53.8 47.7-102.2 91.3-138.1 50.1-41.2 113.4-63.9 178.3-63.9s128.3 22.7 178.3 63.9c43.6 35.9 75.5 84.3 91.3 138.1C885.9 356.8 961 448.9 961 556.5c0 61-23.8 118.3-66.9 161.5-43.1 43.1-100.4 66.9-161.4 66.9z" fill="" p-id="5895"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
1
client/src/components/Icon/icons/light/close.svg
Normal file
1
client/src/components/Icon/icons/light/close.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689496771994" class="icon" viewBox="0 0 1028 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="12388" xmlns:xlink="http://www.w3.org/1999/xlink" ><path d="M646.4 512l345.6-345.6c38.4-38.4 38.4-96 0-134.4-38.4-38.4-96-38.4-134.4 0L512 377.6 166.4 32C128-6.4 70.4-6.4 32 32c-38.4 38.4-38.4 96 0 134.4L377.6 512l-345.6 345.6c-38.4 38.4-38.4 96 0 134.4 19.2 19.2 44.8 25.6 70.4 25.6s51.2-6.4 70.4-25.6L512 646.4l345.6 345.6c19.2 19.2 44.8 25.6 70.4 25.6s51.2-6.4 70.4-25.6c38.4-38.4 38.4-96 0-134.4L646.4 512z" p-id="12389"></path></svg>
|
||||
|
After Width: | Height: | Size: 689 B |
1
client/src/components/Icon/icons/light/import.svg
Normal file
1
client/src/components/Icon/icons/light/import.svg
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1689485689790" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3190" xmlns:xlink="http://www.w3.org/1999/xlink" ><path d="M953.86906337 704.50762194c-19.00036972 0.29457937-34.31849726 15.90728628-34.02391788 34.90765601v87.63736424c0 38.29531883-30.34167568 69.52073264-67.75325638 69.52073264H175.00119435c-37.4115807 0-67.75325639-31.22541381-67.7532564-69.52073264V201.80791751c0-38.29531883 30.34167568-69.52073264 67.7532564-69.52073264h235.66350044l87.49007456 155.6852c7.06990501 12.37233377 20.32597692 18.70579035 33.43475913 17.23289348h320.35507094c37.4115807 0 67.75325639 31.22541381 67.75325639 69.52073264v86.90091578c0 19.2949491 15.31812754 34.907656 34.17120756 34.90765601 18.85308003 0 34.02391789-15.61270692 34.02391788-34.90765601 0-0.88373813 0-1.76747625-0.14728968-2.65121438v-84.2497014c0-76.73792734-60.68335138-139.04146527-135.50651276-139.04146528H547.20223538l-87.34278486-155.09604125c-7.80635344-13.99252034-19.00036972-17.67476253-28.86877881-17.38018316l-0.14728967-0.14728968h-256.28405678c-74.82316139 0-135.50651276 62.30353794-135.50651277 139.04146527v623.47724842c0 76.73792734 60.68335138 139.04146527 135.50651277 139.04146527H852.23917881c74.82316139 0 135.50651276-62.30353794 135.50651276-139.04146527v-83.51325297c0.1472897-0.88373813 0.1472897-1.76747625 0.14728968-2.65121439 0.1472897-9.13196064-3.38766283-17.96934191-9.86840907-24.45008817-6.33345658-6.62803595-15.02354815-10.31027814-24.15550881-10.45756784z" p-id="3191"></path><path d="M473.55739149 451.90580738c-5.44971845-5.89158752-12.96149253-9.27925033-21.06242536-9.27925032-7.95364315 0-15.61270692 3.38766283-21.06242536 9.27925032l-125.49081399 133.59174683-0.73644843 0.73644844h-0.14728971l-0.73644843 0.73644844c-11.34130596 12.51962347-11.34130596 31.66728287 0 44.18690633l125.93268305 134.32819525c5.30242877 5.89158752 12.81420285 9.13196064 20.76784599 9.27925035 7.95364315 0 15.46541722-3.24037312 20.76784598-9.13196064l0.58915876-0.73644844c11.34130596-12.51962347 11.34130596-31.66728287 0-44.18690633l-75.85418922-81.00932829h344.51057975c16.05457598 0 29.16335819-13.99252034 29.16335819-31.07812414 0-17.08560379-13.10878221-31.07812413-29.16335819-31.07812411h-343.18497256l75.70689953-80.86203861c11.63588534-12.51962347 11.63588534-32.10915193 0-44.77606508z" p-id="3192"></path></svg>
|
||||
|
After Width: | Height: | Size: 2.5 KiB |
@@ -54,7 +54,14 @@ const map = {
|
||||
voice: require('./icons/voice.svg').default,
|
||||
html: require('./icons/file/html.svg').default,
|
||||
pdf: require('./icons/file/pdf.svg').default,
|
||||
markdown: require('./icons/file/markdown.svg').default
|
||||
markdown: require('./icons/file/markdown.svg').default,
|
||||
importLight: require('./icons/light/import.svg').default,
|
||||
manualImport: require('./icons/file/manualImport.svg').default,
|
||||
indexImport: require('./icons/file/indexImport.svg').default,
|
||||
csvImport: require('./icons/file/csv.svg').default,
|
||||
qaImport: require('./icons/file/qaImport.svg').default,
|
||||
uploadFile: require('./icons/file/uploadFile.svg').default,
|
||||
closeLight: require('./icons/light/close.svg').default
|
||||
};
|
||||
|
||||
export type IconName = keyof typeof map;
|
||||
|
||||
@@ -1,25 +1,47 @@
|
||||
import React from 'react';
|
||||
import { Stack, Box, Flex, useTheme } from '@chakra-ui/react';
|
||||
import { Box, Flex, useTheme, Grid, type GridProps, theme } from '@chakra-ui/react';
|
||||
import type { StackProps } from '@chakra-ui/react';
|
||||
import MyIcon from '@/components/Icon';
|
||||
|
||||
// @ts-ignore
|
||||
interface Props extends StackProps {
|
||||
list: { label: string; value: string | number }[];
|
||||
interface Props extends GridProps {
|
||||
list: { icon?: string; title: string; desc?: string; value: string | number }[];
|
||||
align?: 'top' | 'center';
|
||||
value: string | number;
|
||||
onChange: (e: string | number) => void;
|
||||
}
|
||||
|
||||
const Radio = ({ list, value, onChange, ...props }: Props) => {
|
||||
const MyRadio = ({ list, value, align = 'center', onChange, ...props }: Props) => {
|
||||
const theme = useTheme();
|
||||
return (
|
||||
<Stack {...props} spacing={5} direction={'row'}>
|
||||
<Grid gridGap={[3, 5]} fontSize={['sm', 'md']} {...props}>
|
||||
{list.map((item) => (
|
||||
<Flex
|
||||
key={item.value}
|
||||
alignItems={'center'}
|
||||
alignItems={align}
|
||||
cursor={'pointer'}
|
||||
userSelect={'none'}
|
||||
_before={{
|
||||
py={3}
|
||||
pl={'14px'}
|
||||
pr={'36px'}
|
||||
border={theme.borders.sm}
|
||||
borderWidth={'1.5px'}
|
||||
borderRadius={'md'}
|
||||
bg={'myWhite.300'}
|
||||
position={'relative'}
|
||||
{...(value === item.value
|
||||
? {
|
||||
borderColor: 'myBlue.700'
|
||||
}
|
||||
: {
|
||||
_hover: {
|
||||
bg: 'white'
|
||||
}
|
||||
})}
|
||||
_after={{
|
||||
content: '""',
|
||||
position: 'absolute',
|
||||
right: '14px',
|
||||
w: '16px',
|
||||
h: '16px',
|
||||
mr: 1,
|
||||
@@ -36,18 +58,21 @@ const Radio = ({ list, value, onChange, ...props }: Props) => {
|
||||
borderColor: 'myGray.200'
|
||||
})
|
||||
}}
|
||||
_hover={{
|
||||
_before: {
|
||||
borderColor: 'myBlue.600'
|
||||
}
|
||||
}}
|
||||
onClick={() => onChange(item.value)}
|
||||
>
|
||||
{item.label}
|
||||
{!!item.icon && <MyIcon mr={'14px'} name={item.icon as any} w={'18px'} />}
|
||||
<Box>
|
||||
<Box>{item.title}</Box>
|
||||
{!!item.desc && (
|
||||
<Box fontSize={'sm'} color={'myGray.500'}>
|
||||
{item.desc}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
</Flex>
|
||||
))}
|
||||
</Stack>
|
||||
</Grid>
|
||||
);
|
||||
};
|
||||
|
||||
export default Radio;
|
||||
export default MyRadio;
|
||||
|
||||
@@ -5,6 +5,14 @@ export enum UserAuthTypeEnum {
|
||||
|
||||
export const PRICE_SCALE = 100000;
|
||||
|
||||
export const fileImgs = [
|
||||
{ reg: /pdf/gi, src: '/imgs/files/pdf.svg' },
|
||||
{ reg: /csv/gi, src: '/imgs/files/csv.svg' },
|
||||
{ reg: /(doc|docs)/gi, src: '/imgs/files/doc.svg' },
|
||||
{ reg: /txt/gi, src: '/imgs/files/txt.svg' },
|
||||
{ reg: /md/gi, src: '/imgs/files/markdown.svg' }
|
||||
];
|
||||
|
||||
export const htmlTemplate = `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
@@ -30,7 +30,7 @@ export const useConfirm = ({ title = '提示', content }: { title?: string; cont
|
||||
() => (
|
||||
<AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
|
||||
<AlertDialogOverlay>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogContent maxW={'min(90vw,400px)'}>
|
||||
<AlertDialogHeader fontSize="lg" fontWeight="bold">
|
||||
{title}
|
||||
</AlertDialogHeader>
|
||||
|
||||
@@ -13,13 +13,15 @@ export const usePagination = <T = any,>({
|
||||
pageSize = 10,
|
||||
params = {},
|
||||
defaultRequest = true,
|
||||
type = 'button'
|
||||
type = 'button',
|
||||
onChange
|
||||
}: {
|
||||
api: (data: any) => any;
|
||||
pageSize?: number;
|
||||
params?: Record<string, any>;
|
||||
defaultRequest?: boolean;
|
||||
type?: 'button' | 'scroll';
|
||||
onChange?: (pageNum: number) => void;
|
||||
}) => {
|
||||
const elementRef = useRef<HTMLDivElement>(null);
|
||||
const { toast } = useToast();
|
||||
@@ -39,6 +41,7 @@ export const usePagination = <T = any,>({
|
||||
setPageNum(num);
|
||||
res.total !== undefined && setTotal(res.total);
|
||||
setData(res.data);
|
||||
onChange && onChange(num);
|
||||
} catch (error: any) {
|
||||
toast({
|
||||
title: error?.message || '获取数据异常',
|
||||
|
||||
@@ -8,9 +8,8 @@ import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import { startQueue } from '@/service/utils/tools';
|
||||
import { PgClient } from '@/service/pg';
|
||||
import { modelToolMap } from '@/utils/plugin';
|
||||
import { OpenAiChatEnum } from '@/constants/model';
|
||||
|
||||
type DateItemType = { a: string; q: string; source?: string };
|
||||
export type DateItemType = { a: string; q: string; source?: string };
|
||||
|
||||
export type Props = {
|
||||
kbId: string;
|
||||
|
||||
@@ -3,28 +3,46 @@ import { jsonRes } from '@/service/response';
|
||||
import { authUser } from '@/service/utils/auth';
|
||||
import { PgClient } from '@/service/pg';
|
||||
import { withNextCors } from '@/service/utils/tools';
|
||||
import { KB, connectToDatabase } from '@/service/mongo';
|
||||
import { getVector } from '../plugin/vector';
|
||||
|
||||
export type Props = {
|
||||
dataId: string;
|
||||
kbId: string;
|
||||
a?: string;
|
||||
q?: string;
|
||||
};
|
||||
|
||||
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
const { dataId, a = '', q = '' } = req.body as { dataId: string; a?: string; q?: string };
|
||||
const { dataId, a = '', q = '', kbId } = req.body as Props;
|
||||
|
||||
if (!dataId) {
|
||||
throw new Error('缺少参数');
|
||||
}
|
||||
|
||||
await connectToDatabase();
|
||||
|
||||
// 凭证校验
|
||||
const { userId } = await authUser({ req });
|
||||
|
||||
// find model
|
||||
const kb = await KB.findById(kbId, 'model');
|
||||
|
||||
if (!kb) {
|
||||
throw new Error("Can't find database");
|
||||
}
|
||||
|
||||
// get vector
|
||||
const vector = await (async () => {
|
||||
const { vectors = [] } = await (async () => {
|
||||
if (q) {
|
||||
return getVector({
|
||||
userId,
|
||||
input: [q]
|
||||
input: [q],
|
||||
model: kb.model
|
||||
});
|
||||
}
|
||||
return [];
|
||||
return { vectors: [[]] };
|
||||
})();
|
||||
|
||||
// 更新 pg 内容.仅修改a，不需要更新向量。
|
||||
@@ -36,7 +54,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
|
||||
...(q
|
||||
? [
|
||||
{ key: 'q', value: q.replace(/'/g, '"') },
|
||||
{ key: 'vector', value: `[${vector[0]}]` }
|
||||
{ key: 'vector', value: `[${vectors[0]}]` }
|
||||
]
|
||||
: [])
|
||||
]
|
||||
|
||||
@@ -18,13 +18,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
|
||||
|
||||
await connectToDatabase();
|
||||
|
||||
const data = await KB.findOne(
|
||||
{
|
||||
_id: id,
|
||||
userId
|
||||
},
|
||||
'_id avatar name userId tags'
|
||||
);
|
||||
const data = await KB.findOne({
|
||||
_id: id,
|
||||
userId
|
||||
});
|
||||
|
||||
if (!data) {
|
||||
throw new Error('kb is not exist');
|
||||
@@ -36,6 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
|
||||
avatar: data.avatar,
|
||||
name: data.name,
|
||||
userId: data.userId,
|
||||
model: data.model,
|
||||
tags: data.tags.join(' ')
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import React from 'react';
|
||||
import React, { useMemo } from 'react';
|
||||
import { Flex, useTheme, Box } from '@chakra-ui/react';
|
||||
import { useGlobalStore } from '@/store/global';
|
||||
import MyIcon from '@/components/Icon';
|
||||
@@ -8,18 +8,20 @@ import ToolMenu from './ToolMenu';
|
||||
import { ChatItemType } from '@/types/chat';
|
||||
|
||||
const ChatHeader = ({
|
||||
title,
|
||||
history,
|
||||
appName,
|
||||
appAvatar,
|
||||
onOpenSlider
|
||||
}: {
|
||||
title: string;
|
||||
history: ChatItemType[];
|
||||
appName: string;
|
||||
appAvatar: string;
|
||||
onOpenSlider: () => void;
|
||||
}) => {
|
||||
const theme = useTheme();
|
||||
const { isPc } = useGlobalStore();
|
||||
const title = useMemo(() => {}, []);
|
||||
|
||||
return (
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
@@ -32,11 +34,11 @@ const ChatHeader = ({
|
||||
{isPc ? (
|
||||
<>
|
||||
<Box mr={3} color={'myGray.1000'}>
|
||||
{title}
|
||||
{appName}
|
||||
</Box>
|
||||
<Tag display={'flex'}>
|
||||
<MyIcon name={'history'} w={'14px'} />
|
||||
<Box ml={1}>{history.length}条记录</Box>
|
||||
<Box ml={1}>{history.length === 0 ? '新的对话' : `${history.length}条记录`}</Box>
|
||||
</Tag>
|
||||
<Box flex={1} />
|
||||
</>
|
||||
@@ -46,7 +48,7 @@ const ChatHeader = ({
|
||||
<Flex px={3} alignItems={'center'} flex={'1 0 0'} w={0} justifyContent={'center'}>
|
||||
<Avatar src={appAvatar} w={'16px'} />
|
||||
<Box ml={1} className="textEllipsis">
|
||||
{title}
|
||||
{appName}
|
||||
</Box>
|
||||
</Flex>
|
||||
</>
|
||||
|
||||
@@ -77,7 +77,7 @@ const ChatHistorySlider = ({
|
||||
}
|
||||
>
|
||||
<Avatar src={appAvatar} />
|
||||
<Box ml={2} fontWeight={'bold'} className={'textEllipsis'}>
|
||||
<Box flex={'1 0 0'} w={0} ml={2} fontWeight={'bold'} className={'textEllipsis'}>
|
||||
{appName}
|
||||
</Box>
|
||||
</Flex>
|
||||
|
||||
@@ -288,7 +288,7 @@ const Chat = () => {
|
||||
{/* header */}
|
||||
<ChatHeader
|
||||
appAvatar={chatData.app.avatar}
|
||||
title={chatData.app.name}
|
||||
appName={chatData.app.name}
|
||||
history={chatData.history}
|
||||
onOpenSlider={onOpenSlider}
|
||||
/>
|
||||
|
||||
@@ -184,7 +184,7 @@ const ShareChat = ({ shareId, historyId }: { shareId: string; historyId: string
|
||||
{/* header */}
|
||||
<ChatHeader
|
||||
appAvatar={shareChatData.app.avatar}
|
||||
title={shareChatData.history.title}
|
||||
appName={shareChatData.history.title}
|
||||
history={shareChatData.history.chats}
|
||||
onOpenSlider={onOpenSlider}
|
||||
/>
|
||||
|
||||
@@ -1,18 +1,5 @@
|
||||
import React, { useCallback, useState, useRef, useEffect } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Card,
|
||||
IconButton,
|
||||
Flex,
|
||||
Button,
|
||||
useDisclosure,
|
||||
Menu,
|
||||
MenuButton,
|
||||
MenuList,
|
||||
MenuItem,
|
||||
Input,
|
||||
Grid
|
||||
} from '@chakra-ui/react';
|
||||
import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react';
|
||||
import type { KbDataItemType } from '@/types/plugin';
|
||||
import { usePagination } from '@/hooks/usePagination';
|
||||
import {
|
||||
@@ -26,15 +13,14 @@ import { fileDownload } from '@/utils/file';
|
||||
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import Papa from 'papaparse';
|
||||
import dynamic from 'next/dynamic';
|
||||
import InputModal, { FormData as InputDataType } from './InputDataModal';
|
||||
import { debounce } from 'lodash';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
|
||||
const SelectFileModal = dynamic(() => import('./SelectFileModal'), { ssr: true });
|
||||
const SelectCsvModal = dynamic(() => import('./SelectCsvModal'), { ssr: true });
|
||||
import MyIcon from '@/components/Icon';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
|
||||
const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
const BoxRef = useRef<HTMLDivElement>(null);
|
||||
const lastSearch = useRef('');
|
||||
const [searchText, setSearchText] = useState('');
|
||||
const { toast } = useToast();
|
||||
@@ -46,74 +32,61 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
Pagination,
|
||||
total,
|
||||
getData,
|
||||
pageNum
|
||||
pageNum,
|
||||
pageSize
|
||||
} = usePagination<KbDataItemType>({
|
||||
api: getKbDataList,
|
||||
pageSize: 24,
|
||||
defaultRequest: false,
|
||||
params: {
|
||||
kbId,
|
||||
searchText
|
||||
},
|
||||
onChange() {
|
||||
if (BoxRef.current) {
|
||||
BoxRef.current.scrollTop = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const [editInputData, setEditInputData] = useState<InputDataType>();
|
||||
|
||||
const {
|
||||
isOpen: isOpenSelectFileModal,
|
||||
onOpen: onOpenSelectFileModal,
|
||||
onClose: onCloseSelectFileModal
|
||||
} = useDisclosure();
|
||||
const {
|
||||
isOpen: isOpenSelectCsvModal,
|
||||
onOpen: onOpenSelectCsvModal,
|
||||
onClose: onCloseSelectCsvModal
|
||||
} = useDisclosure();
|
||||
|
||||
const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch } = useQuery(
|
||||
['getModelSplitDataList', kbId],
|
||||
() => getTrainingData({ kbId, init: false }),
|
||||
{
|
||||
const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch: refetchTrainingData } =
|
||||
useQuery(['getModelSplitDataList', kbId], () => getTrainingData({ kbId, init: false }), {
|
||||
onError(err) {
|
||||
console.log(err);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
const refetchData = useCallback(
|
||||
(num = pageNum) => {
|
||||
getData(num);
|
||||
refetch();
|
||||
refetchTrainingData();
|
||||
return null;
|
||||
},
|
||||
[getData, pageNum, refetch]
|
||||
[getData, pageNum, refetchTrainingData]
|
||||
);
|
||||
|
||||
// get al data and export csv
|
||||
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
|
||||
mutationFn: () => getExportDataList(kbId),
|
||||
onSuccess(res) {
|
||||
try {
|
||||
const text = Papa.unparse({
|
||||
fields: ['question', 'answer', 'source'],
|
||||
data: res
|
||||
});
|
||||
fileDownload({
|
||||
text,
|
||||
type: 'text/csv',
|
||||
filename: 'data.csv'
|
||||
});
|
||||
toast({
|
||||
title: '导出成功，下次导出需要半小时后',
|
||||
status: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
error;
|
||||
}
|
||||
const text = Papa.unparse({
|
||||
fields: ['question', 'answer', 'source'],
|
||||
data: res
|
||||
});
|
||||
fileDownload({
|
||||
text,
|
||||
type: 'text/csv',
|
||||
filename: 'data.csv'
|
||||
});
|
||||
toast({
|
||||
title: '导出成功，下次导出需要半小时后',
|
||||
status: 'success'
|
||||
});
|
||||
},
|
||||
onError(err: any) {
|
||||
toast({
|
||||
title: typeof err === 'string' ? err : err?.message || '导出异常',
|
||||
title: getErrText(err, '导出异常'),
|
||||
status: 'error'
|
||||
});
|
||||
console.log(err);
|
||||
@@ -134,59 +107,39 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
enabled: qaListLen > 0 || vectorListLen > 0
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
setSearchText('');
|
||||
getData(1);
|
||||
}, [kbId]);
|
||||
|
||||
return (
|
||||
<Box position={'relative'} px={5} py={[1, 5]}>
|
||||
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
|
||||
<Flex justifyContent={'space-between'}>
|
||||
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
|
||||
知识库数据: {total}组
|
||||
</Box>
|
||||
<Box>
|
||||
<IconButton
|
||||
icon={<RepeatIcon />}
|
||||
aria-label={'refresh'}
|
||||
variant={'base'}
|
||||
isLoading={isLoading}
|
||||
mr={[2, 4]}
|
||||
size={'sm'}
|
||||
onClick={() => {
|
||||
getData(pageNum);
|
||||
getTrainingData({ kbId, init: true });
|
||||
}}
|
||||
/>
|
||||
<MyTooltip label={'刷新'}>
|
||||
<IconButton
|
||||
icon={<RepeatIcon />}
|
||||
aria-label={'refresh'}
|
||||
variant={'base'}
|
||||
isLoading={isLoading}
|
||||
mr={[2, 4]}
|
||||
size={'sm'}
|
||||
onClick={() => {
|
||||
getData(pageNum);
|
||||
getTrainingData({ kbId, init: true });
|
||||
}}
|
||||
/>
|
||||
</MyTooltip>
|
||||
<Button
|
||||
variant={'base'}
|
||||
mr={2}
|
||||
size={'sm'}
|
||||
variant={'base'}
|
||||
borderColor={'myBlue.600'}
|
||||
color={'myBlue.600'}
|
||||
isLoading={isLoadingExport || isLoading}
|
||||
title={'半小时仅能导出1次'}
|
||||
onClick={() => onclickExport()}
|
||||
>
|
||||
导出csv
|
||||
导出数据
|
||||
</Button>
|
||||
<Menu autoSelect={false}>
|
||||
<MenuButton as={Button} size={'sm'} isLoading={isLoading}>
|
||||
导入
|
||||
</MenuButton>
|
||||
<MenuList>
|
||||
<MenuItem
|
||||
onClick={() =>
|
||||
setEditInputData({
|
||||
a: '',
|
||||
q: ''
|
||||
})
|
||||
}
|
||||
>
|
||||
手动输入
|
||||
</MenuItem>
|
||||
<MenuItem onClick={onOpenSelectFileModal}>文本/文件拆分</MenuItem>
|
||||
<MenuItem onClick={onOpenSelectCsvModal}>csv 问答对导入</MenuItem>
|
||||
</MenuList>
|
||||
</Menu>
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex my={4}>
|
||||
@@ -204,7 +157,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
maxW={['60%', '300px']}
|
||||
size={'sm'}
|
||||
value={searchText}
|
||||
placeholder="根据匹配知识，补充知识和来源搜索"
|
||||
placeholder="根据匹配知识，补充知识和来源进行搜索"
|
||||
onChange={(e) => {
|
||||
setSearchText(e.target.value);
|
||||
getFirstData();
|
||||
@@ -245,7 +198,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
}
|
||||
>
|
||||
<Box
|
||||
h={'100px'}
|
||||
h={'95px'}
|
||||
overflow={'hidden'}
|
||||
wordBreak={'break-all'}
|
||||
px={3}
|
||||
@@ -255,7 +208,9 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
<Box color={'myGray.1000'} mb={2}>
|
||||
{item.q}
|
||||
</Box>
|
||||
<Box color={'myGray.600'}>{item.a}</Box>
|
||||
<Box color={'myGray.600'} className={'textEllipsis3'}>
|
||||
{item.a}
|
||||
</Box>
|
||||
</Box>
|
||||
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
|
||||
<Box className={'textEllipsis'} flex={1}>
|
||||
@@ -292,9 +247,19 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
))}
|
||||
</Grid>
|
||||
|
||||
<Flex mt={2} justifyContent={'center'}>
|
||||
<Pagination />
|
||||
</Flex>
|
||||
{total > pageSize && (
|
||||
<Flex mt={2} justifyContent={'center'}>
|
||||
<Pagination />
|
||||
</Flex>
|
||||
)}
|
||||
{total === 0 && (
|
||||
<Flex h={'100%'} flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
|
||||
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
|
||||
<Box mt={2} color={'myGray.500'}>
|
||||
知识库空空如也
|
||||
</Box>
|
||||
</Flex>
|
||||
)}
|
||||
|
||||
{editInputData !== undefined && (
|
||||
<InputModal
|
||||
@@ -304,12 +269,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
|
||||
onSuccess={() => refetchData()}
|
||||
/>
|
||||
)}
|
||||
{isOpenSelectFileModal && (
|
||||
<SelectFileModal kbId={kbId} onClose={onCloseSelectFileModal} onSuccess={refetchData} />
|
||||
)}
|
||||
{isOpenSelectCsvModal && (
|
||||
<SelectCsvModal kbId={kbId} onClose={onCloseSelectCsvModal} onSuccess={refetchData} />
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
83
client/src/pages/kb/detail/components/Import.tsx
Normal file
83
client/src/pages/kb/detail/components/Import.tsx
Normal file
@@ -0,0 +1,83 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Box, type BoxProps, Flex, Textarea, useTheme } from '@chakra-ui/react';
|
||||
import MyRadio from '@/components/Radio/index';
|
||||
import dynamic from 'next/dynamic';
|
||||
|
||||
import ManualImport from './Import/Manual';
|
||||
|
||||
const ChunkImport = dynamic(() => import('./Import/Chunk'), {
|
||||
ssr: true
|
||||
});
|
||||
const QAImport = dynamic(() => import('./Import/QA'), {
|
||||
ssr: true
|
||||
});
|
||||
const CsvImport = dynamic(() => import('./Import/Csv'), {
|
||||
ssr: true
|
||||
});
|
||||
|
||||
enum ImportTypeEnum {
|
||||
manual = 'manual',
|
||||
index = 'index',
|
||||
qa = 'qa',
|
||||
csv = 'csv'
|
||||
}
|
||||
|
||||
const ImportData = ({ kbId }: { kbId: string }) => {
|
||||
const theme = useTheme();
|
||||
const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.manual);
|
||||
const TitleStyle: BoxProps = {
|
||||
fontWeight: 'bold',
|
||||
fontSize: ['md', 'xl'],
|
||||
mb: [3, 5]
|
||||
};
|
||||
|
||||
return (
|
||||
<Flex flexDirection={'column'} h={'100%'} pt={[1, 5]}>
|
||||
<Box {...TitleStyle} px={[4, 8]}>
|
||||
数据导入方式
|
||||
</Box>
|
||||
<Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
|
||||
<MyRadio
|
||||
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2, 350px)']}
|
||||
list={[
|
||||
{
|
||||
icon: 'manualImport',
|
||||
title: '手动输入',
|
||||
desc: '手动输入问答对，是最精准的数据',
|
||||
value: ImportTypeEnum.manual
|
||||
},
|
||||
{
|
||||
icon: 'indexImport',
|
||||
title: '直接分段',
|
||||
desc: '选择文本文件，直接将其按分段进行处理',
|
||||
value: ImportTypeEnum.index
|
||||
},
|
||||
{
|
||||
icon: 'qaImport',
|
||||
title: 'QA拆分',
|
||||
desc: '选择文本文件，让大模型自动生成问答对',
|
||||
value: ImportTypeEnum.qa
|
||||
},
|
||||
{
|
||||
icon: 'csvImport',
|
||||
title: 'CSV 导入',
|
||||
desc: '批量导入问答对，是最精准的数据',
|
||||
value: ImportTypeEnum.csv
|
||||
}
|
||||
]}
|
||||
value={importType}
|
||||
onChange={(e) => setImportType(e as `${ImportTypeEnum}`)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box flex={'1 0 0'} h={0}>
|
||||
{importType === ImportTypeEnum.manual && <ManualImport kbId={kbId} />}
|
||||
{importType === ImportTypeEnum.index && <ChunkImport kbId={kbId} />}
|
||||
{importType === ImportTypeEnum.qa && <QAImport kbId={kbId} />}
|
||||
{importType === ImportTypeEnum.csv && <CsvImport kbId={kbId} />}
|
||||
</Box>
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export default ImportData;
|
||||
459
client/src/pages/kb/detail/components/Import/Chunk.tsx
Normal file
459
client/src/pages/kb/detail/components/Import/Chunk.tsx
Normal file
@@ -0,0 +1,459 @@
|
||||
import React, { useState, useCallback, useMemo } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
useTheme,
|
||||
NumberInput,
|
||||
NumberInputField,
|
||||
NumberInputStepper,
|
||||
NumberIncrementStepper,
|
||||
NumberDecrementStepper,
|
||||
Image,
|
||||
Textarea
|
||||
} from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import { splitText_token } from '@/utils/file';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
import { formatPrice } from '@/utils/user';
|
||||
import { vectorModelList } from '@/store/static';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import CloseIcon from '@/components/Icon/close';
|
||||
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import { fileImgs } from '@/constants/common';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import FileSelect from './FileSelect';
|
||||
import { useRouter } from 'next/router';
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
|
||||
|
||||
type FileItemType = {
|
||||
id: string;
|
||||
filename: string;
|
||||
text: string;
|
||||
icon: string;
|
||||
chunks: string[];
|
||||
tokens: number;
|
||||
};
|
||||
|
||||
const ChunkImport = ({ kbId }: { kbId: string }) => {
|
||||
const model = vectorModelList[0]?.model;
|
||||
const unitPrice = vectorModelList[0]?.price || 0.2;
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
|
||||
const [chunkLen, setChunkLen] = useState(500);
|
||||
const [showRePreview, setShowRePreview] = useState(false);
|
||||
const [selecting, setSelecting] = useState(false);
|
||||
const [files, setFiles] = useState<FileItemType[]>([]);
|
||||
const [previewFile, setPreviewFile] = useState<FileItemType>();
|
||||
const [successChunks, setSuccessChunks] = useState(0);
|
||||
|
||||
const totalChunk = useMemo(
|
||||
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
|
||||
[files]
|
||||
);
|
||||
const emptyFiles = useMemo(() => files.length === 0, [files]);
|
||||
|
||||
// price count
|
||||
const price = useMemo(() => {
|
||||
return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice);
|
||||
}, [files, unitPrice]);
|
||||
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (files: File[]) => {
|
||||
setSelecting(true);
|
||||
try {
|
||||
let promise = Promise.resolve();
|
||||
files.forEach((file) => {
|
||||
promise = promise.then(async () => {
|
||||
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
|
||||
const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
|
||||
const text = await (async () => {
|
||||
switch (extension) {
|
||||
case 'txt':
|
||||
case 'md':
|
||||
return readTxtContent(file);
|
||||
case 'pdf':
|
||||
return readPdfContent(file);
|
||||
case 'doc':
|
||||
case 'docx':
|
||||
return readDocContent(file);
|
||||
}
|
||||
return '';
|
||||
})();
|
||||
|
||||
if (icon && text) {
|
||||
const splitRes = splitText_token({
|
||||
text: text,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
|
||||
setFiles((state) => [
|
||||
{
|
||||
id: nanoid(),
|
||||
filename: file.name,
|
||||
text,
|
||||
icon,
|
||||
...splitRes
|
||||
},
|
||||
...state
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
await promise;
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: typeof error === 'string' ? error : '解析文件失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setSelecting(false);
|
||||
},
|
||||
[chunkLen, toast]
|
||||
);
|
||||
|
||||
const { mutate: onclickUpload, isLoading: uploading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
const chunks: { a: string; q: string; source: string }[] = [];
|
||||
files.forEach((file) =>
|
||||
file.chunks.forEach((chunk) => {
|
||||
chunks.push({
|
||||
q: chunk,
|
||||
a: '',
|
||||
source: file.filename
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
// subsection import
|
||||
let success = 0;
|
||||
const step = 100;
|
||||
for (let i = 0; i < chunks.length; i += step) {
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
model,
|
||||
data: chunks.slice(i, i + step),
|
||||
mode: TrainingModeEnum.index
|
||||
});
|
||||
|
||||
success += insertLen;
|
||||
setSuccessChunks(success);
|
||||
}
|
||||
|
||||
toast({
|
||||
title: `去重后共导入 ${success} 条数据，请耐心等待训练.`,
|
||||
status: 'success'
|
||||
});
|
||||
|
||||
router.replace({
|
||||
query: {
|
||||
kbId,
|
||||
currentTab: 'data'
|
||||
}
|
||||
});
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err, '导入文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const onRePreview = useCallback(async () => {
|
||||
try {
|
||||
const splitRes = files.map((item) =>
|
||||
splitText_token({
|
||||
text: item.text,
|
||||
maxLen: chunkLen
|
||||
})
|
||||
);
|
||||
|
||||
setFiles((state) =>
|
||||
state.map((file, index) => ({
|
||||
...file,
|
||||
...splitRes[index]
|
||||
}))
|
||||
);
|
||||
setPreviewFile(undefined);
|
||||
setShowRePreview(false);
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(error, '文本分段异常')
|
||||
});
|
||||
}
|
||||
}, [chunkLen, files, toast]);
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<Box flex={1} minW={['auto', '400px']} w={['100%', 0]} p={[4, 8]}>
|
||||
<FileSelect
|
||||
fileExtension={fileExtension}
|
||||
onSelectFile={onSelectFile}
|
||||
isLoading={selecting}
|
||||
py={emptyFiles ? '100px' : 5}
|
||||
/>
|
||||
|
||||
{!emptyFiles && (
|
||||
<>
|
||||
<Box py={4} maxH={'400px'}>
|
||||
{files.map((item) => (
|
||||
<Flex
|
||||
key={item.id}
|
||||
w={'100%'}
|
||||
_notLast={{ mb: 5 }}
|
||||
px={5}
|
||||
py={2}
|
||||
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
|
||||
borderRadius={'md'}
|
||||
cursor={'pointer'}
|
||||
position={'relative'}
|
||||
alignItems={'center'}
|
||||
_hover={{
|
||||
bg: 'myBlue.100',
|
||||
'& .delete': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
onClick={() => setPreviewFile(item)}
|
||||
>
|
||||
<Image src={item.icon} w={'16px'} alt={''} />
|
||||
<Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
|
||||
{item.filename}
|
||||
</Box>
|
||||
<MyIcon
|
||||
position={'absolute'}
|
||||
right={3}
|
||||
className="delete"
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
display={['block', 'none']}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Box>
|
||||
{/* chunk size */}
|
||||
<Flex py={5} alignItems={'center'}>
|
||||
<Box>
|
||||
段落长度
|
||||
<MyTooltip
|
||||
label={'基于 Gpt3.5 的 Token 计算方法进行分段。前后段落会有 30% 的内容重叠。'}
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<NumberInput
|
||||
ml={4}
|
||||
flex={1}
|
||||
defaultValue={chunkLen}
|
||||
min={300}
|
||||
max={1000}
|
||||
step={10}
|
||||
onChange={(e) => {
|
||||
setChunkLen(+e);
|
||||
setShowRePreview(true);
|
||||
}}
|
||||
>
|
||||
<NumberInputField />
|
||||
<NumberInputStepper>
|
||||
<NumberIncrementStepper />
|
||||
<NumberDecrementStepper />
|
||||
</NumberInputStepper>
|
||||
</NumberInput>
|
||||
</Flex>
|
||||
{/* price */}
|
||||
<Flex py={5} alignItems={'center'}>
|
||||
<Box>
|
||||
预估价格
|
||||
<MyTooltip label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Box ml={4}>
|
||||
{}
|
||||
{price}元
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex mt={3}>
|
||||
{showRePreview && (
|
||||
<Button variant={'base'} mr={4} onClick={onRePreview}>
|
||||
重新生成预览
|
||||
</Button>
|
||||
)}
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? (
|
||||
<Box>{Math.round((successChunks / totalChunk) * 100)}%</Box>
|
||||
) : (
|
||||
'确认导入'
|
||||
)}
|
||||
</Button>
|
||||
</Flex>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
{!emptyFiles && (
|
||||
<Box flex={'2 0 0'} w={['100%', 0]} h={'100%'}>
|
||||
{previewFile ? (
|
||||
<Box
|
||||
position={'relative'}
|
||||
display={['block', 'flex']}
|
||||
h={'100%'}
|
||||
flexDirection={'column'}
|
||||
pt={[4, 8]}
|
||||
bg={'myWhite.400'}
|
||||
>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
{previewFile.filename}
|
||||
</Box>
|
||||
<CloseIcon
|
||||
position={'absolute'}
|
||||
right={[4, 8]}
|
||||
top={4}
|
||||
onClick={() => setPreviewFile(undefined)}
|
||||
/>
|
||||
<Box
|
||||
flex={'1 0 0'}
|
||||
h={['auto', 0]}
|
||||
overflow={'overlay'}
|
||||
px={[4, 8]}
|
||||
my={4}
|
||||
contentEditable
|
||||
dangerouslySetInnerHTML={{ __html: previewFile.text }}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
setShowRePreview(true);
|
||||
|
||||
setFiles((state) =>
|
||||
state.map((file) =>
|
||||
file.id === previewFile.id
|
||||
? {
|
||||
...file,
|
||||
text: val
|
||||
}
|
||||
: file
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
) : (
|
||||
<Box h={'100%'} pt={[4, 8]} overflow={'overlay'}>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
分段预览({totalChunk}组)
|
||||
</Box>
|
||||
<Box px={[4, 8]} overflow={'overlay'}>
|
||||
{files.map((file) =>
|
||||
file.chunks.map((item, i) => (
|
||||
<Box
|
||||
key={item}
|
||||
py={4}
|
||||
bg={'myWhite.500'}
|
||||
my={2}
|
||||
borderRadius={'md'}
|
||||
fontSize={'sm'}
|
||||
_hover={{ ...hoverDeleteStyles }}
|
||||
>
|
||||
<Flex mb={1} px={4} userSelect={'none'}>
|
||||
<Box px={3} py={'1px'} border={theme.borders.base} borderRadius={'md'}>
|
||||
# {i + 1}
|
||||
</Box>
|
||||
<Box flex={1} />
|
||||
<DeleteIcon
|
||||
onClick={() => {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [
|
||||
...file.chunks.slice(0, i),
|
||||
...file.chunks.slice(i + 1)
|
||||
]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
<Box
|
||||
px={4}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
contentEditable
|
||||
dangerouslySetInnerHTML={{ __html: item }}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
|
||||
if (val === '') {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [
|
||||
...file.chunks.slice(0, i),
|
||||
...file.chunks.slice(i + 1)
|
||||
]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
} else {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: file.chunks.map((chunk, index) =>
|
||||
i === index ? val : chunk
|
||||
)
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
))
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmChild />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default ChunkImport;
|
||||
241
client/src/pages/kb/detail/components/Import/Csv.tsx
Normal file
241
client/src/pages/kb/detail/components/Import/Csv.tsx
Normal file
@@ -0,0 +1,241 @@
|
||||
import React, { useState, useCallback, useMemo } from 'react';
|
||||
import { Box, Flex, Button, useTheme, Image } from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
import { vectorModelList } from '@/store/static';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import FileSelect from './FileSelect';
|
||||
import { useRouter } from 'next/router';
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
import { readCsvContent } from '@/utils/file';
|
||||
|
||||
const fileExtension = '.csv';
|
||||
|
||||
type FileItemType = {
|
||||
id: string;
|
||||
filename: string;
|
||||
chunks: { q: string; a: string }[];
|
||||
};
|
||||
|
||||
const CsvImport = ({ kbId }: { kbId: string }) => {
|
||||
const model = vectorModelList[0]?.model;
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
|
||||
const [selecting, setSelecting] = useState(false);
|
||||
const [files, setFiles] = useState<FileItemType[]>([]);
|
||||
const [successChunks, setSuccessChunks] = useState(0);
|
||||
|
||||
const totalChunk = useMemo(
|
||||
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
|
||||
[files]
|
||||
);
|
||||
const emptyFiles = useMemo(() => files.length === 0, [files]);
|
||||
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (files: File[]) => {
|
||||
setSelecting(true);
|
||||
try {
|
||||
let promise = Promise.resolve();
|
||||
files.forEach((file) => {
|
||||
promise = promise.then(async () => {
|
||||
const { header, data } = await readCsvContent(file);
|
||||
if (header[0] !== 'question' || header[1] !== 'answer') {
|
||||
throw new Error('csv 文件格式有误');
|
||||
}
|
||||
|
||||
setFiles((state) => [
|
||||
{
|
||||
id: nanoid(),
|
||||
filename: file.name,
|
||||
chunks: data.map((item) => ({
|
||||
q: item[0],
|
||||
a: item[1]
|
||||
}))
|
||||
},
|
||||
...state
|
||||
]);
|
||||
});
|
||||
});
|
||||
await promise;
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: typeof error === 'string' ? error : '解析文件失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setSelecting(false);
|
||||
},
|
||||
[toast]
|
||||
);
|
||||
|
||||
const { mutate: onclickUpload, isLoading: uploading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
const chunks: { a: string; q: string; source: string }[] = [];
|
||||
files.forEach((file) =>
|
||||
file.chunks.forEach((chunk) => {
|
||||
chunks.push({
|
||||
...chunk,
|
||||
source: file.filename
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
// subsection import
|
||||
let success = 0;
|
||||
const step = 100;
|
||||
for (let i = 0; i < chunks.length; i += step) {
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
model,
|
||||
data: chunks.slice(i, i + step),
|
||||
mode: TrainingModeEnum.index
|
||||
});
|
||||
|
||||
success += insertLen;
|
||||
setSuccessChunks(success);
|
||||
}
|
||||
|
||||
toast({
|
||||
title: `去重后共导入 ${success} 条数据，请耐心等待训练.`,
|
||||
status: 'success'
|
||||
});
|
||||
|
||||
router.replace({
|
||||
query: {
|
||||
kbId,
|
||||
currentTab: 'data'
|
||||
}
|
||||
});
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err, '导入文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<Box flex={1} minW={['auto', '400px']} w={['100%', 0]} p={[4, 8]}>
|
||||
<FileSelect
|
||||
fileExtension={fileExtension}
|
||||
onSelectFile={onSelectFile}
|
||||
isLoading={selecting}
|
||||
py={emptyFiles ? '100px' : 5}
|
||||
/>
|
||||
|
||||
{!emptyFiles && (
|
||||
<>
|
||||
<Box py={4} maxH={'400px'}>
|
||||
{files.map((item) => (
|
||||
<Flex
|
||||
key={item.id}
|
||||
w={'100%'}
|
||||
_notLast={{ mb: 5 }}
|
||||
px={5}
|
||||
py={2}
|
||||
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
|
||||
borderRadius={'md'}
|
||||
position={'relative'}
|
||||
alignItems={'center'}
|
||||
_hover={{ ...hoverDeleteStyles }}
|
||||
>
|
||||
<Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} />
|
||||
<Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
|
||||
{item.filename}
|
||||
</Box>
|
||||
<MyIcon
|
||||
position={'absolute'}
|
||||
right={3}
|
||||
className="delete"
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
display={['block', 'none']}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Box>
|
||||
|
||||
<Flex mt={3}>
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? (
|
||||
<Box>{Math.round((successChunks / totalChunk) * 100)}%</Box>
|
||||
) : (
|
||||
'确认导入'
|
||||
)}
|
||||
</Button>
|
||||
</Flex>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
{!emptyFiles && (
|
||||
<Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
数据预览({totalChunk}组)
|
||||
</Box>
|
||||
<Box px={[4, 8]} overflow={'overlay'}>
|
||||
{files.map((file) =>
|
||||
file.chunks.slice(0, 100).map((item, i) => (
|
||||
<Box
|
||||
key={i}
|
||||
py={4}
|
||||
bg={'myWhite.500'}
|
||||
my={2}
|
||||
borderRadius={'md'}
|
||||
fontSize={'sm'}
|
||||
_hover={{ ...hoverDeleteStyles }}
|
||||
>
|
||||
<Flex mb={1} px={4} userSelect={'none'}>
|
||||
<Box px={3} py={'1px'} border={theme.borders.base} borderRadius={'md'}>
|
||||
# {i + 1}
|
||||
</Box>
|
||||
<Box flex={1} />
|
||||
<DeleteIcon
|
||||
onClick={() => {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
<Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
|
||||
{`q: ${item.q}\na: ${item.a}`}
|
||||
</Box>
|
||||
</Box>
|
||||
))
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmChild />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default CsvImport;
|
||||
49
client/src/pages/kb/detail/components/Import/FileSelect.tsx
Normal file
49
client/src/pages/kb/detail/components/Import/FileSelect.tsx
Normal file
@@ -0,0 +1,49 @@
|
||||
import React from 'react';
|
||||
import { Box, Flex, type BoxProps } from '@chakra-ui/react';
|
||||
import { useLoading } from '@/hooks/useLoading';
|
||||
import { useSelectFile } from '@/hooks/useSelectFile';
|
||||
|
||||
import MyIcon from '@/components/Icon';
|
||||
|
||||
interface Props extends BoxProps {
|
||||
fileExtension: string;
|
||||
onSelectFile: (files: File[]) => Promise<void>;
|
||||
isLoading?: boolean;
|
||||
}
|
||||
|
||||
const FileSelect = ({ fileExtension, onSelectFile, isLoading, ...props }: Props) => {
|
||||
const { Loading: FileSelectLoading } = useLoading();
|
||||
|
||||
const { File, onOpen } = useSelectFile({
|
||||
fileType: fileExtension,
|
||||
multiple: true
|
||||
});
|
||||
|
||||
return (
|
||||
<Box
|
||||
display={'inline-block'}
|
||||
textAlign={'center'}
|
||||
bg={'myWhite.400'}
|
||||
p={5}
|
||||
borderRadius={'lg'}
|
||||
border={'1px dashed'}
|
||||
borderColor={'myGray.300'}
|
||||
w={'100%'}
|
||||
position={'relative'}
|
||||
{...props}
|
||||
>
|
||||
<Flex justifyContent={'center'} alignItems={'center'}>
|
||||
<MyIcon mr={1} name={'uploadFile'} w={'16px'} />
|
||||
拖拽文件至此，或{' '}
|
||||
<Box ml={1} as={'span'} cursor={'pointer'} color={'myBlue.700'} onClick={onOpen}>
|
||||
选择文件
|
||||
</Box>
|
||||
</Flex>
|
||||
<Box mt={1}>支持 {fileExtension} 文件</Box>
|
||||
<FileSelectLoading loading={isLoading} fixed={false} />
|
||||
<File onSelect={onSelectFile} />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default FileSelect;
|
||||
100
client/src/pages/kb/detail/components/Import/Manual.tsx
Normal file
100
client/src/pages/kb/detail/components/Import/Manual.tsx
Normal file
@@ -0,0 +1,100 @@
|
||||
import React, { useCallback, useState } from 'react';
|
||||
import { Box, type BoxProps, Flex, Textarea, useTheme, Button } from '@chakra-ui/react';
|
||||
import MyRadio from '@/components/Radio/index';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useRequest } from '@/hooks/useRequest';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
import { vectorModelList } from '@/store/static';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
|
||||
type ManualFormType = { q: string; a: string };
|
||||
|
||||
const ManualImport = ({ kbId }: { kbId: string }) => {
|
||||
const { register, handleSubmit, reset } = useForm({
|
||||
defaultValues: { q: '', a: '' }
|
||||
});
|
||||
const { toast } = useToast();
|
||||
|
||||
const { mutate: onImportData, isLoading } = useRequest({
|
||||
mutationFn: async (e: ManualFormType) => {
|
||||
if (e.a.length + e.q.length >= 3000) {
|
||||
toast({
|
||||
title: '总长度超长了',
|
||||
status: 'warning'
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = {
|
||||
a: e.a,
|
||||
q: e.q,
|
||||
source: '手动录入'
|
||||
};
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
model: vectorModelList[0].model,
|
||||
mode: TrainingModeEnum.index,
|
||||
data: [data]
|
||||
});
|
||||
|
||||
if (insertLen === 0) {
|
||||
toast({
|
||||
title: '已存在完全一致的数据',
|
||||
status: 'warning'
|
||||
});
|
||||
} else {
|
||||
toast({
|
||||
title: '导入数据成功,需要一段时间训练',
|
||||
status: 'success'
|
||||
});
|
||||
reset({
|
||||
a: '',
|
||||
q: ''
|
||||
});
|
||||
}
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: getErrText(err, '出现了点意外~'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Box p={[4, 8]}>
|
||||
<Box display={'flex'} flexDirection={['column', 'row']}>
|
||||
<Box flex={1} mr={[0, 4]} mb={[4, 0]} h={['50%', '100%']}>
|
||||
<Box h={'30px'}>{'匹配的知识点'}</Box>
|
||||
<Textarea
|
||||
placeholder={'匹配的知识点。这部分内容会被搜索，请把控内容的质量。总和最多 3000 字。'}
|
||||
maxLength={3000}
|
||||
h={['250px', '500px']}
|
||||
{...register(`q`, {
|
||||
required: true
|
||||
})}
|
||||
/>
|
||||
</Box>
|
||||
<Box flex={1} h={['50%', '100%']}>
|
||||
<Box h={'30px'}>补充知识</Box>
|
||||
<Textarea
|
||||
placeholder={
|
||||
'补充知识。这部分内容不会被搜索，但会作为"匹配的知识点"的内容补充，你可以讲一些细节的内容填写在这里。总和最多 3000 字。'
|
||||
}
|
||||
h={['250px', '500px']}
|
||||
maxLength={3000}
|
||||
{...register('a')}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
<Button mt={5} isLoading={isLoading} onClick={handleSubmit((data) => onImportData(data))}>
|
||||
确认导入
|
||||
</Button>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(ManualImport);
|
||||
451
client/src/pages/kb/detail/components/Import/QA.tsx
Normal file
451
client/src/pages/kb/detail/components/Import/QA.tsx
Normal file
@@ -0,0 +1,451 @@
|
||||
import React, { useState, useCallback, useMemo } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
useTheme,
|
||||
NumberInput,
|
||||
NumberInputField,
|
||||
NumberInputStepper,
|
||||
NumberIncrementStepper,
|
||||
NumberDecrementStepper,
|
||||
Image,
|
||||
Textarea,
|
||||
Input
|
||||
} from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import { splitText_token } from '@/utils/file';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
import { formatPrice } from '@/utils/user';
|
||||
import { qaModelList } from '@/store/static';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import CloseIcon from '@/components/Icon/close';
|
||||
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import { fileImgs } from '@/constants/common';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import FileSelect from './FileSelect';
|
||||
import { useRouter } from 'next/router';
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
|
||||
|
||||
type FileItemType = {
|
||||
id: string;
|
||||
filename: string;
|
||||
text: string;
|
||||
icon: string;
|
||||
chunks: string[];
|
||||
tokens: number;
|
||||
};
|
||||
|
||||
const QAImport = ({ kbId }: { kbId: string }) => {
|
||||
const model = qaModelList[0]?.model;
|
||||
const unitPrice = qaModelList[0]?.price || 3;
|
||||
const chunkLen = qaModelList[0].maxToken / 2;
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
|
||||
const [selecting, setSelecting] = useState(false);
|
||||
const [files, setFiles] = useState<FileItemType[]>([]);
|
||||
const [showRePreview, setShowRePreview] = useState(false);
|
||||
const [previewFile, setPreviewFile] = useState<FileItemType>();
|
||||
const [successChunks, setSuccessChunks] = useState(0);
|
||||
const [prompt, setPrompt] = useState('');
|
||||
|
||||
const totalChunk = useMemo(
|
||||
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
|
||||
[files]
|
||||
);
|
||||
const emptyFiles = useMemo(() => files.length === 0, [files]);
|
||||
|
||||
// price count
|
||||
const price = useMemo(() => {
|
||||
return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice * 1.3);
|
||||
}, [files, unitPrice]);
|
||||
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: `该任务无法终止！导入后会自动调用大模型生成问答对，会有一些细节丢失，请确认！如果余额不足，未完成的任务会被暂停。`
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (files: File[]) => {
|
||||
setSelecting(true);
|
||||
try {
|
||||
let promise = Promise.resolve();
|
||||
files.forEach((file) => {
|
||||
promise = promise.then(async () => {
|
||||
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
|
||||
const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
|
||||
const text = await (async () => {
|
||||
switch (extension) {
|
||||
case 'txt':
|
||||
case 'md':
|
||||
return readTxtContent(file);
|
||||
case 'pdf':
|
||||
return readPdfContent(file);
|
||||
case 'doc':
|
||||
case 'docx':
|
||||
return readDocContent(file);
|
||||
}
|
||||
return '';
|
||||
})();
|
||||
console.log(extension, text, '=====', icon);
|
||||
|
||||
if (icon && text) {
|
||||
const splitRes = splitText_token({
|
||||
text: text,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
|
||||
setFiles((state) => [
|
||||
{
|
||||
id: nanoid(),
|
||||
filename: file.name,
|
||||
text,
|
||||
icon,
|
||||
...splitRes
|
||||
},
|
||||
...state
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
await promise;
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: typeof error === 'string' ? error : '解析文件失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setSelecting(false);
|
||||
},
|
||||
[chunkLen, toast]
|
||||
);
|
||||
|
||||
const { mutate: onclickUpload, isLoading: uploading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
const chunks: { a: string; q: string; source: string }[] = [];
|
||||
files.forEach((file) =>
|
||||
file.chunks.forEach((chunk) => {
|
||||
chunks.push({
|
||||
q: chunk,
|
||||
a: '',
|
||||
source: file.filename
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
// subsection import
|
||||
let success = 0;
|
||||
const step = 100;
|
||||
for (let i = 0; i < chunks.length; i += step) {
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
model,
|
||||
data: chunks.slice(i, i + step),
|
||||
mode: TrainingModeEnum.qa,
|
||||
prompt: prompt || '下面是一段长文本'
|
||||
});
|
||||
|
||||
success += insertLen;
|
||||
setSuccessChunks(success);
|
||||
}
|
||||
|
||||
toast({
|
||||
title: `共导入 ${success} 条数据，请耐心等待训练.`,
|
||||
status: 'success'
|
||||
});
|
||||
|
||||
router.replace({
|
||||
query: {
|
||||
kbId,
|
||||
currentTab: 'data'
|
||||
}
|
||||
});
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err, '导入文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const onRePreview = useCallback(async () => {
|
||||
try {
|
||||
const splitRes = files.map((item) =>
|
||||
splitText_token({
|
||||
text: item.text,
|
||||
maxLen: chunkLen
|
||||
})
|
||||
);
|
||||
|
||||
setFiles((state) =>
|
||||
state.map((file, index) => ({
|
||||
...file,
|
||||
...splitRes[index]
|
||||
}))
|
||||
);
|
||||
setPreviewFile(undefined);
|
||||
setShowRePreview(false);
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(error, '文本分段异常')
|
||||
});
|
||||
}
|
||||
}, [chunkLen, files, toast]);
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<Box flex={1} minW={['auto', '400px']} w={['100%', 0]} p={[4, 8]}>
|
||||
<FileSelect
|
||||
fileExtension={fileExtension}
|
||||
onSelectFile={onSelectFile}
|
||||
isLoading={selecting}
|
||||
py={emptyFiles ? '100px' : 5}
|
||||
/>
|
||||
|
||||
{!emptyFiles && (
|
||||
<>
|
||||
<Box py={4} maxH={'400px'}>
|
||||
{files.map((item) => (
|
||||
<Flex
|
||||
key={item.id}
|
||||
w={'100%'}
|
||||
_notLast={{ mb: 5 }}
|
||||
px={5}
|
||||
py={2}
|
||||
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
|
||||
borderRadius={'md'}
|
||||
cursor={'pointer'}
|
||||
position={'relative'}
|
||||
alignItems={'center'}
|
||||
_hover={{
|
||||
bg: 'myBlue.100',
|
||||
'& .delete': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
onClick={() => setPreviewFile(item)}
|
||||
>
|
||||
<Image src={item.icon} w={'16px'} alt={''} />
|
||||
<Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
|
||||
{item.filename}
|
||||
</Box>
|
||||
<MyIcon
|
||||
position={'absolute'}
|
||||
right={3}
|
||||
className="delete"
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
display={['block', 'none']}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Box>
|
||||
{/* prompt */}
|
||||
<Box py={5}>
|
||||
<Box mb={2}>
|
||||
QA 拆分引导词{' '}
|
||||
<MyTooltip
|
||||
label={`可输入关于文件内容的范围介绍，例如:\n1. 关于 Laf 的介绍\n2. xxx的简历`}
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Flex alignItems={'center'} fontSize={'sm'}>
|
||||
<Box mr={2}>下面是</Box>
|
||||
<Input
|
||||
flex={1}
|
||||
placeholder={'Laf 云函数的介绍'}
|
||||
bg={'myWhite.500'}
|
||||
defaultValue={prompt}
|
||||
onBlur={(e) => (e.target.value ? setPrompt(`下面是"${e.target.value}"`) : '')}
|
||||
/>
|
||||
</Flex>
|
||||
</Box>
|
||||
{/* price */}
|
||||
<Flex py={5} alignItems={'center'}>
|
||||
<Box>
|
||||
预估价格
|
||||
<MyTooltip label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Box ml={4}>{price}元</Box>
|
||||
</Flex>
|
||||
<Flex mt={3}>
|
||||
{showRePreview && (
|
||||
<Button variant={'base'} mr={4} onClick={onRePreview}>
|
||||
重新生成预览
|
||||
</Button>
|
||||
)}
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? (
|
||||
<Box>{Math.round((successChunks / totalChunk) * 100)}%</Box>
|
||||
) : (
|
||||
'确认导入'
|
||||
)}
|
||||
</Button>
|
||||
</Flex>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
{!emptyFiles && (
|
||||
<Box flex={'2 0 0'} w={['100%', 0]} h={'100%'}>
|
||||
{previewFile ? (
|
||||
<Box
|
||||
position={'relative'}
|
||||
display={['block', 'flex']}
|
||||
h={'100%'}
|
||||
flexDirection={'column'}
|
||||
pt={[4, 8]}
|
||||
bg={'myWhite.400'}
|
||||
>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
{previewFile.filename}
|
||||
</Box>
|
||||
<CloseIcon
|
||||
position={'absolute'}
|
||||
right={[4, 8]}
|
||||
top={4}
|
||||
onClick={() => setPreviewFile(undefined)}
|
||||
/>
|
||||
<Box
|
||||
flex={'1 0 0'}
|
||||
h={['auto', 0]}
|
||||
overflow={'overlay'}
|
||||
px={[4, 8]}
|
||||
my={4}
|
||||
contentEditable
|
||||
dangerouslySetInnerHTML={{ __html: previewFile.text }}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
setShowRePreview(true);
|
||||
setFiles((state) =>
|
||||
state.map((file) =>
|
||||
file.id === previewFile.id
|
||||
? {
|
||||
...file,
|
||||
text: val
|
||||
}
|
||||
: file
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
) : (
|
||||
<Box h={'100%'} pt={[4, 8]} overflow={'overlay'}>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
分段预览({totalChunk}组)
|
||||
</Box>
|
||||
<Box px={[4, 8]} overflow={'overlay'}>
|
||||
{files.map((file) =>
|
||||
file.chunks.map((item, i) => (
|
||||
<Box
|
||||
key={item}
|
||||
py={4}
|
||||
bg={'myWhite.500'}
|
||||
my={2}
|
||||
borderRadius={'md'}
|
||||
fontSize={'sm'}
|
||||
_hover={{ ...hoverDeleteStyles }}
|
||||
>
|
||||
<Flex mb={1} px={4} userSelect={'none'}>
|
||||
<Box px={3} py={'1px'} border={theme.borders.base} borderRadius={'md'}>
|
||||
# {i + 1}
|
||||
</Box>
|
||||
<Box flex={1} />
|
||||
<DeleteIcon
|
||||
onClick={() => {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [
|
||||
...file.chunks.slice(0, i),
|
||||
...file.chunks.slice(i + 1)
|
||||
]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
<Box
|
||||
px={4}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
contentEditable
|
||||
dangerouslySetInnerHTML={{ __html: item }}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
|
||||
if (val === '') {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [
|
||||
...file.chunks.slice(0, i),
|
||||
...file.chunks.slice(i + 1)
|
||||
]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
} else {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: file.chunks.map((chunk, index) =>
|
||||
i === index ? val : chunk
|
||||
)
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
))
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmChild />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default QAImport;
|
||||
@@ -17,6 +17,8 @@ import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { UseFormReturn } from 'react-hook-form';
|
||||
import { compressImg } from '@/utils/file';
|
||||
import type { KbItemType } from '@/types/plugin';
|
||||
import { vectorModelList } from '@/store/static';
|
||||
import MySelect from '@/components/Select';
|
||||
import Avatar from '@/components/Avatar';
|
||||
import Tag from '@/components/Tag';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
@@ -144,18 +146,18 @@ const Info = (
|
||||
}));
|
||||
|
||||
return (
|
||||
<Flex p={5} flexDirection={'column'} alignItems={'center'}>
|
||||
<Flex mt={5} w={'100%'} maxW={'350px'} alignItems={'center'}>
|
||||
<Box flex={'0 0 90px'} w={0}>
|
||||
<Box py={5} px={[5, 10]}>
|
||||
<Flex mt={5} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库 ID
|
||||
</Box>
|
||||
<Box flex={1}>{kbDetail._id}</Box>
|
||||
</Flex>
|
||||
<Flex mt={5} w={'100%'} maxW={'350px'} alignItems={'center'}>
|
||||
<Box flex={'0 0 90px'} w={0}>
|
||||
<Flex mt={5} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库头像
|
||||
</Box>
|
||||
<Box flex={1}>
|
||||
<Box flex={[1, '0 0 300px']}>
|
||||
<Avatar
|
||||
m={'auto'}
|
||||
src={getValues('avatar')}
|
||||
@@ -167,27 +169,44 @@ const Info = (
|
||||
/>
|
||||
</Box>
|
||||
</Flex>
|
||||
<FormControl mt={8} w={'100%'} maxW={'350px'} display={'flex'} alignItems={'center'}>
|
||||
<Box flex={'0 0 90px'} w={0}>
|
||||
<FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库名称
|
||||
</Box>
|
||||
<Input
|
||||
flex={1}
|
||||
flex={[1, '0 0 300px']}
|
||||
{...register('name', {
|
||||
required: '知识库名称不能为空'
|
||||
})}
|
||||
/>
|
||||
</FormControl>
|
||||
<Flex mt={8} alignItems={'center'} w={'100%'} maxW={'350px'} flexWrap={'wrap'}>
|
||||
<Box flex={'0 0 90px'} w={0}>
|
||||
<Flex mt={8} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
索引模型
|
||||
</Box>
|
||||
<Box flex={[1, '0 0 300px']}>
|
||||
<MySelect
|
||||
w={'100%'}
|
||||
value={getValues('model')}
|
||||
list={vectorModelList.map((item) => ({
|
||||
label: item.name,
|
||||
value: item.model
|
||||
}))}
|
||||
onchange={(res) => {
|
||||
setValue('model', res);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
分类标签
|
||||
<MyTooltip label={'用空格隔开多个标签，便于搜索'}>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Input
|
||||
flex={1}
|
||||
maxW={'300px'}
|
||||
flex={[1, '0 0 300px']}
|
||||
ref={InputRef}
|
||||
placeholder={'标签,使用空格分割。'}
|
||||
maxLength={30}
|
||||
@@ -196,7 +215,7 @@ const Info = (
|
||||
setRefresh(!refresh);
|
||||
}}
|
||||
/>
|
||||
<Box pl={'90px'} mt={2} w="100%">
|
||||
<Box w={'100%'} pl={['90px', '160px']} mt={2}>
|
||||
{getValues('tags')
|
||||
.split(' ')
|
||||
.filter((item) => item)
|
||||
@@ -207,8 +226,9 @@ const Info = (
|
||||
))}
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex mt={5} w={'100%'} maxW={'350px'} alignItems={'flex-end'}>
|
||||
<Box flex={'0 0 90px'} w={0}></Box>
|
||||
|
||||
<Flex mt={5} w={'100%'} alignItems={'flex-end'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}></Box>
|
||||
<Button
|
||||
isLoading={btnLoading}
|
||||
mr={4}
|
||||
@@ -232,7 +252,7 @@ const Info = (
|
||||
</Flex>
|
||||
<File onSelect={onSelectFile} />
|
||||
<ConfirmChild />
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -108,12 +108,18 @@ const InputDataModal = ({
|
||||
try {
|
||||
const data = {
|
||||
dataId: e.dataId,
|
||||
kbId,
|
||||
a: e.a,
|
||||
q: e.q === defaultValues.q ? '' : e.q
|
||||
};
|
||||
await putKbDataById(data);
|
||||
onSuccess(data);
|
||||
} catch (error) {}
|
||||
} catch (err) {
|
||||
toast({
|
||||
status: 'error',
|
||||
title: getErrText(err, '更新数据失败')
|
||||
});
|
||||
}
|
||||
setLoading(false);
|
||||
}
|
||||
|
||||
@@ -123,7 +129,7 @@ const InputDataModal = ({
|
||||
});
|
||||
onClose();
|
||||
},
|
||||
[defaultValues, onClose, onSuccess, toast]
|
||||
[defaultValues.a, defaultValues.q, kbId, onClose, onSuccess, toast]
|
||||
);
|
||||
|
||||
return (
|
||||
@@ -194,6 +200,10 @@ const InputDataModal = ({
|
||||
await delOneKbDataByDataId(defaultValues.dataId);
|
||||
onDelete();
|
||||
onClose();
|
||||
toast({
|
||||
status: 'success',
|
||||
title: '记录已删除'
|
||||
});
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
import React, { useState, useCallback } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
Modal,
|
||||
ModalOverlay,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalCloseButton,
|
||||
ModalBody
|
||||
} from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useSelectFile } from '@/hooks/useSelectFile';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { readCsvContent } from '@/utils/file';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import Markdown from '@/components/Markdown';
|
||||
import { useMarkdown } from '@/hooks/useMarkdown';
|
||||
import { fileDownload } from '@/utils/file';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
|
||||
const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
|
||||
|
||||
const SelectJsonModal = ({
|
||||
onClose,
|
||||
onSuccess,
|
||||
kbId
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: () => void;
|
||||
kbId: string;
|
||||
}) => {
|
||||
const [selecting, setSelecting] = useState(false);
|
||||
const { toast } = useToast();
|
||||
const { File, onOpen } = useSelectFile({ fileType: '.csv', multiple: false });
|
||||
const [fileData, setFileData] = useState<{ q: string; a: string }[]>([]);
|
||||
const [fileName, setFileName] = useState('');
|
||||
const [successData, setSuccessData] = useState(0);
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: '确认导入该数据集?'
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (e: File[]) => {
|
||||
const file = e[0];
|
||||
setSelecting(true);
|
||||
setFileName(file.name);
|
||||
try {
|
||||
const { header, data } = await readCsvContent(file);
|
||||
if (header[0] !== 'question' || header[1] !== 'answer') {
|
||||
throw new Error('csv 文件格式有误');
|
||||
}
|
||||
setFileData(
|
||||
data.map((item) => ({
|
||||
q: item[0] || '',
|
||||
a: item[1] || ''
|
||||
}))
|
||||
);
|
||||
} catch (error: any) {
|
||||
toast({
|
||||
title: getErrText(error, 'csv 文件格式有误'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setSelecting(false);
|
||||
},
|
||||
[setSelecting, toast]
|
||||
);
|
||||
|
||||
const { mutate, isLoading: uploading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!fileData || fileData.length === 0) return;
|
||||
|
||||
let success = 0;
|
||||
|
||||
// subsection import
|
||||
const step = 100;
|
||||
for (let i = 0; i < fileData.length; i += step) {
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
data: fileData.slice(i, i + step).map((item) => ({
|
||||
...item,
|
||||
source: fileName
|
||||
})),
|
||||
mode: TrainingModeEnum.index
|
||||
});
|
||||
success += insertLen || 0;
|
||||
setSuccessData((state) => state + step);
|
||||
}
|
||||
|
||||
toast({
|
||||
title: `导入数据成功，最终导入: ${success} 条数据。需要一段时间训练`,
|
||||
status: 'success',
|
||||
duration: 4000
|
||||
});
|
||||
onClose();
|
||||
onSuccess();
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err, '导入文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const { data: intro } = useMarkdown({ url: '/csvSelect.md' });
|
||||
|
||||
return (
|
||||
<Modal isOpen={true} onClose={onClose} isCentered>
|
||||
<ModalOverlay />
|
||||
<ModalContent maxW={'90vw'} position={'relative'} m={0} h={'90vh'}>
|
||||
<ModalHeader>csv 问答对导入</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
|
||||
<ModalBody h={'100%'} display={['block', 'flex']} fontSize={'sm'} overflowY={'auto'}>
|
||||
<Box flex={'2 0 0'} w={['100%', 0]} mr={[0, 4]} mb={[4, 0]}>
|
||||
<Markdown source={intro} />
|
||||
<Box
|
||||
my={3}
|
||||
cursor={'pointer'}
|
||||
textDecoration={'underline'}
|
||||
color={'myBlue.600'}
|
||||
onClick={() =>
|
||||
fileDownload({
|
||||
text: csvTemplate,
|
||||
type: 'text/csv',
|
||||
filename: 'template.csv'
|
||||
})
|
||||
}
|
||||
>
|
||||
点击下载csv模板
|
||||
</Box>
|
||||
<Box>
|
||||
<Button isLoading={selecting} isDisabled={uploading} onClick={onOpen}>
|
||||
选择 csv 问答对
|
||||
</Button>
|
||||
<Box mt={4}>
|
||||
【{fileName}】一共有 {fileData.length} 组数据（下面最多展示100组）
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
<Box flex={'3 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
|
||||
{fileData.slice(0, 100).map((item, index) => (
|
||||
<Box key={index}>
|
||||
<Box>
|
||||
Q{index + 1}. {item.q}
|
||||
</Box>
|
||||
<Box>
|
||||
A{index + 1}. {item.a}
|
||||
</Box>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
</ModalBody>
|
||||
|
||||
<Flex px={6} pt={2} pb={4}>
|
||||
<Box flex={1}></Box>
|
||||
<Button variant={'base'} isLoading={uploading} mr={3} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button isDisabled={fileData.length === 0 || uploading} onClick={openConfirm(mutate)}>
|
||||
{uploading ? (
|
||||
<Box>{Math.round((successData / fileData.length) * 100)}%</Box>
|
||||
) : (
|
||||
'确认导入'
|
||||
)}
|
||||
</Button>
|
||||
</Flex>
|
||||
</ModalContent>
|
||||
<ConfirmChild />
|
||||
<File onSelect={onSelectFile} />
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default SelectJsonModal;
|
||||
@@ -1,349 +0,0 @@
|
||||
import React, { useState, useCallback } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
Modal,
|
||||
ModalOverlay,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalCloseButton,
|
||||
ModalBody,
|
||||
Input,
|
||||
Textarea
|
||||
} from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useSelectFile } from '@/hooks/useSelectFile';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postKbDataFromList } from '@/api/plugins/kb';
|
||||
import Radio from '@/components/Radio';
|
||||
import { splitText_token } from '@/utils/file';
|
||||
import { TrainingModeEnum } from '@/constants/plugin';
|
||||
import { getErrText } from '@/utils/tools';
|
||||
import { formatPrice } from '@/utils/user';
|
||||
import MySlider from '@/components/Slider';
|
||||
import { qaModelList, vectorModelList } from '@/store/static';
|
||||
|
||||
const fileExtension = '.txt,.doc,.docx,.pdf,.md';
|
||||
|
||||
const SelectFileModal = ({
|
||||
onClose,
|
||||
onSuccess,
|
||||
kbId
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: () => void;
|
||||
kbId: string;
|
||||
}) => {
|
||||
const [modeMap, setModeMap] = useState({
|
||||
[TrainingModeEnum.qa]: {
|
||||
model: qaModelList[0].model,
|
||||
maxLen: (qaModelList[0]?.maxToken || 16000) * 0.5,
|
||||
price: qaModelList[0]?.price || 3
|
||||
},
|
||||
[TrainingModeEnum.index]: {
|
||||
model: vectorModelList[0].model,
|
||||
maxLen: 600,
|
||||
price: vectorModelList[0]?.price || 0.2
|
||||
}
|
||||
});
|
||||
const [btnLoading, setBtnLoading] = useState(false);
|
||||
const { toast } = useToast();
|
||||
const [prompt, setPrompt] = useState('');
|
||||
const { File, onOpen } = useSelectFile({
|
||||
fileType: fileExtension,
|
||||
multiple: true
|
||||
});
|
||||
const [mode, setMode] = useState<`${TrainingModeEnum}`>(TrainingModeEnum.index);
|
||||
const [files, setFiles] = useState<{ filename: string; text: string }[]>([
|
||||
{ filename: '文本1', text: '' }
|
||||
]);
|
||||
const [splitRes, setSplitRes] = useState<{
|
||||
price: number;
|
||||
chunks: { filename: string; value: string }[];
|
||||
successChunks: number;
|
||||
}>({
|
||||
price: 0,
|
||||
successChunks: 0,
|
||||
chunks: []
|
||||
});
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: `确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，未完成的任务会被暂停。一共 ${
|
||||
splitRes.chunks.length
|
||||
} 组。${splitRes.price ? `大约 ${splitRes.price} 元。` : ''}`
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (files: File[]) => {
|
||||
setBtnLoading(true);
|
||||
try {
|
||||
let promise = Promise.resolve();
|
||||
files.forEach((file) => {
|
||||
promise = promise.then(async () => {
|
||||
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
|
||||
const text = await (async () => {
|
||||
switch (extension) {
|
||||
case 'txt':
|
||||
case 'md':
|
||||
return readTxtContent(file);
|
||||
case 'pdf':
|
||||
return readPdfContent(file);
|
||||
case 'doc':
|
||||
case 'docx':
|
||||
return readDocContent(file);
|
||||
}
|
||||
return '';
|
||||
})();
|
||||
|
||||
text && setFiles((state) => [{ filename: file.name, text }].concat(state));
|
||||
return;
|
||||
});
|
||||
});
|
||||
await promise;
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: typeof error === 'string' ? error : '解析文件失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setBtnLoading(false);
|
||||
},
|
||||
[toast]
|
||||
);
|
||||
console.log({ model: modeMap[mode].model });
|
||||
|
||||
const { mutate, isLoading: uploading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (splitRes.chunks.length === 0) return;
|
||||
|
||||
// subsection import
|
||||
let success = 0;
|
||||
const step = 100;
|
||||
for (let i = 0; i < splitRes.chunks.length; i += step) {
|
||||
const { insertLen } = await postKbDataFromList({
|
||||
kbId,
|
||||
model: modeMap[mode].model,
|
||||
data: splitRes.chunks
|
||||
.slice(i, i + step)
|
||||
.map((item) => ({ q: item.value, a: '', source: item.filename })),
|
||||
prompt: `下面是"${prompt || '一段长文本'}"`,
|
||||
mode
|
||||
});
|
||||
|
||||
success += insertLen;
|
||||
setSplitRes((state) => ({
|
||||
...state,
|
||||
successChunks: state.successChunks + step
|
||||
}));
|
||||
}
|
||||
|
||||
toast({
|
||||
title: `去重后共导入 ${success} 条数据,需要一段拆解和训练.`,
|
||||
status: 'success'
|
||||
});
|
||||
onClose();
|
||||
onSuccess();
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err, '导入文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const onclickImport = useCallback(async () => {
|
||||
setBtnLoading(true);
|
||||
try {
|
||||
const splitRes = files
|
||||
.map((item) =>
|
||||
splitText_token({
|
||||
text: item.text,
|
||||
...modeMap[mode]
|
||||
})
|
||||
)
|
||||
.map((item, i) => ({
|
||||
...item,
|
||||
filename: files[i].filename
|
||||
}))
|
||||
.filter((item) => item.tokens > 0);
|
||||
|
||||
let price = formatPrice(
|
||||
splitRes.reduce((sum, item) => sum + item.tokens, 0) * modeMap[mode].price
|
||||
);
|
||||
|
||||
if (mode === 'qa') {
|
||||
price *= 1.2;
|
||||
}
|
||||
|
||||
setSplitRes({
|
||||
price,
|
||||
chunks: splitRes
|
||||
.map((item) =>
|
||||
item.chunks.map((chunk) => ({
|
||||
filename: item.filename,
|
||||
value: chunk
|
||||
}))
|
||||
)
|
||||
.flat(),
|
||||
successChunks: 0
|
||||
});
|
||||
|
||||
openConfirm(mutate)();
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(error, '拆分文本异常')
|
||||
});
|
||||
}
|
||||
setBtnLoading(false);
|
||||
}, [files, mode, modeMap, mutate, openConfirm, toast]);
|
||||
|
||||
return (
|
||||
<Modal isOpen={true} onClose={onClose} isCentered>
|
||||
<ModalOverlay />
|
||||
<ModalContent
|
||||
display={'flex'}
|
||||
maxW={'min(1000px, 90vw)'}
|
||||
m={0}
|
||||
position={'relative'}
|
||||
h={'90vh'}
|
||||
>
|
||||
<ModalHeader>文件导入</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
|
||||
<ModalBody
|
||||
flex={1}
|
||||
h={0}
|
||||
display={'flex'}
|
||||
flexDirection={'column'}
|
||||
p={0}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
fontSize={'sm'}
|
||||
>
|
||||
<Box mt={2} px={5} maxW={['100%', '70%']} textAlign={'justify'} color={'blackAlpha.600'}>
|
||||
支持 {fileExtension} 文件。Gpt会自动对文本进行 QA 拆分，需要较长训练时间，拆分需要消耗
|
||||
tokens，账号余额不足时，未拆分的数据会被删除。一个{files.length}
|
||||
个文本。
|
||||
</Box>
|
||||
{/* 拆分模式 */}
|
||||
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
|
||||
<Box flex={'0 0 70px'}>分段模式:</Box>
|
||||
<Radio
|
||||
ml={3}
|
||||
list={[
|
||||
{ label: '直接分段', value: 'index' },
|
||||
{ label: 'QA拆分', value: 'qa' }
|
||||
]}
|
||||
value={mode}
|
||||
onChange={(e) => setMode(e as 'index' | 'qa')}
|
||||
/>
|
||||
</Flex>
|
||||
{/* 内容介绍 */}
|
||||
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
|
||||
{mode === TrainingModeEnum.qa && (
|
||||
<>
|
||||
<Box flex={'0 0 70px'} mr={2}>
|
||||
下面是
|
||||
</Box>
|
||||
<Input
|
||||
placeholder="提示词，例如: Laf的介绍/关于gpt4的论文/一段长文本"
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
size={'sm'}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
{/* chunk size */}
|
||||
{mode === TrainingModeEnum.index && (
|
||||
<Flex mt={5}>
|
||||
<Box w={['70px']} flexShrink={0}>
|
||||
段落长度
|
||||
</Box>
|
||||
<Box flex={1} ml={'10px'}>
|
||||
<MySlider
|
||||
markList={[
|
||||
{ label: '300', value: 300 },
|
||||
{ label: '1000', value: 1000 }
|
||||
]}
|
||||
width={['100%', '260px']}
|
||||
min={300}
|
||||
max={1000}
|
||||
step={50}
|
||||
value={modeMap[TrainingModeEnum.index].maxLen}
|
||||
onChange={(val) => {
|
||||
setModeMap((state) => ({
|
||||
...state,
|
||||
[TrainingModeEnum.index]: {
|
||||
...modeMap[TrainingModeEnum.index],
|
||||
maxLen: val
|
||||
}
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
</Flex>
|
||||
)}
|
||||
</Flex>
|
||||
|
||||
{/* 文本内容 */}
|
||||
<Box flex={'1 0 0'} px={5} h={0} w={'100%'} overflowY={'auto'} mt={4}>
|
||||
{files.slice(0, 100).map((item, i) => (
|
||||
<Box key={i} mb={5}>
|
||||
<Box mb={1}>{item.filename}</Box>
|
||||
<Textarea
|
||||
placeholder="文件内容,空内容会自动忽略"
|
||||
maxLength={-1}
|
||||
rows={10}
|
||||
fontSize={'xs'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
value={item.text}
|
||||
onChange={(e) => {
|
||||
setFiles([
|
||||
...files.slice(0, i),
|
||||
{ ...item, text: e.target.value },
|
||||
...files.slice(i + 1)
|
||||
]);
|
||||
}}
|
||||
onBlur={(e) => {
|
||||
if (files.length > 1 && e.target.value === '') {
|
||||
setFiles((state) => [...state.slice(0, i), ...state.slice(i + 1)]);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
</ModalBody>
|
||||
|
||||
<Flex px={6} pt={2} pb={4}>
|
||||
<Button isLoading={btnLoading} isDisabled={uploading} onClick={onOpen}>
|
||||
选择文件
|
||||
</Button>
|
||||
<Box flex={1}></Box>
|
||||
<Button variant={'base'} isLoading={uploading} mr={3} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button
|
||||
isDisabled={uploading || btnLoading || files[0]?.text === ''}
|
||||
onClick={onclickImport}
|
||||
>
|
||||
{uploading ? (
|
||||
<Box>{Math.round((splitRes.successChunks / splitRes.chunks.length) * 100)}%</Box>
|
||||
) : (
|
||||
'确认导入'
|
||||
)}
|
||||
</Button>
|
||||
</Flex>
|
||||
</ModalContent>
|
||||
<ConfirmChild />
|
||||
<File onSelect={onSelectFile} />
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default SelectFileModal;
|
||||
@@ -5,7 +5,6 @@ import type { KbTestItemType } from '@/types/plugin';
|
||||
import { searchText, getKbDataItemById } from '@/api/plugins/kb';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import { useRequest } from '@/hooks/useRequest';
|
||||
import { useRouter } from 'next/router';
|
||||
import { formatTimeToChatTime } from '@/utils/tools';
|
||||
import InputDataModal, { type FormData } from './InputDataModal';
|
||||
import { useGlobalStore } from '@/store/global';
|
||||
@@ -17,8 +16,7 @@ import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
const Test = () => {
|
||||
const { kbId } = useRouter().query as { kbId: string };
|
||||
const Test = ({ kbId }: { kbId: string }) => {
|
||||
const theme = useTheme();
|
||||
const { toast } = useToast();
|
||||
const { setLoading } = useGlobalStore();
|
||||
|
||||
@@ -17,12 +17,17 @@ import SideTabs from '@/components/SideTabs';
|
||||
import PageContainer from '@/components/PageContainer';
|
||||
import Avatar from '@/components/Avatar';
|
||||
import Info from './components/Info';
|
||||
|
||||
const ImportData = dynamic(() => import('./components/Import'), {
|
||||
ssr: false
|
||||
});
|
||||
const Test = dynamic(() => import('./components/Test'), {
|
||||
ssr: false
|
||||
});
|
||||
|
||||
enum TabEnum {
|
||||
data = 'data',
|
||||
import = 'import',
|
||||
test = 'test',
|
||||
info = 'info'
|
||||
}
|
||||
@@ -35,14 +40,12 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
|
||||
const { isPc } = useScreen();
|
||||
const { kbDetail, getKbDetail } = useUserStore();
|
||||
|
||||
const tabList = useMemo(
|
||||
() => [
|
||||
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' },
|
||||
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
|
||||
{ label: '基本信息', id: TabEnum.info, icon: 'settingLight' }
|
||||
],
|
||||
[]
|
||||
);
|
||||
const tabList = useRef([
|
||||
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' },
|
||||
{ label: '导入数据', id: TabEnum.import, icon: 'importLight' },
|
||||
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
|
||||
{ label: '配置', id: TabEnum.info, icon: 'settingLight' }
|
||||
]);
|
||||
|
||||
const setCurrentTab = useCallback(
|
||||
(tab: `${TabEnum}`) => {
|
||||
@@ -77,70 +80,73 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
|
||||
return (
|
||||
<PageContainer>
|
||||
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
|
||||
{/* pc tab */}
|
||||
<Box
|
||||
display={['none', 'flex']}
|
||||
flexDirection={'column'}
|
||||
p={4}
|
||||
w={'200px'}
|
||||
borderRight={theme.borders.base}
|
||||
>
|
||||
<Flex mb={4} alignItems={'center'}>
|
||||
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
|
||||
<Box ml={2} fontWeight={'bold'}>
|
||||
{kbDetail.name}
|
||||
</Box>
|
||||
</Flex>
|
||||
<SideTabs
|
||||
flex={1}
|
||||
mx={'auto'}
|
||||
mt={2}
|
||||
w={'100%'}
|
||||
list={tabList}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => {
|
||||
setCurrentTab(e);
|
||||
}}
|
||||
/>
|
||||
{isPc ? (
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
cursor={'pointer'}
|
||||
py={2}
|
||||
px={3}
|
||||
borderRadius={'md'}
|
||||
_hover={{ bg: 'myGray.100' }}
|
||||
onClick={() => router.replace('/kb/list')}
|
||||
flexDirection={'column'}
|
||||
p={4}
|
||||
h={'100%'}
|
||||
flex={'0 0 200px'}
|
||||
borderRight={theme.borders.base}
|
||||
>
|
||||
<IconButton
|
||||
mr={3}
|
||||
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
|
||||
bg={'white'}
|
||||
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
|
||||
h={'28px'}
|
||||
size={'sm'}
|
||||
borderRadius={'50%'}
|
||||
aria-label={''}
|
||||
<Flex mb={4} alignItems={'center'}>
|
||||
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
|
||||
<Box ml={2} fontWeight={'bold'}>
|
||||
{kbDetail.name}
|
||||
</Box>
|
||||
</Flex>
|
||||
<SideTabs
|
||||
flex={1}
|
||||
mx={'auto'}
|
||||
mt={2}
|
||||
w={'100%'}
|
||||
list={tabList.current}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => {
|
||||
setCurrentTab(e);
|
||||
}}
|
||||
/>
|
||||
全部知识库
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
cursor={'pointer'}
|
||||
py={2}
|
||||
px={3}
|
||||
borderRadius={'md'}
|
||||
_hover={{ bg: 'myGray.100' }}
|
||||
onClick={() => router.replace('/kb/list')}
|
||||
>
|
||||
<IconButton
|
||||
mr={3}
|
||||
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
|
||||
bg={'white'}
|
||||
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
|
||||
h={'28px'}
|
||||
size={'sm'}
|
||||
borderRadius={'50%'}
|
||||
aria-label={''}
|
||||
/>
|
||||
全部知识库
|
||||
</Flex>
|
||||
</Flex>
|
||||
</Box>
|
||||
<Box mb={3} display={['block', 'none']}>
|
||||
<Tabs
|
||||
m={'auto'}
|
||||
w={'260px'}
|
||||
size={isPc ? 'md' : 'sm'}
|
||||
list={[
|
||||
{ id: TabEnum.data, label: '数据管理' },
|
||||
{ id: TabEnum.test, label: '搜索测试' },
|
||||
{ id: TabEnum.info, label: '基本信息' }
|
||||
]}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => setCurrentTab(e)}
|
||||
/>
|
||||
</Box>
|
||||
<Box flex={'1 0 0'} overflow={'overlay'} pb={[4, 0]}>
|
||||
) : (
|
||||
<Box mb={3}>
|
||||
<Tabs
|
||||
m={'auto'}
|
||||
w={'260px'}
|
||||
size={isPc ? 'md' : 'sm'}
|
||||
list={tabList.current.map((item) => ({
|
||||
id: item.id,
|
||||
label: item.label
|
||||
}))}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => setCurrentTab(e)}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
|
||||
{currentTab === TabEnum.data && <DataCard kbId={kbId} />}
|
||||
{currentTab === TabEnum.test && <Test />}
|
||||
{currentTab === TabEnum.import && <ImportData kbId={kbId} />}
|
||||
{currentTab === TabEnum.test && <Test kbId={kbId} />}
|
||||
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
@@ -56,7 +56,7 @@ export async function generateVector(): Promise<any> {
|
||||
];
|
||||
|
||||
// 生成词向量
|
||||
const vectors = await getVector({
|
||||
const { vectors } = await getVector({
|
||||
model: data.model,
|
||||
input: dataItems.map((item) => item.q),
|
||||
userId
|
||||
|
||||
@@ -19,6 +19,11 @@ const kbSchema = new Schema({
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
model: {
|
||||
type: String,
|
||||
required: true,
|
||||
default: 'text-embedding-ada-002'
|
||||
},
|
||||
tags: {
|
||||
type: [String],
|
||||
default: []
|
||||
|
||||
1
client/src/types/mongoSchema.d.ts
vendored
1
client/src/types/mongoSchema.d.ts
vendored
@@ -137,6 +137,7 @@ export interface kbSchema {
|
||||
updateTime: Date;
|
||||
avatar: string;
|
||||
name: string;
|
||||
model: string;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import { encoding_for_model } from '@dqbd/tiktoken';
|
||||
import type { ChatItemType } from '@/types/chat';
|
||||
import { ChatRoleEnum } from '@/constants/chat';
|
||||
import { ChatCompletionRequestMessageRoleEnum } from 'openai';
|
||||
import { OpenAiChatEnum } from '@/constants/model';
|
||||
import axios from 'axios';
|
||||
import type { MessageItemType } from '@/pages/api/openapi/v1/chat/completions';
|
||||
|
||||
@@ -69,15 +68,7 @@ export function countOpenAIToken({
|
||||
return token;
|
||||
}
|
||||
|
||||
export const openAiSliceTextByToken = ({
|
||||
model = OpenAiChatEnum.GPT35,
|
||||
text,
|
||||
length
|
||||
}: {
|
||||
model: string;
|
||||
text: string;
|
||||
length: number;
|
||||
}) => {
|
||||
export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
|
||||
const enc = getOpenAiEncMap();
|
||||
const encodeText = enc.encode(text);
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
Reference in New Issue
Block a user