ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 21:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: chunk filter

Browse Source
This commit is contained in:
archer
2023-08-07 11:04:32 +08:00
parent 7fe20ef041
commit b6e156db26
3 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
client/src/pages/kb/detail/components/Import/Chunk.tsx
View File

@@ -16,7 +16,7 @@ import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb';
import { splitText_token } from '@/utils/file';
import { splitText2Chunks } from '@/utils/file';
import { getErrText } from '@/utils/tools';
import { formatPrice } from '@/utils/user';
import { vectorModelList } from '@/store/static';
@@ -96,7 +96,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
})();
if (icon && text) {
const splitRes = splitText_token({
const splitRes = splitText2Chunks({
text: text,
maxLen: chunkLen
});
@@ -178,7 +178,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
const onRePreview = useCallback(async () => {
try {
const splitRes = files.map((item) =>
splitText_token({
splitText2Chunks({
text: item.text,
maxLen: chunkLen
})

6
client/src/pages/kb/detail/components/Import/QA.tsx
View File

@@ -5,7 +5,7 @@ import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb';
import { splitText_token } from '@/utils/file';
import { splitText2Chunks } from '@/utils/file';
import { getErrText } from '@/utils/tools';
import { formatPrice } from '@/utils/user';
import { qaModelList } from '@/store/static';
@@ -86,7 +86,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
})();
if (icon && text) {
const splitRes = splitText_token({
const splitRes = splitText2Chunks({
text: text,
maxLen: chunkLen
});
@@ -169,7 +169,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
const onRePreview = useCallback(async () => {
try {
const splitRes = files.map((item) =>
splitText_token({
splitText2Chunks({
text: item.text,
maxLen: chunkLen
})

2
client/src/utils/file.ts
View File

@@ -147,7 +147,7 @@ export const fileDownload = ({
* overlapLen - The size of the before and after Text
* maxLen > overlapLen
*/
export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => {
export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
try {