ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 21:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: stream timeout;feat: hnsw max_scan_tuples config;fix: fulltext search merge error (#4838)

Browse Source 
* perf: stream timeout

* feat: hnsw max_scan_tuples config

* fix: fulltext search merge error

* perf: jieba code
This commit is contained in:
Archer
2025-05-20 09:59:24 +08:00
committed by GitHub
parent 9fef3e15fb
commit 1dac2b70ec
10 changed files with 74 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/service/common/string/jieba/index.ts
View File

@@ -10,6 +10,7 @@ let jieba: Jieba | undefined;
})();
const stopWords = new Set([
'\n',
'--',
'?',
'“',
@@ -1519,8 +1520,7 @@ const stopWords = new Set([
]);
export async function jiebaSplit({ text }: { text: string }) {
text = text.replace(/[#*`_~>[\](){}|]/g, '').replace(/\S*https?\S*/gi, '');
text = text.replace(/[#*`_~>[\](){}|]|\S*https?\S*/g, '').trim();
const tokens = (await jieba!.cutAsync(text, true)) as string[];
return (