ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 05:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: tiktoken count (#1507)

Browse Source 
* perf: tiktoken count

* fix: rerank histories

* fix: rerank histories

* update npmrc
This commit is contained in:
Archer
2024-05-16 15:42:15 +08:00
committed by GitHub
parent c6d9b15897
commit 6067f5aff3
8 changed files with 67 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/service/worker/tiktoken/countGptMessagesTokens.ts
View File

@@ -1,5 +1,5 @@
/* Only the token of gpt-3.5-turbo is used */
import { Tiktoken } from 'fastgpt-js-tiktoken/lite';
import { Tiktoken } from 'tiktoken/lite';
import cl100k_base from './cl100k_base.json';
import {
ChatCompletionMessageParam,
@@ -10,7 +10,7 @@ import {
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { parentPort } from 'worker_threads';
const enc = new Tiktoken(cl100k_base);
const enc = new Tiktoken(cl100k_base.bpe_ranks, cl100k_base.special_tokens, cl100k_base.pat_str);
/* count messages tokens */
parentPort?.on(