ChatGPT/one-api
1
0
Fork 0
mirror of https://github.com/songquanpeng/one-api.git synced 2025-10-20 10:45:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve error logging for token encoder initialization with offline usage guidance

Browse Source
This commit is contained in:
JustSong
2025-02-02 17:24:23 +08:00
parent ececfe5ba0
commit c81ee68dd0
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
relay/adaptor/openai/token.go
View File

@@ -3,14 +3,16 @@ package openai
import ( import (
"errors" "errors"
"fmt" "fmt"
"math"
"strings"
"github.com/pkoukk/tiktoken-go" "github.com/pkoukk/tiktoken-go"
"github.com/songquanpeng/one-api/common/config" "github.com/songquanpeng/one-api/common/config"
"github.com/songquanpeng/one-api/common/image" "github.com/songquanpeng/one-api/common/image"
"github.com/songquanpeng/one-api/common/logger" "github.com/songquanpeng/one-api/common/logger"
billingratio "github.com/songquanpeng/one-api/relay/billing/ratio" billingratio "github.com/songquanpeng/one-api/relay/billing/ratio"
"github.com/songquanpeng/one-api/relay/model" "github.com/songquanpeng/one-api/relay/model"
"math"
"strings"
) )
// tokenEncoderMap won't grow after initialization // tokenEncoderMap won't grow after initialization
@@ -21,7 +23,8 @@ func InitTokenEncoders() {
logger.SysLog("initializing token encoders") logger.SysLog("initializing token encoders")
gpt35TokenEncoder, err := tiktoken.EncodingForModel("gpt-3.5-turbo") gpt35TokenEncoder, err := tiktoken.EncodingForModel("gpt-3.5-turbo")
if err != nil { if err != nil {
logger.FatalLog(fmt.Sprintf("failed to get gpt-3.5-turbo token encoder: %s", err.Error())) logger.FatalLog(fmt.Sprintf("failed to get gpt-3.5-turbo token encoder: %s, "+
"if you are using in offline environment, please set TIKTOKEN_CACHE_DIR to use exsited files, check this link for more information: https://stackoverflow.com/questions/76106366/how-to-use-tiktoken-in-offline-mode-computer ", err.Error()))
} }
defaultTokenEncoder = gpt35TokenEncoder defaultTokenEncoder = gpt35TokenEncoder
gpt4oTokenEncoder, err := tiktoken.EncodingForModel("gpt-4o") gpt4oTokenEncoder, err := tiktoken.EncodingForModel("gpt-4o")