fix: now the input field can be array type now (close #149)

This commit is contained in:
JustSong
2023-06-12 16:11:57 +08:00
parent 8b2ef666ef
commit 7c7eb6b7ec
2 changed files with 16 additions and 2 deletions

View File

@@ -58,6 +58,20 @@ func countTokenMessages(messages []Message, model string) int {
return tokenNum
}
func countTokenInput(input any, model string) int {
switch input.(type) {
case string:
return countTokenText(input.(string), model)
case []string:
text := ""
for _, s := range input.([]string) {
text += s
}
return countTokenText(text, model)
}
return 0
}
func countTokenText(text string, model string) int {
tokenEncoder := getTokenEncoder(model)
token := tokenEncoder.Encode(text, nil, nil)