node/nvm-windows
1
0
Fork 0
mirror of https://github.com/coreybutler/nvm-windows.git synced 2026-01-17 02:01:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move test command to 'debug'. Fixes #955 and #942. Addresses #945 with new encoding for settings (forcing UTF-8).

Browse Source
This commit is contained in:
Corey Butler
2023-04-11 12:53:34 -05:00
parent 16afa15d7d
commit b54163cf78
4 changed files with 124 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
src/encoding/encoding.go
View File

@@ -1,11 +1,9 @@
package encoding
import (
"errors"
"strings"
"unicode/utf8"
"github.com/mushroomsir/iconv"
"github.com/saintfish/chardet"
)
@@ -19,46 +17,56 @@ func DetectCharset(content []byte) (string, error) {
return strings.ToUpper(result.Charset), nil
}
func ToUTF8(content []byte, ignoreInvalidITF8Chars ...bool) (string, error) {
ignore := false
if len(ignoreInvalidITF8Chars) > 0 {
ignore = ignoreInvalidITF8Chars[0]
func ToUTF8(content string) []byte {
b := make([]byte, len(content))
i := 0
for _, r := range content {
i += utf8.EncodeRune(b[i:], r)
}
cs, err := DetectCharset(content)
if err != nil {
if !ignore {
return "", err
}
cs = "UTF-8"
}
bs := string(content)
if ignore {
if !utf8.ValidString(bs) {
v := make([]rune, 0, len(bs))
for i, r := range bs {
if r == utf8.RuneError {
_, size := utf8.DecodeRuneInString(bs[i:])
if size == 1 {
continue
}
}
v = append(v, r)
}
bs = string(v)
}
}
if cs == "UTF-8" {
return bs, nil
}
converter, err := iconv.NewConverter(cs, "UTF-8")
if err != nil {
err = errors.New("Failed to convert " + cs + " to UTF-8: " + err.Error())
return bs, err
}
return converter.ConvertString(bs)
return b[:i]
}
// func ToUTF8(content []byte, ignoreInvalidITF8Chars ...bool) (string, error) {
// ignore := false
// if len(ignoreInvalidITF8Chars) > 0 {
// ignore = ignoreInvalidITF8Chars[0]
// }
// cs, err := DetectCharset(content)
// if err != nil {
// if !ignore {
// return "", err
// }
// cs = "UTF-8"
// }
// bs := string(content)
// if ignore {
// if !utf8.ValidString(bs) {
// v := make([]rune, 0, len(bs))
// for i, r := range bs {
// if r == utf8.RuneError {
// _, size := utf8.DecodeRuneInString(bs[i:])
// if size == 1 {
// continue
// }
// }
// v = append(v, r)
// }
// bs = string(v)
// }
// }
// if cs == "UTF-8" {
// return bs, nil
// }
// converter, err := iconv.NewConverter(cs, "UTF-8")
// if err != nil {
// err = errors.New("Failed to convert " + cs + " to UTF-8: " + err.Error())
// return bs, err
// }
// return converter.ConvertString(bs)
// }