Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 override.go // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package unicode import ( "golang.org/x/text/transform" ) // BOMOverride returns a new decoder transformer that is identical to fallback, // except that the presence of a Byte Order Mark at the start of the input // causes it to switch to the corresponding Unicode decoding. It will only // consider BOMs for UTF-8, UTF-16BE, and UTF-16LE. // // This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not // just UTF-16 variants, and allowing falling back to any encoding scheme. // // This technique is recommended by the W3C for use in HTML 5: "For // compatibility with deployed content, the byte order mark (also known as BOM) // is considered more authoritative than anything else." // http://www.w3.org/TR/encoding/#specification-hooks // // Using BOMOverride is mostly intended for use cases where the first characters // of a fallback encoding are known to not be a BOM, for example, for valid HTML // and most encodings. func BOMOverride(fallback transform.Transformer) transform.Transformer { // TODO: possibly allow a variadic argument of unicode encodings to allow // specifying details of which fallbacks are supported as well as // specifying the details of the implementations. This would also allow for // support for UTF-32, which should not be supported by default. return &bomOverride{fallback: fallback} } type bomOverride struct { fallback transform.Transformer current transform.Transformer } func (d bomOverride) Reset() { d.current = nil d.fallback.Reset() } var ( // TODO: we could use decode functions here, instead of allocating a new // decoder on every NewDecoder as IgnoreBOM decoders can be stateless. utf16le = UTF16(LittleEndian, IgnoreBOM) utf16be = UTF16(BigEndian, IgnoreBOM) ) const utf8BOM = "\ufeff" func (d bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if d.current != nil { return d.current.Transform(dst, src, atEOF) } if len(src) < 3 && !atEOF { return 0, 0, transform.ErrShortSrc } d.current = d.fallback bomSize := 0 if len(src) >= 2 { if src[0] == 0xFF && src[1] == 0xFE { d.current = utf16le.NewDecoder() bomSize = 2 } else if src[0] == 0xFE && src[1] == 0xFF { d.current = utf16be.NewDecoder() bomSize = 2 } else if len(src) >= 3 && src[0] == utf8BOM[0] && src[1] == utf8BOM[1] && src[2] == utf8BOM[2] { d.current = transform.Nop bomSize = 3 } } if bomSize < len(src) { nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF) } return nDst, nSrc + bomSize, err }	2.64 KB	2025-05-25 12:27
📄 unicode.go // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package unicode provides Unicode encodings such as UTF-16. package unicode // import "golang.org/x/text/encoding/unicode" import ( "bytes" "errors" "unicode/utf16" "unicode/utf8" "golang.org/x/text/encoding" "golang.org/x/text/encoding/internal" "golang.org/x/text/encoding/internal/identifier" "golang.org/x/text/internal/utf8internal" "golang.org/x/text/runes" "golang.org/x/text/transform" ) // TODO: I think the Transformers really should return errors on unmatched // surrogate pairs and odd numbers of bytes. This is not required by RFC 2781, // which leaves it open, but is suggested by WhatWG. It will allow for all error // modes as defined by WhatWG: fatal, HTML and Replacement. This would require // the introduction of some kind of error type for conveying the erroneous code // point. // UTF8 is the UTF-8 encoding. It neither removes nor adds byte order marks. var UTF8 encoding.Encoding = utf8enc // UTF8BOM is an UTF-8 encoding where the decoder strips a leading byte order // mark while the encoder adds one. // // Some editors add a byte order mark as a signature to UTF-8 files. Although // the byte order mark is not useful for detecting byte order in UTF-8, it is // sometimes used as a convention to mark UTF-8-encoded files. This relies on // the observation that the UTF-8 byte order mark is either an illegal or at // least very unlikely sequence in any other character encoding. var UTF8BOM encoding.Encoding = utf8bomEncoding{} type utf8bomEncoding struct{} func (utf8bomEncoding) String() string { return "UTF-8-BOM" } func (utf8bomEncoding) ID() (identifier.MIB, string) { return identifier.Unofficial, "x-utf8bom" } func (utf8bomEncoding) NewEncoder() encoding.Encoder { return &encoding.Encoder{ Transformer: &utf8bomEncoder{t: runes.ReplaceIllFormed()}, } } func (utf8bomEncoding) NewDecoder() encoding.Decoder { return &encoding.Decoder{Transformer: &utf8bomDecoder{}} } var utf8enc = &internal.Encoding{ &internal.SimpleEncoding{utf8Decoder{}, runes.ReplaceIllFormed()}, "UTF-8", identifier.UTF8, } type utf8bomDecoder struct { checked bool } func (t utf8bomDecoder) Reset() { t.checked = false } func (t utf8bomDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if !t.checked { if !atEOF && len(src) < len(utf8BOM) { if len(src) == 0 { return 0, 0, nil } return 0, 0, transform.ErrShortSrc } if bytes.HasPrefix(src, []byte(utf8BOM)) { nSrc += len(utf8BOM) src = src[len(utf8BOM):] } t.checked = true } nDst, n, err := utf8Decoder.Transform(utf8Decoder{}, dst[nDst:], src, atEOF) nSrc += n return nDst, nSrc, err } type utf8bomEncoder struct { written bool t transform.Transformer } func (t utf8bomEncoder) Reset() { t.written = false t.t.Reset() } func (t utf8bomEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if !t.written { if len(dst) < len(utf8BOM) { return nDst, 0, transform.ErrShortDst } nDst = copy(dst, utf8BOM) t.written = true } n, nSrc, err := utf8Decoder.Transform(utf8Decoder{}, dst[nDst:], src, atEOF) nDst += n return nDst, nSrc, err } type utf8Decoder struct{ transform.NopResetter } func (utf8Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { var pSrc int // point from which to start copy in src var accept utf8internal.AcceptRange // The decoder can only make the input larger, not smaller. n := len(src) if len(dst) < n { err = transform.ErrShortDst n = len(dst) atEOF = false } for nSrc < n { c := src[nSrc] if c < utf8.RuneSelf { nSrc++ continue } first := utf8internal.First[c] size := int(first & utf8internal.SizeMask) if first == utf8internal.FirstInvalid { goto handleInvalid // invalid starter byte } accept = utf8internal.AcceptRanges[first>>utf8internal.AcceptShift] if nSrc+size > n { if !atEOF { // We may stop earlier than necessary here if the short sequence // has invalid bytes. Not checking for this simplifies the code // and may avoid duplicate computations in certain conditions. if err == nil { err = transform.ErrShortSrc } break } // Determine the maximal subpart of an ill-formed subsequence. switch { case nSrc+1 >= n \|\| src[nSrc+1] < accept.Lo \|\| accept.Hi < src[nSrc+1]: size = 1 case nSrc+2 >= n \|\| src[nSrc+2] < utf8internal.LoCB \|\| utf8internal.HiCB < src[nSrc+2]: size = 2 default: size = 3 // As we are short, the maximum is 3. } goto handleInvalid } if c = src[nSrc+1]; c < accept.Lo \|\| accept.Hi < c { size = 1 goto handleInvalid // invalid continuation byte } else if size == 2 { } else if c = src[nSrc+2]; c < utf8internal.LoCB \|\| utf8internal.HiCB < c { size = 2 goto handleInvalid // invalid continuation byte } else if size == 3 { } else if c = src[nSrc+3]; c < utf8internal.LoCB \|\| utf8internal.HiCB < c { size = 3 goto handleInvalid // invalid continuation byte } nSrc += size continue handleInvalid: // Copy the scanned input so far. nDst += copy(dst[nDst:], src[pSrc:nSrc]) // Append RuneError to the destination. const runeError = "\ufffd" if nDst+len(runeError) > len(dst) { return nDst, nSrc, transform.ErrShortDst } nDst += copy(dst[nDst:], runeError) // Skip the maximal subpart of an ill-formed subsequence according to // the W3C standard way instead of the Go way. This Transform is // probably the only place in the text repo where it is warranted. nSrc += size pSrc = nSrc // Recompute the maximum source length. if sz := len(dst) - nDst; sz < len(src)-nSrc { err = transform.ErrShortDst n = nSrc + sz atEOF = false } } return nDst + copy(dst[nDst:], src[pSrc:nSrc]), nSrc, err } // UTF16 returns a UTF-16 Encoding for the given default endianness and byte // order mark (BOM) policy. // // When decoding from UTF-16 to UTF-8, if the BOMPolicy is IgnoreBOM then // neither BOMs U+FEFF nor noncharacters U+FFFE in the input stream will affect // the endianness used for decoding, and will instead be output as their // standard UTF-8 encodings: "\xef\xbb\xbf" and "\xef\xbf\xbe". If the BOMPolicy // is UseBOM or ExpectBOM a staring BOM is not written to the UTF-8 output. // Instead, it overrides the default endianness e for the remainder of the // transformation. Any subsequent BOMs U+FEFF or noncharacters U+FFFE will not // affect the endianness used, and will instead be output as their standard // UTF-8 encodings. For UseBOM, if there is no starting BOM, it will proceed // with the default Endianness. For ExpectBOM, in that case, the transformation // will return early with an ErrMissingBOM error. // // When encoding from UTF-8 to UTF-16, a BOM will be inserted at the start of // the output if the BOMPolicy is UseBOM or ExpectBOM. Otherwise, a BOM will not // be inserted. The UTF-8 input does not need to contain a BOM. // // There is no concept of a 'native' endianness. If the UTF-16 data is produced // and consumed in a greater context that implies a certain endianness, use // IgnoreBOM. Otherwise, use ExpectBOM and always produce and consume a BOM. // // In the language of https://www.unicode.org/faq/utf_bom.html#bom10, IgnoreBOM // corresponds to "Where the precise type of the data stream is known... the // BOM should not be used" and ExpectBOM corresponds to "A particular // protocol... may require use of the BOM". func UTF16(e Endianness, b BOMPolicy) encoding.Encoding { return utf16Encoding{config{e, b}, mibValue[e][b&bomMask]} } // mibValue maps Endianness and BOMPolicy settings to MIB constants. Note that // some configurations map to the same MIB identifier. RFC 2781 has requirements // and recommendations. Some of the "configurations" are merely recommendations, // so multiple configurations could match. var mibValue = map[Endianness][numBOMValues]identifier.MIB{ BigEndian: [numBOMValues]identifier.MIB{ IgnoreBOM: identifier.UTF16BE, UseBOM: identifier.UTF16, // BigEnding default is preferred by RFC 2781. // TODO: acceptBOM \| strictBOM would map to UTF16BE as well. }, LittleEndian: [numBOMValues]identifier.MIB{ IgnoreBOM: identifier.UTF16LE, UseBOM: identifier.UTF16, // LittleEndian default is allowed and preferred on Windows. // TODO: acceptBOM \| strictBOM would map to UTF16LE as well. }, // ExpectBOM is not widely used and has no valid MIB identifier. } // All lists a configuration for each IANA-defined UTF-16 variant. var All = []encoding.Encoding{ UTF8, UTF16(BigEndian, UseBOM), UTF16(BigEndian, IgnoreBOM), UTF16(LittleEndian, IgnoreBOM), } // BOMPolicy is a UTF-16 encoding's byte order mark policy. type BOMPolicy uint8 const ( writeBOM BOMPolicy = 0x01 acceptBOM BOMPolicy = 0x02 requireBOM BOMPolicy = 0x04 bomMask BOMPolicy = 0x07 // HACK: numBOMValues == 8 triggers a bug in the 1.4 compiler (cannot have a // map of an array of length 8 of a type that is also used as a key or value // in another map). See golang.org/issue/11354. // TODO: consider changing this value back to 8 if the use of 1.4.* has // been minimized. numBOMValues = 8 + 1 // IgnoreBOM means to ignore any byte order marks. IgnoreBOM BOMPolicy = 0 // Common and RFC 2781-compliant interpretation for UTF-16BE/LE. // UseBOM means that the UTF-16 form may start with a byte order mark, which // will be used to override the default encoding. UseBOM BOMPolicy = writeBOM \| acceptBOM // Common and RFC 2781-compliant interpretation for UTF-16. // ExpectBOM means that the UTF-16 form must start with a byte order mark, // which will be used to override the default encoding. ExpectBOM BOMPolicy = writeBOM \| acceptBOM \| requireBOM // Used in Java as Unicode (not to be confused with Java's UTF-16) and // ICU's UTF-16,version=1. Not compliant with RFC 2781. // TODO (maybe): strictBOM: BOM must match Endianness. This would allow: // - UTF-16(B\|L)E,version=1: writeBOM \| acceptBOM \| requireBOM \| strictBOM // (UnicodeBig and UnicodeLittle in Java) // - RFC 2781-compliant, but less common interpretation for UTF-16(B\|L)E: // acceptBOM \| strictBOM (e.g. assigned to CheckBOM). // This addition would be consistent with supporting ExpectBOM. ) // Endianness is a UTF-16 encoding's default endianness. type Endianness bool const ( // BigEndian is UTF-16BE. BigEndian Endianness = false // LittleEndian is UTF-16LE. LittleEndian Endianness = true ) // ErrMissingBOM means that decoding UTF-16 input with ExpectBOM did not find a // starting byte order mark. var ErrMissingBOM = errors.New("encoding: missing byte order mark") type utf16Encoding struct { config mib identifier.MIB } type config struct { endianness Endianness bomPolicy BOMPolicy } func (u utf16Encoding) NewDecoder() encoding.Decoder { return &encoding.Decoder{Transformer: &utf16Decoder{ initial: u.config, current: u.config, }} } func (u utf16Encoding) NewEncoder() encoding.Encoder { return &encoding.Encoder{Transformer: &utf16Encoder{ endianness: u.endianness, initialBOMPolicy: u.bomPolicy, currentBOMPolicy: u.bomPolicy, }} } func (u utf16Encoding) ID() (mib identifier.MIB, other string) { return u.mib, "" } func (u utf16Encoding) String() string { e, b := "B", "" if u.endianness == LittleEndian { e = "L" } switch u.bomPolicy { case ExpectBOM: b = "Expect" case UseBOM: b = "Use" case IgnoreBOM: b = "Ignore" } return "UTF-16" + e + "E (" + b + " BOM)" } type utf16Decoder struct { initial config current config } func (u utf16Decoder) Reset() { u.current = u.initial } func (u utf16Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if len(src) < 2 && atEOF && u.current.bomPolicy&requireBOM != 0 { return 0, 0, ErrMissingBOM } if len(src) == 0 { return 0, 0, nil } if len(src) >= 2 && u.current.bomPolicy&acceptBOM != 0 { switch { case src[0] == 0xfe && src[1] == 0xff: u.current.endianness = BigEndian nSrc = 2 case src[0] == 0xff && src[1] == 0xfe: u.current.endianness = LittleEndian nSrc = 2 default: if u.current.bomPolicy&requireBOM != 0 { return 0, 0, ErrMissingBOM } } u.current.bomPolicy = IgnoreBOM } var r rune var dSize, sSize int for nSrc < len(src) { if nSrc+1 < len(src) { x := uint16(src[nSrc+0])<<8 \| uint16(src[nSrc+1]) if u.current.endianness == LittleEndian { x = x>>8 \| x<<8 } r, sSize = rune(x), 2 if utf16.IsSurrogate(r) { if nSrc+3 < len(src) { x = uint16(src[nSrc+2])<<8 \| uint16(src[nSrc+3]) if u.current.endianness == LittleEndian { x = x>>8 \| x<<8 } // Save for next iteration if it is not a high surrogate. if isHighSurrogate(rune(x)) { r, sSize = utf16.DecodeRune(r, rune(x)), 4 } } else if !atEOF { err = transform.ErrShortSrc break } } if dSize = utf8.RuneLen(r); dSize < 0 { r, dSize = utf8.RuneError, 3 } } else if atEOF { // Single trailing byte. r, dSize, sSize = utf8.RuneError, 3, 1 } else { err = transform.ErrShortSrc break } if nDst+dSize > len(dst) { err = transform.ErrShortDst break } nDst += utf8.EncodeRune(dst[nDst:], r) nSrc += sSize } return nDst, nSrc, err } func isHighSurrogate(r rune) bool { return 0xDC00 <= r && r <= 0xDFFF } type utf16Encoder struct { endianness Endianness initialBOMPolicy BOMPolicy currentBOMPolicy BOMPolicy } func (u utf16Encoder) Reset() { u.currentBOMPolicy = u.initialBOMPolicy } func (u utf16Encoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if u.currentBOMPolicy&writeBOM != 0 { if len(dst) < 2 { return 0, 0, transform.ErrShortDst } dst[0], dst[1] = 0xfe, 0xff u.currentBOMPolicy = IgnoreBOM nDst = 2 } r, size := rune(0), 0 for nSrc < len(src) { r = rune(src[nSrc]) // Decode a 1-byte rune. if r < utf8.RuneSelf { size = 1 } else { // Decode a multi-byte rune. r, size = utf8.DecodeRune(src[nSrc:]) if size == 1 { // All valid runes of size 1 (those below utf8.RuneSelf) were // handled above. We have invalid UTF-8 or we haven't seen the // full character yet. if !atEOF && !utf8.FullRune(src[nSrc:]) { err = transform.ErrShortSrc break } } } if r <= 0xffff { if nDst+2 > len(dst) { err = transform.ErrShortDst break } dst[nDst+0] = uint8(r >> 8) dst[nDst+1] = uint8(r) nDst += 2 } else { if nDst+4 > len(dst) { err = transform.ErrShortDst break } r1, r2 := utf16.EncodeRune(r) dst[nDst+0] = uint8(r1 >> 8) dst[nDst+1] = uint8(r1) dst[nDst+2] = uint8(r2 >> 8) dst[nDst+3] = uint8(r2) nDst += 4 } nSrc += size } if u.endianness == LittleEndian { for i := 0; i < nDst; i += 2 { dst[i], dst[i+1] = dst[i+1], dst[i] } } return nDst, nSrc, err }	14.69 KB	2025-05-25 12:27
📄 unicode_test.go // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package unicode import ( "testing" "golang.org/x/text/encoding" "golang.org/x/text/encoding/charmap" "golang.org/x/text/encoding/internal/enctest" "golang.org/x/text/transform" ) func TestBasics(t testing.T) { testCases := []struct { e encoding.Encoding encPrefix string encSuffix string encoded string utf8 string }{{ e: utf16BEIB, encoded: "\x00\x57\x00\xe4\xd8\x35\xdd\x65", utf8: "\x57\u00e4\U0001d565", }, { e: utf16BEEB, encPrefix: "\xfe\xff", encoded: "\x00\x57\x00\xe4\xd8\x35\xdd\x65", utf8: "\x57\u00e4\U0001d565", }, { e: utf16LEIB, encoded: "\x57\x00\xe4\x00\x35\xd8\x65\xdd", utf8: "\x57\u00e4\U0001d565", }, { e: utf16LEEB, encPrefix: "\xff\xfe", encoded: "\x57\x00\xe4\x00\x35\xd8\x65\xdd", utf8: "\x57\u00e4\U0001d565", }} for _, tc := range testCases { enctest.TestEncoding(t, tc.e, tc.encoded, tc.utf8, tc.encPrefix, tc.encSuffix) } } func TestFiles(t testing.T) { enctest.TestFile(t, UTF8) enctest.TestFile(t, utf16LEIB) } func BenchmarkEncoding(b testing.B) { enctest.Benchmark(b, UTF8) enctest.Benchmark(b, utf16LEIB) } var ( utf16LEIB = UTF16(LittleEndian, IgnoreBOM) // UTF-16LE (atypical interpretation) utf16LEUB = UTF16(LittleEndian, UseBOM) // UTF-16, LE utf16LEEB = UTF16(LittleEndian, ExpectBOM) // UTF-16, LE, Expect utf16BEIB = UTF16(BigEndian, IgnoreBOM) // UTF-16BE (atypical interpretation) utf16BEUB = UTF16(BigEndian, UseBOM) // UTF-16 default utf16BEEB = UTF16(BigEndian, ExpectBOM) // UTF-16 Expect ) func TestUTF16(t testing.T) { testCases := []struct { desc string src string notEOF bool // the inverse of atEOF sizeDst int want string nSrc int err error t transform.Transformer }{{ desc: "utf-16 IgnoreBOM dec: empty string", t: utf16BEIB.NewDecoder(), }, { desc: "utf-16 UseBOM dec: empty string", t: utf16BEUB.NewDecoder(), }, { desc: "utf-16 ExpectBOM dec: empty string", err: ErrMissingBOM, t: utf16BEEB.NewDecoder(), }, { desc: "utf-16 dec: BOM determines encoding BE (RFC 2781:3.3)", src: "\xFE\xFF\xD8\x08\xDF\x45\x00\x3D\x00\x52\x00\x61", sizeDst: 100, want: "\U00012345=Ra", nSrc: 12, t: utf16BEUB.NewDecoder(), }, { desc: "utf-16 dec: BOM determines encoding LE (RFC 2781:3.3)", src: "\xFF\xFE\x08\xD8\x45\xDF\x3D\x00\x52\x00\x61\x00", sizeDst: 100, want: "\U00012345=Ra", nSrc: 12, t: utf16LEUB.NewDecoder(), }, { desc: "utf-16 dec: BOM determines encoding LE, change default (RFC 2781:3.3)", src: "\xFF\xFE\x08\xD8\x45\xDF\x3D\x00\x52\x00\x61\x00", sizeDst: 100, want: "\U00012345=Ra", nSrc: 12, t: utf16BEUB.NewDecoder(), }, { desc: "utf-16 dec: Fail on missing BOM when required", src: "\x08\xD8\x45\xDF\x3D\x00\xFF\xFE\xFE\xFF\x00\x52\x00\x61", sizeDst: 100, want: "", nSrc: 0, err: ErrMissingBOM, t: utf16BEEB.NewDecoder(), }, { desc: "utf-16 dec: Fail on single byte missing BOM when required", src: "\x00", sizeDst: 4, t: utf16BEEB.NewDecoder(), err: ErrMissingBOM, }, { desc: "utf-16 dec: Fail on short src missing BOM when required", src: "\x00", notEOF: true, sizeDst: 4, t: utf16BEEB.NewDecoder(), err: transform.ErrShortSrc, }, { desc: "utf-16 dec: SHOULD interpret text as big-endian when BOM not present (RFC 2781:4.3)", src: "\xD8\x08\xDF\x45\x00\x3D\x00\x52\x00\x61", sizeDst: 100, want: "\U00012345=Ra", nSrc: 10, t: utf16BEUB.NewDecoder(), }, { desc: "utf-16 dec: incorrect UTF-16: odd bytes", src: "\x00", sizeDst: 100, want: "\uFFFD", nSrc: 1, t: utf16BEUB.NewDecoder(), }, { desc: "utf-16 dec: Fail on incorrect UTF-16: short source odd bytes", src: "\x00", notEOF: true, sizeDst: 100, t: utf16BEUB.NewDecoder(), err: transform.ErrShortSrc, }, { // This is an error according to RFC 2781. But errors in RFC 2781 are // open to interpretations, so I guess this is fine. desc: "utf-16le dec: incorrect BOM is an error (RFC 2781:4.1)", src: "\xFE\xFF\x08\xD8\x45\xDF\x3D\x00\x52\x00\x61\x00", sizeDst: 100, want: "\uFFFE\U00012345=Ra", nSrc: 12, t: utf16LEIB.NewDecoder(), }, { desc: "utf-16 enc: SHOULD write BOM (RFC 2781:3.3)", src: "\U00012345=Ra", sizeDst: 100, want: "\xFF\xFE\x08\xD8\x45\xDF\x3D\x00\x52\x00\x61\x00", nSrc: 7, t: utf16LEUB.NewEncoder(), }, { desc: "utf-16 enc: SHOULD write BOM (RFC 2781:3.3)", src: "\U00012345=Ra", sizeDst: 100, want: "\xFE\xFF\xD8\x08\xDF\x45\x00\x3D\x00\x52\x00\x61", nSrc: 7, t: utf16BEUB.NewEncoder(), }, { desc: "utf-16le enc: MUST NOT write BOM (RFC 2781:3.3)", src: "\U00012345=Ra", sizeDst: 100, want: "\x08\xD8\x45\xDF\x3D\x00\x52\x00\x61\x00", nSrc: 7, t: utf16LEIB.NewEncoder(), }, { desc: "utf-16be dec: incorrect UTF-16: odd bytes", src: "\x00", sizeDst: 100, want: "\uFFFD", nSrc: 1, t: utf16BEIB.NewDecoder(), }, { desc: "utf-16be dec: unpaired surrogate, odd bytes", src: "\xD8\x45\x00", sizeDst: 100, want: "\uFFFD\uFFFD", nSrc: 3, t: utf16BEIB.NewDecoder(), }, { desc: "utf-16be dec: unpaired low surrogate + valid text", src: "\xD8\x45\x00a", sizeDst: 100, want: "\uFFFDa", nSrc: 4, t: utf16BEIB.NewDecoder(), }, { desc: "utf-16be dec: unpaired low surrogate + valid text + single byte", src: "\xD8\x45\x00ab", sizeDst: 100, want: "\uFFFDa\uFFFD", nSrc: 5, t: utf16BEIB.NewDecoder(), }, { desc: "utf-16le dec: unpaired high surrogate", src: "\x00\x00\x00\xDC\x12\xD8", sizeDst: 100, want: "\x00\uFFFD\uFFFD", nSrc: 6, t: utf16LEIB.NewDecoder(), }, { desc: "utf-16be dec: two unpaired low surrogates", src: "\xD8\x45\xD8\x12", sizeDst: 100, want: "\uFFFD\uFFFD", nSrc: 4, t: utf16BEIB.NewDecoder(), }, { desc: "utf-16be dec: short dst", src: "\x00a", sizeDst: 0, want: "", nSrc: 0, t: utf16BEIB.NewDecoder(), err: transform.ErrShortDst, }, { desc: "utf-16be dec: short dst surrogate", src: "\xD8\xF5\xDC\x12", sizeDst: 3, want: "", nSrc: 0, t: utf16BEIB.NewDecoder(), err: transform.ErrShortDst, }, { desc: "utf-16be dec: short dst trailing byte", src: "\x00", sizeDst: 2, want: "", nSrc: 0, t: utf16BEIB.NewDecoder(), err: transform.ErrShortDst, }, { desc: "utf-16be dec: short src", src: "\x00", notEOF: true, sizeDst: 3, want: "", nSrc: 0, t: utf16BEIB.NewDecoder(), err: transform.ErrShortSrc, }, { desc: "utf-16 enc", src: "\U00012345=Ra", sizeDst: 100, want: "\xFE\xFF\xD8\x08\xDF\x45\x00\x3D\x00\x52\x00\x61", nSrc: 7, t: utf16BEUB.NewEncoder(), }, { desc: "utf-16 enc: short dst normal", src: "\U00012345=Ra", sizeDst: 9, want: "\xD8\x08\xDF\x45\x00\x3D\x00\x52", nSrc: 6, t: utf16BEIB.NewEncoder(), err: transform.ErrShortDst, }, { desc: "utf-16 enc: short dst surrogate", src: "\U00012345=Ra", sizeDst: 3, want: "", nSrc: 0, t: utf16BEIB.NewEncoder(), err: transform.ErrShortDst, }, { desc: "utf-16 enc: short src", src: "\U00012345=Ra\xC2", notEOF: true, sizeDst: 100, want: "\xD8\x08\xDF\x45\x00\x3D\x00\x52\x00\x61", nSrc: 7, t: utf16BEIB.NewEncoder(), err: transform.ErrShortSrc, }, { desc: "utf-16be dec: don't change byte order mid-stream", src: "\xFE\xFF\xD8\x08\xDF\x45\x00\x3D\xFF\xFE\x00\x52\x00\x61", sizeDst: 100, want: "\U00012345=\ufffeRa", nSrc: 14, t: utf16BEUB.NewDecoder(), }, { desc: "utf-16le dec: don't change byte order mid-stream", src: "\xFF\xFE\x08\xD8\x45\xDF\x3D\x00\xFF\xFE\xFE\xFF\x52\x00\x61\x00", sizeDst: 100, want: "\U00012345=\ufeff\ufffeRa", nSrc: 16, t: utf16LEUB.NewDecoder(), }} for i, tc := range testCases { for j := 0; j < 2; j++ { b := make([]byte, tc.sizeDst) nDst, nSrc, err := tc.t.Transform(b, []byte(tc.src), !tc.notEOF) if err != tc.err { t.Errorf("%d:%s: error was %v; want %v", i, tc.desc, err, tc.err) } if got := string(b[:nDst]); got != tc.want { t.Errorf("%d:%s: result was %q: want %q", i, tc.desc, got, tc.want) } if nSrc != tc.nSrc { t.Errorf("%d:%s: nSrc was %d; want %d", i, tc.desc, nSrc, tc.nSrc) } // Since Transform is stateful, run failures again // to ensure that the same error occurs a second time. if err == nil { break } } } } func TestUTF8Decoder(t testing.T) { testCases := []struct { desc string src string notEOF bool // the inverse of atEOF sizeDst int want string nSrc int err error }{{ desc: "empty string, empty dest buffer", }, { desc: "empty string", sizeDst: 8, }, { desc: "empty string, streaming", notEOF: true, sizeDst: 8, }, { desc: "ascii", src: "abcde", sizeDst: 8, want: "abcde", nSrc: 5, }, { desc: "ascii and error", src: "ab\x80de", sizeDst: 7, want: "ab\ufffdde", nSrc: 5, }, { desc: "valid two-byte sequence", src: "a\u0300bc", sizeDst: 7, want: "a\u0300bc", nSrc: 5, }, { desc: "valid three-byte sequence", src: "a\u0300中", sizeDst: 7, want: "a\u0300中", nSrc: 6, }, { desc: "valid four-byte sequence", src: "a中\U00016F50", sizeDst: 8, want: "a中\U00016F50", nSrc: 8, }, { desc: "short source buffer", src: "abc\xf0\x90", notEOF: true, sizeDst: 10, want: "abc", nSrc: 3, err: transform.ErrShortSrc, }, { // We don't check for the maximal subpart of an ill-formed subsequence // at the end of an open segment. desc: "complete invalid that looks like short at end", src: "abc\xf0\x80", notEOF: true, sizeDst: 10, want: "abc", // instead of "abc\ufffd\ufffd", nSrc: 3, err: transform.ErrShortSrc, }, { desc: "incomplete sequence at end", src: "a\x80bc\xf0\x90", sizeDst: 9, want: "a\ufffdbc\ufffd", nSrc: 6, }, { desc: "invalid second byte", src: "abc\xf0dddd", sizeDst: 10, want: "abc\ufffddddd", nSrc: 8, }, { desc: "invalid second byte at end", src: "abc\xf0d", sizeDst: 10, want: "abc\ufffdd", nSrc: 5, }, { desc: "invalid third byte", src: "a\u0300bc\xf0\x90dddd", sizeDst: 12, want: "a\u0300bc\ufffddddd", nSrc: 11, }, { desc: "invalid third byte at end", src: "a\u0300bc\xf0\x90d", sizeDst: 12, want: "a\u0300bc\ufffdd", nSrc: 8, }, { desc: "invalid fourth byte, tight buffer", src: "a\u0300bc\xf0\x90\x80d", sizeDst: 9, want: "a\u0300bc\ufffdd", nSrc: 9, }, { desc: "invalid fourth byte at end", src: "a\u0300bc\xf0\x90\x80", sizeDst: 8, want: "a\u0300bc\ufffd", nSrc: 8, }, { desc: "invalid fourth byte and short four byte sequence", src: "a\u0300bc\xf0\x90\x80\xf0\x90\x80", notEOF: true, sizeDst: 20, want: "a\u0300bc\ufffd", nSrc: 8, err: transform.ErrShortSrc, }, { desc: "valid four-byte sequence overflowing short buffer", src: "a\u0300bc\xf0\x90\x80\x80", notEOF: true, sizeDst: 8, want: "a\u0300bc", nSrc: 5, err: transform.ErrShortDst, }, { desc: "invalid fourth byte at end short, but short dst", src: "a\u0300bc\xf0\x90\x80\xf0\x90\x80", notEOF: true, sizeDst: 8, // More bytes would fit in the buffer, but this seems to require a more // complicated and slower algorithm. want: "a\u0300bc", // instead of "a\u0300bc" nSrc: 5, err: transform.ErrShortDst, }, { desc: "short dst for error", src: "abc\x80", notEOF: true, sizeDst: 5, want: "abc", nSrc: 3, err: transform.ErrShortDst, }, { desc: "adjusting short dst buffer", src: "abc\x80ef", notEOF: true, sizeDst: 6, want: "abc\ufffd", nSrc: 4, err: transform.ErrShortDst, }} tr := UTF8.NewDecoder() for i, tc := range testCases { b := make([]byte, tc.sizeDst) nDst, nSrc, err := tr.Transform(b, []byte(tc.src), !tc.notEOF) if err != tc.err { t.Errorf("%d:%s: error was %v; want %v", i, tc.desc, err, tc.err) } if got := string(b[:nDst]); got != tc.want { t.Errorf("%d:%s: result was %q: want %q", i, tc.desc, got, tc.want) } if nSrc != tc.nSrc { t.Errorf("%d:%s: nSrc was %d; want %d", i, tc.desc, nSrc, tc.nSrc) } } } func TestUTF8BOMDecoder(t testing.T) { testCases := []struct { desc string src string notEOF bool // the inverse of atEOF sizeDst int want string nSrc int err error wantAll string }{{ desc: "empty string, empty dest buffer", }, { desc: "empty string", sizeDst: 8, }, { desc: "empty string, streaming", notEOF: true, sizeDst: 8, }, { desc: "ascii", src: "abcde", sizeDst: 8, want: "abcde", nSrc: 5, wantAll: "abcde", }, { desc: "ascii with bom", src: utf8BOM + "abcde", sizeDst: 11, want: "abcde", nSrc: 8, wantAll: "abcde", }, { desc: "error with bom", src: utf8BOM + "ab\x80de", sizeDst: 11, want: "ab\ufffdde", nSrc: 8, wantAll: "ab\ufffdde", }, { desc: "short bom", src: utf8BOM[:2], notEOF: true, sizeDst: 7, want: "", nSrc: 0, wantAll: "\ufffd", // needs to be 1 replacement err: transform.ErrShortSrc, }, { desc: "short bom at end", src: utf8BOM[:2], sizeDst: 7, want: "\ufffd", // needs to be 1 replacement nSrc: 2, wantAll: "\ufffd", // needs to be 1 replacement err: nil, }, { desc: "short source buffer", src: "abc\xf0\x90", notEOF: true, sizeDst: 10, want: "abc", nSrc: 3, wantAll: "abc\ufffd", err: transform.ErrShortSrc, }, { desc: "short source buffer with bom", src: utf8BOM + "abc\xf0\x90", notEOF: true, sizeDst: 15, want: "abc", nSrc: 6, wantAll: "abc\ufffd", err: transform.ErrShortSrc, }, { desc: "short dst for error", src: utf8BOM + "abc\x80", notEOF: true, sizeDst: 5, want: "abc", nSrc: 6, wantAll: "abc\ufffd", err: transform.ErrShortDst, }} tr := UTF8BOM.NewDecoder() for i, tc := range testCases { tr.Reset() b := make([]byte, tc.sizeDst) nDst, nSrc, err := tr.Transform(b, []byte(tc.src), !tc.notEOF) if err != tc.err { t.Errorf("%d:%s: error was %v; want %v", i, tc.desc, err, tc.err) } if got := string(b[:nDst]); got != tc.want { t.Errorf("%d:%s: result was %q: want %q", i, tc.desc, got, tc.want) } if nSrc != tc.nSrc { t.Errorf("%d:%s: nSrc was %d; want %d", i, tc.desc, nSrc, tc.nSrc) } if got, _ := tr.String(tc.src); got != tc.wantAll { t.Errorf("%d:%s: String was %s; want %s", i, tc.desc, got, tc.wantAll) } } } func TestUTF8SigEncoder(t testing.T) { testCases := []struct { desc string src string notEOF bool // the inverse of atEOF sizeDst int want string wantAll string // converting all bytes nSrc int err error }{{ desc: "empty string, empty dest buffer", err: transform.ErrShortDst, wantAll: utf8BOM, }, { desc: "empty string", sizeDst: 8, want: utf8BOM, wantAll: utf8BOM, }, { desc: "empty string, streaming", notEOF: true, sizeDst: 8, want: utf8BOM, wantAll: utf8BOM, }, { desc: "ascii", src: "abcde", sizeDst: 8, want: utf8BOM + "abcde", nSrc: 5, wantAll: utf8BOM + "abcde", }, { desc: "short bom at end", src: utf8BOM[:2], sizeDst: 11, want: utf8BOM + "\ufffd", nSrc: 2, wantAll: utf8BOM + "\ufffd", }, { desc: "short bom", src: utf8BOM[:2], notEOF: true, sizeDst: 7, want: utf8BOM, nSrc: 0, err: transform.ErrShortSrc, wantAll: utf8BOM + "\ufffd", }, { desc: "short bom at end", src: utf8BOM[:2], sizeDst: 7, want: utf8BOM + "\ufffd", // needs to be 1 replacement nSrc: 2, err: nil, wantAll: utf8BOM + "\ufffd", }, { desc: "short dst buffer 2", src: "ab", sizeDst: 2, want: "", nSrc: 0, err: transform.ErrShortDst, wantAll: utf8BOM + "ab", }, { desc: "short dst buffer 3", src: "ab", sizeDst: 3, want: utf8BOM, nSrc: 0, err: transform.ErrShortDst, wantAll: utf8BOM + "ab", }, { desc: "short dst buffer 4", src: "ab", sizeDst: 4, want: utf8BOM + "a", nSrc: 1, err: transform.ErrShortDst, wantAll: utf8BOM + "ab", }} tr := UTF8BOM.NewEncoder() for i, tc := range testCases { tr.Reset() b := make([]byte, tc.sizeDst) nDst, nSrc, err := tr.Transform(b, []byte(tc.src), !tc.notEOF) if err != tc.err { t.Errorf("%d:%s: error was %v; want %v", i, tc.desc, err, tc.err) } if got := string(b[:nDst]); got != tc.want { t.Errorf("%d:%s: result was %q: want %q", i, tc.desc, got, tc.want) } if nSrc != tc.nSrc { t.Errorf("%d:%s: nSrc was %d; want %d", i, tc.desc, nSrc, tc.nSrc) } if got, _ := tr.String(tc.src); got != tc.wantAll { t.Errorf("%d:%s: String was %s; want %s", i, tc.desc, got, tc.wantAll) } } } func TestBOMOverride(t testing.T) { dec := BOMOverride(charmap.CodePage437.NewDecoder()) dst := make([]byte, 100) for i, tc := range []struct { src string atEOF bool dst string nSrc int err error }{ 0: {"H\x82ll\x93", true, "Héllô", 5, nil}, 1: {"\uFEFFHéllö", true, "Héllö", 10, nil}, 2: {"\xFE\xFF\x00H\x00e\x00l\x00l\x00o", true, "Hello", 12, nil}, 3: {"\xFF\xFEH\x00e\x00l\x00l\x00o\x00", true, "Hello", 12, nil}, 4: {"\uFEFF", true, "", 3, nil}, 5: {"\xFE\xFF", true, "", 2, nil}, 6: {"\xFF\xFE", true, "", 2, nil}, 7: {"\xEF\xBB", true, "\u2229\u2557", 2, nil}, 8: {"\xEF", true, "\u2229", 1, nil}, 9: {"", true, "", 0, nil}, 10: {"\xFE", true, "\u25a0", 1, nil}, 11: {"\xFF", true, "\u00a0", 1, nil}, 12: {"\xEF\xBB", false, "", 0, transform.ErrShortSrc}, 13: {"\xEF", false, "", 0, transform.ErrShortSrc}, 14: {"", false, "", 0, transform.ErrShortSrc}, 15: {"\xFE", false, "", 0, transform.ErrShortSrc}, 16: {"\xFF", false, "", 0, transform.ErrShortSrc}, 17: {"\xFF\xFE", false, "", 0, transform.ErrShortSrc}, } { dec.Reset() nDst, nSrc, err := dec.Transform(dst, []byte(tc.src), tc.atEOF) got := string(dst[:nDst]) if nSrc != tc.nSrc { t.Errorf("%d: nSrc: got %d; want %d", i, nSrc, tc.nSrc) } if got != tc.dst { t.Errorf("%d: got %+q; want %+q", i, got, tc.dst) } if err != tc.err { t.Errorf("%d: error: got %v; want %v", i, err, tc.err) } } }	18.79 KB	2025-05-25 12:27
📁 utf32	-	2025-05-25 12:27

Terminal