Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 cond.go // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runes import ( "unicode/utf8" "golang.org/x/text/transform" ) // Note: below we pass invalid UTF-8 to the tIn and tNotIn transformers as is. // This is done for various reasons: // - To retain the semantics of the Nop transformer: if input is passed to a Nop // one would expect it to be unchanged. // - It would be very expensive to pass a converted RuneError to a transformer: // a transformer might need more source bytes after RuneError, meaning that // the only way to pass it safely is to create a new buffer and manage the // intermingling of RuneErrors and normal input. // - Many transformers leave ill-formed UTF-8 as is, so this is not // inconsistent. Generally ill-formed UTF-8 is only replaced if it is a // logical consequence of the operation (as for Map) or if it otherwise would // pose security concerns (as for Remove). // - An alternative would be to return an error on ill-formed UTF-8, but this // would be inconsistent with other operations. // If returns a transformer that applies tIn to consecutive runes for which // s.Contains(r) and tNotIn to consecutive runes for which !s.Contains(r). Reset // is called on tIn and tNotIn at the start of each run. A Nop transformer will // substitute a nil value passed to tIn or tNotIn. Invalid UTF-8 is translated // to RuneError to determine which transformer to apply, but is passed as is to // the respective transformer. func If(s Set, tIn, tNotIn transform.Transformer) Transformer { if tIn == nil && tNotIn == nil { return Transformer{transform.Nop} } if tIn == nil { tIn = transform.Nop } if tNotIn == nil { tNotIn = transform.Nop } sIn, ok := tIn.(transform.SpanningTransformer) if !ok { sIn = dummySpan{tIn} } sNotIn, ok := tNotIn.(transform.SpanningTransformer) if !ok { sNotIn = dummySpan{tNotIn} } a := &cond{ tIn: sIn, tNotIn: sNotIn, f: s.Contains, } a.Reset() return Transformer{a} } type dummySpan struct{ transform.Transformer } func (d dummySpan) Span(src []byte, atEOF bool) (n int, err error) { return 0, transform.ErrEndOfSpan } type cond struct { tIn, tNotIn transform.SpanningTransformer f func(rune) bool check func(rune) bool // current check to perform t transform.SpanningTransformer // current transformer to use } // Reset implements transform.Transformer. func (t cond) Reset() { t.check = t.is t.t = t.tIn t.t.Reset() // notIn will be reset on first usage. } func (t cond) is(r rune) bool { if t.f(r) { return true } t.check = t.isNot t.t = t.tNotIn t.tNotIn.Reset() return false } func (t cond) isNot(r rune) bool { if !t.f(r) { return true } t.check = t.is t.t = t.tIn t.tIn.Reset() return false } // This implementation of Span doesn't help all too much, but it needs to be // there to satisfy this package's Transformer interface. // TODO: there are certainly room for improvements, though. For example, if // t.t == transform.Nop (which will a common occurrence) it will save a bundle // to special-case that loop. func (t cond) Span(src []byte, atEOF bool) (n int, err error) { p := 0 for n < len(src) && err == nil { // Don't process too much at a time as the Spanner that will be // called on this block may terminate early. const maxChunk = 4096 max := len(src) if v := n + maxChunk; v < max { max = v } atEnd := false size := 0 current := t.t for ; p < max; p += size { r := rune(src[p]) if r < utf8.RuneSelf { size = 1 } else if r, size = utf8.DecodeRune(src[p:]); size == 1 { if !atEOF && !utf8.FullRune(src[p:]) { err = transform.ErrShortSrc break } } if !t.check(r) { // The next rune will be the start of a new run. atEnd = true break } } n2, err2 := current.Span(src[n:p], atEnd \|\| (atEOF && p == len(src))) n += n2 if err2 != nil { return n, err2 } // At this point either err != nil or t.check will pass for the rune at p. p = n + size } return n, err } func (t *cond) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { p := 0 for nSrc < len(src) && err == nil { // Don't process too much at a time, as the work might be wasted if the // destination buffer isn't large enough to hold the result or a // transform returns an error early. const maxChunk = 4096 max := len(src) if n := nSrc + maxChunk; n < len(src) { max = n } atEnd := false size := 0 current := t.t for ; p < max; p += size { r := rune(src[p]) if r < utf8.RuneSelf { size = 1 } else if r, size = utf8.DecodeRune(src[p:]); size == 1 { if !atEOF && !utf8.FullRune(src[p:]) { err = transform.ErrShortSrc break } } if !t.check(r) { // The next rune will be the start of a new run. atEnd = true break } } nDst2, nSrc2, err2 := current.Transform(dst[nDst:], src[nSrc:p], atEnd \|\| (atEOF && p == len(src))) nDst += nDst2 nSrc += nSrc2 if err2 != nil { return nDst, nSrc, err2 } // At this point either err != nil or t.check will pass for the rune at p. p = nSrc + size } return nDst, nSrc, err }	5.17 KB	2025-05-25 12:27
📄 cond_test.go // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runes import ( "strings" "testing" "unicode" "golang.org/x/text/cases" "golang.org/x/text/language" "golang.org/x/text/transform" ) var ( toUpper = cases.Upper(language.Und) toLower = cases.Lower(language.Und) ) type spanformer interface { transform.SpanningTransformer } func TestPredicate(t testing.T) { testConditional(t, func(rt unicode.RangeTable, t, f spanformer) spanformer { return If(Predicate(func(r rune) bool { return unicode.Is(rt, r) }), t, f) }) } func TestIn(t testing.T) { testConditional(t, func(rt unicode.RangeTable, t, f spanformer) spanformer { return If(In(rt), t, f) }) } func TestNotIn(t testing.T) { testConditional(t, func(rt unicode.RangeTable, t, f spanformer) spanformer { return If(NotIn(rt), f, t) }) } func testConditional(t testing.T, f func(rt unicode.RangeTable, t, f spanformer) spanformer) { lower := f(unicode.Latin, toLower, toLower) for i, tt := range []transformTest{{ desc: "empty", szDst: large, atEOF: true, in: "", out: "", outFull: "", t: lower, }, { desc: "small", szDst: 1, atEOF: true, in: "B", out: "b", outFull: "b", errSpan: transform.ErrEndOfSpan, t: lower, }, { desc: "short dst", szDst: 2, atEOF: true, in: "AAA", out: "aa", outFull: "aaa", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: lower, }, { desc: "short dst writing error", szDst: 1, atEOF: false, in: "A\x80", out: "a", outFull: "a\x80", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: lower, }, { desc: "short dst writing incomplete rune", szDst: 2, atEOF: true, in: "Σ\xc2", out: "Σ", outFull: "Σ\xc2", err: transform.ErrShortDst, t: f(unicode.Latin, toLower, nil), }, { desc: "short dst, longer", szDst: 5, atEOF: true, in: "Hellø", out: "Hell", outFull: "Hellø", err: transform.ErrShortDst, // idem is used to test short buffers by forcing processing of full-rune increments. t: f(unicode.Latin, Map(idem), nil), }, { desc: "short dst, longer, writing error", szDst: 6, atEOF: false, in: "\x80Hello\x80", out: "\x80Hello", outFull: "\x80Hello\x80", err: transform.ErrShortDst, t: f(unicode.Latin, Map(idem), nil), }, { desc: "short src", szDst: 2, atEOF: false, in: "A\xc2", out: "a", outFull: "a\xc2", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: lower, }, { desc: "short src no change", szDst: 2, atEOF: false, in: "a\xc2", out: "a", outFull: "a\xc2", err: transform.ErrShortSrc, errSpan: transform.ErrShortSrc, nSpan: 1, t: lower, }, { desc: "invalid input, atEOF", szDst: large, atEOF: true, in: "\x80", out: "\x80", outFull: "\x80", t: lower, }, { desc: "invalid input, !atEOF", szDst: large, atEOF: false, in: "\x80", out: "\x80", outFull: "\x80", t: lower, }, { desc: "invalid input, incomplete rune atEOF", szDst: large, atEOF: true, in: "\xc2", out: "\xc2", outFull: "\xc2", t: lower, }, { desc: "nop", szDst: large, atEOF: true, in: "Hello World!", out: "Hello World!", outFull: "Hello World!", t: f(unicode.Latin, nil, nil), }, { desc: "nop in", szDst: large, atEOF: true, in: "THIS IS α ΤΕΣΤ", out: "this is α ΤΕΣΤ", outFull: "this is α ΤΕΣΤ", errSpan: transform.ErrEndOfSpan, t: f(unicode.Greek, nil, toLower), }, { desc: "nop in latin", szDst: large, atEOF: true, in: "THIS IS α ΤΕΣΤ", out: "THIS IS α τεστ", outFull: "THIS IS α τεστ", errSpan: transform.ErrEndOfSpan, t: f(unicode.Latin, nil, toLower), }, { desc: "nop not in", szDst: large, atEOF: true, in: "THIS IS α ΤΕΣΤ", out: "this is α ΤΕΣΤ", outFull: "this is α ΤΕΣΤ", errSpan: transform.ErrEndOfSpan, t: f(unicode.Latin, toLower, nil), }, { desc: "pass atEOF is true when at end", szDst: large, atEOF: true, in: "hello", out: "HELLO", outFull: "HELLO", errSpan: transform.ErrEndOfSpan, t: f(unicode.Latin, upperAtEOF{}, nil), }, { desc: "pass atEOF is true when at end of segment", szDst: large, atEOF: true, in: "hello ", out: "HELLO ", outFull: "HELLO ", errSpan: transform.ErrEndOfSpan, t: f(unicode.Latin, upperAtEOF{}, nil), }, { desc: "don't pass atEOF is true when atEOF is false", szDst: large, atEOF: false, in: "hello", out: "", outFull: "HELLO", err: transform.ErrShortSrc, errSpan: transform.ErrShortSrc, t: f(unicode.Latin, upperAtEOF{}, nil), }, { desc: "pass atEOF is true when at end, no change", szDst: large, atEOF: true, in: "HELLO", out: "HELLO", outFull: "HELLO", t: f(unicode.Latin, upperAtEOF{}, nil), }, { desc: "pass atEOF is true when at end of segment, no change", szDst: large, atEOF: true, in: "HELLO ", out: "HELLO ", outFull: "HELLO ", t: f(unicode.Latin, upperAtEOF{}, nil), }, { desc: "large input ASCII", szDst: 12000, atEOF: false, in: strings.Repeat("HELLO", 2000), out: strings.Repeat("hello", 2000), outFull: strings.Repeat("hello", 2000), errSpan: transform.ErrEndOfSpan, err: nil, t: lower, }, { desc: "large input non-ASCII", szDst: 12000, atEOF: false, in: strings.Repeat("\u3333", 2000), out: strings.Repeat("\u3333", 2000), outFull: strings.Repeat("\u3333", 2000), err: nil, t: lower, }} { tt.check(t, i) } } // upperAtEOF is a strange Transformer that converts text to uppercase, but only // if atEOF is true. type upperAtEOF struct{ transform.NopResetter } func (upperAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { if !atEOF { return 0, 0, transform.ErrShortSrc } return toUpper.Transform(dst, src, atEOF) } func (upperAtEOF) Span(src []byte, atEOF bool) (n int, err error) { if !atEOF { return 0, transform.ErrShortSrc } return toUpper.Span(src, atEOF) } func BenchmarkConditional(b *testing.B) { doBench(b, If(In(unicode.Hangul), transform.Nop, transform.Nop)) }	6.54 KB	2025-05-25 12:27
📄 example_test.go // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runes_test import ( "fmt" "unicode" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" "golang.org/x/text/width" ) func ExampleRemove() { t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) s, _, _ := transform.String(t, "résumé") fmt.Println(s) // Output: // resume } func ExampleMap() { replaceHyphens := runes.Map(func(r rune) rune { if unicode.Is(unicode.Hyphen, r) { return '\|' } return r }) s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e") fmt.Println(s) // Output: // a\|b\|c\|d\|e } func ExampleIn() { // Convert Latin characters to their canonical form, while keeping other // width distinctions. t := runes.If(runes.In(unicode.Latin), width.Fold, nil) s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウｔｅｃｈ") fmt.Println(s) // Output: // ｱﾙｱﾉﾘｳ tech / アルアノリウ tech } func ExampleIf() { // Widen everything but ASCII. isASCII := func(r rune) bool { return r <= unicode.MaxASCII } t := runes.If(runes.Predicate(isASCII), nil, width.Widen) s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩") fmt.Println(s) // Output: // アルアノリウ tech / 中國 / 5￦ }	1.41 KB	2025-05-25 12:27
📄 runes.go // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package runes provide transforms for UTF-8 encoded text. package runes // import "golang.org/x/text/runes" import ( "unicode" "unicode/utf8" "golang.org/x/text/transform" ) // A Set is a collection of runes. type Set interface { // Contains returns true if r is contained in the set. Contains(r rune) bool } type setFunc func(rune) bool func (s setFunc) Contains(r rune) bool { return s(r) } // Note: using funcs here instead of wrapping types result in cleaner // documentation and a smaller API. // In creates a Set with a Contains method that returns true for all runes in // the given RangeTable. func In(rt unicode.RangeTable) Set { return setFunc(func(r rune) bool { return unicode.Is(rt, r) }) } // NotIn creates a Set with a Contains method that returns true for all runes not // in the given RangeTable. func NotIn(rt unicode.RangeTable) Set { return setFunc(func(r rune) bool { return !unicode.Is(rt, r) }) } // Predicate creates a Set with a Contains method that returns f(r). func Predicate(f func(rune) bool) Set { return setFunc(f) } // Transformer implements the transform.Transformer interface. type Transformer struct { t transform.SpanningTransformer } func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { return t.t.Transform(dst, src, atEOF) } func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) { return t.t.Span(b, atEOF) } func (t Transformer) Reset() { t.t.Reset() } // Bytes returns a new byte slice with the result of converting b using t. It // calls Reset on t. It returns nil if any error was found. This can only happen // if an error-producing Transformer is passed to If. func (t Transformer) Bytes(b []byte) []byte { b, _, err := transform.Bytes(t, b) if err != nil { return nil } return b } // String returns a string with the result of converting s using t. It calls // Reset on t. It returns the empty string if any error was found. This can only // happen if an error-producing Transformer is passed to If. func (t Transformer) String(s string) string { s, _, err := transform.String(t, s) if err != nil { return "" } return s } // TODO: // - Copy: copying strings and bytes in whole-rune units. // - Validation (maybe) // - Well-formed-ness (maybe) const runeErrorString = string(utf8.RuneError) // Remove returns a Transformer that removes runes r for which s.Contains(r). // Illegal input bytes are replaced by RuneError before being passed to f. func Remove(s Set) Transformer { if f, ok := s.(setFunc); ok { // This little trick cuts the running time of BenchmarkRemove for sets // created by Predicate roughly in half. // TODO: special-case RangeTables as well. return Transformer{remove(f)} } return Transformer{remove(s.Contains)} } // TODO: remove transform.RemoveFunc. type remove func(r rune) bool func (remove) Reset() {} // Span implements transform.Spanner. func (t remove) Span(src []byte, atEOF bool) (n int, err error) { for r, size := rune(0), 0; n < len(src); { if r = rune(src[n]); r < utf8.RuneSelf { size = 1 } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { // Invalid rune. if !atEOF && !utf8.FullRune(src[n:]) { err = transform.ErrShortSrc } else { err = transform.ErrEndOfSpan } break } if t(r) { err = transform.ErrEndOfSpan break } n += size } return } // Transform implements transform.Transformer. func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { for r, size := rune(0), 0; nSrc < len(src); { if r = rune(src[nSrc]); r < utf8.RuneSelf { size = 1 } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { // Invalid rune. if !atEOF && !utf8.FullRune(src[nSrc:]) { err = transform.ErrShortSrc break } // We replace illegal bytes with RuneError. Not doing so might // otherwise turn a sequence of invalid UTF-8 into valid UTF-8. // The resulting byte sequence may subsequently contain runes // for which t(r) is true that were passed unnoticed. if !t(utf8.RuneError) { if nDst+3 > len(dst) { err = transform.ErrShortDst break } dst[nDst+0] = runeErrorString[0] dst[nDst+1] = runeErrorString[1] dst[nDst+2] = runeErrorString[2] nDst += 3 } nSrc++ continue } if t(r) { nSrc += size continue } if nDst+size > len(dst) { err = transform.ErrShortDst break } for i := 0; i < size; i++ { dst[nDst] = src[nSrc] nDst++ nSrc++ } } return } // Map returns a Transformer that maps the runes in the input using the given // mapping. Illegal bytes in the input are converted to utf8.RuneError before // being passed to the mapping func. func Map(mapping func(rune) rune) Transformer { return Transformer{mapper(mapping)} } type mapper func(rune) rune func (mapper) Reset() {} // Span implements transform.Spanner. func (t mapper) Span(src []byte, atEOF bool) (n int, err error) { for r, size := rune(0), 0; n < len(src); n += size { if r = rune(src[n]); r < utf8.RuneSelf { size = 1 } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { // Invalid rune. if !atEOF && !utf8.FullRune(src[n:]) { err = transform.ErrShortSrc } else { err = transform.ErrEndOfSpan } break } if t(r) != r { err = transform.ErrEndOfSpan break } } return n, err } // Transform implements transform.Transformer. func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { var replacement rune var b [utf8.UTFMax]byte for r, size := rune(0), 0; nSrc < len(src); { if r = rune(src[nSrc]); r < utf8.RuneSelf { if replacement = t(r); replacement < utf8.RuneSelf { if nDst == len(dst) { err = transform.ErrShortDst break } dst[nDst] = byte(replacement) nDst++ nSrc++ continue } size = 1 } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { // Invalid rune. if !atEOF && !utf8.FullRune(src[nSrc:]) { err = transform.ErrShortSrc break } if replacement = t(utf8.RuneError); replacement == utf8.RuneError { if nDst+3 > len(dst) { err = transform.ErrShortDst break } dst[nDst+0] = runeErrorString[0] dst[nDst+1] = runeErrorString[1] dst[nDst+2] = runeErrorString[2] nDst += 3 nSrc++ continue } } else if replacement = t(r); replacement == r { if nDst+size > len(dst) { err = transform.ErrShortDst break } for i := 0; i < size; i++ { dst[nDst] = src[nSrc] nDst++ nSrc++ } continue } n := utf8.EncodeRune(b[:], replacement) if nDst+n > len(dst) { err = transform.ErrShortDst break } for i := 0; i < n; i++ { dst[nDst] = b[i] nDst++ } nSrc += size } return } // ReplaceIllFormed returns a transformer that replaces all input bytes that are // not part of a well-formed UTF-8 code sequence with utf8.RuneError. func ReplaceIllFormed() Transformer { return Transformer{&replaceIllFormed{}} } type replaceIllFormed struct{ transform.NopResetter } func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) { for n < len(src) { // ASCII fast path. if src[n] < utf8.RuneSelf { n++ continue } r, size := utf8.DecodeRune(src[n:]) // Look for a valid non-ASCII rune. if r != utf8.RuneError \|\| size != 1 { n += size continue } // Look for short source data. if !atEOF && !utf8.FullRune(src[n:]) { err = transform.ErrShortSrc break } // We have an invalid rune. err = transform.ErrEndOfSpan break } return n, err } func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { for nSrc < len(src) { // ASCII fast path. if r := src[nSrc]; r < utf8.RuneSelf { if nDst == len(dst) { err = transform.ErrShortDst break } dst[nDst] = r nDst++ nSrc++ continue } // Look for a valid non-ASCII rune. if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 { if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { err = transform.ErrShortDst break } nDst += size nSrc += size continue } // Look for short source data. if !atEOF && !utf8.FullRune(src[nSrc:]) { err = transform.ErrShortSrc break } // We have an invalid rune. if nDst+3 > len(dst) { err = transform.ErrShortDst break } dst[nDst+0] = runeErrorString[0] dst[nDst+1] = runeErrorString[1] dst[nDst+2] = runeErrorString[2] nDst += 3 nSrc++ } return nDst, nSrc, err }	8.47 KB	2025-05-25 12:27
📄 runes_test.go // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runes import ( "strings" "testing" "unicode/utf8" "golang.org/x/text/internal/testtext" "golang.org/x/text/transform" ) type transformTest struct { desc string szDst int atEOF bool repl string in string out string // result string of first call to Transform outFull string // transform of entire input string err error errSpan error nSpan int t transform.SpanningTransformer } const large = 10240 func (tt transformTest) check(t testing.T, i int) { if tt.t == nil { return } dst := make([]byte, tt.szDst) src := []byte(tt.in) nDst, nSrc, err := tt.t.Transform(dst, src, tt.atEOF) if err != tt.err { t.Errorf("%d:%s:error: got %v; want %v", i, tt.desc, err, tt.err) } if got := string(dst[:nDst]); got != tt.out { t.Errorf("%d:%s:out: got %q; want %q", i, tt.desc, got, tt.out) } // Calls tt.t.Transform for the remainder of the input. We use this to test // the nSrc return value. out := make([]byte, large) n := copy(out, dst[:nDst]) nDst, _, _ = tt.t.Transform(out[n:], src[nSrc:], true) if got, want := string(out[:n+nDst]), tt.outFull; got != want { t.Errorf("%d:%s:outFull: got %q; want %q", i, tt.desc, got, want) } tt.t.Reset() p := 0 for ; p < len(tt.in) && p < len(tt.outFull) && tt.in[p] == tt.outFull[p]; p++ { } if tt.nSpan != 0 { p = tt.nSpan } if n, err = tt.t.Span([]byte(tt.in), tt.atEOF); n != p \|\| err != tt.errSpan { t.Errorf("%d:%s:span: got %d, %v; want %d, %v", i, tt.desc, n, err, p, tt.errSpan) } } func idem(r rune) rune { return r } func TestMap(t testing.T) { runes := []rune{'a', 'ç', '中', '\U00012345', 'a'} // Default mapper used for this test. rotate := Map(func(r rune) rune { for i, m := range runes { if m == r { return runes[i+1] } } return r }) for i, tt := range []transformTest{{ desc: "empty", szDst: large, atEOF: true, in: "", out: "", outFull: "", t: rotate, }, { desc: "no change", szDst: 1, atEOF: true, in: "b", out: "b", outFull: "b", t: rotate, }, { desc: "short dst", szDst: 2, atEOF: true, in: "aaaa", out: "ç", outFull: "çççç", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "short dst ascii, no change", szDst: 2, atEOF: true, in: "bbb", out: "bb", outFull: "bbb", err: transform.ErrShortDst, t: rotate, }, { desc: "short dst writing error", szDst: 2, atEOF: false, in: "a\x80", out: "ç", outFull: "ç\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "short dst writing incomplete rune", szDst: 2, atEOF: true, in: "a\xc0", out: "ç", outFull: "ç\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "short dst, longer", szDst: 5, atEOF: true, in: "Hellø", out: "Hell", outFull: "Hellø", err: transform.ErrShortDst, t: rotate, }, { desc: "short dst, single", szDst: 1, atEOF: false, in: "ø", out: "", outFull: "ø", err: transform.ErrShortDst, t: Map(idem), }, { desc: "short dst, longer, writing error", szDst: 8, atEOF: false, in: "\x80Hello\x80", out: "\ufffdHello", outFull: "\ufffdHello\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "short src", szDst: 2, atEOF: false, in: "a\xc2", out: "ç", outFull: "ç\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "invalid input, atEOF", szDst: large, atEOF: true, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "invalid input, !atEOF", szDst: large, atEOF: false, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "incomplete rune !atEOF", szDst: large, atEOF: false, in: "\xc2", out: "", outFull: "\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrShortSrc, t: rotate, }, { desc: "invalid input, incomplete rune atEOF", szDst: large, atEOF: true, in: "\xc2", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "misc correct", szDst: large, atEOF: true, in: "a\U00012345 ç!", out: "ça 中!", outFull: "ça 中!", errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "misc correct and invalid", szDst: large, atEOF: true, in: "Hello\x80 w\x80orl\xc0d!\xc0", out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "misc correct and invalid, short src", szDst: large, atEOF: false, in: "Hello\x80 w\x80orl\xc0d!\xc2", out: "Hello\ufffd w\ufffdorl\ufffdd!", outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: rotate, }, { desc: "misc correct and invalid, short src, replacing RuneError", szDst: large, atEOF: false, in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2", out: "Hel?lo? w?orl?d!", outFull: "Hel?lo? w?orl?d!?", errSpan: transform.ErrEndOfSpan, err: transform.ErrShortSrc, t: Map(func(r rune) rune { if r == utf8.RuneError { return '?' } return r }), }} { tt.check(t, i) } } func TestRemove(t testing.T) { remove := Remove(Predicate(func(r rune) bool { return strings.ContainsRune("aeiou\u0300\uFF24\U00012345", r) })) for i, tt := range []transformTest{ 0: { szDst: large, atEOF: true, in: "", out: "", outFull: "", t: remove, }, 1: { szDst: 0, atEOF: true, in: "aaaa", out: "", outFull: "", errSpan: transform.ErrEndOfSpan, t: remove, }, 2: { szDst: 1, atEOF: true, in: "aaaa", out: "", outFull: "", errSpan: transform.ErrEndOfSpan, t: remove, }, 3: { szDst: 1, atEOF: true, in: "baaaa", out: "b", outFull: "b", errSpan: transform.ErrEndOfSpan, t: remove, }, 4: { szDst: 2, atEOF: true, in: "açaaa", out: "ç", outFull: "ç", errSpan: transform.ErrEndOfSpan, t: remove, }, 5: { szDst: 2, atEOF: true, in: "aaaç", out: "ç", outFull: "ç", errSpan: transform.ErrEndOfSpan, t: remove, }, 6: { szDst: 2, atEOF: false, in: "a\x80", out: "", outFull: "\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: remove, }, 7: { szDst: 1, atEOF: true, in: "a\xc0", out: "", outFull: "\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: remove, }, 8: { szDst: 1, atEOF: false, in: "a\xc2", out: "", outFull: "\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: remove, }, 9: { szDst: large, atEOF: true, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: remove, }, 10: { szDst: large, atEOF: false, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: remove, }, 11: { szDst: large, atEOF: true, in: "\xc2", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: remove, }, 12: { szDst: large, atEOF: false, in: "\xc2", out: "", outFull: "\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrShortSrc, t: remove, }, 13: { szDst: large, atEOF: true, in: "Hello \U00012345world!", out: "Hll wrld!", outFull: "Hll wrld!", errSpan: transform.ErrEndOfSpan, t: remove, }, 14: { szDst: large, atEOF: true, in: "Hello\x80 w\x80orl\xc0d!\xc0", out: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", errSpan: transform.ErrEndOfSpan, t: remove, }, 15: { szDst: large, atEOF: false, in: "Hello\x80 w\x80orl\xc0d!\xc2", out: "Hll\ufffd w\ufffdrl\ufffdd!", outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: remove, }, 16: { szDst: large, atEOF: false, in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2", out: "Hello world!", outFull: "Hello world!", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: Remove(Predicate(func(r rune) bool { return r == utf8.RuneError })), }, 17: { szDst: 4, atEOF: true, in: "Hellø", out: "Hll", outFull: "Hllø", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: remove, }, 18: { szDst: 4, atEOF: false, in: "Hellø", out: "Hll", outFull: "Hllø", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: remove, }, 19: { szDst: 8, atEOF: false, in: "\x80Hello\uFF24\x80", out: "\ufffdHll", outFull: "\ufffdHll\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: remove, }, 20: { szDst: 8, atEOF: false, in: "Hllll", out: "Hllll", outFull: "Hllll", t: remove, }} { tt.check(t, i) } } func TestReplaceIllFormed(t testing.T) { replace := ReplaceIllFormed() for i, tt := range []transformTest{ 0: { szDst: large, atEOF: true, in: "", out: "", outFull: "", t: replace, }, 1: { szDst: 1, atEOF: true, in: "aa", out: "a", outFull: "aa", err: transform.ErrShortDst, t: replace, }, 2: { szDst: 1, atEOF: true, in: "a\x80", out: "a", outFull: "a\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: replace, }, 3: { szDst: 1, atEOF: true, in: "a\xc2", out: "a", outFull: "a\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: replace, }, 4: { szDst: large, atEOF: true, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: replace, }, 5: { szDst: large, atEOF: false, in: "\x80", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: replace, }, 6: { szDst: large, atEOF: true, in: "\xc2", out: "\ufffd", outFull: "\ufffd", errSpan: transform.ErrEndOfSpan, t: replace, }, 7: { szDst: large, atEOF: false, in: "\xc2", out: "", outFull: "\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrShortSrc, t: replace, }, 8: { szDst: large, atEOF: true, in: "Hello world!", out: "Hello world!", outFull: "Hello world!", t: replace, }, 9: { szDst: large, atEOF: true, in: "Hello\x80 w\x80orl\xc2d!\xc2", out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", errSpan: transform.ErrEndOfSpan, t: replace, }, 10: { szDst: large, atEOF: false, in: "Hello\x80 w\x80orl\xc2d!\xc2", out: "Hello\ufffd w\ufffdorl\ufffdd!", outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", err: transform.ErrShortSrc, errSpan: transform.ErrEndOfSpan, t: replace, }, 16: { szDst: 10, atEOF: false, in: "\x80Hello\x80", out: "\ufffdHello", outFull: "\ufffdHello\ufffd", err: transform.ErrShortDst, errSpan: transform.ErrEndOfSpan, t: replace, }, 17: { szDst: 10, atEOF: false, in: "\ufffdHello\ufffd", out: "\ufffdHello", outFull: "\ufffdHello\ufffd", err: transform.ErrShortDst, t: replace, }, } { tt.check(t, i) } } func TestMapAlloc(t testing.T) { if n := testtext.AllocsPerRun(3, func() { Map(idem).Transform(nil, nil, false) }); n > 0 { t.Errorf("got %f; want 0", n) } } func rmNop(r rune) bool { return false } func TestRemoveAlloc(t testing.T) { if n := testtext.AllocsPerRun(3, func() { Remove(Predicate(rmNop)).Transform(nil, nil, false) }); n > 0 { t.Errorf("got %f; want 0", n) } } func TestReplaceIllFormedAlloc(t testing.T) { if n := testtext.AllocsPerRun(3, func() { ReplaceIllFormed().Transform(nil, nil, false) }); n > 0 { t.Errorf("got %f; want 0", n) } } func doBench(b testing.B, t Transformer) { for _, bc := range []struct{ name, data string }{ {"ascii", testtext.ASCII}, {"3byte", testtext.ThreeByteUTF8}, } { dst := make([]byte, 2len(bc.data)) src := []byte(bc.data) b.Run(bc.name+"/transform", func(b testing.B) { b.SetBytes(int64(len(src))) for i := 0; i < b.N; i++ { t.Transform(dst, src, true) } }) src = t.Bytes(src) t.Reset() b.Run(bc.name+"/span", func(b testing.B) { b.SetBytes(int64(len(src))) for i := 0; i < b.N; i++ { t.Span(src, true) } }) } } func BenchmarkRemove(b testing.B) { doBench(b, Remove(Predicate(func(r rune) bool { return r == 'e' }))) } func BenchmarkMapAll(b testing.B) { doBench(b, Map(func(r rune) rune { return 'a' })) } func BenchmarkMapNone(b testing.B) { doBench(b, Map(func(r rune) rune { return r })) } func BenchmarkReplaceIllFormed(b testing.B) { doBench(b, ReplaceIllFormed()) } var ( input = strings.Repeat("Thé qüick brøwn føx jumps øver the lazy døg. ", 100) )	14.05 KB	2025-05-25 12:27

Terminal