Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 cldrtree.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package cldrtree builds and generates a CLDR index file, including all // inheritance. package cldrtree //go:generate go test -gen // cldrtree stores CLDR data in a tree-like structure called Tree. In the CLDR // data each branch in the tree is indicated by either an element name or an // attribute value. A Tree does not distinguish between these two cases, but // rather assumes that all branches can be accessed by an enum with a compact // range of positive integer values starting from 0. // // Each Tree consists of three parts: // - a slice mapping compact language identifiers to an offset into a set of // indices, // - a set of indices, stored as a large blob of uint16 values that encode // the actual tree structure of data, and // - a set of buckets that each holds a collection of strings. // each of which is explained in more detail below. // // // Tree lookup // A tree lookup is done by providing a locale and a "path", which is a // sequence of enum values. The search starts with getting the index for the // given locale and then incrementally jumping into the index using the path // values. If an element cannot be found in the index, the search starts anew // for the locale's parent locale. The path may change during lookup by means // of aliasing, described below. // // Buckets // Buckets hold the actual string data of the leaf values of the CLDR tree. // This data is stored in buckets, rather than one large string, for multiple // reasons: // - it allows representing leaf values more compactly, by storing all leaf // values in a single bucket and then needing only needing a uint16 to index // into this bucket for all leaf values, // - (TBD) allow multiple trees to share subsets of buckets, mostly to allow // linking in a smaller amount of data if only a subset of the buckets is // needed, // - to be nice to go fmt and the compiler. // // indices // An index is a slice of uint16 for which the values are interpreted in one of // two ways: as a node or a set of leaf values. // A set of leaf values has the following form: // <max_size>, <bucket>, <offset>... // max_size indicates the maximum enum value for which an offset is defined. // An offset value of 0xFFFF (missingValue) also indicates an undefined value. // If defined offset indicates the offset within the given bucket of the string. // A node value has the following form: // <max_size>, <offset_or_alias>... // max_size indicates the maximum value for which an offset is defined. // A missing offset may also be indicated with 0. If the high bit (0x8000, or // inheritMask) is not set, the offset points to the offset within the index // for the current locale. // An offset with high bit set is an alias. In this case the uint16 has the form // bits: // 15: 1 // 14-12: negative offset into path relative to current position // 0-11: new enum value for path element. // On encountering an alias, the path is modified accordingly and the lookup is // restarted for the given locale. import ( "fmt" "reflect" "regexp" "strings" "unicode/utf8" "golang.org/x/text/internal/gen" "golang.org/x/text/language" "golang.org/x/text/unicode/cldr" ) // TODO: // - allow two Trees to share the same set of buckets. // A Builder allows storing CLDR data in compact form. type Builder struct { table []string rootMeta metaData locales []locale strToBucket map[string]stringInfo buckets [][]byte enums []enum err error // Stats size int sizeAll int bucketWaste int } const ( maxBucketSize = 8 * 1024 // 8K maxStrlen = 254 // allow 0xFF sentinel ) func (b Builder) setError(err error) { if b.err == nil { b.err = err } } func (b Builder) addString(data string) stringInfo { data = b.makeString(data) info, ok := b.strToBucket[data] if !ok { b.size += len(data) x := len(b.buckets) - 1 bucket := b.buckets[x] if len(bucket)+len(data) < maxBucketSize { info.bucket = uint16(x) info.bucketPos = uint16(len(bucket)) b.buckets[x] = append(bucket, data...) } else { info.bucket = uint16(len(b.buckets)) info.bucketPos = 0 b.buckets = append(b.buckets, []byte(data)) } b.strToBucket[data] = info } return info } func (b Builder) addStringToBucket(data string, bucket uint16) stringInfo { data = b.makeString(data) info, ok := b.strToBucket[data] if !ok \|\| info.bucket != bucket { if ok { b.bucketWaste += len(data) } b.size += len(data) bk := b.buckets[bucket] info.bucket = bucket info.bucketPos = uint16(len(bk)) b.buckets[bucket] = append(bk, data...) b.strToBucket[data] = info } return info } func (b Builder) makeString(data string) string { if len(data) > maxStrlen { b.setError(fmt.Errorf("string %q exceeds maximum length of %d", data, maxStrlen)) data = data[:maxStrlen] for i := len(data) - 1; i > len(data)-4; i-- { if utf8.RuneStart(data[i]) { data = data[:i] break } } } data = string([]byte{byte(len(data))}) + data b.sizeAll += len(data) return data } type stringInfo struct { bufferPos uint32 bucket uint16 bucketPos uint16 } // New creates a new Builder. func New(tableName string) Builder { b := &Builder{ strToBucket: map[string]stringInfo{}, buckets: [][]byte{nil}, // initialize with first bucket. } b.rootMeta = &metaData{ b: b, typeInfo: &typeInfo{}, } return b } // Gen writes all the tables and types for the collected data. func (b Builder) Gen(w gen.CodeWriter) error { t, err := build(b) if err != nil { return err } return generate(b, t, w) } // GenTestData generates tables useful for testing data generated with Gen. func (b Builder) GenTestData(w gen.CodeWriter) error { return generateTestData(b, w) } type locale struct { tag language.Tag root Index } // Locale creates an index for the given locale. func (b Builder) Locale(t language.Tag) Index { index := &Index{ meta: b.rootMeta, } b.locales = append(b.locales, locale{tag: t, root: index}) return index } // An Index holds a map of either leaf values or other indices. type Index struct { meta metaData subIndex []Index values []keyValue } func (i Index) setError(err error) { i.meta.b.setError(err) } type keyValue struct { key enumIndex value stringInfo } // Element is a CLDR XML element. type Element interface { GetCommon() cldr.Common } // Index creates a subindex where the type and enum values are not shared // with siblings by default. The name is derived from the elem. If elem is // an alias reference, the alias will be resolved and linked. If elem is nil // Index returns nil. func (i Index) Index(elem Element, opt ...Option) Index { if elem == nil \|\| reflect.ValueOf(elem).IsNil() { return nil } c := elem.GetCommon() o := &options{ parent: i, name: c.GetCommon().Element(), } o.fill(opt) o.setAlias(elem) return i.subIndexForKey(o) } // IndexWithName is like Section but derives the name from the given name. func (i Index) IndexWithName(name string, opt ...Option) Index { o := &options{parent: i, name: name} o.fill(opt) return i.subIndexForKey(o) } // IndexFromType creates a subindex the value of tye type attribute as key. It // will also configure the Index to share the enumeration values with all // sibling values. If elem is an alias, it will be resolved and linked. func (i Index) IndexFromType(elem Element, opts ...Option) Index { o := &options{ parent: i, name: elem.GetCommon().Type, } o.fill(opts) o.setAlias(elem) useSharedType()(o) return i.subIndexForKey(o) } // IndexFromAlt creates a subindex the value of tye alt attribute as key. It // will also configure the Index to share the enumeration values with all // sibling values. If elem is an alias, it will be resolved and linked. func (i Index) IndexFromAlt(elem Element, opts ...Option) Index { o := &options{ parent: i, name: elem.GetCommon().Alt, } o.fill(opts) o.setAlias(elem) useSharedType()(o) return i.subIndexForKey(o) } func (i Index) subIndexForKey(opts options) Index { key := opts.name if len(i.values) > 0 { panic(fmt.Errorf("cldrtree: adding Index for %q when value already exists", key)) } meta := i.meta.sub(key, opts) for _, x := range i.subIndex { if x.meta == meta { return x } } if alias := opts.alias; alias != nil { if a := alias.GetCommon().Alias; a != nil { if a.Source != "locale" { i.setError(fmt.Errorf("cldrtree: non-locale alias not supported %v", a.Path)) } if meta.inheritOffset < 0 { i.setError(fmt.Errorf("cldrtree: alias was already set %v", a.Path)) } path := a.Path for ; strings.HasPrefix(path, "../"); path = path[len("../"):] { meta.inheritOffset-- } m := aliasRe.FindStringSubmatch(path) if m == nil { i.setError(fmt.Errorf("cldrtree: could not parse alias %q", a.Path)) } else { key := m[4] if key == "" { key = m[1] } meta.inheritIndex = key } } } x := &Index{meta: meta} i.subIndex = append(i.subIndex, x) return x } var aliasRe = regexp.MustCompile(`^([a-zA-Z]+)(\[@([a-zA-Z-]+)='([a-zA-Z-]+)'\])?`) // SetValue sets the value, the data from a CLDR XML element, for the given key. func (i Index) SetValue(key string, value Element, opt ...Option) { if len(i.subIndex) > 0 { panic(fmt.Errorf("adding value for key %q when index already exists", key)) } o := &options{parent: i} o.fill(opt) c := value.GetCommon() if c.Alias != nil { i.setError(fmt.Errorf("cldrtree: alias not supported for SetValue %v", c.Alias.Path)) } i.setValue(key, c.Data(), o) } func (i Index) setValue(key, data string, o options) { index, _ := i.meta.typeInfo.lookupSubtype(key, o) kv := keyValue{key: index} if len(i.values) > 0 { // Add string to the same bucket as the other values. bucket := i.values[0].value.bucket kv.value = i.meta.b.addStringToBucket(data, bucket) } else { kv.value = i.meta.b.addString(data) } i.values = append(i.values, kv) }	9.97 KB	2025-05-25 12:27
📄 cldrtree_test.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package cldrtree import ( "bytes" "flag" "log" "math/rand" "os" "path/filepath" "reflect" "regexp" "strconv" "strings" "testing" "golang.org/x/text/internal/gen" "golang.org/x/text/internal/language/compact" "golang.org/x/text/language" "golang.org/x/text/unicode/cldr" ) var genOutput = flag.Bool("gen", false, "generate output files") func TestAliasRegexp(t testing.T) { testCases := []struct { alias string want []string }{{ alias: "miscPatterns[@numberSystem='latn']", want: []string{ "miscPatterns[@numberSystem='latn']", "miscPatterns", "[@numberSystem='latn']", "numberSystem", "latn", }, }, { alias: `calendar[@type='greg-foo']/days/`, want: []string{ "calendar[@type='greg-foo']", "calendar", "[@type='greg-foo']", "type", "greg-foo", }, }, { alias: "eraAbbr", want: []string{ "eraAbbr", "eraAbbr", "", "", "", }, }, { // match must be anchored at beginning. alias: `../calendar[@type='gregorian']/days/`, }} for _, tc := range testCases { t.Run(tc.alias, func(t testing.T) { got := aliasRe.FindStringSubmatch(tc.alias) if !reflect.DeepEqual(got, tc.want) { t.Errorf("got %v; want %v", got, tc.want) } }) } } func TestBuild(t testing.T) { tree1, _ := loadTestdata(t, "test1") tree2, _ := loadTestdata(t, "test2") // Constants for second test const ( calendar = iota field ) const ( month = iota era filler cyclicNameSet ) const ( abbreviated = iota narrow wide ) testCases := []struct { desc string tree Tree locale string path []uint16 isFeature bool result string }{{ desc: "und/chinese month format wide m1", tree: tree1, locale: "und", path: path(calendar, 0, month, 0, wide, 1), result: "cM01", }, { desc: "und/chinese month format wide m12", tree: tree1, locale: "und", path: path(calendar, 0, month, 0, wide, 12), result: "cM12", }, { desc: "und/non-existing value", tree: tree1, locale: "und", path: path(calendar, 0, month, 0, wide, 13), result: "", }, { desc: "und/dangi:chinese month format wide", tree: tree1, locale: "und", path: path(calendar, 1, month, 0, wide, 1), result: "cM01", }, { desc: "und/chinese month format abbreviated:wide", tree: tree1, locale: "und", path: path(calendar, 0, month, 0, abbreviated, 1), result: "cM01", }, { desc: "und/chinese month format narrow:wide", tree: tree1, locale: "und", path: path(calendar, 0, month, 0, narrow, 1), result: "cM01", }, { desc: "und/gregorian month format wide", tree: tree1, locale: "und", path: path(calendar, 2, month, 0, wide, 2), result: "gM02", }, { desc: "und/gregorian month format:stand-alone narrow", tree: tree1, locale: "und", path: path(calendar, 2, month, 0, narrow, 1), result: "1", }, { desc: "und/gregorian month stand-alone:format abbreviated", tree: tree1, locale: "und", path: path(calendar, 2, month, 1, abbreviated, 1), result: "gM01", }, { desc: "und/gregorian month stand-alone:format wide ", tree: tree1, locale: "und", path: path(calendar, 2, month, 1, abbreviated, 1), result: "gM01", }, { desc: "und/dangi:chinese month format narrow:wide ", tree: tree1, locale: "und", path: path(calendar, 1, month, 0, narrow, 4), result: "cM04", }, { desc: "und/field era displayname 0", tree: tree2, locale: "und", path: path(field, 0, 0, 0), result: "Era", }, { desc: "en/field era displayname 0", tree: tree2, locale: "en", path: path(field, 0, 0, 0), result: "era", }, { desc: "und/calendar hebrew format wide 7-leap", tree: tree2, locale: "und", path: path(calendar, 7, month, 0, wide, 0), result: "Adar II", }, { desc: "en-GB:en-001:en:und/calendar hebrew format wide 7-leap", tree: tree2, locale: "en-GB", path: path(calendar, 7, month, 0, wide, 0), result: "Adar II", }, { desc: "und/buddhist month format wide 11", tree: tree2, locale: "und", path: path(calendar, 0, month, 0, wide, 12), result: "genWideM12", }, { desc: "en-GB/gregorian month stand-alone narrow 2", tree: tree2, locale: "en-GB", path: path(calendar, 6, month, 1, narrow, 3), result: "gbNarrowM3", }, { desc: "en-GB/gregorian month format narrow 3/missing in en-GB", tree: tree2, locale: "en-GB", path: path(calendar, 6, month, 0, narrow, 4), result: "enNarrowM4", }, { desc: "en-GB/gregorian month format narrow 3/missing in en and en-GB", tree: tree2, locale: "en-GB", path: path(calendar, 6, month, 0, narrow, 7), result: "gregNarrowM7", }, { desc: "en-GB/gregorian month format narrow 3/missing in en and en-GB", tree: tree2, locale: "en-GB", path: path(calendar, 6, month, 0, narrow, 7), result: "gregNarrowM7", }, { desc: "en-GB/gregorian era narrow", tree: tree2, locale: "en-GB", path: path(calendar, 6, era, abbreviated, 0, 1), isFeature: true, result: "AD", }, { desc: "en-GB/gregorian era narrow", tree: tree2, locale: "en-GB", path: path(calendar, 6, era, narrow, 0, 0), isFeature: true, result: "BC", }, { desc: "en-GB/gregorian era narrow", tree: tree2, locale: "en-GB", path: path(calendar, 6, era, wide, 1, 0), isFeature: true, result: "Before Common Era", }, { desc: "en-GB/dangi:chinese cyclicName, months, format, narrow:abbreviated 2", tree: tree2, locale: "en-GB", path: path(calendar, 1, cyclicNameSet, 3, 0, 1, 2), isFeature: true, result: "year2", }, { desc: "en-GB/field era-narrow ", tree: tree2, locale: "en-GB", path: path(field, 2, 0, 0), result: "era", }, { desc: "en-GB/field month-narrow relativeTime future one", tree: tree2, locale: "en-GB", path: path(field, 5, 2, 0, 1), isFeature: true, result: "001NarrowFutMOne", }, { // Don't fall back to the one of "en". desc: "en-GB/field month-short relativeTime past one:other", tree: tree2, locale: "en-GB", path: path(field, 4, 2, 1, 1), isFeature: true, result: "001ShortPastMOther", }, { desc: "en-GB/field month relativeTime future two:other", tree: tree2, locale: "en-GB", path: path(field, 3, 2, 0, 2), isFeature: true, result: "enFutMOther", }} for _, tc := range testCases { t.Run(tc.desc, func(t testing.T) { tag, _ := compact.RegionalID(compact.Tag(language.MustParse(tc.locale))) s := tc.tree.lookup(tag, tc.isFeature, tc.path...) if s != tc.result { t.Errorf("got %q; want %q", s, tc.result) } }) } } func path(e ...uint16) []uint16 { return e } func TestGen(t testing.T) { testCases := []string{"test1", "test2"} for _, tc := range testCases { t.Run(tc, func(t testing.T) { _, got := loadTestdata(t, tc) // Remove sizes that may vary per architecture. re := regexp.MustCompile("// Size: [0-9]") got = re.ReplaceAllLiteral(got, []byte("// Size: xxxx")) re = regexp.MustCompile("// Total table size [0-9]") got = re.ReplaceAllLiteral(got, []byte("// Total table size: xxxx")) file := filepath.Join("testdata", tc, "output.go") if genOutput { os.WriteFile(file, got, 0700) t.SkipNow() } b, err := os.ReadFile(file) if err != nil { t.Fatalf("failed to open file: %v", err) } if want := string(b); string(got) != want { t.Log(string(got)) t.Errorf("files differ") } }) } } func loadTestdata(t testing.T, test string) (tree Tree, file []byte) { b := New("test") var d cldr.Decoder data, err := d.DecodePath(filepath.Join("testdata", test)) if err != nil { t.Fatalf("error decoding testdata: %v", err) } context := Enum("context") widthMap := func(s string) string { // Align era with width values. if r, ok := map[string]string{ "eraAbbr": "abbreviated", "eraNarrow": "narrow", "eraNames": "wide", }[s]; ok { s = r } return "w" + strings.Title(s) } width := EnumFunc("width", widthMap, "abbreviated", "narrow", "wide") month := Enum("month", "leap7") relative := EnumFunc("relative", func(s string) string { x, err := strconv.ParseInt(s, 10, 8) if err != nil { log.Fatal("Invalid number:", err) } return []string{ "before1", "current", "after1", }[x+1] }) cycleType := EnumFunc("cycleType", func(s string) string { return "cyc" + strings.Title(s) }) r := rand.New(rand.NewSource(0)) for _, loc := range data.Locales() { ldml := data.RawLDML(loc) x := b.Locale(language.Make(loc)) if x := x.Index(ldml.Dates.Calendars); x != nil { for _, cal := range ldml.Dates.Calendars.Calendar { x := x.IndexFromType(cal) if x := x.Index(cal.Months); x != nil { for _, mc := range cal.Months.MonthContext { x := x.IndexFromType(mc, context) for _, mw := range mc.MonthWidth { x := x.IndexFromType(mw, width) for _, m := range mw.Month { x.SetValue(m.Yeartype+m.Type, m, month) } } } } if x := x.Index(cal.CyclicNameSets); x != nil { for _, cns := range cal.CyclicNameSets.CyclicNameSet { x := x.IndexFromType(cns, cycleType) for _, cc := range cns.CyclicNameContext { x := x.IndexFromType(cc, context) for _, cw := range cc.CyclicNameWidth { x := x.IndexFromType(cw, width) for _, c := range cw.CyclicName { x.SetValue(c.Type, c) } } } } } if x := x.Index(cal.Eras); x != nil { opts := []Option{width, SharedType()} if x := x.Index(cal.Eras.EraNames, opts...); x != nil { for _, e := range cal.Eras.EraNames.Era { x.IndexFromAlt(e).SetValue(e.Type, e) } } if x := x.Index(cal.Eras.EraAbbr, opts...); x != nil { for _, e := range cal.Eras.EraAbbr.Era { x.IndexFromAlt(e).SetValue(e.Type, e) } } if x := x.Index(cal.Eras.EraNarrow, opts...); x != nil { for _, e := range cal.Eras.EraNarrow.Era { x.IndexFromAlt(e).SetValue(e.Type, e) } } } { // Ensure having more than 2 buckets. f := x.IndexWithName("filler") b := make([]byte, maxStrlen) opt := &options{parent: x} r.Read(b) f.setValue("0", string(b), opt) } } } if x := x.Index(ldml.Dates.Fields); x != nil { for _, f := range ldml.Dates.Fields.Field { x := x.IndexFromType(f) for _, d := range f.DisplayName { x.Index(d).SetValue("", d) } for _, r := range f.Relative { x.Index(r).SetValue(r.Type, r, relative) } for _, rt := range f.RelativeTime { x := x.Index(rt).IndexFromType(rt) for _, p := range rt.RelativeTimePattern { x.SetValue(p.Count, p) } } for _, rp := range f.RelativePeriod { x.Index(rp).SetValue("", rp) } } } } tree, err = build(b) if err != nil { t.Fatal("error building tree:", err) } w := gen.NewCodeWriter() generate(b, tree, w) generateTestData(b, w) buf := &bytes.Buffer{} if _, err = w.WriteGo(buf, "test", ""); err != nil { t.Log(buf.String()) t.Fatal("error generating code:", err) } return tree, buf.Bytes() }	11.2 KB	2025-05-25 12:27
📄 generate.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package cldrtree import ( "bytes" "fmt" "io" "reflect" "strconv" "strings" "golang.org/x/text/internal/gen" ) func generate(b Builder, t Tree, w gen.CodeWriter) error { fmt.Fprintln(w, `import "golang.org/x/text/internal/cldrtree"`) fmt.Fprintln(w) fmt.Fprintf(w, "var tree = &cldrtree.Tree{locales, indices, buckets}\n\n") w.WriteComment("Path values:\n%s", b.stats()) fmt.Fprintln(w) // Generate enum types. for _, e := range b.enums { // Build enum types. w.WriteComment("%s specifies a property of a CLDR field.", e.name) fmt.Fprintf(w, "type %s uint16\n", e.name) } d, err := getEnumData(b) if err != nil { return err } fmt.Fprintln(w, "const (") for i, k := range d.keys { fmt.Fprintf(w, "%s %s = %d // %s\n", toCamel(k), d.enums[i], d.m[k], k) } fmt.Fprintln(w, ")") w.WriteVar("locales", t.Locales) w.WriteVar("indices", t.Indices) // Generate string buckets. fmt.Fprintln(w, "var buckets = []string{") for i := range t.Buckets { fmt.Fprintf(w, "bucket%d,\n", i) } fmt.Fprint(w, "}\n\n") w.Size += int(reflect.TypeOf("").Size()) len(t.Buckets) // Generate string buckets. for i, bucket := range t.Buckets { w.WriteVar(fmt.Sprint("bucket", i), bucket) } return nil } func generateTestData(b Builder, w gen.CodeWriter) error { d, err := getEnumData(b) if err != nil { return err } fmt.Fprintln(w) fmt.Fprintln(w, "var enumMap = map[string]uint16{") fmt.Fprintln(w, `"": 0,`) for _, k := range d.keys { fmt.Fprintf(w, "%q: %d,\n", k, d.m[k]) } fmt.Fprintln(w, "}") return nil } func toCamel(s string) string { p := strings.Split(s, "-") for i, s := range p[1:] { p[i+1] = strings.Title(s) } return strings.Replace(strings.Join(p, ""), "/", "", -1) } func (b Builder) stats() string { w := &bytes.Buffer{} b.rootMeta.validate() for _, es := range b.enums { fmt.Fprintf(w, "<%s>\n", es.name) printEnumValues(w, es, 1, nil) } fmt.Fprintln(w) printEnums(w, b.rootMeta.typeInfo, 0) fmt.Fprintln(w) fmt.Fprintln(w, "Nr elem: ", len(b.strToBucket)) fmt.Fprintln(w, "uniqued size: ", b.size) fmt.Fprintln(w, "total string size: ", b.sizeAll) fmt.Fprintln(w, "bucket waste: ", b.bucketWaste) return w.String() } func printEnums(w io.Writer, s typeInfo, indent int) { idStr := strings.Repeat(" ", indent) + "- " e := s.enum if e == nil { if len(s.entries) > 0 { panic(fmt.Errorf("has entries but no enum values: %#v", s.entries)) } return } if e.name != "" { fmt.Fprintf(w, "%s<%s>\n", idStr, e.name) } else { printEnumValues(w, e, indent, s) } if s.sharedKeys() { for _, v := range s.entries { printEnums(w, v, indent+1) break } } } func printEnumValues(w io.Writer, e enum, indent int, info typeInfo) { idStr := strings.Repeat(" ", indent) + "- " for i := 0; i < len(e.keys); i++ { fmt.Fprint(w, idStr) k := e.keys[i] if u, err := strconv.ParseUint(k, 10, 16); err == nil { fmt.Fprintf(w, "%s", k) // Skip contiguous integers var v, last uint64 for i++; i < len(e.keys); i++ { k = e.keys[i] if v, err = strconv.ParseUint(k, 10, 16); err != nil { break } last = v } if u < last { fmt.Fprintf(w, `..%d`, last) } fmt.Fprintln(w) if err != nil { fmt.Fprintf(w, "%s%s\n", idStr, k) } } else if k == "" { fmt.Fprintln(w, `""`) } else { fmt.Fprintf(w, "%s\n", k) } if info != nil && !info.sharedKeys() { if e := info.entries[enumIndex(i)]; e != nil { printEnums(w, e, indent+1) } } } } func getEnumData(b Builder) (enumData, error) { d := &enumData{m: map[string]int{}} if errStr := d.insert(b.rootMeta.typeInfo); errStr != "" { // TODO: consider returning the error. return nil, fmt.Errorf("cldrtree: %s", errStr) } return d, nil } type enumData struct { m map[string]int keys []string enums []string } func (d enumData) insert(t typeInfo) (errStr string) { e := t.enum if e == nil { return "" } for i, k := range e.keys { if _, err := strconv.ParseUint(k, 10, 16); err == nil { // We don't include any enum that has integer values. break } if v, ok := d.m[k]; ok { if v != i { return fmt.Sprintf("%q has value %d and %d", k, i, v) } } else { d.m[k] = i if k != "" { d.keys = append(d.keys, k) d.enums = append(d.enums, e.name) } } } for i := range t.enum.keys { if e := t.entries[enumIndex(i)]; e != nil { if errStr := d.insert(e); errStr != "" { return fmt.Sprintf("%q>%v", t.enum.keys[i], errStr) } } } return "" }	4.61 KB	2025-05-25 12:27
📄 option.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package cldrtree import ( "reflect" "golang.org/x/text/unicode/cldr" ) // An Option configures an Index. type Option func(options) type options struct { parent Index name string alias cldr.Common sharedType typeInfo sharedEnums enum } func (o options) fill(opt []Option) { for _, f := range opt { f(o) } } // setAlias sets an alias from the given node, if the node defines one. func (o options) setAlias(n Element) { if n != nil && !reflect.ValueOf(n).IsNil() { o.alias = n.GetCommon() } } // Enum defines an enumeration type. The resulting option may be passed for the // construction of multiple Indexes, which they will share the same enum values. // Calling Gen on a Builder will generate the Enum for the given name. The // optional values fix the values for the given identifier to the argument // position (starting at 0). Other values may still be added and will be // assigned to subsequent values. func Enum(name string, value ...string) Option { return EnumFunc(name, nil, value...) } // EnumFunc is like Enum but also takes a function that allows rewriting keys. func EnumFunc(name string, rename func(string) string, value ...string) Option { enum := &enum{name: name, rename: rename, keyMap: map[string]enumIndex{}} for _, e := range value { enum.lookup(e) } return func(o options) { found := false for _, e := range o.parent.meta.b.enums { if e.name == enum.name { found = true break } } if !found { o.parent.meta.b.enums = append(o.parent.meta.b.enums, enum) } o.sharedEnums = enum } } // SharedType returns an option which causes all Indexes to which this option is // passed to have the same type. func SharedType() Option { info := &typeInfo{} return func(o options) { o.sharedType = info } } func useSharedType() Option { return func(o options) { sub := o.parent.meta.typeInfo.keyTypeInfo if sub == nil { sub = &typeInfo{} o.parent.meta.typeInfo.keyTypeInfo = sub } o.sharedType = sub } }	2.11 KB	2025-05-25 12:27
📁 testdata	-	2025-05-25 12:27
📄 tree.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package cldrtree import ( "golang.org/x/text/internal/language/compact" "golang.org/x/text/language" ) const ( inheritOffsetShift = 12 inheritMask uint16 = 0x8000 inheritValueMask uint16 = 0x0FFF missingValue uint16 = 0xFFFF ) // Tree holds a tree of CLDR data. type Tree struct { Locales []uint32 Indices []uint16 Buckets []string } // Lookup looks up CLDR data for the given path. The lookup adheres to the alias // and locale inheritance rules as defined in CLDR. // // Each subsequent element in path indicates which subtree to select data from. // The last element of the path must select a leaf node. All other elements // of the path select a subindex. func (t Tree) Lookup(tag compact.ID, path ...uint16) string { return t.lookup(tag, false, path...) } // LookupFeature is like Lookup, but will first check whether a value of "other" // as a fallback before traversing the inheritance chain. func (t Tree) LookupFeature(tag compact.ID, path ...uint16) string { return t.lookup(tag, true, path...) } func (t Tree) lookup(tag compact.ID, isFeature bool, path ...uint16) string { origLang := tag outer: for { index := t.Indices[t.Locales[tag]:] k := uint16(0) for i := range path { max := index[k] if i < len(path)-1 { // index (non-leaf) if path[i] >= max { break } k = index[k+1+path[i]] if k == 0 { break } if v := k &^ inheritMask; k != v { offset := v >> inheritOffsetShift value := v & inheritValueMask path[uint16(i)-offset] = value tag = origLang continue outer } } else { // leaf value offset := missingValue if path[i] < max { offset = index[k+2+path[i]] } if offset == missingValue { if !isFeature { break } // "other" feature must exist offset = index[k+2] } data := t.Buckets[index[k+1]] n := uint16(data[offset]) return data[offset+1 : offset+n+1] } } if tag == 0 { break } tag = tag.Parent() } return "" } func build(b Builder) (Tree, error) { var t Tree t.Locales = make([]uint32, language.NumCompactTags) for _, loc := range b.locales { tag, _ := language.CompactIndex(loc.tag) t.Locales[tag] = uint32(len(t.Indices)) var x indexBuilder x.add(loc.root) t.Indices = append(t.Indices, x.index...) } // Set locales for which we don't have data to the parent's data. for i, v := range t.Locales { p := compact.ID(i) for v == 0 && p != 0 { p = p.Parent() v = t.Locales[p] } t.Locales[i] = v } for _, b := range b.buckets { t.Buckets = append(t.Buckets, string(b)) } if b.err != nil { return nil, b.err } return &t, nil } type indexBuilder struct { index []uint16 } func (b indexBuilder) add(i *Index) uint16 { offset := len(b.index) max := enumIndex(0) switch { case len(i.values) > 0: for _, v := range i.values { if v.key > max { max = v.key } } b.index = append(b.index, make([]uint16, max+3)...) b.index[offset] = uint16(max) + 1 b.index[offset+1] = i.values[0].value.bucket for i := offset + 2; i < len(b.index); i++ { b.index[i] = missingValue } for _, v := range i.values { b.index[offset+2+int(v.key)] = v.value.bucketPos } return uint16(offset) case len(i.subIndex) > 0: for _, s := range i.subIndex { if s.meta.index > max { max = s.meta.index } } b.index = append(b.index, make([]uint16, max+2)...) b.index[offset] = uint16(max) + 1 for _, s := range i.subIndex { x := b.add(s) b.index[offset+int(s.meta.index)+1] = x } return uint16(offset) case i.meta.inheritOffset < 0: v := uint16(-(i.meta.inheritOffset + 1)) << inheritOffsetShift p := i.meta for k := i.meta.inheritOffset; k < 0; k++ { p = p.parent } v += uint16(p.typeInfo.enum.lookup(i.meta.inheritIndex)) v \|= inheritMask return v } return 0 }	3.95 KB	2025-05-25 12:27
📄 type.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package cldrtree import ( "log" "strconv" ) // enumIndex is the numerical value of an enum value. type enumIndex int // An enum is a collection of enum values. type enum struct { name string // the Go type of the enum rename func(string) string keyMap map[string]enumIndex keys []string } // lookup returns the index for the enum corresponding to the string. If s // currently does not exist it will add the entry. func (e enum) lookup(s string) enumIndex { if e.rename != nil { s = e.rename(s) } x, ok := e.keyMap[s] if !ok { if e.keyMap == nil { e.keyMap = map[string]enumIndex{} } u, err := strconv.ParseUint(s, 10, 32) if err == nil { for len(e.keys) <= int(u) { x := enumIndex(len(e.keys)) s := strconv.Itoa(int(x)) e.keyMap[s] = x e.keys = append(e.keys, s) } if e.keyMap[s] != enumIndex(u) { // TODO: handle more gracefully. log.Fatalf("cldrtree: mix of integer and non-integer for %q %v", s, e.keys) } return enumIndex(u) } x = enumIndex(len(e.keys)) e.keyMap[s] = x e.keys = append(e.keys, s) } return x } // A typeInfo indicates the set of possible enum values and a mapping from // these values to subtypes. type typeInfo struct { enum enum entries map[enumIndex]typeInfo keyTypeInfo typeInfo shareKeys bool } func (t typeInfo) sharedKeys() bool { return t.shareKeys } func (t typeInfo) lookupSubtype(s string, opts options) (x enumIndex, sub typeInfo) { if t.enum == nil { if t.enum = opts.sharedEnums; t.enum == nil { t.enum = &enum{} } } if opts.sharedEnums != nil && t.enum != opts.sharedEnums { panic("incompatible enums defined") } x = t.enum.lookup(s) if t.entries == nil { t.entries = map[enumIndex]typeInfo{} } sub, ok := t.entries[x] if !ok { sub = opts.sharedType if sub == nil { sub = &typeInfo{} } t.entries[x] = sub } t.shareKeys = opts.sharedType != nil // For analysis purposes. return x, sub } // metaData includes information about subtypes, possibly sharing commonality // with sibling branches, and information about inheritance, which may differ // per branch. type metaData struct { b Builder parent metaData index enumIndex // index into the parent's subtype index key string elem string // XML element corresponding to this type. typeInfo typeInfo lookup map[enumIndex]metaData subs []metaData inheritOffset int // always negative when applicable inheritIndex string // new value for field indicated by inheritOffset // inheritType metaData } func (m metaData) sub(key string, opts options) metaData { if m.lookup == nil { m.lookup = map[enumIndex]metaData{} } enum, info := m.typeInfo.lookupSubtype(key, opts) sub := m.lookup[enum] if sub == nil { sub = &metaData{ b: m.b, parent: m, index: enum, key: key, typeInfo: info, } m.lookup[enum] = sub m.subs = append(m.subs, sub) } return sub } func (m metaData) validate() { for _, s := range m.subs { s.validate() } }	3.13 KB	2025-05-25 12:27

Terminal