Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 extract.go // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package pipeline import ( "bytes" "errors" "fmt" "go/ast" "go/constant" "go/format" "go/token" "go/types" "path/filepath" "sort" "strings" "unicode" "unicode/utf8" fmtparser "golang.org/x/text/internal/format" "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/callgraph/cha" "golang.org/x/tools/go/loader" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" ) const debug = false // TODO: // - merge information into existing files // - handle different file formats (PO, XLIFF) // - handle features (gender, plural) // - message rewriting // - `msg:"etc"` tags // Extract extracts all strings form the package defined in Config. func Extract(c Config) (State, error) { x, err := newExtracter(c) if err != nil { return nil, wrap(err, "") } if err := x.seedEndpoints(); err != nil { return nil, err } x.extractMessages() return &State{ Config: c, program: x.iprog, Extracted: Messages{ Language: c.SourceLanguage, Messages: x.messages, }, }, nil } type extracter struct { conf loader.Config iprog loader.Program prog ssa.Program callGraph callgraph.Graph // Calls and other expressions to collect. globals map[token.Pos]constData funcs map[token.Pos]callData messages []Message } func newExtracter(c Config) (x extracter, err error) { x = &extracter{ conf: loader.Config{}, globals: map[token.Pos]constData{}, funcs: map[token.Pos]callData{}, } x.iprog, err = loadPackages(&x.conf, c.Packages) if err != nil { return nil, wrap(err, "") } x.prog = ssautil.CreateProgram(x.iprog, ssa.GlobalDebug\|ssa.BareInits) x.prog.Build() x.callGraph = cha.CallGraph(x.prog) return x, nil } func (x extracter) globalData(pos token.Pos) constData { cd := x.globals[pos] if cd == nil { cd = &constData{} x.globals[pos] = cd } return cd } func (x extracter) seedEndpoints() error { pkgInfo := x.iprog.Package("golang.org/x/text/message") if pkgInfo == nil { return errors.New("pipeline: golang.org/x/text/message is not imported") } pkg := x.prog.Package(pkgInfo.Pkg) typ := types.NewPointer(pkg.Type("Printer").Type()) x.processGlobalVars() x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Printf"), &callData{ formatPos: 1, argPos: 2, isMethod: true, }) x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Sprintf"), &callData{ formatPos: 1, argPos: 2, isMethod: true, }) x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Fprintf"), &callData{ formatPos: 2, argPos: 3, isMethod: true, }) return nil } // processGlobalVars finds string constants that are assigned to global // variables. func (x extracter) processGlobalVars() { for _, p := range x.prog.AllPackages() { m, ok := p.Members["init"] if !ok { continue } for _, b := range m.(ssa.Function).Blocks { for _, i := range b.Instrs { s, ok := i.(ssa.Store) if !ok { continue } a, ok := s.Addr.(ssa.Global) if !ok { continue } t := a.Type() for { p, ok := t.(types.Pointer) if !ok { break } t = p.Elem() } if b, ok := t.(types.Basic); !ok \|\| b.Kind() != types.String { continue } x.visitInit(a, s.Val) } } } } type constData struct { call callData // to provide a signature for the constants values []constVal others []token.Pos // Assigned to other global data. } func (d constData) visit(x extracter, f func(c constant.Value)) { for _, v := range d.values { f(v.value) } for _, p := range d.others { if od, ok := x.globals[p]; ok { od.visit(x, f) } } } type constVal struct { value constant.Value pos token.Pos } type callData struct { call ssa.CallInstruction expr ast.CallExpr formats []constant.Value callee callData isMethod bool formatPos int argPos int // varargs at this position in the call argTypes []int // arguments extractable from this position } func (c callData) callFormatPos() int { c = c.callee if c.isMethod { return c.formatPos - 1 } return c.formatPos } func (c callData) callArgsStart() int { c = c.callee if c.isMethod { return c.argPos - 1 } return c.argPos } func (c callData) Pos() token.Pos { return c.call.Pos() } func (c callData) Pkg() types.Package { return c.call.Parent().Pkg.Pkg } func (x extracter) handleFunc(f ssa.Function, fd callData) { for _, e := range x.callGraph.Nodes[f].In { if e.Pos() == 0 { continue } call := e.Site caller := x.funcs[call.Pos()] if caller != nil { // TODO: theoretically a format string could be passed to multiple // arguments of a function. Support this eventually. continue } x.debug(call, "CALL", f.String()) caller = &callData{ call: call, callee: fd, formatPos: -1, argPos: -1, } // Offset by one if we are invoking an interface method. offset := 0 if call.Common().IsInvoke() { offset = -1 } x.funcs[call.Pos()] = caller if fd.argPos >= 0 { x.visitArgs(caller, call.Common().Args[fd.argPos+offset]) } x.visitFormats(caller, call.Common().Args[fd.formatPos+offset]) } } type posser interface { Pos() token.Pos Parent() ssa.Function } func (x extracter) debug(v posser, header string, args ...interface{}) { if debug { pos := "" if p := v.Parent(); p != nil { pos = posString(&x.conf, p.Package().Pkg, v.Pos()) } if header != "CALL" && header != "INSERT" { header = " " + header } fmt.Printf("%-32s%-10s%-15T ", pos+fmt.Sprintf("@%d", v.Pos()), header, v) for _, a := range args { fmt.Printf(" %v", a) } fmt.Println() } } // visitInit evaluates and collects values assigned to global variables in an // init function. func (x extracter) visitInit(global ssa.Global, v ssa.Value) { if v == nil { return } x.debug(v, "GLOBAL", v) switch v := v.(type) { case ssa.Phi: for _, e := range v.Edges { x.visitInit(global, e) } case ssa.Const: // Only record strings with letters. if str := constant.StringVal(v.Value); isMsg(str) { cd := x.globalData(global.Pos()) cd.values = append(cd.values, constVal{v.Value, v.Pos()}) } // TODO: handle %m-directive. case ssa.Global: cd := x.globalData(global.Pos()) cd.others = append(cd.others, v.Pos()) case ssa.FieldAddr, ssa.Field: // TODO: mark field index v.Field of v.X.Type() for extraction. extract // an example args as to give parameters for the translator. case ssa.Slice: if v.Low == nil && v.High == nil && v.Max == nil { x.visitInit(global, v.X) } case ssa.Alloc: if ref := v.Referrers(); ref == nil { for _, r := range ref { values := []ssa.Value{} for _, o := range r.Operands(nil) { if o == nil \|\| o == v { continue } values = append(values, o) } // TODO: return something different if we care about multiple // values as well. if len(values) == 1 { x.visitInit(global, values[0]) } } } case ssa.Instruction: rands := v.Operands(nil) if len(rands) == 1 && rands[0] != nil { x.visitInit(global, rands[0]) } } return } // visitFormats finds the original source of the value. The returned index is // position of the argument if originated from a function argument or -1 // otherwise. func (x extracter) visitFormats(call callData, v ssa.Value) { if v == nil { return } x.debug(v, "VALUE", v) switch v := v.(type) { case ssa.Phi: for _, e := range v.Edges { x.visitFormats(call, e) } case ssa.Const: // Only record strings with letters. if isMsg(constant.StringVal(v.Value)) { x.debug(call.call, "FORMAT", v.Value.ExactString()) call.formats = append(call.formats, v.Value) } // TODO: handle %m-directive. case ssa.Global: x.globalData(v.Pos()).call = call case ssa.FieldAddr, ssa.Field: // TODO: mark field index v.Field of v.X.Type() for extraction. extract // an example args as to give parameters for the translator. case ssa.Slice: if v.Low == nil && v.High == nil && v.Max == nil { x.visitFormats(call, v.X) } case ssa.Parameter: // TODO: handle the function for the index parameter. f := v.Parent() for i, p := range f.Params { if p == v { if call.formatPos < 0 { call.formatPos = i // TODO: is there a better way to detect this is calling // a method rather than a function? call.isMethod = len(f.Params) > f.Signature.Params().Len() x.handleFunc(v.Parent(), call) } else if debug && i != call.formatPos { // TODO: support this. fmt.Printf("WARNING:%s: format string passed to arg %d and %d\n", posString(&x.conf, call.Pkg(), call.Pos()), call.formatPos, i) } } } case ssa.Alloc: if ref := v.Referrers(); ref == nil { for _, r := range ref { values := []ssa.Value{} for _, o := range r.Operands(nil) { if o == nil \|\| o == v { continue } values = append(values, o) } // TODO: return something different if we care about multiple // values as well. if len(values) == 1 { x.visitFormats(call, values[0]) } } } // TODO: // case ssa.Index: // // Get all values in the array if applicable // case ssa.IndexAddr: // // Get all values in the slice or array if applicable. // case ssa.Lookup: // // Get all values in the map if applicable. case ssa.FreeVar: // TODO: find the link between free variables and parameters: // // func freeVar(p message.Printer, str string) { // fn := func(p message.Printer) { // p.Printf(str) // } // fn(p) // } case ssa.Call: case ssa.Instruction: rands := v.Operands(nil) if len(rands) == 1 && rands[0] != nil { x.visitFormats(call, rands[0]) } } } // Note: a function may have an argument marked as both format and passthrough. // visitArgs collects information on arguments. For wrapped functions it will // just determine the position of the variable args slice. func (x extracter) visitArgs(fd callData, v ssa.Value) { if v == nil { return } x.debug(v, "ARGV", v) switch v := v.(type) { case ssa.Slice: if v.Low == nil && v.High == nil && v.Max == nil { x.visitArgs(fd, v.X) } case ssa.Parameter: // TODO: handle the function for the index parameter. f := v.Parent() for i, p := range f.Params { if p == v { fd.argPos = i } } case ssa.Alloc: if ref := v.Referrers(); ref == nil { for _, r := range ref { values := []ssa.Value{} for _, o := range r.Operands(nil) { if o == nil \|\| o == v { continue } values = append(values, o) } // TODO: return something different if we care about // multiple values as well. if len(values) == 1 { x.visitArgs(fd, values[0]) } } } case ssa.Instruction: rands := v.Operands(nil) if len(rands) == 1 && rands[0] != nil { x.visitArgs(fd, rands[0]) } } } // print returns Go syntax for the specified node. func (x extracter) print(n ast.Node) string { var buf bytes.Buffer format.Node(&buf, x.conf.Fset, n) return buf.String() } type packageExtracter struct { f ast.File x extracter info loader.PackageInfo cmap ast.CommentMap } func (px packageExtracter) getComment(n ast.Node) string { cs := px.cmap.Filter(n).Comments() if len(cs) > 0 { return strings.TrimSpace(cs[0].Text()) } return "" } func (x extracter) extractMessages() { prog := x.iprog keys := make([]types.Package, 0, len(x.iprog.AllPackages)) for k := range x.iprog.AllPackages { keys = append(keys, k) } sort.Slice(keys, func(i, j int) bool { return keys[i].Path() < keys[j].Path() }) files := []packageExtracter{} for _, k := range keys { info := x.iprog.AllPackages[k] for _, f := range info.Files { // Associate comments with nodes. px := packageExtracter{ f, x, info, ast.NewCommentMap(prog.Fset, f, f.Comments), } files = append(files, px) } } for _, px := range files { ast.Inspect(px.f, func(n ast.Node) bool { switch v := n.(type) { case ast.CallExpr: if d := x.funcs[v.Lparen]; d != nil { d.expr = v } } return true }) } for _, px := range files { ast.Inspect(px.f, func(n ast.Node) bool { switch v := n.(type) { case ast.CallExpr: return px.handleCall(v) case ast.ValueSpec: return px.handleGlobal(v) } return true }) } } func (px packageExtracter) handleGlobal(spec ast.ValueSpec) bool { comment := px.getComment(spec) for _, ident := range spec.Names { data, ok := px.x.globals[ident.Pos()] if !ok { continue } name := ident.Name var arguments []argument if data.call != nil { arguments = px.getArguments(data.call) } else if !strings.HasPrefix(name, "msg") && !strings.HasPrefix(name, "Msg") { continue } data.visit(px.x, func(c constant.Value) { px.addMessage(spec.Pos(), []string{name}, c, comment, arguments) }) } return true } func (px packageExtracter) handleCall(call ast.CallExpr) bool { x := px.x data := x.funcs[call.Lparen] if data == nil \|\| len(data.formats) == 0 { return true } if data.expr != call { panic("invariant `data.call != call` failed") } x.debug(data.call, "INSERT", data.formats) argn := data.callFormatPos() if argn >= len(call.Args) { return true } format := call.Args[argn] arguments := px.getArguments(data) comment := "" key := []string{} if ident, ok := format.(ast.Ident); ok { key = append(key, ident.Name) if v, ok := ident.Obj.Decl.(ast.ValueSpec); ok && v.Comment != nil { // TODO: get comment above ValueSpec as well comment = v.Comment.Text() } } if c := px.getComment(call.Args[0]); c != "" { comment = c } formats := data.formats for _, c := range formats { px.addMessage(call.Lparen, key, c, comment, arguments) } return true } func (px packageExtracter) getArguments(data callData) []argument { arguments := []argument{} x := px.x info := px.info if data.callArgsStart() >= 0 { args := data.expr.Args[data.callArgsStart():] for i, arg := range args { expr := x.print(arg) val := "" if v := info.Types[arg].Value; v != nil { val = v.ExactString() switch arg.(type) { case ast.BinaryExpr, ast.UnaryExpr: expr = val } } arguments = append(arguments, argument{ ArgNum: i + 1, Type: info.Types[arg].Type.String(), UnderlyingType: info.Types[arg].Type.Underlying().String(), Expr: expr, Value: val, Comment: px.getComment(arg), Position: posString(&x.conf, info.Pkg, arg.Pos()), // TODO report whether it implements // interfaces plural.Interface, // gender.Interface. }) } } return arguments } func (px packageExtracter) addMessage( pos token.Pos, key []string, c constant.Value, comment string, arguments []argument) { x := px.x fmtMsg := constant.StringVal(c) ph := placeholders{index: map[string]string{}} trimmed, _, _ := trimWS(fmtMsg) p := fmtparser.Parser{} simArgs := make([]interface{}, len(arguments)) for i, v := range arguments { simArgs[i] = v } msg := "" p.Reset(simArgs) for p.SetFormat(trimmed); p.Scan(); { name := "" var arg argument switch p.Status { case fmtparser.StatusText: msg += p.Text() continue case fmtparser.StatusSubstitution, fmtparser.StatusBadWidthSubstitution, fmtparser.StatusBadPrecSubstitution: arguments[p.ArgNum-1].used = true arg = &arguments[p.ArgNum-1] name = getID(arg) case fmtparser.StatusBadArgNum, fmtparser.StatusMissingArg: arg = &argument{ ArgNum: p.ArgNum, Position: posString(&x.conf, px.info.Pkg, pos), } name, arg.UnderlyingType = verbToPlaceholder(p.Text(), p.ArgNum) } sub := p.Text() if !p.HasIndex { r, sz := utf8.DecodeLastRuneInString(sub) sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r) } msg += fmt.Sprintf("{%s}", ph.addArg(arg, name, sub)) } key = append(key, msg) // Add additional Placeholders that can be used in translations // that are not present in the string. for _, arg := range arguments { if arg.used { continue } ph.addArg(&arg, getID(&arg), fmt.Sprintf("%%[%d]v", arg.ArgNum)) } x.messages = append(x.messages, Message{ ID: key, Key: fmtMsg, Message: Text{Msg: msg}, // TODO(fix): this doesn't get the before comment. Comment: comment, Placeholders: ph.slice, Position: posString(&x.conf, px.info.Pkg, pos), }) } func posString(conf loader.Config, pkg types.Package, pos token.Pos) string { p := conf.Fset.Position(pos) file := fmt.Sprintf("%s:%d:%d", filepath.Base(p.Filename), p.Line, p.Column) return filepath.Join(pkg.Path(), file) } func getID(arg argument) string { s := getLastComponent(arg.Expr) s = strip(s) s = strings.Replace(s, " ", "", -1) // For small variable names, use user-defined types for more info. if len(s) <= 2 && arg.UnderlyingType != arg.Type { s = getLastComponent(arg.Type) } return strings.Title(s) } // strip is a dirty hack to convert function calls to placeholder IDs. func strip(s string) string { s = strings.Map(func(r rune) rune { if unicode.IsSpace(r) \|\| r == '-' { return '_' } if !unicode.In(r, unicode.Letter, unicode.Mark, unicode.Number) { return -1 } return r }, s) // Strip "Get" from getter functions. if strings.HasPrefix(s, "Get") \|\| strings.HasPrefix(s, "get") { if len(s) > len("get") { r, _ := utf8.DecodeRuneInString(s) if !unicode.In(r, unicode.Ll, unicode.M) { // not lower or mark s = s[len("get"):] } } } return s } // verbToPlaceholder gives a name for a placeholder based on the substitution // verb. This is only to be used if there is otherwise no other type information // available. func verbToPlaceholder(sub string, pos int) (name, underlying string) { r, _ := utf8.DecodeLastRuneInString(sub) name = fmt.Sprintf("Arg_%d", pos) switch r { case 's', 'q': underlying = "string" case 'd': name = "Integer" underlying = "int" case 'e', 'f', 'g': name = "Number" underlying = "float64" case 'm': name = "Message" underlying = "string" default: underlying = "interface{}" } return name, underlying } type placeholders struct { index map[string]string slice []Placeholder } func (p placeholders) addArg(arg argument, name, sub string) (id string) { id = name alt, ok := p.index[id] for i := 1; ok && alt != sub; i++ { id = fmt.Sprintf("%s_%d", name, i) alt, ok = p.index[id] } p.index[id] = sub p.slice = append(p.slice, Placeholder{ ID: id, String: sub, Type: arg.Type, UnderlyingType: arg.UnderlyingType, ArgNum: arg.ArgNum, Expr: arg.Expr, Comment: arg.Comment, }) return id } func getLastComponent(s string) string { return s[1+strings.LastIndexByte(s, '.'):] } // isMsg returns whether s should be translated. func isMsg(s string) bool { // TODO: parse as format string and omit strings that contain letters // coming from format verbs. for _, r := range s { if unicode.In(r, unicode.L) { return true } } return false }	18.85 KB	2025-05-25 12:27
📄 generate.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package pipeline import ( "fmt" "go/build" "io" "os" "path/filepath" "regexp" "sort" "strings" "text/template" "golang.org/x/text/collate" "golang.org/x/text/feature/plural" "golang.org/x/text/internal" "golang.org/x/text/internal/catmsg" "golang.org/x/text/internal/gen" "golang.org/x/text/language" "golang.org/x/tools/go/loader" ) var transRe = regexp.MustCompile(`messages\.(.)\.json`) // Generate writes a Go file that defines a Catalog with translated messages. // Translations are retrieved from s.Messages, not s.Translations, so it // is assumed Merge has been called. func (s State) Generate() error { path := s.Config.GenPackage if path == "" { path = "." } isDir := path[0] == '.' prog, err := loadPackages(&loader.Config{}, []string{path}) if err != nil { return wrap(err, "could not load package") } pkgs := prog.InitialPackages() if len(pkgs) != 1 { return errorf("more than one package selected: %v", pkgs) } pkg := pkgs[0].Pkg.Name() cw, err := s.generate() if err != nil { return err } if !isDir { gopath := filepath.SplitList(build.Default.GOPATH)[0] path = filepath.Join(gopath, "src", filepath.FromSlash(pkgs[0].Pkg.Path())) } if len(s.Config.GenFile) == 0 { cw.WriteGo(os.Stdout, pkg, "") return nil } if filepath.IsAbs(s.Config.GenFile) { path = s.Config.GenFile } else { path = filepath.Join(path, s.Config.GenFile) } cw.WriteGoFile(path, pkg) // TODO: WriteGoFile should return error. return err } // WriteGen writes a Go file with the given package name to w that defines a // Catalog with translated messages. Translations are retrieved from s.Messages, // not s.Translations, so it is assumed Merge has been called. func (s State) WriteGen(w io.Writer, pkg string) error { cw, err := s.generate() if err != nil { return err } _, err = cw.WriteGo(w, pkg, "") return err } // Generate is deprecated; use (State).Generate(). func Generate(w io.Writer, pkg string, extracted Messages, trans ...Messages) (n int, err error) { s := State{ Extracted: extracted, Translations: trans, } cw, err := s.generate() if err != nil { return 0, err } return cw.WriteGo(w, pkg, "") } func (s State) generate() (gen.CodeWriter, error) { // Build up index of translations and original messages. translations := map[language.Tag]map[string]Message{} languages := []language.Tag{} usedKeys := map[string]int{} for _, loc := range s.Messages { tag := loc.Language if _, ok := translations[tag]; !ok { translations[tag] = map[string]Message{} languages = append(languages, tag) } for _, m := range loc.Messages { if !m.Translation.IsEmpty() { for _, id := range m.ID { if _, ok := translations[tag][id]; ok { warnf("Duplicate translation in locale %q for message %q", tag, id) } translations[tag][id] = m } } } } // Verify completeness and register keys. internal.SortTags(languages) langVars := []string{} for _, tag := range languages { langVars = append(langVars, strings.Replace(tag.String(), "-", "_", -1)) dict := translations[tag] for _, msg := range s.Extracted.Messages { for _, id := range msg.ID { if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() { if _, ok := usedKeys[msg.Key]; !ok { usedKeys[msg.Key] = len(usedKeys) } break } // TODO: log missing entry. warnf("%s: Missing entry for %q.", tag, id) } } } cw := gen.NewCodeWriter() x := &struct { Fallback language.Tag Languages []string }{ Fallback: s.Extracted.Language, Languages: langVars, } if err := lookup.Execute(cw, x); err != nil { return nil, wrap(err, "error") } keyToIndex := []string{} for k := range usedKeys { keyToIndex = append(keyToIndex, k) } sort.Strings(keyToIndex) fmt.Fprint(cw, "var messageKeyToIndex = map[string]int{\n") for _, k := range keyToIndex { fmt.Fprintf(cw, "%q: %d,\n", k, usedKeys[k]) } fmt.Fprint(cw, "}\n\n") for i, tag := range languages { dict := translations[tag] a := make([]string, len(usedKeys)) for _, msg := range s.Extracted.Messages { for _, id := range msg.ID { if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() { m, err := assemble(&msg, &trans.Translation) if err != nil { return nil, wrap(err, "error") } _, leadWS, trailWS := trimWS(msg.Key) if leadWS != "" \|\| trailWS != "" { m = catmsg.Affix{ Message: m, Prefix: leadWS, Suffix: trailWS, } } // TODO: support macros. data, err := catmsg.Compile(tag, nil, m) if err != nil { return nil, wrap(err, "error") } key := usedKeys[msg.Key] if d := a[key]; d != "" && d != data { warnf("Duplicate non-consistent translation for key %q, picking the one for message %q", msg.Key, id) } a[key] = string(data) break } } } index := []uint32{0} p := 0 for _, s := range a { p += len(s) index = append(index, uint32(p)) } cw.WriteVar(langVars[i]+"Index", index) cw.WriteConst(langVars[i]+"Data", strings.Join(a, "")) } return cw, nil } func assemble(m Message, t Text) (msg catmsg.Message, err error) { keys := []string{} for k := range t.Var { keys = append(keys, k) } sort.Strings(keys) var a []catmsg.Message for _, k := range keys { t := t.Var[k] m, err := assemble(m, &t) if err != nil { return nil, err } a = append(a, &catmsg.Var{Name: k, Message: m}) } if t.Select != nil { s, err := assembleSelect(m, t.Select) if err != nil { return nil, err } a = append(a, s) } if t.Msg != "" { sub, err := m.Substitute(t.Msg) if err != nil { return nil, err } a = append(a, catmsg.String(sub)) } switch len(a) { case 0: return nil, errorf("generate: empty message") case 1: return a[0], nil default: return catmsg.FirstOf(a), nil } } func assembleSelect(m Message, s Select) (msg catmsg.Message, err error) { cases := []string{} for c := range s.Cases { cases = append(cases, c) } sortCases(cases) caseMsg := []interface{}{} for _, c := range cases { cm := s.Cases[c] m, err := assemble(m, &cm) if err != nil { return nil, err } caseMsg = append(caseMsg, c, m) } ph := m.Placeholder(s.Arg) switch s.Feature { case "plural": // TODO: only printf-style selects are supported as of yet. return plural.Selectf(ph.ArgNum, ph.String, caseMsg...), nil } return nil, errorf("unknown feature type %q", s.Feature) } func sortCases(cases []string) { // TODO: implement full interface. sort.Slice(cases, func(i, j int) bool { switch { case cases[i] != "other" && cases[j] == "other": return true case cases[i] == "other" && cases[j] != "other": return false } // the following code relies on '<' < '=' < any letter. return cmpNumeric(cases[i], cases[j]) == -1 }) } var cmpNumeric = collate.New(language.Und, collate.Numeric).CompareString var lookup = template.Must(template.New("gen").Parse(` import ( "golang.org/x/text/language" "golang.org/x/text/message" "golang.org/x/text/message/catalog" ) type dictionary struct { index []uint32 data string } func (d *dictionary) Lookup(key string) (data string, ok bool) { p, ok := messageKeyToIndex[key] if !ok { return "", false } start, end := d.index[p], d.index[p+1] if start == end { return "", false } return d.data[start:end], true } func init() { dict := map[string]catalog.Dictionary{ {{range .Languages}}"{{.}}": &dictionary{index: {{.}}Index, data: {{.}}Data }, {{end}} } fallback := language.MustParse("{{.Fallback}}") cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback)) if err != nil { panic(err) } message.DefaultCatalog = cat } `))	7.72 KB	2025-05-25 12:27
📄 go19_test.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. //go:build go1.9 package pipeline import "testing" func init() { setHelper = (*testing.T).Helper }	263 B	2025-05-25 12:27
📄 message.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package pipeline import ( "encoding/json" "errors" "strings" "golang.org/x/text/language" ) // TODO: these definitions should be moved to a package so that the can be used // by other tools. // The file contains the structures used to define translations of a certain // messages. // // A translation may have multiple translations strings, or messages, depending // on the feature values of the various arguments. For instance, consider // a hypothetical translation from English to English, where the source defines // the format string "%d file(s) remaining". // See the examples directory for examples of extracted messages. // Messages is used to store translations for a single language. type Messages struct { Language language.Tag `json:"language"` Messages []Message `json:"messages"` Macros map[string]Text `json:"macros,omitempty"` } // A Message describes a message to be translated. type Message struct { // ID contains a list of identifiers for the message. ID IDList `json:"id"` // Key is the string that is used to look up the message at runtime. Key string `json:"key,omitempty"` Meaning string `json:"meaning,omitempty"` Message Text `json:"message"` Translation Text `json:"translation"` Comment string `json:"comment,omitempty"` TranslatorComment string `json:"translatorComment,omitempty"` Placeholders []Placeholder `json:"placeholders,omitempty"` // Fuzzy indicates that the provide translation needs review by a // translator, for instance because it was derived from automated // translation. Fuzzy bool `json:"fuzzy,omitempty"` // TODO: default placeholder syntax is {foo}. Allow alternative escaping // like `foo`. // Extraction information. Position string `json:"position,omitempty"` // filePosition:line } // Placeholder reports the placeholder for the given ID if it is defined or nil // otherwise. func (m Message) Placeholder(id string) Placeholder { for _, p := range m.Placeholders { if p.ID == id { return &p } } return nil } // Substitute replaces placeholders in msg with their original value. func (m Message) Substitute(msg string) (sub string, err error) { last := 0 for i := 0; i < len(msg); { pLeft := strings.IndexByte(msg[i:], '{') if pLeft == -1 { break } pLeft += i pRight := strings.IndexByte(msg[pLeft:], '}') if pRight == -1 { return "", errorf("unmatched '}'") } pRight += pLeft id := strings.TrimSpace(msg[pLeft+1 : pRight]) i = pRight + 1 if id != "" && id[0] == '$' { continue } sub += msg[last:pLeft] last = i ph := m.Placeholder(id) if ph == nil { return "", errorf("unknown placeholder %q in message %q", id, msg) } sub += ph.String } sub += msg[last:] return sub, err } var errIncompatibleMessage = errors.New("messages incompatible") func checkEquivalence(a, b Message) error { for _, v := range a.ID { for _, w := range b.ID { if v == w { return nil } } } // TODO: canonicalize placeholders and check for type equivalence. return errIncompatibleMessage } // A Placeholder is a part of the message that should not be changed by a // translator. It can be used to hide or prettify format strings (e.g. %d or // {{.Count}}), hide HTML, or mark common names that should not be translated. type Placeholder struct { // ID is the placeholder identifier without the curly braces. ID string `json:"id"` // String is the string with which to replace the placeholder. This may be a // formatting string (for instance "%d" or "{{.Count}}") or a literal string // (<div>). String string `json:"string"` Type string `json:"type"` UnderlyingType string `json:"underlyingType"` // ArgNum and Expr are set if the placeholder is a substitution of an // argument. ArgNum int `json:"argNum,omitempty"` Expr string `json:"expr,omitempty"` Comment string `json:"comment,omitempty"` Example string `json:"example,omitempty"` // Features contains the features that are available for the implementation // of this argument. Features []Feature `json:"features,omitempty"` } // An argument contains information about the arguments passed to a message. type argument struct { // ArgNum corresponds to the number that should be used for explicit argument indexes (e.g. // "%[1]d"). ArgNum int `json:"argNum,omitempty"` used bool // Used by Placeholder Type string `json:"type"` UnderlyingType string `json:"underlyingType"` Expr string `json:"expr"` Value string `json:"value,omitempty"` Comment string `json:"comment,omitempty"` Position string `json:"position,omitempty"` } // Feature holds information about a feature that can be implemented by // an Argument. type Feature struct { Type string `json:"type"` // Right now this is only gender and plural. // TODO: possible values and examples for the language under consideration. } // Text defines a message to be displayed. type Text struct { // Msg and Select contains the message to be displayed. Msg may be used as // a fallback value if none of the select cases match. Msg string `json:"msg,omitempty"` Select Select `json:"select,omitempty"` // Var defines a map of variables that may be substituted in the selected // message. Var map[string]Text `json:"var,omitempty"` // Example contains an example message formatted with default values. Example string `json:"example,omitempty"` } // IsEmpty reports whether this Text can generate anything. func (t Text) IsEmpty() bool { return t.Msg == "" && t.Select == nil && t.Var == nil } // rawText erases the UnmarshalJSON method. type rawText Text // UnmarshalJSON implements json.Unmarshaler. func (t Text) UnmarshalJSON(b []byte) error { if b[0] == '"' { return json.Unmarshal(b, &t.Msg) } return json.Unmarshal(b, (rawText)(t)) } // MarshalJSON implements json.Marshaler. func (t Text) MarshalJSON() ([]byte, error) { if t.Select == nil && t.Var == nil && t.Example == "" { return json.Marshal(t.Msg) } return json.Marshal((rawText)(t)) } // IDList is a set identifiers that each may refer to possibly different // versions of the same message. When looking up a messages, the first // identifier in the list takes precedence. type IDList []string // UnmarshalJSON implements json.Unmarshaler. func (id IDList) UnmarshalJSON(b []byte) error { if b[0] == '"' { id = []string{""} return json.Unmarshal(b, &((id)[0])) } return json.Unmarshal(b, ([]string)(id)) } // MarshalJSON implements json.Marshaler. func (id IDList) MarshalJSON() ([]byte, error) { if len(id) == 1 { return json.Marshal((id)[0]) } return json.Marshal(([]string)(id)) } // Select selects a Text based on the feature value associated with a feature of // a certain argument. type Select struct { Feature string `json:"feature"` // Name of Feature type (e.g plural) Arg string `json:"arg"` // The placeholder ID Cases map[string]Text `json:"cases"` } // TODO: order matters, but can we derive the ordering from the case keys? // type Case struct { // Key string `json:"key"` // Value Text `json:"value"` // }	7.18 KB	2025-05-25 12:27
📄 pipeline.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package pipeline provides tools for creating translation pipelines. // // NOTE: UNDER DEVELOPMENT. API MAY CHANGE. package pipeline import ( "bytes" "encoding/json" "fmt" "go/build" "go/parser" "io/ioutil" "log" "os" "path/filepath" "regexp" "strings" "text/template" "unicode" "golang.org/x/text/internal" "golang.org/x/text/language" "golang.org/x/text/runes" "golang.org/x/tools/go/loader" ) const ( extractFile = "extracted.gotext.json" outFile = "out.gotext.json" gotextSuffix = "gotext.json" ) // Config contains configuration for the translation pipeline. type Config struct { // Supported indicates the languages for which data should be generated. // The default is to support all locales for which there are matching // translation files. Supported []language.Tag // --- Extraction SourceLanguage language.Tag Packages []string // --- File structure // Dir is the root dir for all operations. Dir string // TranslationsPattern is a regular expression to match incoming translation // files. These files may appear in any directory rooted at Dir. // language for the translation files is determined as follows: // 1. From the Language field in the file. // 2. If not present, from a valid language tag in the filename, separated // by dots (e.g. "en-US.json" or "incoming.pt_PT.xmb"). // 3. If not present, from the closest subdirectory in which the file // is contained that parses as a valid language tag. TranslationsPattern string // OutPattern defines the location for translation files for a certain // language. The default is "{{.Dir}}/{{.Language}}/out.{{.Ext}}" OutPattern string // Format defines the file format for generated translation files. // The default is XMB. Alternatives are GetText, XLIFF, L20n, GoText. Format string Ext string // TODO: // Actions are additional actions to be performed after the initial extract // and merge. // Actions []struct { // Name string // Options map[string]string // } // --- Generation // GenFile may be in a different package. It is not defined, it will // be written to stdout. GenFile string // GenPackage is the package or relative path into which to generate the // file. If not specified it is relative to the current directory. GenPackage string // DeclareVar defines a variable to which to assign the generated Catalog. DeclareVar string // SetDefault determines whether to assign the generated Catalog to // message.DefaultCatalog. The default for this is true if DeclareVar is // not defined, false otherwise. SetDefault bool // TODO: // - Printf-style configuration // - Template-style configuration // - Extraction options // - Rewrite options // - Generation options } // Operations: // - extract: get the strings // - disambiguate: find messages with the same key, but possible different meaning. // - create out: create a list of messages that need translations // - load trans: load the list of current translations // - merge: assign list of translations as done // - (action)expand: analyze features and create example sentences for each version. // - (action)googletrans: pre-populate messages with automatic translations. // - (action)export: send out messages somewhere non-standard // - (action)import: load messages from somewhere non-standard // - vet program: don't pass "foo" + var + "bar" strings. Not using funcs for translated strings. // - vet trans: coverage: all translations/ all features. // - generate: generate Go code // State holds all accumulated information on translations during processing. type State struct { Config Config Package string program loader.Program Extracted Messages `json:"messages"` // Messages includes all messages for which there need to be translations. // Duplicates may be eliminated. Generation will be done from these messages // (usually after merging). Messages []Messages // Translations are incoming translations for the application messages. Translations []Messages } func (s State) dir() string { if d := s.Config.Dir; d != "" { return d } return "./locales" } func outPattern(s State) (string, error) { c := s.Config pat := c.OutPattern if pat == "" { pat = "{{.Dir}}/{{.Language}}/out.{{.Ext}}" } ext := c.Ext if ext == "" { ext = c.Format } if ext == "" { ext = gotextSuffix } t, err := template.New("").Parse(pat) if err != nil { return "", wrap(err, "error parsing template") } buf := bytes.Buffer{} err = t.Execute(&buf, map[string]string{ "Dir": s.dir(), "Language": "%s", "Ext": ext, }) return filepath.FromSlash(buf.String()), wrap(err, "incorrect OutPattern") } var transRE = regexp.MustCompile(`.\.` + gotextSuffix) // Import loads existing translation files. func (s State) Import() error { outPattern, err := outPattern(s) if err != nil { return err } re := transRE if pat := s.Config.TranslationsPattern; pat != "" { if re, err = regexp.Compile(pat); err != nil { return wrapf(err, "error parsing regexp %q", s.Config.TranslationsPattern) } } x := importer{s, outPattern, re} return x.walkImport(s.dir(), s.Config.SourceLanguage) } type importer struct { state State outPattern string transFile regexp.Regexp } func (i importer) walkImport(path string, tag language.Tag) error { files, err := ioutil.ReadDir(path) if err != nil { return nil } for _, f := range files { name := f.Name() tag := tag if f.IsDir() { if t, err := language.Parse(name); err == nil { tag = t } // We ignore errors if err := i.walkImport(filepath.Join(path, name), tag); err != nil { return err } continue } for _, l := range strings.Split(name, ".") { if t, err := language.Parse(l); err == nil { tag = t } } file := filepath.Join(path, name) // TODO: Should we skip files that match output files? if fmt.Sprintf(i.outPattern, tag) == file { continue } // TODO: handle different file formats. if !i.transFile.MatchString(name) { continue } b, err := ioutil.ReadFile(file) if err != nil { return wrap(err, "read file failed") } var translations Messages if err := json.Unmarshal(b, &translations); err != nil { return wrap(err, "parsing translation file failed") } i.state.Translations = append(i.state.Translations, translations) } return nil } // Merge merges the extracted messages with the existing translations. func (s State) Merge() error { if s.Messages != nil { panic("already merged") } // Create an index for each unique message. // Duplicates are okay as long as the substitution arguments are okay as // well. // Top-level messages are okay to appear in multiple substitution points. // Collect key equivalence. msgs := []Message{} keyToIDs := map[string]Message{} for _, m := range s.Extracted.Messages { m := m if prev, ok := keyToIDs[m.Key]; ok { if err := checkEquivalence(&m, prev); err != nil { warnf("Key %q matches conflicting messages: %v and %v", m.Key, prev.ID, m.ID) // TODO: track enough information so that the rewriter can // suggest/disambiguate messages. } // TODO: add position to message. continue } i := len(msgs) msgs = append(msgs, &m) keyToIDs[m.Key] = msgs[i] } // Messages with different keys may still refer to the same translated // message (e.g. different whitespace). Filter these. idMap := map[string]bool{} filtered := []Message{} for _, m := range msgs { found := false for _, id := range m.ID { found = found \|\| idMap[id] } if !found { filtered = append(filtered, m) } for _, id := range m.ID { idMap[id] = true } } // Build index of translations. translations := map[language.Tag]map[string]Message{} languages := append([]language.Tag{}, s.Config.Supported...) for _, t := range s.Translations { tag := t.Language if _, ok := translations[tag]; !ok { translations[tag] = map[string]Message{} languages = append(languages, tag) } for _, m := range t.Messages { if !m.Translation.IsEmpty() { for _, id := range m.ID { if _, ok := translations[tag][id]; ok { warnf("Duplicate translation in locale %q for message %q", tag, id) } translations[tag][id] = m } } } } languages = internal.UniqueTags(languages) for _, tag := range languages { ms := Messages{Language: tag} for _, orig := range filtered { m := orig m.Key = "" m.Position = "" for _, id := range m.ID { if t, ok := translations[tag][id]; ok { m.Translation = t.Translation if t.TranslatorComment != "" { m.TranslatorComment = t.TranslatorComment m.Fuzzy = t.Fuzzy } break } } if tag == s.Config.SourceLanguage && m.Translation.IsEmpty() { m.Translation = m.Message if m.TranslatorComment == "" { m.TranslatorComment = "Copied from source." m.Fuzzy = true } } // TODO: if translation is empty: pre-expand based on available // linguistic features. This may also be done as a plugin. ms.Messages = append(ms.Messages, m) } s.Messages = append(s.Messages, ms) } return nil } // Export writes out the messages to translation out files. func (s State) Export() error { path, err := outPattern(s) if err != nil { return wrap(err, "export failed") } for _, out := range s.Messages { // TODO: inject translations from existing files to avoid retranslation. data, err := json.MarshalIndent(out, "", " ") if err != nil { return wrap(err, "JSON marshal failed") } file := fmt.Sprintf(path, out.Language) if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { return wrap(err, "dir create failed") } if err := ioutil.WriteFile(file, data, 0644); err != nil { return wrap(err, "write failed") } } return nil } var ( ws = runes.In(unicode.White_Space).Contains notWS = runes.NotIn(unicode.White_Space).Contains ) func trimWS(s string) (trimmed, leadWS, trailWS string) { trimmed = strings.TrimRightFunc(s, ws) trailWS = s[len(trimmed):] if i := strings.IndexFunc(trimmed, notWS); i > 0 { leadWS = trimmed[:i] trimmed = trimmed[i:] } return trimmed, leadWS, trailWS } // NOTE: The command line tool already prefixes with "gotext:". var ( wrap = func(err error, msg string) error { if err == nil { return nil } return fmt.Errorf("%s: %v", msg, err) } wrapf = func(err error, msg string, args ...interface{}) error { if err == nil { return nil } return wrap(err, fmt.Sprintf(msg, args...)) } errorf = fmt.Errorf ) func warnf(format string, args ...interface{}) { // TODO: don't log. log.Printf(format, args...) } func loadPackages(conf loader.Config, args []string) (loader.Program, error) { if len(args) == 0 { args = []string{"."} } conf.Build = &build.Default conf.ParserMode = parser.ParseComments // Use the initial packages from the command line. args, err := conf.FromArgs(args, false) if err != nil { return nil, wrap(err, "loading packages failed") } // Load, parse and type-check the whole program. return conf.Load() }	11.06 KB	2025-05-25 12:27
📄 pipeline_test.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package pipeline import ( "bufio" "bytes" "encoding/json" "flag" "fmt" "go/build" "os" "os/exec" "path" "path/filepath" "reflect" "runtime" "strings" "testing" "golang.org/x/text/language" ) var genFiles = flag.Bool("gen", false, "generate output files instead of comparing") // setHelper is testing.T.Helper on Go 1.9+, overridden by go19_test.go. var setHelper = func(t testing.T) {} func TestFullCycle(t testing.T) { if runtime.GOOS == "android" { t.Skip("cannot load outside packages on android") } if b := os.Getenv("GO_BUILDER_NAME"); b == "plan9-arm" { t.Skipf("skipping: test frequently times out on %s", b) } if _, err := exec.LookPath("go"); err != nil { t.Skipf("skipping because 'go' command is unavailable: %v", err) } GOPATH, err := os.MkdirTemp("", "pipeline_test") if err != nil { t.Fatal(err) } defer os.RemoveAll(GOPATH) testdata := filepath.Join(GOPATH, "src", "testdata") // Copy the testdata contents into a new module. copyTestdata(t, testdata) initTestdataModule(t, testdata) // Several places hard-code the use of build.Default. // Adjust it to match the test's temporary GOPATH. defer func(prev string) { build.Default.GOPATH = prev }(build.Default.GOPATH) build.Default.GOPATH = GOPATH + string(filepath.ListSeparator) + build.Default.GOPATH if wd := reflect.ValueOf(&build.Default).Elem().FieldByName("WorkingDir"); wd.IsValid() { defer func(prev string) { wd.SetString(prev) }(wd.String()) wd.SetString(testdata) } // To work around https://golang.org/issue/34860, execute the commands // that (transitively) use go/build in the working directory of the // corresponding module. wd, _ := os.Getwd() defer os.Chdir(wd) dirs, err := os.ReadDir(testdata) if err != nil { t.Fatal(err) } for _, f := range dirs { if !f.IsDir() { continue } t.Run(f.Name(), func(t testing.T) { chk := func(t testing.T, err error) { setHelper(t) if err != nil { t.Fatal(err) } } dir := filepath.Join(testdata, f.Name()) pkgPath := "testdata/" + f.Name() config := Config{ SourceLanguage: language.AmericanEnglish, Packages: []string{pkgPath}, Dir: filepath.Join(dir, "locales"), GenFile: "catalog_gen.go", GenPackage: pkgPath, } os.Chdir(dir) // TODO: load config if available. s, err := Extract(&config) chk(t, err) chk(t, s.Import()) chk(t, s.Merge()) // TODO: // for range s.Config.Actions { // // TODO: do the actions. // } chk(t, s.Export()) chk(t, s.Generate()) os.Chdir(wd) writeJSON(t, filepath.Join(dir, "extracted.gotext.json"), s.Extracted) checkOutput(t, dir, f.Name()) }) } } func copyTestdata(t testing.T, dst string) { err := filepath.Walk("testdata", func(p string, f os.FileInfo, err error) error { if p == "testdata" \|\| strings.HasSuffix(p, ".want") { return nil } rel := strings.TrimPrefix(p, "testdata"+string(filepath.Separator)) if f.IsDir() { return os.MkdirAll(filepath.Join(dst, rel), 0755) } data, err := os.ReadFile(p) if err != nil { return err } return os.WriteFile(filepath.Join(dst, rel), data, 0644) }) if err != nil { t.Fatal(err) } } func initTestdataModule(t testing.T, dst string) { xTextDir, err := filepath.Abs("../..") if err != nil { t.Fatal(err) } goMod := fmt.Sprintf(`module testdata replace golang.org/x/text => %s `, xTextDir) if err := os.WriteFile(filepath.Join(dst, "go.mod"), []byte(goMod), 0644); err != nil { t.Fatal(err) } // Copy in the checksums from the parent module so that we won't // need to re-fetch them from the checksum database. data, err := os.ReadFile(filepath.Join(xTextDir, "go.sum")) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dst, "go.sum"), data, 0644); err != nil { t.Fatal(err) } // We've added a replacement for the parent version of x/text, // but now we need to populate the correct version. // (We can't just replace the zero-version because x/text // may indirectly depend on some nonzero version of itself.) // // We use 'go get' instead of 'go mod tidy' to avoid the old-release // compatibility check when graph pruning is enabled, and to avoid doing // more work than necessary for test dependencies of imported packages // (we're not going to run those tests here anyway). // // We 'go get' the packages in the testdata module — not specific dependencies // of those packages — so that they will resolve to whatever version is // already required in the (replaced) x/text go.mod file. getCmd := exec.Command("go", "get", "-d", "./...") getCmd.Dir = dst getCmd.Env = append(os.Environ(), "PWD="+dst, "GOPROXY=off", "GOCACHE=off") if out, err := getCmd.CombinedOutput(); err != nil { t.Logf("%s", out) t.Fatal(err) } } func checkOutput(t testing.T, gen string, testdataDir string) { err := filepath.Walk(gen, func(gotFile string, f os.FileInfo, err error) error { if f.IsDir() { return nil } rel := strings.TrimPrefix(gotFile, gen+string(filepath.Separator)) wantFile := filepath.Join("testdata", testdataDir, rel+".want") if _, err := os.Stat(wantFile); os.IsNotExist(err) { return nil } got, err := os.ReadFile(gotFile) if err != nil { t.Errorf("failed to read %q", gotFile) return nil } if genFiles { if err := os.WriteFile(wantFile, got, 0644); err != nil { t.Fatal(err) } } want, err := os.ReadFile(wantFile) if err != nil { t.Errorf("failed to read %q", wantFile) } else { scanGot := bufio.NewScanner(bytes.NewReader(got)) scanWant := bufio.NewScanner(bytes.NewReader(want)) line := 0 clean := func(s string) string { if i := strings.LastIndex(s, "//"); i != -1 { s = s[:i] } return path.Clean(filepath.ToSlash(s)) } for scanGot.Scan() && scanWant.Scan() { got := clean(scanGot.Text()) want := clean(scanWant.Text()) if got != want { t.Errorf("file %q differs from .want file at line %d:\n\t%s\n\t%s", gotFile, line, got, want) break } line++ } if scanGot.Scan() \|\| scanWant.Scan() { t.Errorf("file %q differs from .want file at line %d.", gotFile, line) } } return nil }) if err != nil { t.Fatal(err) } } func writeJSON(t *testing.T, path string, x interface{}) { data, err := json.MarshalIndent(x, "", " ") if err != nil { t.Fatal(err) } if err := os.WriteFile(path, data, 0644); err != nil { t.Fatal(err) } }	6.5 KB	2025-05-25 12:27
📄 rewrite.go // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package pipeline import ( "bytes" "fmt" "go/ast" "go/constant" "go/format" "go/token" "io" "os" "strings" "golang.org/x/tools/go/loader" ) const printerType = "golang.org/x/text/message.Printer" // Rewrite rewrites the Go files in a single package to use the localization // machinery and rewrites strings to adopt best practices when possible. // If w is not nil the generated files are written to it, each files with a // "--- <filename>" header. Otherwise the files are overwritten. func Rewrite(w io.Writer, args ...string) error { conf := &loader.Config{ AllowErrors: true, // Allow unused instances of message.Printer. } prog, err := loadPackages(conf, args) if err != nil { return wrap(err, "") } for _, info := range prog.InitialPackages() { for _, f := range info.Files { // Associate comments with nodes. // Pick up initialized Printers at the package level. r := rewriter{info: info, conf: conf} for _, n := range info.InitOrder { if t := r.info.Types[n.Rhs].Type.String(); strings.HasSuffix(t, printerType) { r.printerVar = n.Lhs[0].Name() } } ast.Walk(&r, f) w := w if w == nil { var err error if w, err = os.Create(conf.Fset.File(f.Pos()).Name()); err != nil { return wrap(err, "open failed") } } else { fmt.Fprintln(w, "---", conf.Fset.File(f.Pos()).Name()) } if err := format.Node(w, conf.Fset, f); err != nil { return wrap(err, "go format failed") } } } return nil } type rewriter struct { info loader.PackageInfo conf loader.Config printerVar string } // print returns Go syntax for the specified node. func (r rewriter) print(n ast.Node) string { var buf bytes.Buffer format.Node(&buf, r.conf.Fset, n) return buf.String() } func (r rewriter) Visit(n ast.Node) ast.Visitor { // Save the state by scope. if _, ok := n.(ast.BlockStmt); ok { r := r return &r } // Find Printers created by assignment. stmt, ok := n.(ast.AssignStmt) if ok { for _, v := range stmt.Lhs { if r.printerVar == r.print(v) { r.printerVar = "" } } for i, v := range stmt.Rhs { if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) { r.printerVar = r.print(stmt.Lhs[i]) return r } } } // Find Printers created by variable declaration. spec, ok := n.(ast.ValueSpec) if ok { for _, v := range spec.Names { if r.printerVar == r.print(v) { r.printerVar = "" } } for i, v := range spec.Values { if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) { r.printerVar = r.print(spec.Names[i]) return r } } } if r.printerVar == "" { return r } call, ok := n.(ast.CallExpr) if !ok { return r } // TODO: Handle literal values? sel, ok := call.Fun.(ast.SelectorExpr) if !ok { return r } meth := r.info.Selections[sel] source := r.print(sel.X) fun := r.print(sel.Sel) if meth != nil { source = meth.Recv().String() fun = meth.Obj().Name() } // TODO: remove cheap hack and check if the type either // implements some interface or is specifically of type // "golang.org/x/text/message".Printer. m, ok := rewriteFuncs[source] if !ok { return r } rewriteType, ok := m[fun] if !ok { return r } ident := ast.NewIdent(r.printerVar) ident.NamePos = sel.X.Pos() sel.X = ident if rewriteType.method != "" { sel.Sel.Name = rewriteType.method } // Analyze arguments. argn := rewriteType.arg if rewriteType.format \|\| argn >= len(call.Args) { return r } hasConst := false for _, a := range call.Args[argn:] { if v := r.info.Types[a].Value; v != nil && v.Kind() == constant.String { hasConst = true break } } if !hasConst { return r } sel.Sel.Name = rewriteType.methodf // We are done if there is only a single string that does not need to be // escaped. if len(call.Args) == 1 { s, ok := constStr(r.info, call.Args[0]) if ok && !strings.Contains(s, "%") && !rewriteType.newLine { return r } } // Rewrite arguments as format string. expr := &ast.BasicLit{ ValuePos: call.Lparen, Kind: token.STRING, } newArgs := append(call.Args[:argn:argn], expr) newStr := []string{} for i, a := range call.Args[argn:] { if s, ok := constStr(r.info, a); ok { newStr = append(newStr, strings.Replace(s, "%", "%%", -1)) } else { newStr = append(newStr, "%v") newArgs = append(newArgs, call.Args[argn+i]) } } s := strings.Join(newStr, rewriteType.sep) if rewriteType.newLine { s += "\n" } expr.Value = fmt.Sprintf("%q", s) call.Args = newArgs // TODO: consider creating an expression instead of a constant string and // then wrapping it in an escape function or so: // call.Args[argn+i] = &ast.CallExpr{ // Fun: &ast.SelectorExpr{ // X: ast.NewIdent("message"), // Sel: ast.NewIdent("Lookup"), // }, // Args: []ast.Expr{a}, // } // } return r } type rewriteType struct { // method is the name of the equivalent method on a printer, or "" if it is // the same. method string // methodf is the method to use if the arguments can be rewritten as a // arguments to a printf-style call. methodf string // format is true if the method takes a formatting string followed by // substitution arguments. format bool // arg indicates the position of the argument to extract. If all is // positive, all arguments from this argument onwards needs to be extracted. arg int sep string newLine bool } // rewriteFuncs list functions that can be directly mapped to the printer // functions of the message package. var rewriteFuncs = map[string]map[string]rewriteType{ // TODO: Printer -> golang.org/x/text/message.Printer "fmt": { "Print": rewriteType{methodf: "Printf"}, "Sprint": rewriteType{methodf: "Sprintf"}, "Fprint": rewriteType{methodf: "Fprintf"}, "Println": rewriteType{methodf: "Printf", sep: " ", newLine: true}, "Sprintln": rewriteType{methodf: "Sprintf", sep: " ", newLine: true}, "Fprintln": rewriteType{methodf: "Fprintf", sep: " ", newLine: true}, "Printf": rewriteType{method: "Printf", format: true}, "Sprintf": rewriteType{method: "Sprintf", format: true}, "Fprintf": rewriteType{method: "Fprintf", format: true}, }, } func constStr(info loader.PackageInfo, e ast.Expr) (s string, ok bool) { v := info.Types[e].Value if v == nil \|\| v.Kind() != constant.String { return "", false } return constant.StringVal(v), true }	6.44 KB	2025-05-25 12:27
📁 testdata	-	2025-05-25 12:27

Terminal