Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 cache.go // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Parallel cache. // This file is copied from cmd/go/internal/par. package sumdb import ( "sync" "sync/atomic" ) // parCache runs an action once per key and caches the result. type parCache struct { m sync.Map } type cacheEntry struct { done uint32 mu sync.Mutex result interface{} } // Do calls the function f if and only if Do is being called for the first time with this key. // No call to Do with a given key returns until the one call to f returns. // Do returns the value returned by the one call to f. func (c parCache) Do(key interface{}, f func() interface{}) interface{} { entryIface, ok := c.m.Load(key) if !ok { entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry)) } e := entryIface.(cacheEntry) if atomic.LoadUint32(&e.done) == 0 { e.mu.Lock() if atomic.LoadUint32(&e.done) == 0 { e.result = f() atomic.StoreUint32(&e.done, 1) } e.mu.Unlock() } return e.result } // Get returns the cached result associated with key. // It returns nil if there is no such result. // If the result for key is being computed, Get does not wait for the computation to finish. func (c parCache) Get(key interface{}) interface{} { entryIface, ok := c.m.Load(key) if !ok { return nil } e := entryIface.(cacheEntry) if atomic.LoadUint32(&e.done) == 0 { return nil } return e.result }	1.46 KB	2025-05-25 12:27
📄 client.go // Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sumdb import ( "bytes" "errors" "fmt" "strings" "sync" "sync/atomic" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/note" "golang.org/x/mod/sumdb/tlog" ) // A ClientOps provides the external operations // (file caching, HTTP fetches, and so on) needed by the [Client]. // The methods must be safe for concurrent use by multiple goroutines. type ClientOps interface { // ReadRemote reads and returns the content served at the given path // on the remote database server. The path begins with "/lookup" or "/tile/", // and there is no need to parse the path in any way. // It is the implementation's responsibility to turn that path into a full URL // and make the HTTP request. ReadRemote should return an error for // any non-200 HTTP response status. ReadRemote(path string) ([]byte, error) // ReadConfig reads and returns the content of the named configuration file. // There are only a fixed set of configuration files. // // "key" returns a file containing the verifier key for the server. // // serverName + "/latest" returns a file containing the latest known // signed tree from the server. // To signal that the client wishes to start with an "empty" signed tree, // ReadConfig can return a successful empty result (0 bytes of data). ReadConfig(file string) ([]byte, error) // WriteConfig updates the content of the named configuration file, // changing it from the old []byte to the new []byte. // If the old []byte does not match the stored configuration, // WriteConfig must return ErrWriteConflict. // Otherwise, WriteConfig should atomically replace old with new. // The "key" configuration file is never written using WriteConfig. WriteConfig(file string, old, new []byte) error // ReadCache reads and returns the content of the named cache file. // Any returned error will be treated as equivalent to the file not existing. // There can be arbitrarily many cache files, such as: // serverName/lookup/pkg@version // serverName/tile/8/1/x123/456 ReadCache(file string) ([]byte, error) // WriteCache writes the named cache file. WriteCache(file string, data []byte) // Log prints the given log message (such as with log.Print) Log(msg string) // SecurityError prints the given security error log message. // The Client returns ErrSecurity from any operation that invokes SecurityError, // but the return value is mainly for testing. In a real program, // SecurityError should typically print the message and call log.Fatal or os.Exit. SecurityError(msg string) } // ErrWriteConflict signals a write conflict during Client.WriteConfig. var ErrWriteConflict = errors.New("write conflict") // ErrSecurity is returned by [Client] operations that invoke Client.SecurityError. var ErrSecurity = errors.New("security error: misbehaving server") // A Client is a client connection to a checksum database. // All the methods are safe for simultaneous use by multiple goroutines. type Client struct { ops ClientOps // access to operations in the external world didLookup uint32 // one-time initialized data initOnce sync.Once initErr error // init error, if any name string // name of accepted verifier verifiers note.Verifiers // accepted verifiers (just one, but Verifiers for note.Open) tileReader tileReader tileHeight int nosumdb string record parCache // cache of record lookup, keyed by path@vers tileCache parCache // cache of c.readTile, keyed by tile latestMu sync.Mutex latest tlog.Tree // latest known tree head latestMsg []byte // encoded signed note for latest tileSavedMu sync.Mutex tileSaved map[tlog.Tile]bool // which tiles have been saved using c.ops.WriteCache already } // NewClient returns a new [Client] using the given [ClientOps]. func NewClient(ops ClientOps) Client { return &Client{ ops: ops, } } // init initializes the client (if not already initialized) // and returns any initialization error. func (c Client) init() error { c.initOnce.Do(c.initWork) return c.initErr } // initWork does the actual initialization work. func (c Client) initWork() { defer func() { if c.initErr != nil { c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr) } }() c.tileReader.c = c if c.tileHeight == 0 { c.tileHeight = 8 } c.tileSaved = make(map[tlog.Tile]bool) vkey, err := c.ops.ReadConfig("key") if err != nil { c.initErr = err return } verifier, err := note.NewVerifier(strings.TrimSpace(string(vkey))) if err != nil { c.initErr = err return } c.verifiers = note.VerifierList(verifier) c.name = verifier.Name() if c.latest.N == 0 { c.latest.Hash, err = tlog.TreeHash(0, nil) if err != nil { c.initErr = err return } } data, err := c.ops.ReadConfig(c.name + "/latest") if err != nil { c.initErr = err return } if err := c.mergeLatest(data); err != nil { c.initErr = err return } } // SetTileHeight sets the tile height for the Client. // Any call to SetTileHeight must happen before the first call to [Client.Lookup]. // If SetTileHeight is not called, the Client defaults to tile height 8. // SetTileHeight can be called at most once, // and if so it must be called before the first call to Lookup. func (c Client) SetTileHeight(height int) { if atomic.LoadUint32(&c.didLookup) != 0 { panic("SetTileHeight used after Lookup") } if height <= 0 { panic("invalid call to SetTileHeight") } if c.tileHeight != 0 { panic("multiple calls to SetTileHeight") } c.tileHeight = height } // SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Client. // For any module path matching one of the patterns, // [Client.Lookup] will return ErrGONOSUMDB. // SetGONOSUMDB can be called at most once, // and if so it must be called before the first call to Lookup. func (c Client) SetGONOSUMDB(list string) { if atomic.LoadUint32(&c.didLookup) != 0 { panic("SetGONOSUMDB used after Lookup") } if c.nosumdb != "" { panic("multiple calls to SetGONOSUMDB") } c.nosumdb = list } // ErrGONOSUMDB is returned by [Client.Lookup] for paths that match // a pattern listed in the GONOSUMDB list (set by [Client.SetGONOSUMDB], // usually from the environment variable). var ErrGONOSUMDB = errors.New("skipped (listed in GONOSUMDB)") func (c Client) skip(target string) bool { return module.MatchPrefixPatterns(c.nosumdb, target) } // Lookup returns the go.sum lines for the given module path and version. // The version may end in a /go.mod suffix, in which case Lookup returns // the go.sum lines for the module's go.mod-only hash. func (c Client) Lookup(path, vers string) (lines []string, err error) { atomic.StoreUint32(&c.didLookup, 1) if c.skip(path) { return nil, ErrGONOSUMDB } defer func() { if err != nil { err = fmt.Errorf("%s@%s: %v", path, vers, err) } }() if err := c.init(); err != nil { return nil, err } // Prepare encoded cache filename / URL. epath, err := module.EscapePath(path) if err != nil { return nil, err } evers, err := module.EscapeVersion(strings.TrimSuffix(vers, "/go.mod")) if err != nil { return nil, err } remotePath := "/lookup/" + epath + "@" + evers file := c.name + remotePath // Fetch the data. // The lookupCache avoids redundant ReadCache/GetURL operations // (especially since go.sum lines tend to come in pairs for a given // path and version) and also avoids having multiple of the same // request in flight at once. type cached struct { data []byte err error } result := c.record.Do(file, func() interface{} { // Try the on-disk cache, or else get from web. writeCache := false data, err := c.ops.ReadCache(file) if err != nil { data, err = c.ops.ReadRemote(remotePath) if err != nil { return cached{nil, err} } writeCache = true } // Validate the record before using it for anything. id, text, treeMsg, err := tlog.ParseRecord(data) if err != nil { return cached{nil, err} } if err := c.mergeLatest(treeMsg); err != nil { return cached{nil, err} } if err := c.checkRecord(id, text); err != nil { return cached{nil, err} } // Now that we've validated the record, // save it to the on-disk cache (unless that's where it came from). if writeCache { c.ops.WriteCache(file, data) } return cached{data, nil} }).(cached) if result.err != nil { return nil, result.err } // Extract the lines for the specific version we want // (with or without /go.mod). prefix := path + " " + vers + " " var hashes []string for _, line := range strings.Split(string(result.data), "\n") { if strings.HasPrefix(line, prefix) { hashes = append(hashes, line) } } return hashes, nil } // mergeLatest merges the tree head in msg // with the Client's current latest tree head, // ensuring the result is a consistent timeline. // If the result is inconsistent, mergeLatest calls c.ops.SecurityError // with a detailed security error message and then // (only if c.ops.SecurityError does not exit the program) returns ErrSecurity. // If the Client's current latest tree head moves forward, // mergeLatest updates the underlying configuration file as well, // taking care to merge any independent updates to that configuration. func (c Client) mergeLatest(msg []byte) error { // Merge msg into our in-memory copy of the latest tree head. when, err := c.mergeLatestMem(msg) if err != nil { return err } if when != msgFuture { // msg matched our present or was in the past. // No change to our present, so no update of config file. return nil } // Flush our extended timeline back out to the configuration file. // If the configuration file has been updated in the interim, // we need to merge any updates made there as well. // Note that writeConfig is an atomic compare-and-swap. for { msg, err := c.ops.ReadConfig(c.name + "/latest") if err != nil { return err } when, err := c.mergeLatestMem(msg) if err != nil { return err } if when != msgPast { // msg matched our present or was from the future, // and now our in-memory copy matches. return nil } // msg (== config) is in the past, so we need to update it. c.latestMu.Lock() latestMsg := c.latestMsg c.latestMu.Unlock() if err := c.ops.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict { // Success or a non-write-conflict error. return err } } } const ( msgPast = 1 + iota msgNow msgFuture ) // mergeLatestMem is like mergeLatest but is only concerned with // updating the in-memory copy of the latest tree head (c.latest) // not the configuration file. // The when result explains when msg happened relative to our // previous idea of c.latest: // msgPast means msg was from before c.latest, // msgNow means msg was exactly c.latest, and // msgFuture means msg was from after c.latest, which has now been updated. func (c Client) mergeLatestMem(msg []byte) (when int, err error) { if len(msg) == 0 { // Accept empty msg as the unsigned, empty timeline. c.latestMu.Lock() latest := c.latest c.latestMu.Unlock() if latest.N == 0 { return msgNow, nil } return msgPast, nil } note, err := note.Open(msg, c.verifiers) if err != nil { return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg) } tree, err := tlog.ParseTree([]byte(note.Text)) if err != nil { return 0, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text) } // Other lookups may be calling mergeLatest with other heads, // so c.latest is changing underfoot. We don't want to hold the // c.mu lock during tile fetches, so loop trying to update c.latest. c.latestMu.Lock() latest := c.latest latestMsg := c.latestMsg c.latestMu.Unlock() for { // If the tree head looks old, check that it is on our timeline. if tree.N <= latest.N { if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil { return 0, err } if tree.N < latest.N { return msgPast, nil } return msgNow, nil } // The tree head looks new. Check that we are on its timeline and try to move our timeline forward. if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil { return 0, err } // Install our msg if possible. // Otherwise we will go around again. c.latestMu.Lock() installed := false if c.latest == latest { installed = true c.latest = tree c.latestMsg = msg } else { latest = c.latest latestMsg = c.latestMsg } c.latestMu.Unlock() if installed { return msgFuture, nil } } } // checkTrees checks that older (from olderNote) is contained in newer (from newerNote). // If an error occurs, such as malformed data or a network problem, checkTrees returns that error. // If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed // message and calls log.Fatal. func (c Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error { thr := tlog.TileHashReader(newer, &c.tileReader) h, err := tlog.TreeHash(older.N, thr) if err != nil { if older.N == newer.N { return fmt.Errorf("checking tree#%d: %v", older.N, err) } return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err) } if h == older.Hash { return nil } // Detected a fork in the tree timeline. // Start by reporting the inconsistent signed tree notes. var buf bytes.Buffer fmt.Fprintf(&buf, "SECURITY ERROR\n") fmt.Fprintf(&buf, "go.sum database server misbehavior detected!\n\n") indent := func(b []byte) []byte { return bytes.Replace(b, []byte("\n"), []byte("\n\t"), -1) } fmt.Fprintf(&buf, "old database:\n\t%s\n", indent(olderNote)) fmt.Fprintf(&buf, "new database:\n\t%s\n", indent(newerNote)) // The notes alone are not enough to prove the inconsistency. // We also need to show that the newer note's tree hash for older.N // does not match older.Hash. The consumer of this report could // of course consult the server to try to verify the inconsistency, // but we are holding all the bits we need to prove it right now, // so we might as well print them and make the report not depend // on the continued availability of the misbehaving server. // Preparing this data only reuses the tiled hashes needed for // tlog.TreeHash(older.N, thr) above, so assuming thr is caching tiles, // there are no new access to the server here, and these operations cannot fail. fmt.Fprintf(&buf, "proof of misbehavior:\n\t%v", h) if p, err := tlog.ProveTree(newer.N, older.N, thr); err != nil { fmt.Fprintf(&buf, "\tinternal error: %v\n", err) } else if err := tlog.CheckTree(p, newer.N, newer.Hash, older.N, h); err != nil { fmt.Fprintf(&buf, "\tinternal error: generated inconsistent proof\n") } else { for _, h := range p { fmt.Fprintf(&buf, "\n\t%v", h) } } c.ops.SecurityError(buf.String()) return ErrSecurity } // checkRecord checks that record #id's hash matches data. func (c Client) checkRecord(id int64, data []byte) error { c.latestMu.Lock() latest := c.latest c.latestMu.Unlock() if id >= latest.N { return fmt.Errorf("cannot validate record %d in tree of size %d", id, latest.N) } hashes, err := tlog.TileHashReader(latest, &c.tileReader).ReadHashes([]int64{tlog.StoredHashIndex(0, id)}) if err != nil { return err } if hashes[0] == tlog.RecordHash(data) { return nil } return fmt.Errorf("cannot authenticate record data in server response") } // tileReader is a Client wrapper that implements tlog.TileReader. // The separate type avoids exposing the ReadTiles and SaveTiles // methods on Client itself. type tileReader struct { c Client } func (r tileReader) Height() int { return r.c.tileHeight } // ReadTiles reads and returns the requested tiles, // either from the on-disk cache or the server. func (r tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) { // Read all the tiles in parallel. data := make([][]byte, len(tiles)) errs := make([]error, len(tiles)) var wg sync.WaitGroup for i, tile := range tiles { wg.Add(1) go func(i int, tile tlog.Tile) { defer wg.Done() defer func() { if e := recover(); e != nil { errs[i] = fmt.Errorf("panic: %v", e) } }() data[i], errs[i] = r.c.readTile(tile) }(i, tile) } wg.Wait() for _, err := range errs { if err != nil { return nil, err } } return data, nil } // tileCacheKey returns the cache key for the tile. func (c Client) tileCacheKey(tile tlog.Tile) string { return c.name + "/" + tile.Path() } // tileRemotePath returns the remote path for the tile. func (c Client) tileRemotePath(tile tlog.Tile) string { return "/" + tile.Path() } // readTile reads a single tile, either from the on-disk cache or the server. func (c Client) readTile(tile tlog.Tile) ([]byte, error) { type cached struct { data []byte err error } result := c.tileCache.Do(tile, func() interface{} { // Try the requested tile in on-disk cache. data, err := c.ops.ReadCache(c.tileCacheKey(tile)) if err == nil { c.markTileSaved(tile) return cached{data, nil} } // Try the full tile in on-disk cache (if requested tile not already full). // We only save authenticated tiles to the on-disk cache, // so the recreated prefix is equally authenticated. full := tile full.W = 1 << uint(tile.H) if tile != full { data, err := c.ops.ReadCache(c.tileCacheKey(full)) if err == nil { c.markTileSaved(tile) // don't save tile later; we already have full return cached{data[:len(data)/full.Wtile.W], nil} } } // Try requested tile from server. data, err = c.ops.ReadRemote(c.tileRemotePath(tile)) if err == nil { return cached{data, nil} } // Try full tile on server. // If the partial tile does not exist, it should be because // the tile has been completed and only the complete one // is available. if tile != full { data, err := c.ops.ReadRemote(c.tileRemotePath(full)) if err == nil { // Note: We could save the full tile in the on-disk cache here, // but we don't know if it is valid yet, and we will only find out // about the partial data, not the full data. So let SaveTiles // save the partial tile, and we'll just refetch the full tile later // once we can validate more (or all) of it. return cached{data[:len(data)/full.Wtile.W], nil} } } // Nothing worked. // Return the error from the server fetch for the requested (not full) tile. return cached{nil, err} }).(cached) return result.data, result.err } // markTileSaved records that tile is already present in the on-disk cache, // so that a future SaveTiles for that tile can be ignored. func (c Client) markTileSaved(tile tlog.Tile) { c.tileSavedMu.Lock() c.tileSaved[tile] = true c.tileSavedMu.Unlock() } // SaveTiles saves the now validated tiles. func (r tileReader) SaveTiles(tiles []tlog.Tile, data [][]byte) { c := r.c // Determine which tiles need saving. // (Tiles that came from the cache need not be saved back.) save := make([]bool, len(tiles)) c.tileSavedMu.Lock() for i, tile := range tiles { if !c.tileSaved[tile] { save[i] = true c.tileSaved[tile] = true } } c.tileSavedMu.Unlock() for i, tile := range tiles { if save[i] { // If WriteCache fails here (out of disk space? i/o error?), // c.tileSaved[tile] is still true and we will not try to write it again. // Next time we run maybe we'll redownload it again and be // more successful. c.ops.WriteCache(c.name+"/"+tile.Path(), data[i]) } } }	19.2 KB	2025-05-25 12:27
📄 client_test.go // Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sumdb import ( "bytes" "fmt" "strings" "sync" "testing" "golang.org/x/mod/sumdb/note" "golang.org/x/mod/sumdb/tlog" ) const ( testName = "localhost.localdev/sumdb" testVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6" testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk" ) func TestClientLookup(t testing.T) { tc := newTestClient(t) tc.mustHaveLatest(1) // Basic lookup. tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") tc.mustHaveLatest(3) // Everything should now be cached, both for the original package and its /go.mod. tc.getOK = false tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") tc.mustLookup("rsc.io/sampler", "v1.3.0/go.mod", "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=") tc.mustHaveLatest(3) tc.getOK = true tc.getTileOK = false // the cache has what we need // Lookup with multiple returned lines. tc.mustLookup("rsc.io/quote", "v1.5.2", "rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=\nrsc.io/quote v1.5.2 h2:xyzzy") tc.mustHaveLatest(3) // Lookup with need for !-encoding. // rsc.io/Quote is the only record written after rsc.io/samper, // so it is the only one that should need more tiles. tc.getTileOK = true tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=") tc.mustHaveLatest(4) } func TestClientBadTiles(t testing.T) { tc := newTestClient(t) flipBits := func() { for url, data := range tc.remote { if strings.Contains(url, "/tile/") { for i := range data { data[i] ^= 0x80 } } } } // Bad tiles in initial download. tc.mustHaveLatest(1) flipBits() _, err := tc.client.Lookup("rsc.io/sampler", "v1.3.0") tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile") flipBits() tc.newClient() tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") // Bad tiles after initial download. flipBits() _, err = tc.client.Lookup("rsc.io/Quote", "v1.5.2") tc.mustError(err, "rsc.io/Quote@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile") flipBits() tc.newClient() tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=") // Bad starting tree hash looks like bad tiles. tc.newClient() text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}}) data, err := note.Sign(&note.Note{Text: string(text)}, tc.signer) if err != nil { tc.t.Fatal(err) } tc.config[testName+"/latest"] = data _, err = tc.client.Lookup("rsc.io/sampler", "v1.3.0") tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile") } func TestClientFork(t testing.T) { tc := newTestClient(t) tc2 := tc.fork() tc.addRecord("rsc.io/pkg1@v1.5.2", `rsc.io/pkg1 v1.5.2 h1:hash!= `) tc.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!= `) tc.mustLookup("rsc.io/pkg1", "v1.5.2", "rsc.io/pkg1 v1.5.2 h1:hash!=") tc2.addRecord("rsc.io/pkg1@v1.5.3", `rsc.io/pkg1 v1.5.3 h1:hash!= `) tc2.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!= `) tc2.mustLookup("rsc.io/pkg1", "v1.5.4", "rsc.io/pkg1 v1.5.4 h1:hash!=") key := "/lookup/rsc.io/pkg1@v1.5.2" tc2.remote[key] = tc.remote[key] _, err := tc2.client.Lookup("rsc.io/pkg1", "v1.5.2") tc2.mustError(err, ErrSecurity.Error()) / SECURITY ERROR go.sum database server misbehavior detected! old database: go.sum database tree! 5 nWzN20+pwMt62p7jbv1/NlN95ePTlHijabv5zO/s36w= — localhost.localdev/sumdb AAAMZ5/2FVAdMH58kmnz/0h299pwyskEbzDzoa2/YaPdhvLya4YWDFQQxu2TQb5GpwAH4NdWnTwuhILafisyf3CNbgg= new database: go.sum database tree 6 wc4SkQt52o5W2nQ8To2ARs+mWuUJjss+sdleoiqxMmM= — localhost.localdev/sumdb AAAMZ6oRNswlEZ6ZZhxrCvgl1MBy+nusq4JU+TG6Fe2NihWLqOzb+y2c2kzRLoCr4tvw9o36ucQEnhc20e4nA4Qc/wc= proof of misbehavior: T7i+H/8ER4nXOiw4Bj0koZOkGjkxoNvlI34GpvhHhQg= Nsuejv72de9hYNM5bqFv8rv3gm3zJQwv/DT/WNbLDLA= mOmqqZ1aI/lzS94oq/JSbj7pD8Rv9S+xDyi12BtVSHo= /7Aw5jVSMM9sFjQhaMg+iiDYPMk6decH7QLOGrL9Lx0= / wants := []string{ "SECURITY ERROR", "go.sum database server misbehavior detected!", "old database:\n\tgo.sum database tree\n\t5\n", "— localhost.localdev/sumdb AAAMZ5/2FVAd", "new database:\n\tgo.sum database tree\n\t6\n", "— localhost.localdev/sumdb AAAMZ6oRNswl", "proof of misbehavior:\n\tT7i+H/8ER4nXOiw4Bj0k", } text := tc2.security.String() for _, want := range wants { if !strings.Contains(text, want) { t.Fatalf("cannot find %q in security text:\n%s", want, text) } } } func TestClientGONOSUMDB(t testing.T) { tc := newTestClient(t) tc.client.SetGONOSUMDB("p,/q") tc.client.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network tc.getOK = false ok := []string{ "abc", "a/p", "pq", "q", "n/o/p/q", } skip := []string{ "p", "p/x", "x/q", "x/q/z", } for _, path := range ok { _, err := tc.client.Lookup(path, "v1.0.0") if err == ErrGONOSUMDB { t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path) } } for _, path := range skip { _, err := tc.client.Lookup(path, "v1.0.0") if err != ErrGONOSUMDB { t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err) } } } // A testClient is a self-contained client-side testing environment. type testClient struct { t testing.T // active test client Client // client being tested tileHeight int // tile height to use (default 2) getOK bool // should tc.GetURL succeed? getTileOK bool // should tc.GetURL of tiles succeed? treeSize int64 hashes []tlog.Hash remote map[string][]byte signer note.Signer // mu protects config, cache, log, security // during concurrent use of the exported methods // by the client itself (testClient is the Client's ClientOps, // and the Client methods can both read and write these fields). // Unexported methods invoked directly by the test // (for example, addRecord) need not hold the mutex: // for proper test execution those methods should only // be called when the Client is idle and not using its ClientOps. // Not holding the mutex in those methods ensures // that if a mistake is made, go test -race will report it. // (Holding the mutex would eliminate the race report but // not the underlying problem.) // Similarly, the get map is not protected by the mutex, // because the Client methods only read it. mu sync.Mutex // prot config map[string][]byte cache map[string][]byte security bytes.Buffer } // newTestClient returns a new testClient that will call t.Fatal on error // and has a few records already available on the remote server. func newTestClient(t testing.T) testClient { tc := &testClient{ t: t, tileHeight: 2, getOK: true, getTileOK: true, config: make(map[string][]byte), cache: make(map[string][]byte), remote: make(map[string][]byte), } tc.config["key"] = []byte(testVerifierKey + "\n") var err error tc.signer, err = note.NewSigner(testSignerKey) if err != nil { t.Fatal(err) } tc.newClient() tc.addRecord("rsc.io/quote@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= rsc.io/quote v1.5.2 h2:xyzzy `) tc.addRecord("golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c", `golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= `) tc.addRecord("rsc.io/sampler@v1.3.0", `rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= `) tc.config[testName+"/latest"] = tc.signTree(1) tc.addRecord("rsc.io/!quote@v1.5.2", `rsc.io/Quote v1.5.2 h1:uppercase!= `) return tc } // newClient resets the Client associated with tc. // This clears any in-memory cache from the Client // but not tc's on-disk cache. func (tc testClient) newClient() { tc.client = NewClient(tc) tc.client.SetTileHeight(tc.tileHeight) } // mustLookup does a lookup for path@vers and checks that the lines that come back match want. func (tc testClient) mustLookup(path, vers, want string) { tc.t.Helper() lines, err := tc.client.Lookup(path, vers) if err != nil { tc.t.Fatal(err) } if strings.Join(lines, "\n") != want { tc.t.Fatalf("Lookup(%q, %q):\n\t%s\nwant:\n\t%s", path, vers, strings.Join(lines, "\n\t"), strings.Replace(want, "\n", "\n\t", -1)) } } // mustHaveLatest checks that the on-disk configuration // for latest is a tree of size n. func (tc testClient) mustHaveLatest(n int64) { tc.t.Helper() latest := tc.config[testName+"/latest"] lines := strings.Split(string(latest), "\n") if len(lines) < 2 \|\| lines[1] != fmt.Sprint(n) { tc.t.Fatalf("/latest should have tree %d, but has:\n%s", n, latest) } } // mustError checks that err's error string contains the text. func (tc testClient) mustError(err error, text string) { tc.t.Helper() if err == nil \|\| !strings.Contains(err.Error(), text) { tc.t.Fatalf("err = %v, want %q", err, text) } } // fork returns a copy of tc. // Changes made to the new copy or to tc are not reflected in the other. func (tc testClient) fork() testClient { tc2 := &testClient{ t: tc.t, getOK: tc.getOK, getTileOK: tc.getTileOK, tileHeight: tc.tileHeight, treeSize: tc.treeSize, hashes: append([]tlog.Hash{}, tc.hashes...), signer: tc.signer, config: copyMap(tc.config), cache: copyMap(tc.cache), remote: copyMap(tc.remote), } tc2.newClient() return tc2 } func copyMap(m map[string][]byte) map[string][]byte { m2 := make(map[string][]byte) for k, v := range m { m2[k] = v } return m2 } // ReadHashes is tc's implementation of tlog.HashReader, for use with // tlog.TreeHash and so on. func (tc testClient) ReadHashes(indexes []int64) ([]tlog.Hash, error) { var list []tlog.Hash for _, id := range indexes { list = append(list, tc.hashes[id]) } return list, nil } // addRecord adds a log record using the given (!-encoded) key and data. func (tc testClient) addRecord(key, data string) { tc.t.Helper() // Create record, add hashes to log tree. id := tc.treeSize tc.treeSize++ rec, err := tlog.FormatRecord(id, []byte(data)) if err != nil { tc.t.Fatal(err) } hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), tc) if err != nil { tc.t.Fatal(err) } tc.hashes = append(tc.hashes, hashes...) // Create lookup result. tc.remote["/lookup/"+key] = append(rec, tc.signTree(tc.treeSize)...) // Create new tiles. tiles := tlog.NewTiles(tc.tileHeight, id, tc.treeSize) for _, tile := range tiles { data, err := tlog.ReadTileData(tile, tc) if err != nil { tc.t.Fatal(err) } tc.remote["/"+tile.Path()] = data // TODO delete old partial tiles } } // signTree returns the signed head for the tree of the given size. func (tc testClient) signTree(size int64) []byte { h, err := tlog.TreeHash(size, tc) if err != nil { tc.t.Fatal(err) } text := tlog.FormatTree(tlog.Tree{N: size, Hash: h}) data, err := note.Sign(&note.Note{Text: string(text)}, tc.signer) if err != nil { tc.t.Fatal(err) } return data } // ReadRemote is for tc's implementation of Client. func (tc testClient) ReadRemote(path string) ([]byte, error) { // No mutex here because only the Client should be running // and the Client cannot change tc.get. if !tc.getOK { return nil, fmt.Errorf("disallowed remote read %s", path) } if strings.Contains(path, "/tile/") && !tc.getTileOK { return nil, fmt.Errorf("disallowed remote tile read %s", path) } data, ok := tc.remote[path] if !ok { return nil, fmt.Errorf("no remote path %s", path) } return data, nil } // ReadConfig is for tc's implementation of Client. func (tc testClient) ReadConfig(file string) ([]byte, error) { tc.mu.Lock() defer tc.mu.Unlock() data, ok := tc.config[file] if !ok { return nil, fmt.Errorf("no config %s", file) } return data, nil } // WriteConfig is for tc's implementation of Client. func (tc testClient) WriteConfig(file string, old, new []byte) error { tc.mu.Lock() defer tc.mu.Unlock() data := tc.config[file] if !bytes.Equal(old, data) { return ErrWriteConflict } tc.config[file] = new return nil } // ReadCache is for tc's implementation of Client. func (tc testClient) ReadCache(file string) ([]byte, error) { tc.mu.Lock() defer tc.mu.Unlock() data, ok := tc.cache[file] if !ok { return nil, fmt.Errorf("no cache %s", file) } return data, nil } // WriteCache is for tc's implementation of Client. func (tc testClient) WriteCache(file string, data []byte) { tc.mu.Lock() defer tc.mu.Unlock() tc.cache[file] = data } // Log is for tc's implementation of Client. func (tc testClient) Log(msg string) { tc.t.Log(msg) } // SecurityError is for tc's implementation of Client. func (tc *testClient) SecurityError(msg string) { tc.mu.Lock() defer tc.mu.Unlock() fmt.Fprintf(&tc.security, "%s\n", strings.TrimRight(msg, "\n")) }	13.38 KB	2025-05-25 12:27
📁 dirhash	-	2025-05-25 12:27
📁 note	-	2025-05-25 12:27
📄 server.go // Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package sumdb implements the HTTP protocols for serving or accessing a module checksum database. package sumdb import ( "bytes" "context" "net/http" "os" "strings" "golang.org/x/mod/internal/lazyregexp" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/tlog" ) // A ServerOps provides the external operations // (underlying database access and so on) needed by the [Server]. type ServerOps interface { // Signed returns the signed hash of the latest tree. Signed(ctx context.Context) ([]byte, error) // ReadRecords returns the content for the n records id through id+n-1. ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) // Lookup looks up a record for the given module, // returning the record ID. Lookup(ctx context.Context, m module.Version) (int64, error) // ReadTileData reads the content of tile t. // It is only invoked for hash tiles (t.L ≥ 0). ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) } // A Server is the checksum database HTTP server, // which implements http.Handler and should be invoked // to serve the paths listed in [ServerPaths]. type Server struct { ops ServerOps } // NewServer returns a new Server using the given operations. func NewServer(ops ServerOps) Server { return &Server{ops: ops} } // ServerPaths are the URL paths the Server can (and should) serve. // // Typically a server will do: // // srv := sumdb.NewServer(ops) // for _, path := range sumdb.ServerPaths { // http.Handle(path, srv) // } var ServerPaths = []string{ "/lookup/", "/latest", "/tile/", } var modVerRE = lazyregexp.New(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@])?(\+incompatible)?$`) func (s Server) ServeHTTP(w http.ResponseWriter, r http.Request) { ctx := r.Context() switch { default: http.NotFound(w, r) case strings.HasPrefix(r.URL.Path, "/lookup/"): mod := strings.TrimPrefix(r.URL.Path, "/lookup/") if !modVerRE.MatchString(mod) { http.Error(w, "invalid module@version syntax", http.StatusBadRequest) return } i := strings.Index(mod, "@") escPath, escVers := mod[:i], mod[i+1:] path, err := module.UnescapePath(escPath) if err != nil { reportError(w, err) return } vers, err := module.UnescapeVersion(escVers) if err != nil { reportError(w, err) return } id, err := s.ops.Lookup(ctx, module.Version{Path: path, Version: vers}) if err != nil { reportError(w, err) return } records, err := s.ops.ReadRecords(ctx, id, 1) if err != nil { // This should never happen - the lookup says the record exists. http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(records) != 1 { http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError) return } msg, err := tlog.FormatRecord(id, records[0]) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } signed, err := s.ops.Signed(ctx) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.Write(msg) w.Write(signed) case r.URL.Path == "/latest": data, err := s.ops.Signed(ctx) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.Write(data) case strings.HasPrefix(r.URL.Path, "/tile/"): t, err := tlog.ParseTilePath(r.URL.Path[1:]) if err != nil { http.Error(w, "invalid tile syntax", http.StatusBadRequest) return } if t.L == -1 { // Record data. start := t.N << uint(t.H) records, err := s.ops.ReadRecords(ctx, start, int64(t.W)) if err != nil { reportError(w, err) return } if len(records) != t.W { http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError) return } var data []byte for i, text := range records { msg, err := tlog.FormatRecord(start+int64(i), text) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Data tiles contain formatted records without the first line with record ID. _, msg, _ = bytes.Cut(msg, []byte{'\n'}) data = append(data, msg...) } w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.Write(data) return } data, err := s.ops.ReadTileData(ctx, t) if err != nil { reportError(w, err) return } w.Header().Set("Content-Type", "application/octet-stream") w.Write(data) } } // reportError reports err to w. // If it's a not-found, the reported error is 404. // Otherwise it is an internal server error. // The caller must only call reportError in contexts where // a not-found err should be reported as 404. func reportError(w http.ResponseWriter, err error) { if os.IsNotExist(err) { http.Error(w, err.Error(), http.StatusNotFound) return } http.Error(w, err.Error(), http.StatusInternalServerError) }	5.01 KB	2025-05-25 12:27
📁 storage	-	2025-05-25 12:27
📄 test.go // Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sumdb import ( "context" "fmt" "sync" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/note" "golang.org/x/mod/sumdb/tlog" ) // NewTestServer constructs a new [TestServer] // that will sign its tree with the given signer key // (see [golang.org/x/mod/sumdb/note]) // and fetch new records as needed by calling gosum. func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) TestServer { return &TestServer{signer: signer, gosum: gosum} } // A TestServer is an in-memory implementation of [ServerOps] for testing. type TestServer struct { signer string gosum func(path, vers string) ([]byte, error) mu sync.Mutex hashes testHashes records [][]byte lookup map[string]int64 } // testHashes implements tlog.HashReader, reading from a slice. type testHashes []tlog.Hash func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) { var list []tlog.Hash for _, id := range indexes { list = append(list, h[id]) } return list, nil } func (s TestServer) Signed(ctx context.Context) ([]byte, error) { s.mu.Lock() defer s.mu.Unlock() size := int64(len(s.records)) h, err := tlog.TreeHash(size, s.hashes) if err != nil { return nil, err } text := tlog.FormatTree(tlog.Tree{N: size, Hash: h}) signer, err := note.NewSigner(s.signer) if err != nil { return nil, err } return note.Sign(&note.Note{Text: string(text)}, signer) } func (s TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) { s.mu.Lock() defer s.mu.Unlock() var list [][]byte for i := int64(0); i < n; i++ { if id+i >= int64(len(s.records)) { return nil, fmt.Errorf("missing records") } list = append(list, s.records[id+i]) } return list, nil } func (s TestServer) Lookup(ctx context.Context, m module.Version) (int64, error) { key := m.String() s.mu.Lock() id, ok := s.lookup[key] s.mu.Unlock() if ok { return id, nil } // Look up module and compute go.sum lines. data, err := s.gosum(m.Path, m.Version) if err != nil { return 0, err } s.mu.Lock() defer s.mu.Unlock() // We ran the fetch without the lock. // If another fetch happened and committed, use it instead. id, ok = s.lookup[key] if ok { return id, nil } // Add record. id = int64(len(s.records)) s.records = append(s.records, data) if s.lookup == nil { s.lookup = make(map[string]int64) } s.lookup[key] = id hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash(data), s.hashes) if err != nil { panic(err) } s.hashes = append(s.hashes, hashes...) return id, nil } func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) { s.mu.Lock() defer s.mu.Unlock() return tlog.ReadTileData(t, s.hashes) }	2.82 KB	2025-05-25 12:27
📁 tlog	-	2025-05-25 12:27

Terminal