Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-05-25 12:27
📁 .github	-	2025-05-25 12:27
📄 .gitignore # Compiled Object files, Static and Dynamic libs (Shared Objects) .o .a .so # Folders _obj _test # Architecture specific extensions/prefixes .[568vq] [568vq].out .cgo1.go .cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go .exe .test .vagrant/	270 B	2025-05-25 12:27
📄 .golangci.yml # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: MPL-2.0 linters: disable-all: true enable: # (TODO) uncomment after fixing the lint issues # - gofmt - govet # - unconvert # - staticcheck # - ineffassign # - unparam - forbidigo issues: # Disable the default exclude list so that all excludes are explicitly # defined in this file. exclude-use-default: false exclude-rules: # Temp Ignore SA9004: only the first constant in this group has an explicit type # https://staticcheck.io/docs/checks#SA9004 - linters: [staticcheck] text: 'SA9004:' - linters: [staticcheck] text: 'SA1019: Package github.com/golang/protobuf/jsonpb is deprecated' - linters: [staticcheck] text: 'SA1019: Package github.com/golang/protobuf/proto is deprecated' - linters: [staticcheck] text: 'SA1019: ptypes.MarshalAny is deprecated' - linters: [staticcheck] text: 'SA1019: ptypes.UnmarshalAny is deprecated' - linters: [staticcheck] text: 'SA1019: package github.com/golang/protobuf/ptypes is deprecated' # An argument that always receives the same value is often not a problem. - linters: [unparam] text: 'always receives' # Often functions will implement an interface that returns an error without # needing to return an error. Sometimes the error return value is unnecessary # but a linter can not tell the difference. - linters: [unparam] text: 'result \d+ $error$ is always nil' # Allow unused parameters to start with an underscore. Arguments with a name # of '_' are already ignored. # Ignoring longer names that start with underscore allow for better # self-documentation than a single underscore by itself. Underscore arguments # should generally only be used when a function is implementing an interface. - linters: [unparam] text: '`_[^`]*` is unused' # Temp ignore some common unused parameters so that unparam can be added # incrementally. - linters: [unparam] text: '`(t\|resp\|req\|entMeta)` is unused' linters-settings: gofmt: simplify: true forbidigo: # Forbid the following identifiers (list of regexp). forbid: - '\brequire\.New\b(# Use package-level functions with explicit TestingT)?' - '\bassert\.New\b(# Use package-level functions with explicit TestingT)?' - '\bmetrics\.IncrCounter\b(# Use labeled metrics)?' - '\bmetrics\.AddSample\b(# Use labeled metrics)?' - '\bmetrics\.MeasureSince\b(# Use labeled metrics)?' - '\bmetrics\.SetGauge\b(# Use labeled metrics)?' # Exclude godoc examples from forbidigo checks. # Default: true exclude_godoc_examples: false run: timeout: 10m concurrency: 4	2.71 KB	2025-05-25 12:27
📄 LICENSE Copyright (c) 2013 HashiCorp, Inc. Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.	15.64 KB	2025-05-25 12:27
📄 Makefile SHELL := bash GOFILES ?= $(shell go list ./... \| grep -v /vendor/) default: test test: vet subnet go test ./... integ: subnet INTEG_TESTS=yes go test ./... subnet: ./test/setup_subnet.sh cov: go test ./... -coverprofile=coverage.out go tool cover -html=coverage.out format: @echo "--> Running go fmt" @go fmt $(GOFILES) vet: @echo "--> Running go vet" @go vet -tags '$(GOTAGS)' $(GOFILES); if [ $$? -eq 1 ]; then \ echo ""; \ echo "Vet found suspicious constructs. Please check the reported constructs"; \ echo "and fix them if necessary before submitting the code for review."; \ exit 1; \ fi .PHONY: default test integ subnet cov format vet	670 B	2025-05-25 12:27
📄 README.md # memberlist [![GoDoc](https://godoc.org/github.com/hashicorp/memberlist?status.png)](https://godoc.org/github.com/hashicorp/memberlist) [![CircleCI](https://circleci.com/gh/hashicorp/memberlist.svg?style=svg)](https://circleci.com/gh/hashicorp/memberlist) memberlist is a [Go](http://www.golang.org) library that manages cluster membership and member failure detection using a gossip based protocol. The use cases for such a library are far-reaching: all distributed systems require membership, and memberlist is a re-usable solution to managing cluster membership and node failure detection. memberlist is eventually consistent but converges quickly on average. The speed at which it converges can be heavily tuned via various knobs on the protocol. Node failures are detected and network partitions are partially tolerated by attempting to communicate to potentially dead nodes through multiple routes. ## Building If you wish to build memberlist you'll need Go version 1.2+ installed. Please check your installation with: ``` go version ``` ## Usage Memberlist is surprisingly simple to use. An example is shown below: ```go /* Create the initial memberlist from a safe configuration. Please reference the godoc for other default config types. http://godoc.org/github.com/hashicorp/memberlist#Config / list, err := memberlist.Create(memberlist.DefaultLocalConfig()) if err != nil { panic("Failed to create memberlist: " + err.Error()) } // Join an existing cluster by specifying at least one known member. n, err := list.Join([]string{"1.2.3.4"}) if err != nil { panic("Failed to join cluster: " + err.Error()) } // Ask for members of the cluster for _, member := range list.Members() { fmt.Printf("Member: %s %s\n", member.Name, member.Addr) } // Continue doing whatever you need, memberlist will maintain membership // information in the background. Delegates can be used for receiving // events when members join or leave. ``` The most difficult part of memberlist is configuring it since it has many available knobs in order to tune state propagation delay and convergence times. Memberlist provides a default configuration that offers a good starting point, but errs on the side of caution, choosing values that are optimized for higher convergence at the cost of higher bandwidth usage. For complete documentation, see the associated [Godoc](http://godoc.org/github.com/hashicorp/memberlist). ## Protocol memberlist is based on ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol"](http://ieeexplore.ieee.org/document/1028914/). However, we extend the protocol in a number of ways: Several extensions are made to increase propagation speed and convergence rate. * Another set of extensions, that we call Lifeguard, are made to make memberlist more robust in the presence of slow message processing (due to factors such as CPU starvation, and network delay or loss). For details on all of these extensions, please read our paper "[Lifeguard : SWIM-ing with Situational Awareness](https://arxiv.org/abs/1707.00788)", along with the memberlist source. We welcome any questions related to the protocol on our issue tracker.	3.12 KB	2025-05-25 12:27
📄 alive_delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist // AliveDelegate is used to involve a client in processing // a node "alive" message. When a node joins, either through // a UDP gossip or TCP push/pull, we update the state of // that node via an alive message. This can be used to filter // a node out and prevent it from being considered a peer // using application specific logic. type AliveDelegate interface { // NotifyAlive is invoked when a message about a live // node is received from the network. Returning a non-nil // error prevents the node from being considered a peer. NotifyAlive(peer *Node) error }	661 B	2025-05-25 12:27
📄 awareness.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "sync" "time" "github.com/armon/go-metrics" ) // awareness manages a simple metric for tracking the estimated health of the // local node. Health is primary the node's ability to respond in the soft // real-time manner required for correct health checking of other nodes in the // cluster. type awareness struct { sync.RWMutex // max is the upper threshold for the timeout scale (the score will be // constrained to be from 0 <= score < max). max int // score is the current awareness score. Lower values are healthier and // zero is the minimum value. score int // metricLabels is the slice of labels to put on all emitted metrics metricLabels []metrics.Label } // newAwareness returns a new awareness object. func newAwareness(max int, metricLabels []metrics.Label) awareness { return &awareness{ max: max, score: 0, metricLabels: metricLabels, } } // ApplyDelta takes the given delta and applies it to the score in a thread-safe // manner. It also enforces a floor of zero and a max of max, so deltas may not // change the overall score if it's railed at one of the extremes. func (a awareness) ApplyDelta(delta int) { a.Lock() initial := a.score a.score += delta if a.score < 0 { a.score = 0 } else if a.score > (a.max - 1) { a.score = (a.max - 1) } final := a.score a.Unlock() if initial != final { metrics.SetGaugeWithLabels([]string{"memberlist", "health", "score"}, float32(final), a.metricLabels) } } // GetHealthScore returns the raw health score. func (a awareness) GetHealthScore() int { a.RLock() score := a.score a.RUnlock() return score } // ScaleTimeout takes the given duration and scales it based on the current // score. Less healthyness will lead to longer timeouts. func (a awareness) ScaleTimeout(timeout time.Duration) time.Duration { a.RLock() score := a.score a.RUnlock() return timeout * (time.Duration(score) + 1) }	1.97 KB	2025-05-25 12:27
📄 awareness_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "testing" "time" ) func TestAwareness(t testing.T) { cases := []struct { delta int score int timeout time.Duration }{ {0, 0, 1 time.Second}, {-1, 0, 1 * time.Second}, {-10, 0, 1 * time.Second}, {1, 1, 2 * time.Second}, {-1, 0, 1 * time.Second}, {10, 7, 8 * time.Second}, {-1, 6, 7 * time.Second}, {-1, 5, 6 * time.Second}, {-1, 4, 5 * time.Second}, {-1, 3, 4 * time.Second}, {-1, 2, 3 * time.Second}, {-1, 1, 2 * time.Second}, {-1, 0, 1 * time.Second}, {-1, 0, 1 * time.Second}, } a := newAwareness(8, nil) for i, c := range cases { a.ApplyDelta(c.delta) if a.GetHealthScore() != c.score { t.Errorf("case %d: score mismatch %d != %d", i, a.score, c.score) } if timeout := a.ScaleTimeout(1 * time.Second); timeout != c.timeout { t.Errorf("case %d: scaled timeout mismatch %9.6f != %9.6f", i, timeout.Seconds(), c.timeout.Seconds()) } } }	1007 B	2025-05-25 12:27
📄 broadcast.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist /* The broadcast mechanism works by maintaining a sorted list of messages to be sent out. When a message is to be broadcast, the retransmit count is set to zero and appended to the queue. The retransmit count serves as the "priority", ensuring that newer messages get sent first. Once a message hits the retransmit limit, it is removed from the queue. Additionally, older entries can be invalidated by new messages that are contradictory. For example, if we send "{suspect M1 inc: 1}, then a following {alive M1 inc: 2} will invalidate that message / type memberlistBroadcast struct { node string msg []byte notify chan struct{} } func (b memberlistBroadcast) Invalidates(other Broadcast) bool { // Check if that broadcast is a memberlist type mb, ok := other.(memberlistBroadcast) if !ok { return false } // Invalidates any message about the same node return b.node == mb.node } // memberlist.NamedBroadcast optional interface func (b memberlistBroadcast) Name() string { return b.node } func (b memberlistBroadcast) Message() []byte { return b.msg } func (b memberlistBroadcast) Finished() { select { case b.notify <- struct{}{}: default: } } // encodeAndBroadcast encodes a message and enqueues it for broadcast. Fails // silently if there is an encoding error. func (m Memberlist) encodeAndBroadcast(node string, msgType messageType, msg interface{}) { m.encodeBroadcastNotify(node, msgType, msg, nil) } // encodeBroadcastNotify encodes a message and enqueues it for broadcast // and notifies the given channel when transmission is finished. Fails // silently if there is an encoding error. func (m Memberlist) encodeBroadcastNotify(node string, msgType messageType, msg interface{}, notify chan struct{}) { buf, err := encode(msgType, msg, m.config.MsgpackUseNewTimeFormat) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to encode message for broadcast: %s", err) } else { m.queueBroadcast(node, buf.Bytes(), notify) } } // queueBroadcast is used to start dissemination of a message. It will be // sent up to a configured number of times. The message could potentially // be invalidated by a future message about the same node func (m Memberlist) queueBroadcast(node string, msg []byte, notify chan struct{}) { b := &memberlistBroadcast{node, msg, notify} m.broadcasts.QueueBroadcast(b) } // getBroadcasts is used to return a slice of broadcasts to send up to // a maximum byte size, while imposing a per-broadcast overhead. This is used // to fill a UDP packet with piggybacked data func (m Memberlist) getBroadcasts(overhead, limit int) [][]byte { // Get memberlist messages first toSend := m.broadcasts.GetBroadcasts(overhead, limit) // Check if the user has anything to broadcast d := m.config.Delegate if d != nil { // Determine the bytes used already bytesUsed := 0 for _, msg := range toSend { bytesUsed += len(msg) + overhead } // Check space remaining for user messages avail := limit - bytesUsed if avail > overhead+userMsgOverhead { userMsgs := d.GetBroadcasts(overhead+userMsgOverhead, avail) // Frame each user message for _, msg := range userMsgs { buf := make([]byte, 1, len(msg)+1) buf[0] = byte(userMsg) buf = append(buf, msg...) toSend = append(toSend, buf) } } } return toSend }	3.33 KB	2025-05-25 12:27
📄 broadcast_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "reflect" "testing" ) func TestMemberlistBroadcast_Invalidates(t testing.T) { m1 := &memberlistBroadcast{"test", nil, nil} m2 := &memberlistBroadcast{"foo", nil, nil} if m1.Invalidates(m2) \|\| m2.Invalidates(m1) { t.Fatalf("unexpected invalidation") } if !m1.Invalidates(m1) { t.Fatalf("expected invalidation") } } func TestMemberlistBroadcast_Message(t testing.T) { m1 := &memberlistBroadcast{"test", []byte("test"), nil} msg := m1.Message() if !reflect.DeepEqual(msg, []byte("test")) { t.Fatalf("messages do not match") } }	648 B	2025-05-25 12:27
📄 config.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "io" "log" "net" "os" "strings" "time" "github.com/armon/go-metrics" "github.com/hashicorp/go-multierror" ) type Config struct { // The name of this node. This must be unique in the cluster. Name string // Transport is a hook for providing custom code to communicate with // other nodes. If this is left nil, then memberlist will by default // make a NetTransport using BindAddr and BindPort from this structure. Transport Transport // Label is an optional set of bytes to include on the outside of each // packet and stream. // // If gossip encryption is enabled and this is set it is treated as GCM // authenticated data. Label string // SkipInboundLabelCheck skips the check that inbound packets and gossip // streams need to be label prefixed. SkipInboundLabelCheck bool // Configuration related to what address to bind to and ports to // listen on. The port is used for both UDP and TCP gossip. It is // assumed other nodes are running on this port, but they do not need // to. BindAddr string BindPort int // Configuration related to what address to advertise to other // cluster members. Used for nat traversal. AdvertiseAddr string AdvertisePort int // ProtocolVersion is the configured protocol version that we // will _speak_. This must be between ProtocolVersionMin and // ProtocolVersionMax. ProtocolVersion uint8 // TCPTimeout is the timeout for establishing a stream connection with // a remote node for a full state sync, and for stream read and write // operations. This is a legacy name for backwards compatibility, but // should really be called StreamTimeout now that we have generalized // the transport. TCPTimeout time.Duration // IndirectChecks is the number of nodes that will be asked to perform // an indirect probe of a node in the case a direct probe fails. Memberlist // waits for an ack from any single indirect node, so increasing this // number will increase the likelihood that an indirect probe will succeed // at the expense of bandwidth. IndirectChecks int // RetransmitMult is the multiplier for the number of retransmissions // that are attempted for messages broadcasted over gossip. The actual // count of retransmissions is calculated using the formula: // // Retransmits = RetransmitMult * log(N+1) // // This allows the retransmits to scale properly with cluster size. The // higher the multiplier, the more likely a failed broadcast is to converge // at the expense of increased bandwidth. RetransmitMult int // SuspicionMult is the multiplier for determining the time an // inaccessible node is considered suspect before declaring it dead. // The actual timeout is calculated using the formula: // // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval // // This allows the timeout to scale properly with expected propagation // delay with a larger cluster size. The higher the multiplier, the longer // an inaccessible node is considered part of the cluster before declaring // it dead, giving that suspect node more time to refute if it is indeed // still alive. SuspicionMult int // SuspicionMaxTimeoutMult is the multiplier applied to the // SuspicionTimeout used as an upper bound on detection time. This max // timeout is calculated using the formula: // // SuspicionMaxTimeout = SuspicionMaxTimeoutMult * SuspicionTimeout // // If everything is working properly, confirmations from other nodes will // accelerate suspicion timers in a manner which will cause the timeout // to reach the base SuspicionTimeout before that elapses, so this value // will typically only come into play if a node is experiencing issues // communicating with other nodes. It should be set to a something fairly // large so that a node having problems will have a lot of chances to // recover before falsely declaring other nodes as failed, but short // enough for a legitimately isolated node to still make progress marking // nodes failed in a reasonable amount of time. SuspicionMaxTimeoutMult int // PushPullInterval is the interval between complete state syncs. // Complete state syncs are done with a single node over TCP and are // quite expensive relative to standard gossiped messages. Setting this // to zero will disable state push/pull syncs completely. // // Setting this interval lower (more frequent) will increase convergence // speeds across larger clusters at the expense of increased bandwidth // usage. PushPullInterval time.Duration // ProbeInterval and ProbeTimeout are used to configure probing // behavior for memberlist. // // ProbeInterval is the interval between random node probes. Setting // this lower (more frequent) will cause the memberlist cluster to detect // failed nodes more quickly at the expense of increased bandwidth usage. // // ProbeTimeout is the timeout to wait for an ack from a probed node // before assuming it is unhealthy. This should be set to 99-percentile // of RTT (round-trip time) on your network. ProbeInterval time.Duration ProbeTimeout time.Duration // DisableTcpPings will turn off the fallback TCP pings that are attempted // if the direct UDP ping fails. These get pipelined along with the // indirect UDP pings. DisableTcpPings bool // DisableTcpPingsForNode is like DisableTcpPings, but lets you control // whether to perform TCP pings on a node-by-node basis. DisableTcpPingsForNode func(nodeName string) bool // AwarenessMaxMultiplier will increase the probe interval if the node // becomes aware that it might be degraded and not meeting the soft real // time requirements to reliably probe other nodes. AwarenessMaxMultiplier int // GossipInterval and GossipNodes are used to configure the gossip // behavior of memberlist. // // GossipInterval is the interval between sending messages that need // to be gossiped that haven't been able to piggyback on probing messages. // If this is set to zero, non-piggyback gossip is disabled. By lowering // this value (more frequent) gossip messages are propagated across // the cluster more quickly at the expense of increased bandwidth. // // GossipNodes is the number of random nodes to send gossip messages to // per GossipInterval. Increasing this number causes the gossip messages // to propagate across the cluster more quickly at the expense of // increased bandwidth. // // GossipToTheDeadTime is the interval after which a node has died that // we will still try to gossip to it. This gives it a chance to refute. GossipInterval time.Duration GossipNodes int GossipToTheDeadTime time.Duration // GossipVerifyIncoming controls whether to enforce encryption for incoming // gossip. It is used for upshifting from unencrypted to encrypted gossip on // a running cluster. GossipVerifyIncoming bool // GossipVerifyOutgoing controls whether to enforce encryption for outgoing // gossip. It is used for upshifting from unencrypted to encrypted gossip on // a running cluster. GossipVerifyOutgoing bool // EnableCompression is used to control message compression. This can // be used to reduce bandwidth usage at the cost of slightly more CPU // utilization. This is only available starting at protocol version 1. EnableCompression bool // SecretKey is used to initialize the primary encryption key in a keyring. // The primary encryption key is the only key used to encrypt messages and // the first key used while attempting to decrypt messages. Providing a // value for this primary key will enable message-level encryption and // verification, and automatically install the key onto the keyring. // The value should be either 16, 24, or 32 bytes to select AES-128, // AES-192, or AES-256. SecretKey []byte // The keyring holds all of the encryption keys used internally. It is // automatically initialized using the SecretKey and SecretKeys values. Keyring Keyring // Delegate and Events are delegates for receiving and providing // data to memberlist via callback mechanisms. For Delegate, see // the Delegate interface. For Events, see the EventDelegate interface. // // The DelegateProtocolMin/Max are used to guarantee protocol-compatibility // for any custom messages that the delegate might do (broadcasts, // local/remote state, etc.). If you don't set these, then the protocol // versions will just be zero, and version compliance won't be done. Delegate Delegate DelegateProtocolVersion uint8 DelegateProtocolMin uint8 DelegateProtocolMax uint8 Events EventDelegate Conflict ConflictDelegate Merge MergeDelegate Ping PingDelegate Alive AliveDelegate // DNSConfigPath points to the system's DNS config file, usually located // at /etc/resolv.conf. It can be overridden via config for easier testing. DNSConfigPath string // LogOutput is the writer where logs should be sent. If this is not // set, logging will go to stderr by default. You cannot specify both LogOutput // and Logger at the same time. LogOutput io.Writer // Logger is a custom logger which you provide. If Logger is set, it will use // this for the internal logger. If Logger is not set, it will fall back to the // behavior for using LogOutput. You cannot specify both LogOutput and Logger // at the same time. Logger log.Logger // Size of Memberlist's internal channel which handles UDP messages. The // size of this determines the size of the queue which Memberlist will keep // while UDP messages are handled. HandoffQueueDepth int // Maximum number of bytes that memberlist will put in a packet (this // will be for UDP packets by default with a NetTransport). A safe value // for this is typically 1400 bytes (which is the default). However, // depending on your network's MTU (Maximum Transmission Unit) you may // be able to increase this to get more content into each gossip packet. // This is a legacy name for backward compatibility but should really be // called PacketBufferSize now that we have generalized the transport. UDPBufferSize int // DeadNodeReclaimTime controls the time before a dead node's name can be // reclaimed by one with a different address or port. By default, this is 0, // meaning nodes cannot be reclaimed this way. DeadNodeReclaimTime time.Duration // RequireNodeNames controls if the name of a node is required when sending // a message to that node. RequireNodeNames bool // CIDRsAllowed If nil, allow any connection (default), otherwise specify all networks // allowed to connect (you must specify IPv6/IPv4 separately) // Using [] will block all connections. CIDRsAllowed []net.IPNet // MetricLabels is a map of optional labels to apply to all metrics emitted. MetricLabels []metrics.Label // QueueCheckInterval is the interval at which we check the message // queue to apply the warning and max depth. QueueCheckInterval time.Duration // MsgpackUseNewTimeFormat when set to true, force the underlying msgpack // codec to use the new format of time.Time when encoding (used in // go-msgpack v1.1.5 by default). Decoding is not affected, as all // go-msgpack v2.1.0+ decoders know how to decode both formats. MsgpackUseNewTimeFormat bool } // ParseCIDRs return a possible empty list of all Network that have been parsed // In case of error, it returns succesfully parsed CIDRs and the last error found func ParseCIDRs(v []string) ([]net.IPNet, error) { nets := make([]net.IPNet, 0) if v == nil { return nets, nil } var errs error hasErrors := false for _, p := range v { _, net, err := net.ParseCIDR(strings.TrimSpace(p)) if err != nil { err = fmt.Errorf("invalid cidr: %s", p) errs = multierror.Append(errs, err) hasErrors = true } else { nets = append(nets, net) } } if !hasErrors { errs = nil } return nets, errs } // DefaultLANConfig returns a sane set of configurations for Memberlist. // It uses the hostname as the node name, and otherwise sets very conservative // values that are sane for most LAN environments. The default configuration // errs on the side of caution, choosing values that are optimized // for higher convergence at the cost of higher bandwidth usage. Regardless, // these values are a good starting point when getting started with memberlist. func DefaultLANConfig() Config { hostname, _ := os.Hostname() return &Config{ Name: hostname, BindAddr: "0.0.0.0", BindPort: 7946, AdvertiseAddr: "", AdvertisePort: 7946, ProtocolVersion: ProtocolVersion2Compatible, TCPTimeout: 10 * time.Second, // Timeout after 10 seconds IndirectChecks: 3, // Use 3 nodes for the indirect ping RetransmitMult: 4, // Retransmit a message 4 * log(N+1) nodes SuspicionMult: 4, // Suspect a node for 4 * log(N+1) * Interval SuspicionMaxTimeoutMult: 6, // For 10k nodes this will give a max timeout of 120 seconds PushPullInterval: 30 * time.Second, // Low frequency ProbeTimeout: 500 * time.Millisecond, // Reasonable RTT time for LAN ProbeInterval: 1 * time.Second, // Failure check every second DisableTcpPings: false, // TCP pings are safe, even with mixed versions AwarenessMaxMultiplier: 8, // Probe interval backs off to 8 seconds GossipNodes: 3, // Gossip to 3 nodes GossipInterval: 200 * time.Millisecond, // Gossip more rapidly GossipToTheDeadTime: 30 * time.Second, // Same as push/pull GossipVerifyIncoming: true, GossipVerifyOutgoing: true, EnableCompression: true, // Enable compression by default SecretKey: nil, Keyring: nil, DNSConfigPath: "/etc/resolv.conf", HandoffQueueDepth: 1024, UDPBufferSize: 1400, CIDRsAllowed: nil, // same as allow all QueueCheckInterval: 30 * time.Second, } } // DefaultWANConfig works like DefaultConfig, however it returns a configuration // that is optimized for most WAN environments. The default configuration is // still very conservative and errs on the side of caution. func DefaultWANConfig() Config { conf := DefaultLANConfig() conf.TCPTimeout = 30 time.Second conf.SuspicionMult = 6 conf.PushPullInterval = 60 * time.Second conf.ProbeTimeout = 3 * time.Second conf.ProbeInterval = 5 * time.Second conf.GossipNodes = 4 // Gossip less frequently, but to an additional node conf.GossipInterval = 500 * time.Millisecond conf.GossipToTheDeadTime = 60 * time.Second return conf } // IPMustBeChecked return true if IPAllowed must be called func (c Config) IPMustBeChecked() bool { return len(c.CIDRsAllowed) > 0 } // IPAllowed return an error if access to memberlist is denied func (c Config) IPAllowed(ip net.IP) error { if !c.IPMustBeChecked() { return nil } for _, n := range c.CIDRsAllowed { if n.Contains(ip) { return nil } } return fmt.Errorf("%s is not allowed", ip) } // DefaultLocalConfig works like DefaultConfig, however it returns a configuration // that is optimized for a local loopback environments. The default configuration is // still very conservative and errs on the side of caution. func DefaultLocalConfig() Config { conf := DefaultLANConfig() conf.TCPTimeout = time.Second conf.IndirectChecks = 1 conf.RetransmitMult = 2 conf.SuspicionMult = 3 conf.PushPullInterval = 15 time.Second conf.ProbeTimeout = 200 * time.Millisecond conf.ProbeInterval = time.Second conf.GossipInterval = 100 * time.Millisecond conf.GossipToTheDeadTime = 15 * time.Second return conf } // Returns whether or not encryption is enabled func (c *Config) EncryptionEnabled() bool { return c.Keyring != nil && len(c.Keyring.GetKeys()) > 0 }	15.68 KB	2025-05-25 12:27
📄 config_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "net" "testing" ) func Test_IsValidAddressDefaults(t testing.T) { tests := []string{ "127.0.0.1", "127.0.0.5", "10.0.0.9", "172.16.0.7", "192.168.2.1", "fe80::aede:48ff:fe00:1122", "::1", } config := DefaultLANConfig() for _, ip := range tests { localV4 := net.ParseIP(ip) if err := config.IPAllowed(localV4); err != nil { t.Fatalf("IP %s Localhost Should be accepted for LAN", ip) } } config = DefaultWANConfig() for _, ip := range tests { localV4 := net.ParseIP(ip) if err := config.IPAllowed(localV4); err != nil { t.Fatalf("IP %s Localhost Should be accepted for WAN", ip) } } } func Test_IsValidAddressOverride(t testing.T) { t.Parallel() cases := []struct { name string allow []string success []string fail []string }{ { name: "Default, nil allows all", allow: nil, success: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7", "::1"}, fail: []string{}, }, { name: "Only IPv4", allow: []string{"0.0.0.0/0"}, success: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7"}, fail: []string{"fe80::38bc:4dff:fe62:b1ae", "::1"}, }, { name: "Only IPv6", allow: []string{"::0/0"}, success: []string{"fe80::38bc:4dff:fe62:b1ae", "::1"}, fail: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7"}, }, { name: "Only 127.0.0.0/8 and ::1", allow: []string{"::1/128", "127.0.0.0/8"}, success: []string{"127.0.0.5", "::1"}, fail: []string{"::2", "178.250.0.187", "10.0.0.9", "192.168.1.7", "fe80::38bc:4dff:fe62:b1ae"}, }, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { config := DefaultLANConfig() var err error config.CIDRsAllowed, err = ParseCIDRs(testCase.allow) if err != nil { t.Fatalf("failed parsing %s", testCase.allow) } for _, ips := range testCase.success { ip := net.ParseIP(ips) if err := config.IPAllowed(ip); err != nil { t.Fatalf("Test case with %s should pass", ip) } } for _, ips := range testCase.fail { ip := net.ParseIP(ips) if err := config.IPAllowed(ip); err == nil { t.Fatalf("Test case with %s should fail", ip) } } }) } }	2.24 KB	2025-05-25 12:27
📄 conflict_delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist // ConflictDelegate is a used to inform a client that // a node has attempted to join which would result in a // name conflict. This happens if two clients are configured // with the same name but different addresses. type ConflictDelegate interface { // NotifyConflict is invoked when a name conflict is detected NotifyConflict(existing, other *Node) }	446 B	2025-05-25 12:27
📄 delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist // Delegate is the interface that clients must implement if they want to hook // into the gossip layer of Memberlist. All the methods must be thread-safe, // as they can and generally will be called concurrently. type Delegate interface { // NodeMeta is used to retrieve meta-data about the current node // when broadcasting an alive message. It's length is limited to // the given byte size. This metadata is available in the Node structure. NodeMeta(limit int) []byte // NotifyMsg is called when a user-data message is received. // Care should be taken that this method does not block, since doing // so would block the entire UDP packet receive loop. Additionally, the byte // slice may be modified after the call returns, so it should be copied if needed NotifyMsg([]byte) // GetBroadcasts is called when user data messages can be broadcast. // It can return a list of buffers to send. Each buffer should assume an // overhead as provided with a limit on the total byte size allowed. // The total byte size of the resulting data to send must not exceed // the limit. Care should be taken that this method does not block, // since doing so would block the entire UDP packet receive loop. GetBroadcasts(overhead, limit int) [][]byte // LocalState is used for a TCP Push/Pull. This is sent to // the remote side in addition to the membership information. Any // data can be sent here. See MergeRemoteState as well. The `join` // boolean indicates this is for a join instead of a push/pull. LocalState(join bool) []byte // MergeRemoteState is invoked after a TCP Push/Pull. This is the // state received from the remote side and is the result of the // remote side's LocalState call. The 'join' // boolean indicates this is for a join instead of a push/pull. MergeRemoteState(buf []byte, join bool) }	1.88 KB	2025-05-25 12:27
📄 event_delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist // EventDelegate is a simpler delegate that is used only to receive // notifications about members joining and leaving. The methods in this // delegate may be called by multiple goroutines, but never concurrently. // This allows you to reason about ordering. type EventDelegate interface { // NotifyJoin is invoked when a node is detected to have joined. // The Node argument must not be modified. NotifyJoin(Node) // NotifyLeave is invoked when a node is detected to have left. // The Node argument must not be modified. NotifyLeave(Node) // NotifyUpdate is invoked when a node is detected to have // updated, usually involving the meta data. The Node argument // must not be modified. NotifyUpdate(Node) } // ChannelEventDelegate is used to enable an application to receive // events about joins and leaves over a channel instead of a direct // function call. // // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type ChannelEventDelegate struct { Ch chan<- NodeEvent } // NodeEventType are the types of events that can be sent from the // ChannelEventDelegate. type NodeEventType int const ( NodeJoin NodeEventType = iota NodeLeave NodeUpdate ) // NodeEvent is a single event related to node activity in the memberlist. // The Node member of this struct must not be directly modified. It is passed // as a pointer to avoid unnecessary copies. If you wish to modify the node, // make a copy first. type NodeEvent struct { Event NodeEventType Node Node } func (c ChannelEventDelegate) NotifyJoin(n Node) { node := n c.Ch <- NodeEvent{NodeJoin, &node} } func (c ChannelEventDelegate) NotifyLeave(n Node) { node := n c.Ch <- NodeEvent{NodeLeave, &node} } func (c ChannelEventDelegate) NotifyUpdate(n Node) { node := *n c.Ch <- NodeEvent{NodeUpdate, &node} }	1.94 KB	2025-05-25 12:27
📄 go.mod module github.com/hashicorp/memberlist go 1.20 require ( github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c github.com/hashicorp/go-msgpack/v2 v2.1.1 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-sockaddr v1.0.0 github.com/miekg/dns v1.1.26 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 github.com/stretchr/testify v1.2.2 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.0 // indirect github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.16.0 // indirect golang.org/x/sys v0.13.0 // indirect )	900 B	2025-05-25 12:27
📄 go.sum github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I= github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=	4.77 KB	2025-05-25 12:27
📄 integ_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "log" "os" "testing" "time" ) // CheckInteg will skip a test if integration testing is not enabled. func CheckInteg(t testing.T) { if !IsInteg() { t.SkipNow() } } // IsInteg returns a boolean telling you if we're in integ testing mode. func IsInteg() bool { return os.Getenv("INTEG_TESTS") != "" } // Tests the memberlist by creating a cluster of 100 nodes // and checking that we get strong convergence of changes. func TestMemberlist_Integ(t testing.T) { CheckInteg(t) num := 16 var members []Memberlist secret := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} eventCh := make(chan NodeEvent, num) addr := "127.0.0.1" for i := 0; i < num; i++ { c := DefaultLANConfig() c.Name = fmt.Sprintf("%s:%d", addr, 12345+i) c.BindAddr = addr c.BindPort = 12345 + i c.ProbeInterval = 20 time.Millisecond c.ProbeTimeout = 100 * time.Millisecond c.GossipInterval = 20 * time.Millisecond c.PushPullInterval = 200 * time.Millisecond c.SecretKey = secret c.Logger = log.New(os.Stderr, c.Name, log.LstdFlags) if i == 0 { c.Events = &ChannelEventDelegate{eventCh} } m, err := Create(c) if err != nil { t.Fatalf("unexpected err: %s", err) } defer m.Shutdown() members = append(members, m) if i > 0 { last := members[i-1] num, err := m.Join([]string{last.config.Name + "/" + last.config.Name}) if num == 0 \|\| err != nil { t.Fatalf("unexpected err: %s", err) } } } // Wait and print debug info breakTimer := time.After(250 * time.Millisecond) WAIT: for { select { case e := <-eventCh: if e.Event == NodeJoin { t.Logf("[DEBUG] Node join: %v (%d)", e.Node, members[0].NumMembers()) } else { t.Logf("[DEBUG] Node leave: %v (%d)", e.Node, members[0].NumMembers()) } case <-breakTimer: break WAIT } } for idx, m := range members { got := m.NumMembers() if got != num { t.Errorf("bad num members at idx %d. Expected %d. Got %d.", idx, num, got) } } }	2.04 KB	2025-05-25 12:27
📁 internal	-	2025-05-25 12:27
📄 keyring.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "sync" ) type Keyring struct { // Keys stores the key data used during encryption and decryption. It is // ordered in such a way where the first key (index 0) is the primary key, // which is used for encrypting messages, and is the first key tried during // message decryption. keys [][]byte // The keyring lock is used while performing IO operations on the keyring. l sync.Mutex } // Init allocates substructures func (k Keyring) init() { k.keys = make([][]byte, 0) } // NewKeyring constructs a new container for a set of encryption keys. The // keyring contains all key data used internally by memberlist. // // While creating a new keyring, you must do one of: // - Omit keys and primary key, effectively disabling encryption // - Pass a set of keys plus the primary key // - Pass only a primary key // // If only a primary key is passed, then it will be automatically added to the // keyring. If creating a keyring with multiple keys, one key must be designated // primary by passing it as the primaryKey. If the primaryKey does not exist in // the list of secondary keys, it will be automatically added at position 0. // // A key should be either 16, 24, or 32 bytes to select AES-128, // AES-192, or AES-256. func NewKeyring(keys [][]byte, primaryKey []byte) (Keyring, error) { keyring := &Keyring{} keyring.init() if len(keys) > 0 \|\| len(primaryKey) > 0 { if len(primaryKey) == 0 { return nil, fmt.Errorf("Empty primary key not allowed") } if err := keyring.AddKey(primaryKey); err != nil { return nil, err } for _, key := range keys { if err := keyring.AddKey(key); err != nil { return nil, err } } } return keyring, nil } // ValidateKey will check to see if the key is valid and returns an error if not. // // key should be either 16, 24, or 32 bytes to select AES-128, // AES-192, or AES-256. func ValidateKey(key []byte) error { if l := len(key); l != 16 && l != 24 && l != 32 { return fmt.Errorf("key size must be 16, 24 or 32 bytes") } return nil } // AddKey will install a new key on the ring. Adding a key to the ring will make // it available for use in decryption. If the key already exists on the ring, // this function will just return noop. // // key should be either 16, 24, or 32 bytes to select AES-128, // AES-192, or AES-256. func (k Keyring) AddKey(key []byte) error { if err := ValidateKey(key); err != nil { return err } // No-op if key is already installed for _, installedKey := range k.keys { if bytes.Equal(installedKey, key) { return nil } } keys := append(k.keys, key) primaryKey := k.GetPrimaryKey() if primaryKey == nil { primaryKey = key } k.installKeys(keys, primaryKey) return nil } // UseKey changes the key used to encrypt messages. This is the only key used to // encrypt messages, so peers should know this key before this method is called. func (k Keyring) UseKey(key []byte) error { for _, installedKey := range k.keys { if bytes.Equal(key, installedKey) { k.installKeys(k.keys, key) return nil } } return fmt.Errorf("Requested key is not in the keyring") } // RemoveKey drops a key from the keyring. This will return an error if the key // requested for removal is currently at position 0 (primary key). func (k Keyring) RemoveKey(key []byte) error { if bytes.Equal(key, k.keys[0]) { return fmt.Errorf("Removing the primary key is not allowed") } for i, installedKey := range k.keys { if bytes.Equal(key, installedKey) { keys := append(k.keys[:i], k.keys[i+1:]...) k.installKeys(keys, k.keys[0]) } } return nil } // installKeys will take out a lock on the keyring, and replace the keys with a // new set of keys. The key indicated by primaryKey will be installed as the new // primary key. func (k Keyring) installKeys(keys [][]byte, primaryKey []byte) { k.l.Lock() defer k.l.Unlock() newKeys := [][]byte{primaryKey} for _, key := range keys { if !bytes.Equal(key, primaryKey) { newKeys = append(newKeys, key) } } k.keys = newKeys } // GetKeys returns the current set of keys on the ring. func (k Keyring) GetKeys() [][]byte { k.l.Lock() defer k.l.Unlock() return k.keys } // GetPrimaryKey returns the key on the ring at position 0. This is the key used // for encrypting messages, and is the first key tried for decrypting messages. func (k Keyring) GetPrimaryKey() (key []byte) { k.l.Lock() defer k.l.Unlock() if len(k.keys) > 0 { key = k.keys[0] } return }	4.47 KB	2025-05-25 12:27
📄 keyring_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "testing" ) var TestKeys [][]byte = [][]byte{ []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, []byte{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, []byte{8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}, } func TestKeyring_EmptyRing(t testing.T) { // Keyrings can be created with no encryption keys (disabled encryption) keyring, err := NewKeyring(nil, nil) if err != nil { t.Fatalf("err: %s", err) } keys := keyring.GetKeys() if len(keys) != 0 { t.Fatalf("Expected 0 keys but have %d", len(keys)) } } func TestKeyring_PrimaryOnly(t testing.T) { // Keyrings can be created using only a primary key keyring, err := NewKeyring(nil, TestKeys[0]) if err != nil { t.Fatalf("err: %s", err) } keys := keyring.GetKeys() if len(keys) != 1 { t.Fatalf("Expected 1 key but have %d", len(keys)) } } func TestKeyring_GetPrimaryKey(t testing.T) { keyring, err := NewKeyring(TestKeys, TestKeys[1]) if err != nil { t.Fatalf("err: %s", err) } // GetPrimaryKey returns correct key primaryKey := keyring.GetPrimaryKey() if !bytes.Equal(primaryKey, TestKeys[1]) { t.Fatalf("Unexpected primary key: %v", primaryKey) } } func TestKeyring_AddRemoveUse(t testing.T) { keyring, err := NewKeyring(nil, TestKeys[1]) if err != nil { t.Fatalf("err :%s", err) } // Use non-existent key throws error if err := keyring.UseKey(TestKeys[2]); err == nil { t.Fatalf("Expected key not installed error") } // Add key to ring if err := keyring.AddKey(TestKeys[2]); err != nil { t.Fatalf("err: %s", err) } keys := keyring.GetKeys() if !bytes.Equal(keys[0], TestKeys[1]) { t.Fatalf("Unexpected primary key change") } if len(keys) != 2 { t.Fatalf("Expected 2 keys but have %d", len(keys)) } // Use key that exists should succeed if err := keyring.UseKey(TestKeys[2]); err != nil { t.Fatalf("err: %s", err) } primaryKey := keyring.GetPrimaryKey() if !bytes.Equal(primaryKey, TestKeys[2]) { t.Fatalf("Unexpected primary key: %v", primaryKey) } // Removing primary key should fail if err := keyring.RemoveKey(TestKeys[2]); err == nil { t.Fatalf("Expected primary key removal error") } // Removing non-primary key should succeed if err := keyring.RemoveKey(TestKeys[1]); err != nil { t.Fatalf("err: %s", err) } keys = keyring.GetKeys() if len(keys) != 1 { t.Fatalf("Expected 1 key but have %d", len(keys)) } } func TestKeyRing_MultiKeyEncryptDecrypt(t *testing.T) { plaintext := []byte("this is a plain text message") extra := []byte("random data") keyring, err := NewKeyring(TestKeys, TestKeys[0]) if err != nil { t.Fatalf("err: %s", err) } // First encrypt using the primary key and make sure we can decrypt var buf bytes.Buffer err = encryptPayload(1, TestKeys[0], plaintext, extra, &buf) if err != nil { t.Fatalf("err: %v", err) } msg, err := decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) if err != nil { t.Fatalf("err: %v", err) } if !bytes.Equal(msg, plaintext) { t.Fatalf("bad: %v", msg) } // Now encrypt with a secondary key and try decrypting again. buf.Reset() err = encryptPayload(1, TestKeys[2], plaintext, extra, &buf) if err != nil { t.Fatalf("err: %v", err) } msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) if err != nil { t.Fatalf("err: %v", err) } if !bytes.Equal(msg, plaintext) { t.Fatalf("bad: %v", msg) } // Remove a key from the ring, and then try decrypting again if err := keyring.RemoveKey(TestKeys[2]); err != nil { t.Fatalf("err: %s", err) } msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) if err == nil { t.Fatalf("Expected no keys to decrypt message") } }	3.69 KB	2025-05-25 12:27
📄 label.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bufio" "fmt" "io" "net" ) // General approach is to prefix all packets and streams with the same structure: // // magic type byte (244): uint8 // length of label name: uint8 (because labels can't be longer than 255 bytes) // label name: []uint8 // LabelMaxSize is the maximum length of a packet or stream label. const LabelMaxSize = 255 // AddLabelHeaderToPacket prefixes outgoing packets with the correct header if // the label is not empty. func AddLabelHeaderToPacket(buf []byte, label string) ([]byte, error) { if label == "" { return buf, nil } if len(label) > LabelMaxSize { return nil, fmt.Errorf("label %q is too long", label) } return makeLabelHeader(label, buf), nil } // RemoveLabelHeaderFromPacket removes any label header from the provided // packet and returns it along with the remaining packet contents. func RemoveLabelHeaderFromPacket(buf []byte) (newBuf []byte, label string, err error) { if len(buf) == 0 { return buf, "", nil // can't possibly be labeled } // [type:byte] [size:byte] [size bytes] msgType := messageType(buf[0]) if msgType != hasLabelMsg { return buf, "", nil } if len(buf) < 2 { return nil, "", fmt.Errorf("cannot decode label; packet has been truncated") } size := int(buf[1]) if size < 1 { return nil, "", fmt.Errorf("label header cannot be empty when present") } if len(buf) < 2+size { return nil, "", fmt.Errorf("cannot decode label; packet has been truncated") } label = string(buf[2 : 2+size]) newBuf = buf[2+size:] return newBuf, label, nil } // AddLabelHeaderToStream prefixes outgoing streams with the correct header if // the label is not empty. func AddLabelHeaderToStream(conn net.Conn, label string) error { if label == "" { return nil } if len(label) > LabelMaxSize { return fmt.Errorf("label %q is too long", label) } header := makeLabelHeader(label, nil) _, err := conn.Write(header) return err } // RemoveLabelHeaderFromStream removes any label header from the beginning of // the stream if present and returns it along with an updated conn with that // header removed. // // Note that on error it is the caller's responsibility to close the // connection. func RemoveLabelHeaderFromStream(conn net.Conn) (net.Conn, string, error) { br := bufio.NewReader(conn) // First check for the type byte. peeked, err := br.Peek(1) if err != nil { if err == io.EOF { // It is safe to return the original net.Conn at this point because // it never contained any data in the first place so we don't have // to splice the buffer into the conn because both are empty. return conn, "", nil } return nil, "", err } msgType := messageType(peeked[0]) if msgType != hasLabelMsg { conn, err = newPeekedConnFromBufferedReader(conn, br, 0) return conn, "", err } // We are guaranteed to get a size byte as well. peeked, err = br.Peek(2) if err != nil { if err == io.EOF { return nil, "", fmt.Errorf("cannot decode label; stream has been truncated") } return nil, "", err } size := int(peeked[1]) if size < 1 { return nil, "", fmt.Errorf("label header cannot be empty when present") } // NOTE: we don't have to check this against LabelMaxSize because a byte // already has a max value of 255. // Once we know the size we can peek the label as well. Note that since we // are using the default bufio.Reader size of 4096, the entire label header // fits in the initial buffer fill so this should be free. peeked, err = br.Peek(2 + size) if err != nil { if err == io.EOF { return nil, "", fmt.Errorf("cannot decode label; stream has been truncated") } return nil, "", err } label := string(peeked[2 : 2+size]) conn, err = newPeekedConnFromBufferedReader(conn, br, 2+size) if err != nil { return nil, "", err } return conn, label, nil } // newPeekedConnFromBufferedReader will splice the buffer contents after the // offset into the provided net.Conn and return the result so that the rest of // the buffer contents are returned first when reading from the returned // peekedConn before moving on to the unbuffered conn contents. func newPeekedConnFromBufferedReader(conn net.Conn, br bufio.Reader, offset int) (peekedConn, error) { // Extract any of the readahead buffer. peeked, err := br.Peek(br.Buffered()) if err != nil { return nil, err } return &peekedConn{ Peeked: peeked[offset:], Conn: conn, }, nil } func makeLabelHeader(label string, rest []byte) []byte { newBuf := make([]byte, 2, 2+len(label)+len(rest)) newBuf[0] = byte(hasLabelMsg) newBuf[1] = byte(len(label)) newBuf = append(newBuf, []byte(label)...) if len(rest) > 0 { newBuf = append(newBuf, []byte(rest)...) } return newBuf } func labelOverhead(label string) int { if label == "" { return 0 } return 2 + len(label) }	4.79 KB	2025-05-25 12:27
📄 label_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "io" "net" "strings" "testing" "github.com/stretchr/testify/require" ) func TestAddLabelHeaderToPacket(t testing.T) { type testcase struct { buf []byte label string expectPacket []byte expectErr string } run := func(t testing.T, tc testcase) { got, err := AddLabelHeaderToPacket(tc.buf, tc.label) if tc.expectErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErr) } else { require.NoError(t, err) require.Equal(t, tc.expectPacket, got) } } longLabel := strings.Repeat("a", 255) cases := map[string]testcase{ "nil buf with no label": testcase{ buf: nil, label: "", expectPacket: nil, }, "nil buf with label": testcase{ buf: nil, label: "foo", expectPacket: append([]byte{byte(hasLabelMsg), 3}, []byte("foo")...), }, "message with label": testcase{ buf: []byte("something"), label: "foo", expectPacket: append([]byte{byte(hasLabelMsg), 3}, []byte("foosomething")...), }, "message with no label": testcase{ buf: []byte("something"), label: "", expectPacket: []byte("something"), }, "message with almost too long label": testcase{ buf: []byte("something"), label: longLabel, expectPacket: append([]byte{byte(hasLabelMsg), 255}, []byte(longLabel+"something")...), }, "label too long by one byte": testcase{ buf: []byte("something"), label: longLabel + "x", expectErr: `label "` + longLabel + `x" is too long`, }, } for name, tc := range cases { t.Run(name, func(t testing.T) { run(t, tc) }) } } func TestRemoveLabelHeaderFromPacket(t testing.T) { type testcase struct { buf []byte expectLabel string expectPacket []byte expectErr string } run := func(t testing.T, tc testcase) { gotBuf, gotLabel, err := RemoveLabelHeaderFromPacket(tc.buf) if tc.expectErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErr) } else { require.NoError(t, err) require.Equal(t, tc.expectPacket, gotBuf) require.Equal(t, tc.expectLabel, gotLabel) } } cases := map[string]testcase{ "empty buf": testcase{ buf: []byte{}, expectLabel: "", expectPacket: []byte{}, }, "ping with no label": testcase{ buf: buildBuffer(t, pingMsg, "blah"), expectLabel: "", expectPacket: buildBuffer(t, pingMsg, "blah"), }, "error with no label": testcase{ // 2021-10: largest standard message type buf: buildBuffer(t, errMsg, "blah"), expectLabel: "", expectPacket: buildBuffer(t, errMsg, "blah"), }, "v1 encrypt with no label": testcase{ // 2021-10: highest encryption version buf: buildBuffer(t, maxEncryptionVersion, "blah"), expectLabel: "", expectPacket: buildBuffer(t, maxEncryptionVersion, "blah"), }, "buf too small for label": testcase{ buf: buildBuffer(t, hasLabelMsg, "x"), expectErr: `cannot decode label; packet has been truncated`, }, "buf too small for label size": testcase{ buf: buildBuffer(t, hasLabelMsg), expectErr: `cannot decode label; packet has been truncated`, }, "label empty": testcase{ buf: buildBuffer(t, hasLabelMsg, 0, "x"), expectErr: `label header cannot be empty when present`, }, "label truncated": testcase{ buf: buildBuffer(t, hasLabelMsg, 2, "x"), expectErr: `cannot decode label; packet has been truncated`, }, "ping with label": testcase{ buf: buildBuffer(t, hasLabelMsg, 3, "abc", pingMsg, "blah"), expectLabel: "abc", expectPacket: buildBuffer(t, pingMsg, "blah"), }, "error with label": testcase{ // 2021-10: largest standard message type buf: buildBuffer(t, hasLabelMsg, 3, "abc", errMsg, "blah"), expectLabel: "abc", expectPacket: buildBuffer(t, errMsg, "blah"), }, "v1 encrypt with label": testcase{ // 2021-10: highest encryption version buf: buildBuffer(t, hasLabelMsg, 3, "abc", maxEncryptionVersion, "blah"), expectLabel: "abc", expectPacket: buildBuffer(t, maxEncryptionVersion, "blah"), }, } for name, tc := range cases { t.Run(name, func(t testing.T) { run(t, tc) }) } } func TestAddLabelHeaderToStream(t testing.T) { type testcase struct { label string expectData []byte expectErr string } suffixData := "EXTRA DATA" run := func(t testing.T, tc testcase) { server, client := net.Pipe() defer server.Close() defer client.Close() var ( dataCh = make(chan []byte, 1) errCh = make(chan error, 1) ) go func() { var buf bytes.Buffer _, err := io.Copy(&buf, server) if err != nil { errCh <- err } dataCh <- buf.Bytes() }() err := AddLabelHeaderToStream(client, tc.label) if tc.expectErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErr) return } require.NoError(t, err) client.Write([]byte(suffixData)) client.Close() expect := make([]byte, 0, len(suffixData)+len(tc.expectData)) expect = append(expect, tc.expectData...) expect = append(expect, suffixData...) select { case err := <-errCh: require.NoError(t, err) case got := <-dataCh: require.Equal(t, expect, got) } } longLabel := strings.Repeat("a", 255) cases := map[string]testcase{ "no label": testcase{ label: "", expectData: nil, }, "with label": testcase{ label: "foo", expectData: buildBuffer(t, hasLabelMsg, 3, "foo"), }, "almost too long label": testcase{ label: longLabel, expectData: buildBuffer(t, hasLabelMsg, 255, longLabel), }, "label too long by one byte": testcase{ label: longLabel + "x", expectErr: `label "` + longLabel + `x" is too long`, }, } for name, tc := range cases { t.Run(name, func(t testing.T) { run(t, tc) }) } } func TestRemoveLabelHeaderFromStream(t testing.T) { type testcase struct { buf []byte expectLabel string expectData []byte expectErr string } run := func(t testing.T, tc testcase) { server, client := net.Pipe() defer server.Close() defer client.Close() var ( errCh = make(chan error, 1) ) go func() { _, err := server.Write(tc.buf) if err != nil { errCh <- err } server.Close() }() newConn, gotLabel, err := RemoveLabelHeaderFromStream(client) if tc.expectErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErr) return } require.NoError(t, err) gotBuf, err := io.ReadAll(newConn) require.NoError(t, err) require.Equal(t, tc.expectData, gotBuf) require.Equal(t, tc.expectLabel, gotLabel) } cases := map[string]testcase{ "empty buf": testcase{ buf: []byte{}, expectLabel: "", expectData: []byte{}, }, "ping with no label": testcase{ buf: buildBuffer(t, pingMsg, "blah"), expectLabel: "", expectData: buildBuffer(t, pingMsg, "blah"), }, "error with no label": testcase{ // 2021-10: largest standard message type buf: buildBuffer(t, errMsg, "blah"), expectLabel: "", expectData: buildBuffer(t, errMsg, "blah"), }, "v1 encrypt with no label": testcase{ // 2021-10: highest encryption version buf: buildBuffer(t, maxEncryptionVersion, "blah"), expectLabel: "", expectData: buildBuffer(t, maxEncryptionVersion, "blah"), }, "buf too small for label": testcase{ buf: buildBuffer(t, hasLabelMsg, "x"), expectErr: `cannot decode label; stream has been truncated`, }, "buf too small for label size": testcase{ buf: buildBuffer(t, hasLabelMsg), expectErr: `cannot decode label; stream has been truncated`, }, "label empty": testcase{ buf: buildBuffer(t, hasLabelMsg, 0, "x"), expectErr: `label header cannot be empty when present`, }, "label truncated": testcase{ buf: buildBuffer(t, hasLabelMsg, 2, "x"), expectErr: `cannot decode label; stream has been truncated`, }, "ping with label": testcase{ buf: buildBuffer(t, hasLabelMsg, 3, "abc", pingMsg, "blah"), expectLabel: "abc", expectData: buildBuffer(t, pingMsg, "blah"), }, "error with label": testcase{ // 2021-10: largest standard message type buf: buildBuffer(t, hasLabelMsg, 3, "abc", errMsg, "blah"), expectLabel: "abc", expectData: buildBuffer(t, errMsg, "blah"), }, "v1 encrypt with label": testcase{ // 2021-10: highest encryption version buf: buildBuffer(t, hasLabelMsg, 3, "abc", maxEncryptionVersion, "blah"), expectLabel: "abc", expectData: buildBuffer(t, maxEncryptionVersion, "blah"), }, } for name, tc := range cases { t.Run(name, func(t testing.T) { run(t, tc) }) } } func buildBuffer(t testing.T, stuff ...interface{}) []byte { var buf bytes.Buffer for _, item := range stuff { switch x := item.(type) { case int: x2 := uint(x) if x2 > 255 { t.Fatalf("int is too big") } buf.WriteByte(byte(x2)) case byte: buf.WriteByte(byte(x)) case messageType: buf.WriteByte(byte(x)) case encryptionVersion: buf.WriteByte(byte(x)) case string: buf.Write([]byte(x)) case []byte: buf.Write(x) default: t.Fatalf("unexpected type %T", item) } } return buf.Bytes() } func TestLabelOverhead(t testing.T) { require.Equal(t, 0, labelOverhead("")) require.Equal(t, 3, labelOverhead("a")) require.Equal(t, 9, labelOverhead("abcdefg")) }	9.36 KB	2025-05-25 12:27
📄 logging.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "net" ) func LogAddress(addr net.Addr) string { if addr == nil { return "from=<unknown address>" } return fmt.Sprintf("from=%s", addr.String()) } func LogStringAddress(addr string) string { if addr == "" { return "from=<unknown address>" } return fmt.Sprintf("from=%s", addr) } func LogConn(conn net.Conn) string { if conn == nil { return LogAddress(nil) } return LogAddress(conn.RemoteAddr()) }	524 B	2025-05-25 12:27
📄 logging_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "net" "testing" ) func TestLogging_Address(t testing.T) { s := LogAddress(nil) if s != "from=<unknown address>" { t.Fatalf("bad: %s", s) } addr, err := net.ResolveIPAddr("ip4", "127.0.0.1") if err != nil { t.Fatalf("err: %v", err) } s = LogAddress(addr) if s != "from=127.0.0.1" { t.Fatalf("bad: %s", s) } } func TestLogging_Conn(t testing.T) { s := LogConn(nil) if s != "from=<unknown address>" { t.Fatalf("bad: %s", s) } ln, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("err: %v", err) } conn, err := net.Dial("tcp", ln.Addr().String()) if err != nil { t.Fatalf("err: %v", err) } defer conn.Close() s = LogConn(conn) if s != fmt.Sprintf("from=%s", conn.RemoteAddr().String()) { t.Fatalf("bad: %s", s) } }	871 B	2025-05-25 12:27
📄 memberlist.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 /* memberlist is a library that manages cluster membership and member failure detection using a gossip based protocol. The use cases for such a library are far-reaching: all distributed systems require membership, and memberlist is a re-usable solution to managing cluster membership and node failure detection. memberlist is eventually consistent but converges quickly on average. The speed at which it converges can be heavily tuned via various knobs on the protocol. Node failures are detected and network partitions are partially tolerated by attempting to communicate to potentially dead nodes through multiple routes. / package memberlist import ( "container/list" "errors" "fmt" "log" "net" "os" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/armon/go-metrics" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-sockaddr" "github.com/miekg/dns" ) var errNodeNamesAreRequired = errors.New("memberlist: node names are required by configuration but one was not provided") type Memberlist struct { sequenceNum uint32 // Local sequence number incarnation uint32 // Local incarnation number numNodes uint32 // Number of known nodes (estimate) pushPullReq uint32 // Number of push/pull requests advertiseLock sync.RWMutex advertiseAddr net.IP advertisePort uint16 config Config shutdown int32 // Used as an atomic boolean value shutdownCh chan struct{} leave int32 // Used as an atomic boolean value leaveBroadcast chan struct{} shutdownLock sync.Mutex // Serializes calls to Shutdown leaveLock sync.Mutex // Serializes calls to Leave transport NodeAwareTransport handoffCh chan struct{} highPriorityMsgQueue list.List lowPriorityMsgQueue list.List msgQueueLock sync.Mutex nodeLock sync.RWMutex nodes []nodeState // Known nodes nodeMap map[string]nodeState // Maps Node.Name -> NodeState nodeTimers map[string]suspicion // Maps Node.Name -> suspicion timer awareness awareness tickerLock sync.Mutex tickers []time.Ticker stopTick chan struct{} probeIndex int ackLock sync.Mutex ackHandlers map[uint32]ackHandler broadcasts TransmitLimitedQueue logger log.Logger // metricLabels is the slice of labels to put on all emitted metrics metricLabels []metrics.Label } // BuildVsnArray creates the array of Vsn func (conf Config) BuildVsnArray() []uint8 { return []uint8{ ProtocolVersionMin, ProtocolVersionMax, conf.ProtocolVersion, conf.DelegateProtocolMin, conf.DelegateProtocolMax, conf.DelegateProtocolVersion, } } // newMemberlist creates the network listeners. // Does not schedule execution of background maintenance. func newMemberlist(conf Config) (Memberlist, error) { if conf.ProtocolVersion < ProtocolVersionMin { return nil, fmt.Errorf("Protocol version '%d' too low. Must be in range: [%d, %d]", conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax) } else if conf.ProtocolVersion > ProtocolVersionMax { return nil, fmt.Errorf("Protocol version '%d' too high. Must be in range: [%d, %d]", conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax) } if len(conf.SecretKey) > 0 { if conf.Keyring == nil { keyring, err := NewKeyring(nil, conf.SecretKey) if err != nil { return nil, err } conf.Keyring = keyring } else { if err := conf.Keyring.AddKey(conf.SecretKey); err != nil { return nil, err } if err := conf.Keyring.UseKey(conf.SecretKey); err != nil { return nil, err } } } if conf.LogOutput != nil && conf.Logger != nil { return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") } logDest := conf.LogOutput if logDest == nil { logDest = os.Stderr } logger := conf.Logger if logger == nil { logger = log.New(logDest, "", log.LstdFlags) } // Set up a network transport by default if a custom one wasn't given // by the config. transport := conf.Transport if transport == nil { nc := &NetTransportConfig{ BindAddrs: []string{conf.BindAddr}, BindPort: conf.BindPort, Logger: logger, MetricLabels: conf.MetricLabels, } // See comment below for details about the retry in here. makeNetRetry := func(limit int) (NetTransport, error) { var err error for try := 0; try < limit; try++ { var nt NetTransport if nt, err = NewNetTransport(nc); err == nil { return nt, nil } if strings.Contains(err.Error(), "address already in use") { logger.Printf("[DEBUG] memberlist: Got bind error: %v", err) continue } } return nil, fmt.Errorf("failed to obtain an address: %v", err) } // The dynamic bind port operation is inherently racy because // even though we are using the kernel to find a port for us, we // are attempting to bind multiple protocols (and potentially // multiple addresses) with the same port number. We build in a // few retries here since this often gets transient errors in // busy unit tests. limit := 1 if conf.BindPort == 0 { limit = 10 } nt, err := makeNetRetry(limit) if err != nil { return nil, fmt.Errorf("Could not set up network transport: %v", err) } if conf.BindPort == 0 { port := nt.GetAutoBindPort() conf.BindPort = port conf.AdvertisePort = port logger.Printf("[DEBUG] memberlist: Using dynamic bind port %d", port) } transport = nt } nodeAwareTransport, ok := transport.(NodeAwareTransport) if !ok { logger.Printf("[DEBUG] memberlist: configured Transport is not a NodeAwareTransport and some features may not work as desired") nodeAwareTransport = &shimNodeAwareTransport{transport} } if len(conf.Label) > LabelMaxSize { return nil, fmt.Errorf("could not use %q as a label: too long", conf.Label) } if conf.Label != "" { nodeAwareTransport = &labelWrappedTransport{ label: conf.Label, NodeAwareTransport: nodeAwareTransport, } } m := &Memberlist{ config: conf, shutdownCh: make(chan struct{}), leaveBroadcast: make(chan struct{}, 1), transport: nodeAwareTransport, handoffCh: make(chan struct{}, 1), highPriorityMsgQueue: list.New(), lowPriorityMsgQueue: list.New(), nodeMap: make(map[string]nodeState), nodeTimers: make(map[string]suspicion), awareness: newAwareness(conf.AwarenessMaxMultiplier, conf.MetricLabels), ackHandlers: make(map[uint32]ackHandler), broadcasts: &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult}, logger: logger, metricLabels: conf.MetricLabels, } m.broadcasts.NumNodes = func() int { return m.estNumNodes() } // Get the final advertise address from the transport, which may need // to see which address we bound to. We'll refresh this each time we // send out an alive message. if _, _, err := m.refreshAdvertise(); err != nil { return nil, err } go m.streamListen() go m.packetListen() go m.packetHandler() go m.checkBroadcastQueueDepth() return m, nil } // Create will create a new Memberlist using the given configuration. // This will not connect to any other node (see Join) yet, but will start // all the listeners to allow other nodes to join this memberlist. // After creating a Memberlist, the configuration given should not be // modified by the user anymore. func Create(conf Config) (Memberlist, error) { m, err := newMemberlist(conf) if err != nil { return nil, err } if err := m.setAlive(); err != nil { m.Shutdown() return nil, err } m.schedule() return m, nil } // Join is used to take an existing Memberlist and attempt to join a cluster // by contacting all the given hosts and performing a state sync. Initially, // the Memberlist only contains our own state, so doing this will cause // remote nodes to become aware of the existence of this node, effectively // joining the cluster. // // This returns the number of hosts successfully contacted and an error if // none could be reached. If an error is returned, the node did not successfully // join the cluster. func (m Memberlist) Join(existing []string) (int, error) { numSuccess := 0 var errs error for _, exist := range existing { addrs, err := m.resolveAddr(exist) if err != nil { err = fmt.Errorf("Failed to resolve %s: %v", exist, err) errs = multierror.Append(errs, err) m.logger.Printf("[WARN] memberlist: %v", err) continue } for _, addr := range addrs { hp := joinHostPort(addr.ip.String(), addr.port) a := Address{Addr: hp, Name: addr.nodeName} if err := m.pushPullNode(a, true); err != nil { err = fmt.Errorf("Failed to join %s: %v", a.Addr, err) errs = multierror.Append(errs, err) m.logger.Printf("[DEBUG] memberlist: %v", err) continue } numSuccess++ } } if numSuccess > 0 { errs = nil } return numSuccess, errs } // ipPort holds information about a node we want to try to join. type ipPort struct { ip net.IP port uint16 nodeName string // optional } // tcpLookupIP is a helper to initiate a TCP-based DNS lookup for the given host. // The built-in Go resolver will do a UDP lookup first, and will only use TCP if // the response has the truncate bit set, which isn't common on DNS servers like // Consul's. By doing the TCP lookup directly, we get the best chance for the // largest list of hosts to join. Since joins are relatively rare events, it's ok // to do this rather expensive operation. func (m Memberlist) tcpLookupIP(host string, defaultPort uint16, nodeName string) ([]ipPort, error) { // Don't attempt any TCP lookups against non-fully qualified domain // names, since those will likely come from the resolv.conf file. if !strings.Contains(host, ".") { return nil, nil } // Make sure the domain name is terminated with a dot (we know there's // at least one character at this point). dn := host if dn[len(dn)-1] != '.' { dn = dn + "." } // See if we can find a server to try. cc, err := dns.ClientConfigFromFile(m.config.DNSConfigPath) if err != nil { return nil, err } if len(cc.Servers) > 0 { // We support host:port in the DNS config, but need to add the // default port if one is not supplied. server := cc.Servers[0] if !hasPort(server) { server = net.JoinHostPort(server, cc.Port) } // Do the lookup. c := new(dns.Client) c.Net = "tcp" msg := new(dns.Msg) msg.SetQuestion(dn, dns.TypeANY) in, _, err := c.Exchange(msg, server) if err != nil { return nil, err } // Handle any IPs we get back that we can attempt to join. var ips []ipPort for _, r := range in.Answer { switch rr := r.(type) { case (dns.A): ips = append(ips, ipPort{ip: rr.A, port: defaultPort, nodeName: nodeName}) case (dns.AAAA): ips = append(ips, ipPort{ip: rr.AAAA, port: defaultPort, nodeName: nodeName}) case (dns.CNAME): m.logger.Printf("[DEBUG] memberlist: Ignoring CNAME RR in TCP-first answer for '%s'", host) } } return ips, nil } return nil, nil } // resolveAddr is used to resolve the address into an address, // port, and error. If no port is given, use the default func (m Memberlist) resolveAddr(hostStr string) ([]ipPort, error) { // First peel off any leading node name. This is optional. nodeName := "" if slashIdx := strings.Index(hostStr, "/"); slashIdx >= 0 { if slashIdx == 0 { return nil, fmt.Errorf("empty node name provided") } nodeName = hostStr[0:slashIdx] hostStr = hostStr[slashIdx+1:] } // This captures the supplied port, or the default one. hostStr = ensurePort(hostStr, m.config.BindPort) host, sport, err := net.SplitHostPort(hostStr) if err != nil { return nil, err } lport, err := strconv.ParseUint(sport, 10, 16) if err != nil { return nil, err } port := uint16(lport) // If it looks like an IP address we are done. The SplitHostPort() above // will make sure the host part is in good shape for parsing, even for // IPv6 addresses. if ip := net.ParseIP(host); ip != nil { return []ipPort{ ipPort{ip: ip, port: port, nodeName: nodeName}, }, nil } // First try TCP so we have the best chance for the largest list of // hosts to join. If this fails it's not fatal since this isn't a standard // way to query DNS, and we have a fallback below. ips, err := m.tcpLookupIP(host, port, nodeName) if err != nil { m.logger.Printf("[DEBUG] memberlist: TCP-first lookup failed for '%s', falling back to UDP: %s", hostStr, err) } if len(ips) > 0 { return ips, nil } // If TCP didn't yield anything then use the normal Go resolver which // will try UDP, then might possibly try TCP again if the UDP response // indicates it was truncated. ans, err := net.LookupIP(host) if err != nil { return nil, err } ips = make([]ipPort, 0, len(ans)) for _, ip := range ans { ips = append(ips, ipPort{ip: ip, port: port, nodeName: nodeName}) } return ips, nil } // setAlive is used to mark this node as being alive. This is the same // as if we received an alive notification our own network channel for // ourself. func (m Memberlist) setAlive() error { // Get the final advertise address from the transport, which may need // to see which address we bound to. addr, port, err := m.refreshAdvertise() if err != nil { return err } // Check if this is a public address without encryption ipAddr, err := sockaddr.NewIPAddr(addr.String()) if err != nil { return fmt.Errorf("Failed to parse interface addresses: %v", err) } ifAddrs := []sockaddr.IfAddr{ sockaddr.IfAddr{ SockAddr: ipAddr, }, } _, publicIfs, err := sockaddr.IfByRFC("6890", ifAddrs) if len(publicIfs) > 0 && !m.config.EncryptionEnabled() { m.logger.Printf("[WARN] memberlist: Binding to public address without encryption!") } // Set any metadata from the delegate. var meta []byte if m.config.Delegate != nil { meta = m.config.Delegate.NodeMeta(MetaMaxSize) if len(meta) > MetaMaxSize { panic("Node meta data provided is longer than the limit") } } a := alive{ Incarnation: m.nextIncarnation(), Node: m.config.Name, Addr: addr, Port: uint16(port), Meta: meta, Vsn: m.config.BuildVsnArray(), } m.aliveNode(&a, nil, true) return nil } func (m Memberlist) getAdvertise() (net.IP, uint16) { m.advertiseLock.RLock() defer m.advertiseLock.RUnlock() return m.advertiseAddr, m.advertisePort } func (m Memberlist) setAdvertise(addr net.IP, port int) { m.advertiseLock.Lock() defer m.advertiseLock.Unlock() m.advertiseAddr = addr m.advertisePort = uint16(port) } func (m Memberlist) refreshAdvertise() (net.IP, int, error) { addr, port, err := m.transport.FinalAdvertiseAddr( m.config.AdvertiseAddr, m.config.AdvertisePort) if err != nil { return nil, 0, fmt.Errorf("Failed to get final advertise address: %v", err) } m.setAdvertise(addr, port) return addr, port, nil } // LocalNode is used to return the local Node func (m Memberlist) LocalNode() Node { m.nodeLock.RLock() defer m.nodeLock.RUnlock() state := m.nodeMap[m.config.Name] return &state.Node } // UpdateNode is used to trigger re-advertising the local node. This is // primarily used with a Delegate to support dynamic updates to the local // meta data. This will block until the update message is successfully // broadcasted to a member of the cluster, if any exist or until a specified // timeout is reached. func (m Memberlist) UpdateNode(timeout time.Duration) error { // Get the node meta data var meta []byte if m.config.Delegate != nil { meta = m.config.Delegate.NodeMeta(MetaMaxSize) if len(meta) > MetaMaxSize { panic("Node meta data provided is longer than the limit") } } // Get the existing node m.nodeLock.RLock() state := m.nodeMap[m.config.Name] m.nodeLock.RUnlock() // Format a new alive message a := alive{ Incarnation: m.nextIncarnation(), Node: m.config.Name, Addr: state.Addr, Port: state.Port, Meta: meta, Vsn: m.config.BuildVsnArray(), } notifyCh := make(chan struct{}) m.aliveNode(&a, notifyCh, true) // Wait for the broadcast or a timeout if m.anyAlive() { var timeoutCh <-chan time.Time if timeout > 0 { timeoutCh = time.After(timeout) } select { case <-notifyCh: case <-timeoutCh: return fmt.Errorf("timeout waiting for update broadcast") } } return nil } // Deprecated: SendTo is deprecated in favor of SendBestEffort, which requires a node to // target. If you don't have a node then use SendToAddress. func (m Memberlist) SendTo(to net.Addr, msg []byte) error { a := Address{Addr: to.String(), Name: ""} return m.SendToAddress(a, msg) } func (m Memberlist) SendToAddress(a Address, msg []byte) error { // Encode as a user message buf := make([]byte, 1, len(msg)+1) buf[0] = byte(userMsg) buf = append(buf, msg...) // Send the message return m.rawSendMsgPacket(a, nil, buf) } // Deprecated: SendToUDP is deprecated in favor of SendBestEffort. func (m Memberlist) SendToUDP(to Node, msg []byte) error { return m.SendBestEffort(to, msg) } // Deprecated: SendToTCP is deprecated in favor of SendReliable. func (m Memberlist) SendToTCP(to Node, msg []byte) error { return m.SendReliable(to, msg) } // SendBestEffort uses the unreliable packet-oriented interface of the transport // to target a user message at the given node (this does not use the gossip // mechanism). The maximum size of the message depends on the configured // UDPBufferSize for this memberlist instance. func (m Memberlist) SendBestEffort(to Node, msg []byte) error { // Encode as a user message buf := make([]byte, 1, len(msg)+1) buf[0] = byte(userMsg) buf = append(buf, msg...) // Send the message a := Address{Addr: to.Address(), Name: to.Name} return m.rawSendMsgPacket(a, to, buf) } // SendReliable uses the reliable stream-oriented interface of the transport to // target a user message at the given node (this does not use the gossip // mechanism). Delivery is guaranteed if no error is returned, and there is no // limit on the size of the message. func (m Memberlist) SendReliable(to Node, msg []byte) error { return m.sendUserMsg(to.FullAddress(), msg) } // Members returns a list of all known live nodes. The node structures // returned must not be modified. If you wish to modify a Node, make a // copy first. func (m Memberlist) Members() []Node { m.nodeLock.RLock() defer m.nodeLock.RUnlock() nodes := make([]Node, 0, len(m.nodes)) for _, n := range m.nodes { if !n.DeadOrLeft() { nodes = append(nodes, &n.Node) } } return nodes } // NumMembers returns the number of alive nodes currently known. Between // the time of calling this and calling Members, the number of alive nodes // may have changed, so this shouldn't be used to determine how many // members will be returned by Members. func (m Memberlist) NumMembers() (alive int) { m.nodeLock.RLock() defer m.nodeLock.RUnlock() for _, n := range m.nodes { if !n.DeadOrLeft() { alive++ } } return } // Leave will broadcast a leave message but will not shutdown the background // listeners, meaning the node will continue participating in gossip and state // updates. // // This will block until the leave message is successfully broadcasted to // a member of the cluster, if any exist or until a specified timeout // is reached. // // This method is safe to call multiple times, but must not be called // after the cluster is already shut down. func (m Memberlist) Leave(timeout time.Duration) error { m.leaveLock.Lock() defer m.leaveLock.Unlock() if m.hasShutdown() { panic("leave after shutdown") } if !m.hasLeft() { atomic.StoreInt32(&m.leave, 1) m.nodeLock.Lock() state, ok := m.nodeMap[m.config.Name] m.nodeLock.Unlock() if !ok { m.logger.Printf("[WARN] memberlist: Leave but we're not in the node map.") return nil } // This dead message is special, because Node and From are the // same. This helps other nodes figure out that a node left // intentionally. When Node equals From, other nodes know for // sure this node is gone. d := dead{ Incarnation: state.Incarnation, Node: state.Name, From: state.Name, } m.deadNode(&d) // Block until the broadcast goes out if m.anyAlive() { var timeoutCh <-chan time.Time if timeout > 0 { timeoutCh = time.After(timeout) } select { case <-m.leaveBroadcast: case <-timeoutCh: return fmt.Errorf("timeout waiting for leave broadcast") } } } return nil } // Check for any other alive node. func (m Memberlist) anyAlive() bool { m.nodeLock.RLock() defer m.nodeLock.RUnlock() for _, n := range m.nodes { if !n.DeadOrLeft() && n.Name != m.config.Name { return true } } return false } // GetHealthScore gives this instance's idea of how well it is meeting the soft // real-time requirements of the protocol. Lower numbers are better, and zero // means "totally healthy". func (m Memberlist) GetHealthScore() int { return m.awareness.GetHealthScore() } // ProtocolVersion returns the protocol version currently in use by // this memberlist. func (m Memberlist) ProtocolVersion() uint8 { // NOTE: This method exists so that in the future we can control // any locking if necessary, if we change the protocol version at // runtime, etc. return m.config.ProtocolVersion } // Shutdown will stop any background maintenance of network activity // for this memberlist, causing it to appear "dead". A leave message // will not be broadcasted prior, so the cluster being left will have // to detect this node's shutdown using probing. If you wish to more // gracefully exit the cluster, call Leave prior to shutting down. // // This method is safe to call multiple times. func (m Memberlist) Shutdown() error { m.shutdownLock.Lock() defer m.shutdownLock.Unlock() if m.hasShutdown() { return nil } // Shut down the transport first, which should block until it's // completely torn down. If we kill the memberlist-side handlers // those I/O handlers might get stuck. if err := m.transport.Shutdown(); err != nil { m.logger.Printf("[ERR] Failed to shutdown transport: %v", err) } // Now tear down everything else. atomic.StoreInt32(&m.shutdown, 1) close(m.shutdownCh) m.deschedule() return nil } func (m Memberlist) hasShutdown() bool { return atomic.LoadInt32(&m.shutdown) == 1 } func (m Memberlist) hasLeft() bool { return atomic.LoadInt32(&m.leave) == 1 } func (m Memberlist) getNodeState(addr string) NodeStateType { m.nodeLock.RLock() defer m.nodeLock.RUnlock() n := m.nodeMap[addr] return n.State } func (m Memberlist) getNodeStateChange(addr string) time.Time { m.nodeLock.RLock() defer m.nodeLock.RUnlock() n := m.nodeMap[addr] return n.StateChange } func (m Memberlist) changeNode(addr string, f func(nodeState)) { m.nodeLock.Lock() defer m.nodeLock.Unlock() n := m.nodeMap[addr] f(n) } // checkBroadcastQueueDepth periodically checks the size of the broadcast queue // to see if it is too large func (m *Memberlist) checkBroadcastQueueDepth() { for { select { case <-time.After(m.config.QueueCheckInterval): numq := m.broadcasts.NumQueued() metrics.AddSampleWithLabels([]string{"memberlist", "queue", "broadcasts"}, float32(numq), m.metricLabels) case <-m.shutdownCh: return } } }	22.93 KB	2025-05-25 12:27
📄 memberlist_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "io/ioutil" "log" "net" "os" "reflect" "strconv" "strings" "sync" "testing" "time" iretry "github.com/hashicorp/memberlist/internal/retry" "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var bindLock sync.Mutex var bindNum byte = 10 func getBindAddrNet(network byte) net.IP { bindLock.Lock() defer bindLock.Unlock() result := net.IPv4(127, 0, network, bindNum) bindNum++ if bindNum > 255 { bindNum = 10 } return result } func getBindAddr() net.IP { return getBindAddrNet(0) } func testConfigNet(tb testing.TB, network byte) Config { tb.Helper() config := DefaultLANConfig() config.BindAddr = getBindAddrNet(network).String() config.Name = config.BindAddr config.BindPort = 0 // choose free port config.RequireNodeNames = true config.Logger = log.New(os.Stderr, config.Name, log.LstdFlags) return config } func testConfig(tb testing.TB) Config { return testConfigNet(tb, 0) } func yield() { time.Sleep(250 * time.Millisecond) } type MockDelegate struct { mu sync.Mutex meta []byte msgs [][]byte broadcasts [][]byte state []byte remoteState []byte } func (m MockDelegate) setMeta(meta []byte) { m.mu.Lock() defer m.mu.Unlock() m.meta = meta } func (m MockDelegate) setState(state []byte) { m.mu.Lock() defer m.mu.Unlock() m.state = state } func (m MockDelegate) setBroadcasts(broadcasts [][]byte) { m.mu.Lock() defer m.mu.Unlock() m.broadcasts = broadcasts } func (m MockDelegate) getRemoteState() []byte { m.mu.Lock() defer m.mu.Unlock() out := make([]byte, len(m.remoteState)) copy(out, m.remoteState) return out } func (m MockDelegate) getMessages() [][]byte { m.mu.Lock() defer m.mu.Unlock() out := make([][]byte, len(m.msgs)) for i, msg := range m.msgs { out[i] = make([]byte, len(msg)) copy(out[i], msg) } return out } func (m MockDelegate) NodeMeta(limit int) []byte { m.mu.Lock() defer m.mu.Unlock() return m.meta } func (m MockDelegate) NotifyMsg(msg []byte) { m.mu.Lock() defer m.mu.Unlock() cp := make([]byte, len(msg)) copy(cp, msg) m.msgs = append(m.msgs, cp) } func (m MockDelegate) GetBroadcasts(overhead, limit int) [][]byte { m.mu.Lock() defer m.mu.Unlock() b := m.broadcasts m.broadcasts = nil return b } func (m MockDelegate) LocalState(join bool) []byte { m.mu.Lock() defer m.mu.Unlock() return m.state } func (m MockDelegate) MergeRemoteState(s []byte, join bool) { m.mu.Lock() defer m.mu.Unlock() m.remoteState = s } func GetMemberlist(tb testing.TB, f func(c Config)) Memberlist { c := testConfig(tb) c.BindPort = 0 // assign a free port if f != nil { f(c) } m, err := newMemberlist(c) require.NoError(tb, err) return m } func TestDefaultLANConfig_protocolVersion(t testing.T) { c := DefaultLANConfig() if c.ProtocolVersion != ProtocolVersion2Compatible { t.Fatalf("should be max: %d", c.ProtocolVersion) } } func TestCreate_protocolVersion(t testing.T) { cases := []struct { name string version uint8 err bool }{ {"min", ProtocolVersionMin, false}, {"max", ProtocolVersionMax, false}, // TODO(mitchellh): uncommon when we're over 0 //{"uncommon", ProtocolVersionMin - 1, true}, {"max+1", ProtocolVersionMax + 1, true}, {"min-1", ProtocolVersionMax - 1, false}, } for _, tc := range cases { t.Run(tc.name, func(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() c.ProtocolVersion = tc.version m, err := Create(c) if err == nil { require.NoError(t, m.Shutdown()) } if tc.err && err == nil { t.Fatalf("Should've failed with version: %d", tc.version) } else if !tc.err && err != nil { t.Fatalf("Version '%d' error: %s", tc.version, err) } }) } } func TestCreate_secretKey(t testing.T) { cases := []struct { name string key []byte err bool }{ {"size-0", make([]byte, 0), false}, {"abc", []byte("abc"), true}, {"size-16", make([]byte, 16), false}, {"size-38", make([]byte, 38), true}, } for _, tc := range cases { t.Run(tc.name, func(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() c.SecretKey = tc.key m, err := Create(c) if err == nil { require.NoError(t, m.Shutdown()) } if tc.err && err == nil { t.Fatalf("Should've failed with key: %#v", tc.key) } else if !tc.err && err != nil { t.Fatalf("Key '%#v' error: %s", tc.key, err) } }) } } func TestCreate_secretKeyEmpty(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() c.SecretKey = make([]byte, 0) m, err := Create(c) require.NoError(t, err) defer m.Shutdown() if m.config.EncryptionEnabled() { t.Fatalf("Expected encryption to be disabled") } } func TestCreate_checkBroadcastQueueMetrics(t testing.T) { sink := registerInMemorySink(t) c := DefaultLANConfig() c.QueueCheckInterval = 1 time.Second c.BindAddr = getBindAddr().String() c.SecretKey = make([]byte, 0) m, err := Create(c) require.NoError(t, err) defer m.Shutdown() time.Sleep(3 * time.Second) sampleName := "consul.usage.test.memberlist.queue.broadcasts" verifySampleExists(t, sampleName, sink) } func TestCreate_keyringOnly(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() keyring, err := NewKeyring(nil, make([]byte, 16)) require.NoError(t, err) c.Keyring = keyring m, err := Create(c) require.NoError(t, err) defer m.Shutdown() if !m.config.EncryptionEnabled() { t.Fatalf("Expected encryption to be enabled") } } func TestCreate_keyringAndSecretKey(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() keyring, err := NewKeyring(nil, make([]byte, 16)) require.NoError(t, err) c.Keyring = keyring c.SecretKey = []byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} m, err := Create(c) require.NoError(t, err) defer m.Shutdown() if !m.config.EncryptionEnabled() { t.Fatalf("Expected encryption to be enabled") } ringKeys := c.Keyring.GetKeys() if !bytes.Equal(c.SecretKey, ringKeys[0]) { t.Fatalf("Unexpected primary key %v", ringKeys[0]) } } func TestCreate_invalidLoggerSettings(t testing.T) { c := DefaultLANConfig() c.BindAddr = getBindAddr().String() c.Logger = log.New(ioutil.Discard, "", log.LstdFlags) c.LogOutput = ioutil.Discard m, err := Create(c) if err == nil { require.NoError(t, m.Shutdown()) t.Fatal("Memberlist should not allow both LogOutput and Logger to be set, but it did not raise an error") } } func TestCreate(t testing.T) { c := testConfig(t) c.ProtocolVersion = ProtocolVersionMin c.DelegateProtocolVersion = 13 c.DelegateProtocolMin = 12 c.DelegateProtocolMax = 24 m, err := Create(c) require.NoError(t, err) defer m.Shutdown() yield() members := m.Members() if len(members) != 1 { t.Fatalf("bad number of members") } if members[0].PMin != ProtocolVersionMin { t.Fatalf("bad: %#v", members[0]) } if members[0].PMax != ProtocolVersionMax { t.Fatalf("bad: %#v", members[0]) } if members[0].PCur != c.ProtocolVersion { t.Fatalf("bad: %#v", members[0]) } if members[0].DMin != c.DelegateProtocolMin { t.Fatalf("bad: %#v", members[0]) } if members[0].DMax != c.DelegateProtocolMax { t.Fatalf("bad: %#v", members[0]) } if members[0].DCur != c.DelegateProtocolVersion { t.Fatalf("bad: %#v", members[0]) } } func TestMemberList_CreateShutdown(t testing.T) { m := GetMemberlist(t, nil) m.schedule() require.NoError(t, m.Shutdown()) } func TestMemberList_ResolveAddr(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() defaultPort := uint16(m.config.BindPort) type testCase struct { name string in string expectErr bool ignoreExpectIP bool expect []ipPort } baseCases := []testCase{ { name: "localhost", in: "localhost", ignoreExpectIP: true, expect: []ipPort{ {port: defaultPort}, }, }, { name: "ipv6 pair", in: "[::1]:80", expect: []ipPort{ {ip: net.IPv6loopback, port: 80}, }, }, { name: "ipv6 non-pair", in: "[::1]", expect: []ipPort{ {ip: net.IPv6loopback, port: defaultPort}, }, }, { name: "hostless port", in: ":80", expectErr: true, }, { name: "hostname port combo", in: "localhost:80", ignoreExpectIP: true, expect: []ipPort{ {port: 80}, }, }, { name: "too high port", in: "localhost:80000", expectErr: true, }, { name: "ipv4 port combo", in: "127.0.0.1:80", expect: []ipPort{ {ip: net.IPv4(127, 0, 0, 1), port: 80}, }, }, { name: "ipv6 port combo", in: "[2001:db8:a0b:12f0::1]:80", expect: []ipPort{ { ip: net.IP{0x20, 0x01, 0x0d, 0xb8, 0x0a, 0x0b, 0x12, 0xf0, 0, 0, 0, 0, 0, 0, 0, 0x1}, port: 80, }, }, }, { name: "ipv4 port combo with empty tag", in: "/127.0.0.1:80", expectErr: true, }, { name: "ipv4 only", in: "127.0.0.1", expect: []ipPort{ {ip: net.IPv4(127, 0, 0, 1), port: defaultPort}, }, }, { name: "ipv6 only", in: "[2001:db8:a0b:12f0::1]", expect: []ipPort{ { ip: net.IP{0x20, 0x01, 0x0d, 0xb8, 0x0a, 0x0b, 0x12, 0xf0, 0, 0, 0, 0, 0, 0, 0, 0x1}, port: defaultPort, }, }, }, } // explode the cases to include tagged versions of everything var cases []testCase for _, tc := range baseCases { cases = append(cases, tc) if !strings.Contains(tc.in, "/") { // don't double tag already tagged cases tc2 := testCase{ name: tc.name + " (tagged)", in: "foo.bar/" + tc.in, expectErr: tc.expectErr, ignoreExpectIP: tc.ignoreExpectIP, } for _, ipp := range tc.expect { tc2.expect = append(tc2.expect, ipPort{ ip: ipp.ip, port: ipp.port, nodeName: "foo.bar", }) } cases = append(cases, tc2) } } for _, tc := range cases { tc := tc t.Run(tc.name, func(t testing.T) { t.Parallel() got, err := m.resolveAddr(tc.in) if tc.expectErr { require.Error(t, err) } else { require.NoError(t, err) if tc.ignoreExpectIP { if len(got) > 1 { got = got[0:1] } for i := 0; i < len(got); i++ { got[i].ip = nil } } require.Equal(t, tc.expect, got) } }) } } type dnsHandler struct { t testing.T } func (h dnsHandler) ServeDNS(w dns.ResponseWriter, r dns.Msg) { if len(r.Question) != 1 { h.t.Fatalf("bad: %#v", r.Question) } name := "join.service.consul." question := r.Question[0] if question.Name != name \|\| question.Qtype != dns.TypeANY { h.t.Fatalf("bad: %#v", question) } m := new(dns.Msg) m.SetReply(r) m.Authoritative = true m.RecursionAvailable = false m.Answer = append(m.Answer, &dns.A{ Hdr: dns.RR_Header{ Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("127.0.0.1"), }) m.Answer = append(m.Answer, &dns.AAAA{ Hdr: dns.RR_Header{ Name: name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("2001:db8:a0b:12f0::1"), }) if err := w.WriteMsg(m); err != nil { h.t.Fatalf("err: %v", err) } } func TestMemberList_ResolveAddr_TCP_First(t testing.T) { bind := "127.0.0.1:8600" var wg sync.WaitGroup wg.Add(1) server := &dns.Server{ Addr: bind, Handler: dnsHandler{t}, Net: "tcp", NotifyStartedFunc: wg.Done, } defer server.Shutdown() go func() { if err := server.ListenAndServe(); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { t.Errorf("err: %v", err) } }() wg.Wait() tmpFile, err := ioutil.TempFile("", "") if err != nil { t.Fatalf("err: %v", err) } defer os.Remove(tmpFile.Name()) content := []byte(fmt.Sprintf("nameserver %s", bind)) if _, err := tmpFile.Write(content); err != nil { t.Fatalf("err: %v", err) } if err := tmpFile.Close(); err != nil { t.Fatalf("err: %v", err) } m := GetMemberlist(t, func(c Config) { c.DNSConfigPath = tmpFile.Name() }) m.setAlive() m.schedule() defer m.Shutdown() // Try with and without the trailing dot. hosts := []string{ "join.service.consul.", "join.service.consul", } for _, host := range hosts { ips, err := m.resolveAddr(host) if err != nil { t.Fatalf("err: %v", err) } port := uint16(m.config.BindPort) expected := []ipPort{ // Go now parses IPs like this and returns IP4-mapped IPv6 address. // Confusingly if you print it you see the same as the input since // IP.String converts IP4-mapped addresses back to dotted decimal notation // but the underlying IP bytes don't compare as equal to the actual IPv4 // bytes the resolver will get from DNS. ipPort{ip: net.ParseIP("127.0.0.1").To4(), port: port, nodeName: ""}, ipPort{ip: net.ParseIP("2001:db8:a0b:12f0::1"), port: port, nodeName: ""}, } require.Equal(t, expected, ips) } } func TestMemberList_Members(t testing.T) { n1 := &Node{Name: "test"} n2 := &Node{Name: "test2"} n3 := &Node{Name: "test3"} m := &Memberlist{} nodes := []nodeState{ &nodeState{Node: n1, State: StateAlive}, &nodeState{Node: n2, State: StateDead}, &nodeState{Node: n3, State: StateSuspect}, } m.nodes = nodes members := m.Members() if !reflect.DeepEqual(members, []Node{n1, n3}) { t.Fatalf("bad members") } } func TestMemberlist_Join(t testing.T) { c1 := testConfig(t) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfig(t) c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 1 { t.Fatalf("unexpected 1: %d", num) } if err != nil { t.Fatalf("unexpected err: %s", err) } // Check the hosts if len(m2.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } if m2.estNumNodes() != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } } func TestMemberlist_Join_with_Labels(t testing.T) { testMemberlist_Join_with_Labels(t, nil) } func TestMemberlist_Join_with_Labels_and_Encryption(t testing.T) { secretKey := TestKeys[0] testMemberlist_Join_with_Labels(t, secretKey) } func testMemberlist_Join_with_Labels(t testing.T, secretKey []byte) { c1 := testConfig(t) c1.Label = "blah" c1.SecretKey = secretKey m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfig(t) c2.Label = "blah" c2.BindPort = bindPort c2.SecretKey = secretKey m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() checkHost := func(t testing.T, m Memberlist, expected int) { assert.Equal(t, expected, len(m.Members())) assert.Equal(t, expected, m.estNumNodes()) } runStep(t, "same label can join", func(t testing.T) { num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) // Check the hosts checkHost(t, m2, 2) checkHost(t, m1, 2) }) // Create a third node that uses no label c3 := testConfig(t) c3.Label = "" c3.BindPort = bindPort c3.SecretKey = secretKey m3, err := Create(c3) require.NoError(t, err) defer m3.Shutdown() runStep(t, "no label cannot join", func(t testing.T) { _, err := m3.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.Error(t, err) // Check the failed host checkHost(t, m3, 1) // Check the existing hosts checkHost(t, m2, 2) checkHost(t, m1, 2) }) // Create a fourth node that uses a mismatched label c4 := testConfig(t) c4.Label = "not-blah" c4.BindPort = bindPort c4.SecretKey = secretKey m4, err := Create(c4) require.NoError(t, err) defer m4.Shutdown() runStep(t, "mismatched label cannot join", func(t testing.T) { _, err := m4.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.Error(t, err) // Check the failed host checkHost(t, m4, 1) // Check the previous failed host checkHost(t, m3, 1) // Check the existing hosts checkHost(t, m2, 2) checkHost(t, m1, 2) }) } func TestMemberlist_JoinDifferentNetworksUniqueMask(t testing.T) { c1 := testConfigNet(t, 0) c1.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/8"}) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfigNet(t, 1) c2.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/8"}) c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 1 { t.Fatalf("unexpected 1: %d", num) } if err != nil { t.Fatalf("unexpected err: %s", err) } // Check the hosts if len(m2.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } if m2.estNumNodes() != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } } func TestMemberlist_JoinDifferentNetworksMultiMasks(t testing.T) { c1 := testConfigNet(t, 0) c1.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/24", "127.0.1.0/24"}) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfigNet(t, 1) c2.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/24", "127.0.1.0/24"}) c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() err = joinAndTestMemberShip(t, m2, []string{m1.config.Name + "/" + m1.config.BindAddr}, 2) if err != nil { t.Fatalf("unexpected err: %s", err) } // Create a rogue node that allows all networks // It should see others, but will not be seen by others c3 := testConfigNet(t, 2) c3.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/8"}) c3.BindPort = bindPort m3, err := Create(c3) require.NoError(t, err) defer m3.Shutdown() // The rogue can see others, but others cannot see it err = joinAndTestMemberShip(t, m3, []string{m1.config.Name + "/" + m1.config.BindAddr}, 3) // For the node itself, everything seems fine, it should see others if err != nil { t.Fatalf("unexpected err: %s", err) } // m1 and m2 should not see newcomer however if len(m1.Members()) != 2 { t.Fatalf("m1 should have 2 nodes! %v", m1.Members()) } if m1.estNumNodes() != 2 { t.Fatalf("m1 should have 2 est. nodes! %v", m1.estNumNodes()) } if len(m2.Members()) != 2 { t.Fatalf("m2 should have 2 nodes! %v", m2.Members()) } if m2.estNumNodes() != 2 { t.Fatalf("m2 should have 2 est. nodes! %v", m2.estNumNodes()) } // Another rogue, this time with a config that denies itself // Create a rogue node that allows all networks // It should see others, but will not be seen by others c4 := testConfigNet(t, 2) c4.CIDRsAllowed, _ = ParseCIDRs([]string{"127.0.0.0/24", "127.0.1.0/24"}) c4.BindPort = bindPort m4, err := Create(c4) require.NoError(t, err) defer m4.Shutdown() // This time, the node should not even see itself, so 2 expected nodes err = joinAndTestMemberShip(t, m4, []string{m1.config.BindAddr, m2.config.BindAddr}, 2) // m1 and m2 should not see newcomer however if len(m1.Members()) != 2 { t.Fatalf("m1 should have 2 nodes! %v", m1.Members()) } if m1.estNumNodes() != 2 { t.Fatalf("m1 should have 2 est. nodes! %v", m1.estNumNodes()) } if len(m2.Members()) != 2 { t.Fatalf("m2 should have 2 nodes! %v", m2.Members()) } if m2.estNumNodes() != 2 { t.Fatalf("m2 should have 2 est. nodes! %v", m2.estNumNodes()) } } type CustomMergeDelegate struct { invoked bool t testing.T } func (c CustomMergeDelegate) NotifyMerge(nodes []Node) error { c.t.Logf("Cancel merge") c.invoked = true return fmt.Errorf("Custom merge canceled") } func TestMemberlist_Join_Cancel(t testing.T) { c1 := testConfig(t) merge1 := &CustomMergeDelegate{t: t} c1.Merge = merge1 m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfig(t) c2.BindPort = bindPort merge2 := &CustomMergeDelegate{t: t} c2.Merge = merge2 m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 0 { t.Fatalf("unexpected 0: %d", num) } if !strings.Contains(err.Error(), "Custom merge canceled") { t.Fatalf("unexpected err: %s", err) } // Check the hosts if len(m2.Members()) != 1 { t.Fatalf("should have 1 nodes! %v", m2.Members()) } if len(m1.Members()) != 1 { t.Fatalf("should have 1 nodes! %v", m1.Members()) } // Check delegate invocation if !merge1.invoked { t.Fatalf("should invoke delegate") } if !merge2.invoked { t.Fatalf("should invoke delegate") } } type CustomAliveDelegate struct { Ignore string count int t testing.T } func (c CustomAliveDelegate) NotifyAlive(peer Node) error { c.count++ if peer.Name == c.Ignore { return nil } c.t.Logf("Cancel alive") return fmt.Errorf("Custom alive canceled") } func TestMemberlist_Join_Cancel_Passive(t testing.T) { c1 := testConfig(t) alive1 := &CustomAliveDelegate{ Ignore: c1.Name, t: t, } c1.Alive = alive1 m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := testConfig(t) c2.BindPort = bindPort alive2 := &CustomAliveDelegate{ Ignore: c2.Name, t: t, } c2.Alive = alive2 m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 1 { t.Fatalf("unexpected 1: %d", num) } if err != nil { t.Fatalf("err: %s", err) } // Check the hosts if len(m2.Members()) != 1 { t.Fatalf("should have 1 nodes! %v", m2.Members()) } if len(m1.Members()) != 1 { t.Fatalf("should have 1 nodes! %v", m1.Members()) } // Check delegate invocation if alive1.count == 0 { t.Fatalf("should invoke delegate: %d", alive1.count) } if alive2.count == 0 { t.Fatalf("should invoke delegate: %d", alive2.count) } } func TestMemberlist_Join_protocolVersions(t testing.T) { c1 := testConfig(t) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2 := testConfig(t) c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() c3 := testConfig(t) c3.BindPort = bindPort c3.ProtocolVersion = ProtocolVersionMax m3, err := Create(c3) require.NoError(t, err) defer m3.Shutdown() _, err = m1.Join([]string{c2.Name + "/" + c2.BindAddr}) require.NoError(t, err) yield() _, err = m1.Join([]string{c3.Name + "/" + c3.BindAddr}) require.NoError(t, err) } func joinAndTestMemberShip(t testing.T, self Memberlist, membersToJoin []string, expectedMembers int) error { t.Helper() num, err := self.Join(membersToJoin) if err != nil { return err } if num != len(membersToJoin) { t.Fatalf("unexpected %d, was expecting %d to be joined", num, len(membersToJoin)) } if err != nil { t.Fatalf("unexpected err: %s", err) } // Check the hosts if len(self.Members()) != expectedMembers { t.Fatalf("should have 2 nodes! %v", self.Members()) } if len(self.Members()) != expectedMembers { t.Fatalf("should have 2 nodes! %v", self.Members()) } return nil } func TestMemberlist_Leave(t testing.T) { newConfig := func() Config { c := testConfig(t) c.GossipInterval = time.Millisecond return c } c1 := newConfig() m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := newConfig() c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() err = joinAndTestMemberShip(t, m2, []string{m1.config.Name + "/" + m1.config.BindAddr}, 2) if err != nil { t.Fatalf("unexpected err: %s", err) } // Leave err = m1.Leave(time.Second) require.NoError(t, err) // Wait for leave time.Sleep(10 time.Millisecond) // m1 should think dead if len(m1.Members()) != 1 { t.Fatalf("should have 1 node") } if len(m2.Members()) != 1 { t.Fatalf("should have 1 node") } if m2.nodeMap[c1.Name].State != StateLeft { t.Fatalf("bad state") } } func TestMemberlist_JoinShutdown(t testing.T) { newConfig := func() Config { c := testConfig(t) c.ProbeInterval = time.Millisecond c.ProbeTimeout = 100 * time.Microsecond c.SuspicionMaxTimeoutMult = 1 return c } c1 := newConfig() m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := newConfig() c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 1 { t.Fatalf("unexpected 1: %d", num) } if err != nil { t.Fatalf("unexpected err: %s", err) } // Check the hosts if len(m2.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } require.NoError(t, m1.Shutdown()) waitForCondition(t, func() (bool, string) { n := len(m2.Members()) return n == 1, fmt.Sprintf("expected 1 node, got %d", n) }) } func TestMemberlist_delegateMeta(t testing.T) { c1 := testConfig(t) c1.Delegate = &MockDelegate{meta: []byte("web")} m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2 := testConfig(t) c2.BindPort = bindPort c2.Delegate = &MockDelegate{meta: []byte("lb")} m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() _, err = m1.Join([]string{c2.Name + "/" + c2.BindAddr}) require.NoError(t, err) yield() var roles map[string]string // Check the roles of members of m1 m1m := m1.Members() if len(m1m) != 2 { t.Fatalf("bad: %#v", m1m) } roles = make(map[string]string) for _, m := range m1m { roles[m.Name] = string(m.Meta) } if r := roles[c1.Name]; r != "web" { t.Fatalf("bad role for %s: %s", c1.Name, r) } if r := roles[c2.Name]; r != "lb" { t.Fatalf("bad role for %s: %s", c2.Name, r) } // Check the roles of members of m2 m2m := m2.Members() if len(m2m) != 2 { t.Fatalf("bad: %#v", m2m) } roles = make(map[string]string) for _, m := range m2m { roles[m.Name] = string(m.Meta) } if r := roles[c1.Name]; r != "web" { t.Fatalf("bad role for %s: %s", c1.Name, r) } if r := roles[c2.Name]; r != "lb" { t.Fatalf("bad role for %s: %s", c2.Name, r) } } func TestMemberlist_delegateMeta_Update(t testing.T) { c1 := testConfig(t) mock1 := &MockDelegate{meta: []byte("web")} c1.Delegate = mock1 m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2 := testConfig(t) c2.BindPort = bindPort mock2 := &MockDelegate{meta: []byte("lb")} c2.Delegate = mock2 m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() _, err = m1.Join([]string{c2.Name + "/" + c2.BindAddr}) require.NoError(t, err) yield() // Update the meta data roles mock1.setMeta([]byte("api")) mock2.setMeta([]byte("db")) err = m1.UpdateNode(0) require.NoError(t, err) err = m2.UpdateNode(0) require.NoError(t, err) yield() // Check the updates have propagated var roles map[string]string // Check the roles of members of m1 m1m := m1.Members() if len(m1m) != 2 { t.Fatalf("bad: %#v", m1m) } roles = make(map[string]string) for _, m := range m1m { roles[m.Name] = string(m.Meta) } if r := roles[c1.Name]; r != "api" { t.Fatalf("bad role for %s: %s", c1.Name, r) } if r := roles[c2.Name]; r != "db" { t.Fatalf("bad role for %s: %s", c2.Name, r) } // Check the roles of members of m2 m2m := m2.Members() if len(m2m) != 2 { t.Fatalf("bad: %#v", m2m) } roles = make(map[string]string) for _, m := range m2m { roles[m.Name] = string(m.Meta) } if r := roles[c1.Name]; r != "api" { t.Fatalf("bad role for %s: %s", c1.Name, r) } if r := roles[c2.Name]; r != "db" { t.Fatalf("bad role for %s: %s", c2.Name, r) } } func TestMemberlist_UserData(t testing.T) { newConfig := func() (Config, MockDelegate) { d := &MockDelegate{} c := testConfig(t) // Set the gossip/pushpull intervals fast enough to get a reasonable test, // but slow enough to avoid "sendto: operation not permitted" c.GossipInterval = 100 time.Millisecond c.PushPullInterval = 100 * time.Millisecond c.Delegate = d return c, d } c1, d1 := newConfig() d1.setState([]byte("something")) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort bcasts := make([][]byte, 256) for i := range bcasts { bcasts[i] = []byte(fmt.Sprintf("%d", i)) } // Create a second node c2, d2 := newConfig() c2.BindPort = bindPort // Second delegate has things to send d2.setBroadcasts(bcasts) d2.setState([]byte("my state")) m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) if num != 1 { t.Fatalf("unexpected 1: %d", num) } require.NoError(t, err) // Check the hosts if m2.NumMembers() != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } // Wait for a little while iretry.Run(t, func(r iretry.R) { msgs1 := d1.getMessages() // Ensure we got the messages. Ordering of messages is not guaranteed so just // check we got them both in either order. require.ElementsMatch(r, bcasts, msgs1) rs1 := d1.getRemoteState() rs2 := d2.getRemoteState() // Check the push/pull state if !reflect.DeepEqual(rs1, []byte("my state")) { r.Fatalf("bad state %s", rs1) } if !reflect.DeepEqual(rs2, []byte("something")) { r.Fatalf("bad state %s", rs2) } }) } func TestMemberlist_SendTo(t testing.T) { newConfig := func() (Config, MockDelegate, net.IP) { d := &MockDelegate{} c := testConfig(t) c.GossipInterval = time.Millisecond c.PushPullInterval = time.Millisecond c.Delegate = d return c, d, net.ParseIP(c.BindAddr) } c1, d1, _ := newConfig() m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2, d2, addr2 := newConfig() c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) // Check the hosts require.Equal(t, 2, m2.NumMembers(), "should have 2 nodes! %v", m2.Members()) // Try to do a direct send m2Addr := &net.UDPAddr{ IP: addr2, Port: bindPort, } m2Address := Address{ Addr: m2Addr.String(), Name: m2.config.Name, } if err := m1.SendToAddress(m2Address, []byte("ping")); err != nil { t.Fatalf("err: %v", err) } m1Addr := &net.UDPAddr{ IP: net.ParseIP(m1.config.BindAddr), Port: bindPort, } m1Address := Address{ Addr: m1Addr.String(), Name: m1.config.Name, } if err := m2.SendToAddress(m1Address, []byte("pong")); err != nil { t.Fatalf("err: %v", err) } waitForCondition(t, func() (bool, string) { msgs := d1.getMessages() return len(msgs) == 1, fmt.Sprintf("expected 1 message, got %d", len(msgs)) }) msgs1 := d1.getMessages() if !reflect.DeepEqual(msgs1[0], []byte("pong")) { t.Fatalf("bad msg %v", msgs1[0]) } waitForCondition(t, func() (bool, string) { msgs := d2.getMessages() return len(msgs) == 1, fmt.Sprintf("expected 1 message, got %d", len(msgs)) }) msgs2 := d2.getMessages() if !reflect.DeepEqual(msgs2[0], []byte("ping")) { t.Fatalf("bad msg %v", msgs2[0]) } } func waitForCondition(t testing.T, fn func() (bool, string)) { start := time.Now() var msg string for time.Since(start) < 20time.Second { var done bool done, msg = fn() if done { return } time.Sleep(5 * time.Millisecond) } t.Fatalf("timeout waiting for condition: %v", msg) } func TestMemberlistProtocolVersion(t testing.T) { c := testConfig(t) c.ProtocolVersion = ProtocolVersionMax m, err := Create(c) require.NoError(t, err) defer m.Shutdown() result := m.ProtocolVersion() if result != ProtocolVersionMax { t.Fatalf("bad: %d", result) } } func TestMemberlist_Join_DeadNode(t testing.T) { c1 := testConfig(t) c1.TCPTimeout = 50 * time.Millisecond m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second "node", which is just a TCP listener that // does not ever respond. This is to test our deadlines addr2 := getBindAddr() list, err := net.Listen("tcp", net.JoinHostPort(addr2.String(), strconv.Itoa(bindPort))) if err != nil { t.Fatalf("err: %v", err) } defer list.Close() // Ensure we don't hang forever timer := time.AfterFunc(100time.Millisecond, func() { panic("should have timed out by now") }) defer timer.Stop() num, err := m1.Join([]string{"fake/" + addr2.String()}) if num != 0 { t.Fatalf("unexpected 0: %d", num) } if err == nil { t.Fatal("expect err") } } // Tests that nodes running different versions of the protocol can successfully // discover each other and add themselves to their respective member lists. func TestMemberlist_Join_Protocol_Compatibility(t testing.T) { testProtocolVersionPair := func(t testing.T, pv1 uint8, pv2 uint8) { t.Helper() c1 := testConfig(t) c1.ProtocolVersion = pv1 m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2 := testConfig(t) c2.BindPort = bindPort c2.ProtocolVersion = pv2 m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) // Check the hosts if len(m2.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } // Check the hosts if len(m1.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m1.Members()) } } t.Run("2,1", func(t testing.T) { testProtocolVersionPair(t, 2, 1) }) t.Run("2,3", func(t testing.T) { testProtocolVersionPair(t, 2, 3) }) t.Run("3,2", func(t testing.T) { testProtocolVersionPair(t, 3, 2) }) t.Run("3,1", func(t testing.T) { testProtocolVersionPair(t, 3, 1) }) } var ( ipv6LoopbackAvailableOnce sync.Once ipv6LoopbackAvailable bool ) func isIPv6LoopbackAvailable(t testing.T) bool { const ipv6LoopbackAddress = "::1" ipv6LoopbackAvailableOnce.Do(func() { ifaces, err := net.Interfaces() require.NoError(t, err) for _, iface := range ifaces { if iface.Flags&net.FlagLoopback == 0 { continue } addrs, err := iface.Addrs() require.NoError(t, err) for _, addr := range addrs { ipaddr := addr.(net.IPNet) if ipaddr.IP.String() == ipv6LoopbackAddress { ipv6LoopbackAvailable = true return } } } ipv6LoopbackAvailable = false t.Logf("IPv6 loopback address %q not found, disabling tests that require it", ipv6LoopbackAddress) }) return ipv6LoopbackAvailable } func TestMemberlist_Join_IPv6(t testing.T) { if !isIPv6LoopbackAvailable(t) { t.SkipNow() return } // Since this binds to all interfaces we need to exclude other tests // from grabbing an interface. bindLock.Lock() defer bindLock.Unlock() c1 := DefaultLANConfig() c1.Name = "A" c1.BindAddr = "[::1]" c1.BindPort = 0 // choose free c1.Logger = log.New(os.Stderr, c1.Name, log.LstdFlags) m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() // Create a second node c2 := DefaultLANConfig() c2.Name = "B" c2.BindAddr = "[::1]" c2.BindPort = 0 // choose free c2.Logger = log.New(os.Stderr, c2.Name, log.LstdFlags) m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{fmt.Sprintf("%s/%s:%d", m1.config.Name, m1.config.BindAddr, m1.config.BindPort)}) require.NoError(t, err) require.Equal(t, 1, num) // Check the hosts if len(m2.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } if len(m1.Members()) != 2 { t.Fatalf("should have 2 nodes! %v", m2.Members()) } } func reservePort(t testing.T, ip net.IP, purpose string) int { for i := 0; i < 10; i++ { tcpAddr := &net.TCPAddr{IP: ip, Port: 0} tcpLn, err := net.ListenTCP("tcp", tcpAddr) if err != nil { if strings.Contains(err.Error(), "address already in use") { continue } t.Fatalf("unexpected error: %v", err) } port := tcpLn.Addr().(net.TCPAddr).Port udpAddr := &net.UDPAddr{IP: ip, Port: port} udpLn, err := net.ListenUDP("udp", udpAddr) if err != nil { _ = tcpLn.Close() if strings.Contains(err.Error(), "address already in use") { continue } t.Fatalf("unexpected error: %v", err) } t.Logf("Using dynamic bind port %d for %s", port, purpose) _ = tcpLn.Close() _ = udpLn.Close() return port } t.Fatalf("could not find a free TCP+UDP port to listen on for %s", purpose) panic("IMPOSSIBLE") } func TestAdvertiseAddr(t testing.T) { bindAddr := getBindAddr() advertiseAddr := getBindAddr() bindPort := reservePort(t, bindAddr, "BIND") advertisePort := reservePort(t, advertiseAddr, "ADVERTISE") c := DefaultLANConfig() c.BindAddr = bindAddr.String() c.BindPort = bindPort c.Name = c.BindAddr c.AdvertiseAddr = advertiseAddr.String() c.AdvertisePort = advertisePort m, err := Create(c) require.NoError(t, err) defer m.Shutdown() yield() members := m.Members() require.Equal(t, 1, len(members)) require.Equal(t, advertiseAddr.String(), members[0].Addr.String()) require.Equal(t, advertisePort, int(members[0].Port)) } type MockConflict struct { existing Node other Node } func (m MockConflict) NotifyConflict(existing, other Node) { m.existing = existing m.other = other } func TestMemberlist_conflictDelegate(t testing.T) { c1 := testConfig(t) mock := &MockConflict{} c1.Conflict = mock m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Ensure name conflict c2 := testConfig(t) c2.Name = c1.Name c2.BindPort = bindPort m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m1.Join([]string{c2.Name + "/" + c2.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) yield() // Ensure we were notified if mock.existing == nil \|\| mock.other == nil { t.Fatalf("should get notified mock.existing=%v VS mock.other=%v", mock.existing, mock.other) } if mock.existing.Name != mock.other.Name { t.Fatalf("bad: %v %v", mock.existing, mock.other) } } type MockPing struct { mu sync.Mutex other Node rtt time.Duration payload []byte } func (m MockPing) NotifyPingComplete(other Node, rtt time.Duration, payload []byte) { m.mu.Lock() defer m.mu.Unlock() m.other = other m.rtt = rtt m.payload = payload } func (m MockPing) getContents() (Node, time.Duration, []byte) { m.mu.Lock() defer m.mu.Unlock() return m.other, m.rtt, m.payload } const DEFAULT_PAYLOAD = "whatever" func (m MockPing) AckPayload() []byte { return []byte(DEFAULT_PAYLOAD) } func TestMemberlist_PingDelegate(t testing.T) { newConfig := func() Config { c := testConfig(t) c.ProbeInterval = 100 * time.Millisecond c.Ping = &MockPing{} return c } c1 := newConfig() m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort // Create a second node c2 := newConfig() c2.BindPort = bindPort mock := c2.Ping.(MockPing) m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() num, err := m2.Join([]string{m1.config.Name + "/" + m1.config.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) waitUntilSize(t, m1, 2) waitUntilSize(t, m2, 2) time.Sleep(2 c1.ProbeInterval) require.NoError(t, m1.Shutdown()) require.NoError(t, m2.Shutdown()) mOther, mRTT, mPayload := mock.getContents() // Ensure we were notified if mOther == nil { t.Fatalf("should get notified") } if !reflect.DeepEqual(mOther, m1.LocalNode()) { t.Fatalf("not notified about the correct node; expected: %+v; actual: %+v", m2.LocalNode(), mOther) } if mRTT <= 0 { t.Fatalf("rtt should be greater than 0") } if bytes.Compare(mPayload, []byte(DEFAULT_PAYLOAD)) != 0 { t.Fatalf("incorrect payload. expected: %v; actual: %v", []byte(DEFAULT_PAYLOAD), mPayload) } } func waitUntilSize(t testing.T, m Memberlist, expected int) { t.Helper() retry(t, 15, 500time.Millisecond, func(failf func(string, ...interface{})) { t.Helper() if m.NumMembers() != expected { failf("%s expected to have %d members but had: %v", m.config.Name, expected, m.Members()) } }) } func isPortFree(t testing.T, addr string, port int) error { t.Helper() ip := net.ParseIP(addr) tcpAddr := &net.TCPAddr{IP: ip, Port: port} tcpLn, err := net.ListenTCP("tcp", tcpAddr) if err != nil { return err } if err := tcpLn.Close(); err != nil { return err } udpAddr := &net.UDPAddr{IP: ip, Port: port} udpLn, err := net.ListenUDP("udp", udpAddr) if err != nil { return err } return udpLn.Close() } func waitUntilPortIsFree(t testing.T, m Memberlist) { t.Helper() // wait until we know for certain that m1 is dead dead addr := m.config.BindAddr port := m.config.BindPort retry(t, 15, 250time.Millisecond, func(failf func(string, ...interface{})) { t.Helper() if err := isPortFree(t, addr, port); err != nil { failf("%s port is not yet free", m.config.Name) } }) } // This test should follow the recommended upgrade guide: // https://www.consul.io/docs/agent/encryption.html#configuring-gossip-encryption-on-an-existing-cluster // // We will use two nodes for this: m0 and m1 // // 0. Start with nodes without encryption. // 1. Set an encryption key and set GossipVerifyIncoming=false and GossipVerifyOutgoing=false to all nodes. // 2. Change GossipVerifyOutgoing=true to all nodes. // 3. Change GossipVerifyIncoming=true to all nodes. func TestMemberlist_EncryptedGossipTransition(t testing.T) { // ensure these all get the same general set of customizations pretty := make(map[string]string) // addr->shortName newConfig := func(shortName string, addr string) Config { t.Helper() conf := DefaultLANConfig() if addr == "" { addr = getBindAddr().String() } conf.Name = addr // conf.Name = shortName conf.BindAddr = addr conf.BindPort = 0 // Set the gossip interval fast enough to get a reasonable test, // but slow enough to avoid "sendto: operation not permitted" conf.GossipInterval = 100 time.Millisecond conf.Logger = log.New(os.Stderr, shortName, log.LstdFlags) pretty[conf.Name] = shortName return conf } var bindPort int createOK := func(conf Config) Memberlist { t.Helper() if bindPort > 0 { conf.BindPort = bindPort } else { // try a range of port numbers until something sticks } m, err := Create(conf) require.NoError(t, err) if bindPort == 0 { bindPort = m.config.BindPort } return m } joinOK := func(src, dst Memberlist, numNodes int) { t.Helper() srcName, dstName := pretty[src.config.Name], pretty[dst.config.Name] t.Logf("Node %s[%s] joining node %s[%s]", srcName, src.config.Name, dstName, dst.config.Name) num, err := src.Join([]string{dst.config.Name + "/" + dst.config.BindAddr}) require.NoError(t, err) require.Equal(t, 1, num) waitUntilSize(t, src, numNodes) waitUntilSize(t, dst, numNodes) // Check the hosts require.Equal(t, numNodes, len(src.Members()), "nodes: %v", src.Members()) require.Equal(t, numNodes, src.estNumNodes(), "nodes: %v", src.Members()) require.Equal(t, numNodes, len(dst.Members()), "nodes: %v", dst.Members()) require.Equal(t, numNodes, dst.estNumNodes(), "nodes: %v", dst.Members()) } leaveOK := func(m Memberlist, why string) { t.Helper() name := pretty[m.config.Name] t.Logf("Node %s[%s] is leaving %s", name, m.config.Name, why) err := m.Leave(time.Second) require.NoError(t, err) } shutdownOK := func(m Memberlist, why string) { t.Helper() name := pretty[m.config.Name] t.Logf("Node %s[%s] is shutting down %s", name, m.config.Name, why) err := m.Shutdown() require.NoError(t, err) // Double check that it genuinely shutdown. waitUntilPortIsFree(t, m) } leaveAndShutdown := func(leaver, bystander Memberlist, why string) { t.Helper() leaveOK(leaver, why) waitUntilSize(t, bystander, 1) shutdownOK(leaver, why) waitUntilSize(t, bystander, 1) } // ==== STEP 0 ==== // Create a first cluster of 2 nodes with no gossip encryption settings. conf0 := newConfig("m0", "") m0 := createOK(conf0) defer m0.Shutdown() conf1 := newConfig("m1", "") m1 := createOK(conf1) defer m1.Shutdown() joinOK(m1, m0, 2) t.Logf("==== STEP 0 complete: two node unencrypted cluster ====") // ==== STEP 1 ==== // Take down m0, upgrade to first stage of gossip transition settings. leaveAndShutdown(m0, m1, "to upgrade gossip to first stage") // Resurrect the first node with the first stage of gossip transition settings. conf0 = newConfig("m0", m0.config.BindAddr) conf0.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") conf0.GossipVerifyIncoming = false conf0.GossipVerifyOutgoing = false m0 = createOK(conf0) defer m0.Shutdown() // Join the second node. m1 has no encryption while m0 has encryption configured and // can receive encrypted gossip, but will not encrypt outgoing gossip. joinOK(m0, m1, 2) leaveAndShutdown(m1, m0, "to upgrade gossip to first stage") // Resurrect the second node with the first stage of gossip transition settings. conf1 = newConfig("m1", m1.config.BindAddr) conf1.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") conf1.GossipVerifyIncoming = false conf1.GossipVerifyOutgoing = false m1 = createOK(conf1) defer m1.Shutdown() // Join the first node. Both have encryption configured and can receive // encrypted gossip, but will not encrypt outgoing gossip. joinOK(m1, m0, 2) t.Logf("==== STEP 1 complete: two node encryption-aware cluster ====") // ==== STEP 2 ==== // Take down m0, upgrade to second stage of gossip transition settings. leaveAndShutdown(m0, m1, "to upgrade gossip to second stage") // Resurrect the first node with the second stage of gossip transition settings. conf0 = newConfig("m0", m0.config.BindAddr) conf0.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") conf0.GossipVerifyIncoming = false m0 = createOK(conf0) defer m0.Shutdown() // Join the second node. At this step, both nodes have encryption // configured but only m0 is sending encrypted gossip. joinOK(m0, m1, 2) leaveAndShutdown(m1, m0, "to upgrade gossip to second stage") // Resurrect the second node with the second stage of gossip transition settings. conf1 = newConfig("m1", m1.config.BindAddr) conf1.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") conf1.GossipVerifyIncoming = false m1 = createOK(conf1) defer m1.Shutdown() // Join the first node. Both have encryption configured and can receive // encrypted gossip, and encrypt outgoing gossip, but aren't forcing // incoming gossip is encrypted. joinOK(m1, m0, 2) t.Logf("==== STEP 2 complete: two node encryption-aware cluster being encrypted ====") // ==== STEP 3 ==== // Take down m0, upgrade to final stage of gossip transition settings. leaveAndShutdown(m0, m1, "to upgrade gossip to final stage") // Resurrect the first node with the final stage of gossip transition settings. conf0 = newConfig("m0", m0.config.BindAddr) conf0.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") m0 = createOK(conf0) defer m0.Shutdown() // Join the second node. At this step, both nodes have encryption // configured and are sending it, bu tonly m0 is verifying inbound gossip // is encrypted. joinOK(m0, m1, 2) leaveAndShutdown(m1, m0, "to upgrade gossip to final stage") // Resurrect the second node with the final stage of gossip transition settings. conf1 = newConfig("m1", m1.config.BindAddr) conf1.SecretKey = []byte("Hi16ZXu2lNCRVwtr20khAg==") m1 = createOK(conf1) defer m1.Shutdown() // Join the first node. Both have encryption configured and fully in // enforcement. joinOK(m1, m0, 2) t.Logf("==== STEP 3 complete: two node encrypted cluster locked down ====") } // Consul bug, rapid restart (before failure detection), // with an updated meta data. Should be at incarnation 1 for // both. // // This test is uncommented because it requires that either we // can rebind the socket (SO_REUSEPORT) which Go does not allow, // OR we must disable the address conflict checking in memberlist. // I just comment out that code to test this case. // //func TestMemberlist_Restart_delegateMeta_Update(t testing.T) { // c1 := testConfig() // c2 := testConfig() // mock1 := &MockDelegate{meta: []byte("web")} // mock2 := &MockDelegate{meta: []byte("lb")} // c1.Delegate = mock1 // c2.Delegate = mock2 // m1, err := Create(c1) // if err != nil { // t.Fatalf("err: %s", err) // } // defer m1.Shutdown() // m2, err := Create(c2) // if err != nil { // t.Fatalf("err: %s", err) // } // defer m2.Shutdown() // _, err = m1.Join([]string{c2.BindAddr}) // if err != nil { // t.Fatalf("err: %s", err) // } // yield() // // Recreate m1 with updated meta // m1.Shutdown() // c3 := testConfig() // c3.Name = c1.Name // c3.Delegate = mock1 // c3.GossipInterval = time.Millisecond // mock1.meta = []byte("api") // m1, err = Create(c3) // if err != nil { // t.Fatalf("err: %s", err) // } // defer m1.Shutdown() // _, err = m1.Join([]string{c2.BindAddr}) // if err != nil { // t.Fatalf("err: %s", err) // } // yield() // yield() // // Check the updates have propagated // var roles map[string]string // // Check the roles of members of m1 // m1m := m1.Members() // if len(m1m) != 2 { // t.Fatalf("bad: %#v", m1m) // } // roles = make(map[string]string) // for _, m := range m1m { // roles[m.Name] = string(m.Meta) // } // if r := roles[c1.Name]; r != "api" { // t.Fatalf("bad role for %s: %s", c1.Name, r) // } // if r := roles[c2.Name]; r != "lb" { // t.Fatalf("bad role for %s: %s", c2.Name, r) // } // // Check the roles of members of m2 // m2m := m2.Members() // if len(m2m) != 2 { // t.Fatalf("bad: %#v", m2m) // } // roles = make(map[string]string) // for _, m := range m2m { // roles[m.Name] = string(m.Meta) // } // if r := roles[c1.Name]; r != "api" { // t.Fatalf("bad role for %s: %s", c1.Name, r) // } // if r := roles[c2.Name]; r != "lb" { // t.Fatalf("bad role for %s: %s", c2.Name, r) // } //} func runStep(t testing.T, name string, fn func(t *testing.T)) { t.Helper() if !t.Run(name, fn) { t.FailNow() } }	49.83 KB	2025-05-25 12:27
📄 merge_delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist // MergeDelegate is used to involve a client in // a potential cluster merge operation. Namely, when // a node does a TCP push/pull (as part of a join), // the delegate is involved and allowed to cancel the join // based on custom logic. The merge delegate is NOT invoked // as part of the push-pull anti-entropy. type MergeDelegate interface { // NotifyMerge is invoked when a merge could take place. // Provides a list of the nodes known by the peer. If // the return value is non-nil, the merge is canceled. NotifyMerge(peers []*Node) error }	640 B	2025-05-25 12:27
📄 mock_transport.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "io" "net" "strconv" "time" ) // MockNetwork is used as a factory that produces MockTransport instances which // are uniquely addressed and wired up to talk to each other. type MockNetwork struct { transportsByAddr map[string]MockTransport transportsByName map[string]MockTransport port int } // NewTransport returns a new MockTransport with a unique address, wired up to // talk to the other transports in the MockNetwork. func (n MockNetwork) NewTransport(name string) MockTransport { n.port += 1 addr := fmt.Sprintf("127.0.0.1:%d", n.port) transport := &MockTransport{ net: n, addr: &MockAddress{addr, name}, packetCh: make(chan Packet), streamCh: make(chan net.Conn), } if n.transportsByAddr == nil { n.transportsByAddr = make(map[string]MockTransport) } n.transportsByAddr[addr] = transport if n.transportsByName == nil { n.transportsByName = make(map[string]MockTransport) } n.transportsByName[name] = transport return transport } // MockAddress is a wrapper which adds the net.Addr interface to our mock // address scheme. type MockAddress struct { addr string name string } // See net.Addr. func (a MockAddress) Network() string { return "mock" } // See net.Addr. func (a MockAddress) String() string { return a.addr } // MockTransport directly plumbs messages to other transports its MockNetwork. type MockTransport struct { net MockNetwork addr MockAddress packetCh chan Packet streamCh chan net.Conn } var _ NodeAwareTransport = (MockTransport)(nil) // See Transport. func (t MockTransport) FinalAdvertiseAddr(string, int) (net.IP, int, error) { host, portStr, err := net.SplitHostPort(t.addr.String()) if err != nil { return nil, 0, err } ip := net.ParseIP(host) if ip == nil { return nil, 0, fmt.Errorf("Failed to parse IP %q", host) } port, err := strconv.ParseInt(portStr, 10, 16) if err != nil { return nil, 0, err } return ip, int(port), nil } // See Transport. func (t MockTransport) WriteTo(b []byte, addr string) (time.Time, error) { a := Address{Addr: addr, Name: ""} return t.WriteToAddress(b, a) } // See NodeAwareTransport. func (t MockTransport) WriteToAddress(b []byte, a Address) (time.Time, error) { dest, err := t.getPeer(a) if err != nil { return time.Time{}, err } now := time.Now() dest.packetCh <- &Packet{ Buf: b, From: t.addr, Timestamp: now, } return now, nil } // See Transport. func (t MockTransport) PacketCh() <-chan Packet { return t.packetCh } // See NodeAwareTransport. func (t MockTransport) IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error { if shouldClose { defer conn.Close() } // Copy everything from the stream into packet buffer. var buf bytes.Buffer if _, err := io.Copy(&buf, conn); err != nil { return fmt.Errorf("failed to read packet: %v", err) } // Check the length - it needs to have at least one byte to be a proper // message. This is checked elsewhere for writes coming in directly from // the UDP socket. if n := buf.Len(); n < 1 { return fmt.Errorf("packet too short (%d bytes) %s", n, LogAddress(addr)) } // Inject the packet. t.packetCh <- &Packet{ Buf: buf.Bytes(), From: addr, Timestamp: now, } return nil } // See Transport. func (t MockTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { a := Address{Addr: addr, Name: ""} return t.DialAddressTimeout(a, timeout) } // See NodeAwareTransport. func (t MockTransport) DialAddressTimeout(a Address, timeout time.Duration) (net.Conn, error) { dest, err := t.getPeer(a) if err != nil { return nil, err } p1, p2 := net.Pipe() dest.streamCh <- p1 return p2, nil } // See Transport. func (t MockTransport) StreamCh() <-chan net.Conn { return t.streamCh } // See NodeAwareTransport. func (t MockTransport) IngestStream(conn net.Conn) error { t.streamCh <- conn return nil } // See Transport. func (t MockTransport) Shutdown() error { return nil } func (t MockTransport) getPeer(a Address) (MockTransport, error) { var ( dest *MockTransport ok bool ) if a.Name != "" { dest, ok = t.net.transportsByName[a.Name] } else { dest, ok = t.net.transportsByAddr[a.Addr] } if !ok { return nil, fmt.Errorf("No route to %s", a) } return dest, nil }	4.36 KB	2025-05-25 12:27
📄 net.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bufio" "bytes" "encoding/binary" "fmt" "hash/crc32" "io" "math" "net" "sync/atomic" "time" "github.com/armon/go-metrics" "github.com/hashicorp/go-msgpack/v2/codec" ) // This is the minimum and maximum protocol version that we can // _understand_. We're allowed to speak at any version within this // range. This range is inclusive. const ( ProtocolVersionMin uint8 = 1 // Version 3 added support for TCP pings but we kept the default // protocol version at 2 to ease transition to this new feature. // A memberlist speaking version 2 of the protocol will attempt // to TCP ping another memberlist who understands version 3 or // greater. // // Version 4 added support for nacks as part of indirect probes. // A memberlist speaking version 2 of the protocol will expect // nacks from another memberlist who understands version 4 or // greater, and likewise nacks will be sent to memberlists who // understand version 4 or greater. ProtocolVersion2Compatible = 2 ProtocolVersionMax = 5 ) // messageType is an integer ID of a type of message that can be received // on network channels from other members. type messageType uint8 // The list of available message types. // // WARNING: ONLY APPEND TO THIS LIST! The numeric values are part of the // protocol itself. const ( pingMsg messageType = iota indirectPingMsg ackRespMsg suspectMsg aliveMsg deadMsg pushPullMsg compoundMsg userMsg // User mesg, not handled by us compressMsg encryptMsg nackRespMsg hasCrcMsg errMsg ) const ( // hasLabelMsg has a deliberately high value so that you can disambiguate // it from the encryptionVersion header which is either 0/1 right now and // also any of the existing messageTypes hasLabelMsg messageType = 244 ) // compressionType is used to specify the compression algorithm type compressionType uint8 const ( lzwAlgo compressionType = iota ) const ( MetaMaxSize = 512 // Maximum size for node meta data compoundHeaderOverhead = 2 // Assumed header overhead compoundOverhead = 2 // Assumed overhead per entry in compoundHeader userMsgOverhead = 1 blockingWarning = 10 * time.Millisecond // Warn if a UDP packet takes this long to process maxPushStateBytes = 20 * 1024 * 1024 maxPushPullRequests = 128 // Maximum number of concurrent push/pull requests ) // ping request sent directly to node type ping struct { SeqNo uint32 // Node is sent so the target can verify they are // the intended recipient. This is to protect again an agent // restart with a new name. Node string SourceAddr []byte `codec:",omitempty"` // Source address, used for a direct reply SourcePort uint16 `codec:",omitempty"` // Source port, used for a direct reply SourceNode string `codec:",omitempty"` // Source name, used for a direct reply } // indirect ping sent to an indirect node type indirectPingReq struct { SeqNo uint32 Target []byte Port uint16 // Node is sent so the target can verify they are // the intended recipient. This is to protect against an agent // restart with a new name. Node string Nack bool // true if we'd like a nack back SourceAddr []byte `codec:",omitempty"` // Source address, used for a direct reply SourcePort uint16 `codec:",omitempty"` // Source port, used for a direct reply SourceNode string `codec:",omitempty"` // Source name, used for a direct reply } // ack response is sent for a ping type ackResp struct { SeqNo uint32 Payload []byte } // nack response is sent for an indirect ping when the pinger doesn't hear from // the ping-ee within the configured timeout. This lets the original node know // that the indirect ping attempt happened but didn't succeed. type nackResp struct { SeqNo uint32 } // err response is sent to relay the error from the remote end type errResp struct { Error string } // suspect is broadcast when we suspect a node is dead type suspect struct { Incarnation uint32 Node string From string // Include who is suspecting } // alive is broadcast when we know a node is alive. // Overloaded for nodes joining type alive struct { Incarnation uint32 Node string Addr []byte Port uint16 Meta []byte // The versions of the protocol/delegate that are being spoken, order: // pmin, pmax, pcur, dmin, dmax, dcur Vsn []uint8 } // dead is broadcast when we confirm a node is dead // Overloaded for nodes leaving type dead struct { Incarnation uint32 Node string From string // Include who is suspecting } // pushPullHeader is used to inform the // otherside how many states we are transferring type pushPullHeader struct { Nodes int UserStateLen int // Encodes the byte lengh of user state Join bool // Is this a join request or a anti-entropy run } // userMsgHeader is used to encapsulate a userMsg type userMsgHeader struct { UserMsgLen int // Encodes the byte lengh of user state } // pushNodeState is used for pushPullReq when we are // transferring out node states type pushNodeState struct { Name string Addr []byte Port uint16 Meta []byte Incarnation uint32 State NodeStateType Vsn []uint8 // Protocol versions } // compress is used to wrap an underlying payload // using a specified compression algorithm type compress struct { Algo compressionType Buf []byte } // msgHandoff is used to transfer a message between goroutines type msgHandoff struct { msgType messageType buf []byte from net.Addr } // encryptionVersion returns the encryption version to use func (m Memberlist) encryptionVersion() encryptionVersion { switch m.ProtocolVersion() { case 1: return 0 default: return 1 } } // streamListen is a long running goroutine that pulls incoming streams from the // transport and hands them off for processing. func (m Memberlist) streamListen() { for { select { case conn := <-m.transport.StreamCh(): go m.handleConn(conn) case <-m.shutdownCh: return } } } // handleConn handles a single incoming stream connection from the transport. func (m Memberlist) handleConn(conn net.Conn) { defer conn.Close() m.logger.Printf("[DEBUG] memberlist: Stream connection %s", LogConn(conn)) metrics.IncrCounterWithLabels([]string{"memberlist", "tcp", "accept"}, 1, m.metricLabels) conn.SetDeadline(time.Now().Add(m.config.TCPTimeout)) var ( streamLabel string err error ) conn, streamLabel, err = RemoveLabelHeaderFromStream(conn) if err != nil { m.logger.Printf("[ERR] memberlist: failed to receive and remove the stream label header: %s %s", err, LogConn(conn)) return } if m.config.SkipInboundLabelCheck { if streamLabel != "" { m.logger.Printf("[ERR] memberlist: unexpected double stream label header: %s", LogConn(conn)) return } // Set this from config so that the auth data assertions work below. streamLabel = m.config.Label } if m.config.Label != streamLabel { m.logger.Printf("[ERR] memberlist: discarding stream with unacceptable label %q: %s", streamLabel, LogConn(conn)) return } msgType, bufConn, dec, err := m.readStream(conn, streamLabel) if err != nil { if err != io.EOF { m.logger.Printf("[ERR] memberlist: failed to receive: %s %s", err, LogConn(conn)) resp := errResp{err.Error()} out, err := encode(errMsg, &resp, m.config.MsgpackUseNewTimeFormat) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to encode error response: %s", err) return } err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to send error: %s %s", err, LogConn(conn)) return } } return } switch msgType { case userMsg: if err := m.readUserMsg(bufConn, dec); err != nil { m.logger.Printf("[ERR] memberlist: Failed to receive user message: %s %s", err, LogConn(conn)) } case pushPullMsg: // Increment counter of pending push/pulls numConcurrent := atomic.AddUint32(&m.pushPullReq, 1) defer atomic.AddUint32(&m.pushPullReq, ^uint32(0)) // Check if we have too many open push/pull requests if numConcurrent >= maxPushPullRequests { m.logger.Printf("[ERR] memberlist: Too many pending push/pull requests") return } join, remoteNodes, userState, err := m.readRemoteState(bufConn, dec) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to read remote state: %s %s", err, LogConn(conn)) return } if err := m.sendLocalState(conn, join, streamLabel); err != nil { m.logger.Printf("[ERR] memberlist: Failed to push local state: %s %s", err, LogConn(conn)) return } if err := m.mergeRemoteState(join, remoteNodes, userState); err != nil { m.logger.Printf("[ERR] memberlist: Failed push/pull merge: %s %s", err, LogConn(conn)) return } case pingMsg: var p ping if err := dec.Decode(&p); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode ping: %s %s", err, LogConn(conn)) return } if p.Node != "" && p.Node != m.config.Name { m.logger.Printf("[WARN] memberlist: Got ping for unexpected node %s %s", p.Node, LogConn(conn)) return } ack := ackResp{p.SeqNo, nil} out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to encode ack: %s", err) return } err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to send ack: %s %s", err, LogConn(conn)) return } default: m.logger.Printf("[ERR] memberlist: Received invalid msgType (%d) %s", msgType, LogConn(conn)) } } // packetListen is a long running goroutine that pulls packets out of the // transport and hands them off for processing. func (m Memberlist) packetListen() { for { select { case packet := <-m.transport.PacketCh(): m.ingestPacket(packet.Buf, packet.From, packet.Timestamp) case <-m.shutdownCh: return } } } func (m Memberlist) ingestPacket(buf []byte, from net.Addr, timestamp time.Time) { var ( packetLabel string err error ) buf, packetLabel, err = RemoveLabelHeaderFromPacket(buf) if err != nil { m.logger.Printf("[ERR] memberlist: %v %s", err, LogAddress(from)) return } if m.config.SkipInboundLabelCheck { if packetLabel != "" { m.logger.Printf("[ERR] memberlist: unexpected double packet label header: %s", LogAddress(from)) return } // Set this from config so that the auth data assertions work below. packetLabel = m.config.Label } if m.config.Label != packetLabel { m.logger.Printf("[ERR] memberlist: discarding packet with unacceptable label %q: %s", packetLabel, LogAddress(from)) return } // Check if encryption is enabled if m.config.EncryptionEnabled() { // Decrypt the payload authData := []byte(packetLabel) plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, authData) if err != nil { if !m.config.GossipVerifyIncoming { // Treat the message as plaintext plain = buf } else { m.logger.Printf("[ERR] memberlist: Decrypt packet failed: %v %s", err, LogAddress(from)) return } } // Continue processing the plaintext buffer buf = plain } // See if there's a checksum included to verify the contents of the message if len(buf) >= 5 && messageType(buf[0]) == hasCrcMsg { crc := crc32.ChecksumIEEE(buf[5:]) expected := binary.BigEndian.Uint32(buf[1:5]) if crc != expected { m.logger.Printf("[WARN] memberlist: Got invalid checksum for UDP packet: %x, %x", crc, expected) return } m.handleCommand(buf[5:], from, timestamp) } else { m.handleCommand(buf, from, timestamp) } } func (m Memberlist) handleCommand(buf []byte, from net.Addr, timestamp time.Time) { if len(buf) < 1 { m.logger.Printf("[ERR] memberlist: missing message type byte %s", LogAddress(from)) return } // Decode the message type msgType := messageType(buf[0]) buf = buf[1:] // Switch on the msgType switch msgType { case compoundMsg: m.handleCompound(buf, from, timestamp) case compressMsg: m.handleCompressed(buf, from, timestamp) case pingMsg: m.handlePing(buf, from) case indirectPingMsg: m.handleIndirectPing(buf, from) case ackRespMsg: m.handleAck(buf, from, timestamp) case nackRespMsg: m.handleNack(buf, from) case suspectMsg: fallthrough case aliveMsg: fallthrough case deadMsg: fallthrough case userMsg: // Determine the message queue, prioritize alive queue := m.lowPriorityMsgQueue if msgType == aliveMsg { queue = m.highPriorityMsgQueue } // Check for overflow and append if not full m.msgQueueLock.Lock() if queue.Len() >= m.config.HandoffQueueDepth { m.logger.Printf("[WARN] memberlist: handler queue full, dropping message (%d) %s", msgType, LogAddress(from)) } else { queue.PushBack(msgHandoff{msgType, buf, from}) } m.msgQueueLock.Unlock() // Notify of pending message select { case m.handoffCh <- struct{}{}: default: } default: m.logger.Printf("[ERR] memberlist: msg type (%d) not supported %s", msgType, LogAddress(from)) } } // getNextMessage returns the next message to process in priority order, using LIFO func (m Memberlist) getNextMessage() (msgHandoff, bool) { m.msgQueueLock.Lock() defer m.msgQueueLock.Unlock() if el := m.highPriorityMsgQueue.Back(); el != nil { m.highPriorityMsgQueue.Remove(el) msg := el.Value.(msgHandoff) return msg, true } else if el := m.lowPriorityMsgQueue.Back(); el != nil { m.lowPriorityMsgQueue.Remove(el) msg := el.Value.(msgHandoff) return msg, true } return msgHandoff{}, false } // packetHandler is a long running goroutine that processes messages received // over the packet interface, but is decoupled from the listener to avoid // blocking the listener which may cause ping/ack messages to be delayed. func (m Memberlist) packetHandler() { for { select { case <-m.handoffCh: for { msg, ok := m.getNextMessage() if !ok { break } msgType := msg.msgType buf := msg.buf from := msg.from switch msgType { case suspectMsg: m.handleSuspect(buf, from) case aliveMsg: m.handleAlive(buf, from) case deadMsg: m.handleDead(buf, from) case userMsg: m.handleUser(buf, from) default: m.logger.Printf("[ERR] memberlist: Message type (%d) not supported %s (packet handler)", msgType, LogAddress(from)) } } case <-m.shutdownCh: return } } } func (m Memberlist) handleCompound(buf []byte, from net.Addr, timestamp time.Time) { // Decode the parts trunc, parts, err := decodeCompoundMessage(buf) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode compound request: %s %s", err, LogAddress(from)) return } // Log any truncation if trunc > 0 { m.logger.Printf("[WARN] memberlist: Compound request had %d truncated messages %s", trunc, LogAddress(from)) } // Handle each message for _, part := range parts { m.handleCommand(part, from, timestamp) } } func (m Memberlist) handlePing(buf []byte, from net.Addr) { var p ping if err := decode(buf, &p); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode ping request: %s %s", err, LogAddress(from)) return } // If node is provided, verify that it is for us if p.Node != "" && p.Node != m.config.Name { m.logger.Printf("[WARN] memberlist: Got ping for unexpected node '%s' %s", p.Node, LogAddress(from)) return } var ack ackResp ack.SeqNo = p.SeqNo if m.config.Ping != nil { ack.Payload = m.config.Ping.AckPayload() } addr := "" if len(p.SourceAddr) > 0 && p.SourcePort > 0 { addr = joinHostPort(net.IP(p.SourceAddr).String(), p.SourcePort) } else { addr = from.String() } a := Address{ Addr: addr, Name: p.SourceNode, } if err := m.encodeAndSendMsg(a, ackRespMsg, &ack); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send ack: %s %s", err, LogAddress(from)) } } func (m Memberlist) handleIndirectPing(buf []byte, from net.Addr) { var ind indirectPingReq if err := decode(buf, &ind); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode indirect ping request: %s %s", err, LogAddress(from)) return } // For proto versions < 2, there is no port provided. Mask old // behavior by using the configured port. if m.ProtocolVersion() < 2 \|\| ind.Port == 0 { ind.Port = uint16(m.config.BindPort) } // Send a ping to the correct host. localSeqNo := m.nextSeqNo() selfAddr, selfPort := m.getAdvertise() ping := ping{ SeqNo: localSeqNo, Node: ind.Node, // The outbound message is addressed FROM us. SourceAddr: selfAddr, SourcePort: selfPort, SourceNode: m.config.Name, } // Forward the ack back to the requestor. If the request encodes an origin // use that otherwise assume that the other end of the UDP socket is // usable. indAddr := "" if len(ind.SourceAddr) > 0 && ind.SourcePort > 0 { indAddr = joinHostPort(net.IP(ind.SourceAddr).String(), ind.SourcePort) } else { indAddr = from.String() } // Setup a response handler to relay the ack cancelCh := make(chan struct{}) respHandler := func(payload []byte, timestamp time.Time) { // Try to prevent the nack if we've caught it in time. close(cancelCh) ack := ackResp{ind.SeqNo, nil} a := Address{ Addr: indAddr, Name: ind.SourceNode, } if err := m.encodeAndSendMsg(a, ackRespMsg, &ack); err != nil { m.logger.Printf("[ERR] memberlist: Failed to forward ack: %s %s", err, LogStringAddress(indAddr)) } } m.setAckHandler(localSeqNo, respHandler, m.config.ProbeTimeout) // Send the ping. addr := joinHostPort(net.IP(ind.Target).String(), ind.Port) a := Address{ Addr: addr, Name: ind.Node, } if err := m.encodeAndSendMsg(a, pingMsg, &ping); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send indirect ping: %s %s", err, LogStringAddress(indAddr)) } // Setup a timer to fire off a nack if no ack is seen in time. if ind.Nack { go func() { select { case <-cancelCh: return case <-time.After(m.config.ProbeTimeout): nack := nackResp{ind.SeqNo} a := Address{ Addr: indAddr, Name: ind.SourceNode, } if err := m.encodeAndSendMsg(a, nackRespMsg, &nack); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send nack: %s %s", err, LogStringAddress(indAddr)) } } }() } } func (m Memberlist) handleAck(buf []byte, from net.Addr, timestamp time.Time) { var ack ackResp if err := decode(buf, &ack); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode ack response: %s %s", err, LogAddress(from)) return } m.invokeAckHandler(ack, timestamp) } func (m Memberlist) handleNack(buf []byte, from net.Addr) { var nack nackResp if err := decode(buf, &nack); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode nack response: %s %s", err, LogAddress(from)) return } m.invokeNackHandler(nack) } func (m Memberlist) handleSuspect(buf []byte, from net.Addr) { var sus suspect if err := decode(buf, &sus); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode suspect message: %s %s", err, LogAddress(from)) return } m.suspectNode(&sus) } // ensureCanConnect return the IP from a RemoteAddress // return error if this client must not connect func (m Memberlist) ensureCanConnect(from net.Addr) error { if !m.config.IPMustBeChecked() { return nil } source := from.String() if source == "pipe" { return nil } host, _, err := net.SplitHostPort(source) if err != nil { return err } ip := net.ParseIP(host) if ip == nil { return fmt.Errorf("Cannot parse IP from %s", host) } return m.config.IPAllowed(ip) } func (m Memberlist) handleAlive(buf []byte, from net.Addr) { if err := m.ensureCanConnect(from); err != nil { m.logger.Printf("[DEBUG] memberlist: Blocked alive message: %s %s", err, LogAddress(from)) return } var live alive if err := decode(buf, &live); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode alive message: %s %s", err, LogAddress(from)) return } if m.config.IPMustBeChecked() { innerIP := net.IP(live.Addr) if innerIP != nil { if err := m.config.IPAllowed(innerIP); err != nil { m.logger.Printf("[DEBUG] memberlist: Blocked alive.Addr=%s message from: %s %s", innerIP.String(), err, LogAddress(from)) return } } } // For proto versions < 2, there is no port provided. Mask old // behavior by using the configured port if m.ProtocolVersion() < 2 \|\| live.Port == 0 { live.Port = uint16(m.config.BindPort) } m.aliveNode(&live, nil, false) } func (m Memberlist) handleDead(buf []byte, from net.Addr) { var d dead if err := decode(buf, &d); err != nil { m.logger.Printf("[ERR] memberlist: Failed to decode dead message: %s %s", err, LogAddress(from)) return } m.deadNode(&d) } // handleUser is used to notify channels of incoming user data func (m Memberlist) handleUser(buf []byte, from net.Addr) { d := m.config.Delegate if d != nil { d.NotifyMsg(buf) } } // handleCompressed is used to unpack a compressed message func (m Memberlist) handleCompressed(buf []byte, from net.Addr, timestamp time.Time) { // Try to decode the payload payload, err := decompressPayload(buf) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to decompress payload: %v %s", err, LogAddress(from)) return } // Recursively handle the payload m.handleCommand(payload, from, timestamp) } // encodeAndSendMsg is used to combine the encoding and sending steps func (m Memberlist) encodeAndSendMsg(a Address, msgType messageType, msg interface{}) error { out, err := encode(msgType, msg, m.config.MsgpackUseNewTimeFormat) if err != nil { return err } if err := m.sendMsg(a, out.Bytes()); err != nil { return err } return nil } // sendMsg is used to send a message via packet to another host. It will // opportunistically create a compoundMsg and piggy back other broadcasts. func (m Memberlist) sendMsg(a Address, msg []byte) error { // Check if we can piggy back any messages bytesAvail := m.config.UDPBufferSize - len(msg) - compoundHeaderOverhead - labelOverhead(m.config.Label) if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing { bytesAvail -= encryptOverhead(m.encryptionVersion()) } extra := m.getBroadcasts(compoundOverhead, bytesAvail) // Fast path if nothing to piggypack if len(extra) == 0 { return m.rawSendMsgPacket(a, nil, msg) } // Join all the messages msgs := make([][]byte, 0, 1+len(extra)) msgs = append(msgs, msg) msgs = append(msgs, extra...) // Create a compound message compound := makeCompoundMessage(msgs) // Send the message return m.rawSendMsgPacket(a, nil, compound.Bytes()) } // rawSendMsgPacket is used to send message via packet to another host without // modification, other than compression or encryption if enabled. func (m Memberlist) rawSendMsgPacket(a Address, node Node, msg []byte) error { if a.Name == "" && m.config.RequireNodeNames { return errNodeNamesAreRequired } // Check if we have compression enabled if m.config.EnableCompression { buf, err := compressPayload(msg, m.config.MsgpackUseNewTimeFormat) if err != nil { m.logger.Printf("[WARN] memberlist: Failed to compress payload: %v", err) } else { // Only use compression if it reduced the size if buf.Len() < len(msg) { msg = buf.Bytes() } } } // Try to look up the destination node. Note this will only work if the // bare ip address is used as the node name, which is not guaranteed. if node == nil { toAddr, _, err := net.SplitHostPort(a.Addr) if err != nil { m.logger.Printf("[ERR] memberlist: Failed to parse address %q: %v", a.Addr, err) return err } m.nodeLock.RLock() nodeState, ok := m.nodeMap[toAddr] m.nodeLock.RUnlock() if ok { node = &nodeState.Node } } // Add a CRC to the end of the payload if the recipient understands // ProtocolVersion >= 5 if node != nil && node.PMax >= 5 { crc := crc32.ChecksumIEEE(msg) header := make([]byte, 5, 5+len(msg)) header[0] = byte(hasCrcMsg) binary.BigEndian.PutUint32(header[1:], crc) msg = append(header, msg...) } // Check if we have encryption enabled if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing { // Encrypt the payload var ( primaryKey = m.config.Keyring.GetPrimaryKey() packetLabel = []byte(m.config.Label) buf bytes.Buffer ) err := encryptPayload(m.encryptionVersion(), primaryKey, msg, packetLabel, &buf) if err != nil { m.logger.Printf("[ERR] memberlist: Encryption of message failed: %v", err) return err } msg = buf.Bytes() } metrics.IncrCounterWithLabels([]string{"memberlist", "udp", "sent"}, float32(len(msg)), m.metricLabels) _, err := m.transport.WriteToAddress(msg, a) return err } // rawSendMsgStream is used to stream a message to another host without // modification, other than applying compression and encryption if enabled. func (m Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte, streamLabel string) error { // Check if compression is enabled if m.config.EnableCompression { compBuf, err := compressPayload(sendBuf, m.config.MsgpackUseNewTimeFormat) if err != nil { m.logger.Printf("[ERROR] memberlist: Failed to compress payload: %v", err) } else { sendBuf = compBuf.Bytes() } } // Check if encryption is enabled if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing { crypt, err := m.encryptLocalState(sendBuf, streamLabel) if err != nil { m.logger.Printf("[ERROR] memberlist: Failed to encrypt local state: %v", err) return err } sendBuf = crypt } // Write out the entire send buffer metrics.IncrCounterWithLabels([]string{"memberlist", "tcp", "sent"}, float32(len(sendBuf)), m.metricLabels) if n, err := conn.Write(sendBuf); err != nil { return err } else if n != len(sendBuf) { return fmt.Errorf("only %d of %d bytes written", n, len(sendBuf)) } return nil } // sendUserMsg is used to stream a user message to another host. func (m Memberlist) sendUserMsg(a Address, sendBuf []byte) error { if a.Name == "" && m.config.RequireNodeNames { return errNodeNamesAreRequired } conn, err := m.transport.DialAddressTimeout(a, m.config.TCPTimeout) if err != nil { return err } defer conn.Close() bufConn := bytes.NewBuffer(nil) if err := bufConn.WriteByte(byte(userMsg)); err != nil { return err } header := userMsgHeader{UserMsgLen: len(sendBuf)} hd := codec.MsgpackHandle{ BasicHandle: codec.BasicHandle{ TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat, }, } enc := codec.NewEncoder(bufConn, &hd) if err := enc.Encode(&header); err != nil { return err } if _, err := bufConn.Write(sendBuf); err != nil { return err } return m.rawSendMsgStream(conn, bufConn.Bytes(), m.config.Label) } // sendAndReceiveState is used to initiate a push/pull over a stream with a // remote host. func (m Memberlist) sendAndReceiveState(a Address, join bool) ([]pushNodeState, []byte, error) { if a.Name == "" && m.config.RequireNodeNames { return nil, nil, errNodeNamesAreRequired } // Attempt to connect conn, err := m.transport.DialAddressTimeout(a, m.config.TCPTimeout) if err != nil { return nil, nil, err } defer conn.Close() m.logger.Printf("[DEBUG] memberlist: Initiating push/pull sync with: %s %s", a.Name, conn.RemoteAddr()) metrics.IncrCounterWithLabels([]string{"memberlist", "tcp", "connect"}, 1, m.metricLabels) // Send our state if err := m.sendLocalState(conn, join, m.config.Label); err != nil { return nil, nil, err } conn.SetDeadline(time.Now().Add(m.config.TCPTimeout)) msgType, bufConn, dec, err := m.readStream(conn, m.config.Label) if err != nil { return nil, nil, err } if msgType == errMsg { var resp errResp if err := dec.Decode(&resp); err != nil { return nil, nil, err } return nil, nil, fmt.Errorf("remote error: %v", resp.Error) } // Quit if not push/pull if msgType != pushPullMsg { err := fmt.Errorf("received invalid msgType (%d), expected pushPullMsg (%d) %s", msgType, pushPullMsg, LogConn(conn)) return nil, nil, err } // Read remote state _, remoteNodes, userState, err := m.readRemoteState(bufConn, dec) return remoteNodes, userState, err } // sendLocalState is invoked to send our local state over a stream connection. func (m Memberlist) sendLocalState(conn net.Conn, join bool, streamLabel string) error { // Setup a deadline conn.SetDeadline(time.Now().Add(m.config.TCPTimeout)) // Prepare the local node state m.nodeLock.RLock() localNodes := make([]pushNodeState, len(m.nodes)) for idx, n := range m.nodes { localNodes[idx].Name = n.Name localNodes[idx].Addr = n.Addr localNodes[idx].Port = n.Port localNodes[idx].Incarnation = n.Incarnation localNodes[idx].State = n.State localNodes[idx].Meta = n.Meta localNodes[idx].Vsn = []uint8{ n.PMin, n.PMax, n.PCur, n.DMin, n.DMax, n.DCur, } } m.nodeLock.RUnlock() nodeStateCounts := make(map[string]int) nodeStateCounts[StateAlive.metricsString()] = 0 nodeStateCounts[StateLeft.metricsString()] = 0 nodeStateCounts[StateDead.metricsString()] = 0 nodeStateCounts[StateSuspect.metricsString()] = 0 for _, n := range localNodes { nodeStateCounts[n.State.metricsString()]++ } for nodeState, cnt := range nodeStateCounts { metrics.SetGaugeWithLabels([]string{"memberlist", "node", "instances"}, float32(cnt), append(m.metricLabels, metrics.Label{Name: "node_state", Value: nodeState})) } // Get the delegate state var userData []byte if m.config.Delegate != nil { userData = m.config.Delegate.LocalState(join) } // Create a bytes buffer writer bufConn := bytes.NewBuffer(nil) // Send our node state header := pushPullHeader{Nodes: len(localNodes), UserStateLen: len(userData), Join: join} hd := codec.MsgpackHandle{} enc := codec.NewEncoder(bufConn, &hd) // Begin state push if _, err := bufConn.Write([]byte{byte(pushPullMsg)}); err != nil { return err } if err := enc.Encode(&header); err != nil { return err } for i := 0; i < header.Nodes; i++ { if err := enc.Encode(&localNodes[i]); err != nil { return err } } // Write the user state as well if userData != nil { if _, err := bufConn.Write(userData); err != nil { return err } } moreBytes := binary.BigEndian.Uint32(bufConn.Bytes()[1:5]) metrics.SetGaugeWithLabels([]string{"memberlist", "size", "local"}, float32(moreBytes), m.metricLabels) // Get the send buffer return m.rawSendMsgStream(conn, bufConn.Bytes(), streamLabel) } // encryptLocalState is used to help encrypt local state before sending func (m Memberlist) encryptLocalState(sendBuf []byte, streamLabel string) ([]byte, error) { var buf bytes.Buffer // Write the encryptMsg byte buf.WriteByte(byte(encryptMsg)) // Write the size of the message sizeBuf := make([]byte, 4) encVsn := m.encryptionVersion() encLen := encryptedLength(encVsn, len(sendBuf)) binary.BigEndian.PutUint32(sizeBuf, uint32(encLen)) buf.Write(sizeBuf) // Authenticated Data is: // // [messageType; byte] [messageLength; uint32] [stream_label; optional] // dataBytes := appendBytes(buf.Bytes()[:5], []byte(streamLabel)) // Write the encrypted cipher text to the buffer key := m.config.Keyring.GetPrimaryKey() err := encryptPayload(encVsn, key, sendBuf, dataBytes, &buf) if err != nil { return nil, err } return buf.Bytes(), nil } // decryptRemoteState is used to help decrypt the remote state func (m Memberlist) decryptRemoteState(bufConn io.Reader, streamLabel string) ([]byte, error) { // Read in enough to determine message length cipherText := bytes.NewBuffer(nil) cipherText.WriteByte(byte(encryptMsg)) _, err := io.CopyN(cipherText, bufConn, 4) if err != nil { return nil, err } // Ensure we aren't asked to download too much. This is to guard against // an attack vector where a huge amount of state is sent moreBytes := binary.BigEndian.Uint32(cipherText.Bytes()[1:5]) metrics.AddSampleWithLabels([]string{"memberlist", "size", "remote"}, float32(moreBytes), m.metricLabels) if moreBytes > maxPushStateBytes { return nil, fmt.Errorf("Remote node state is larger than limit (%d)", moreBytes) } //Start reporting the size before you cross the limit if moreBytes > uint32(math.Floor(.6maxPushStateBytes)) { m.logger.Printf("[WARN] memberlist: Remote node state size is (%d) limit is (%d)", moreBytes, maxPushStateBytes) } // Read in the rest of the payload _, err = io.CopyN(cipherText, bufConn, int64(moreBytes)) if err != nil { return nil, err } // Decrypt the cipherText with some authenticated data // // Authenticated Data is: // // [messageType; byte] [messageLength; uint32] [label_data; optional] // dataBytes := appendBytes(cipherText.Bytes()[:5], []byte(streamLabel)) cipherBytes := cipherText.Bytes()[5:] // Decrypt the payload keys := m.config.Keyring.GetKeys() return decryptPayload(keys, cipherBytes, dataBytes) } // readStream is used to read messages from a stream connection, decrypting and // decompressing the stream if necessary. // // The provided streamLabel if present will be authenticated during decryption // of each message. func (m Memberlist) readStream(conn net.Conn, streamLabel string) (messageType, io.Reader, codec.Decoder, error) { // Created a buffered reader var bufConn io.Reader = bufio.NewReader(conn) // Read the message type buf := [1]byte{0} if _, err := io.ReadFull(bufConn, buf[:]); err != nil { return 0, nil, nil, err } msgType := messageType(buf[0]) // Check if the message is encrypted if msgType == encryptMsg { if !m.config.EncryptionEnabled() { return 0, nil, nil, fmt.Errorf("Remote state is encrypted and encryption is not configured") } plain, err := m.decryptRemoteState(bufConn, streamLabel) if err != nil { return 0, nil, nil, err } // Reset message type and bufConn msgType = messageType(plain[0]) bufConn = bytes.NewReader(plain[1:]) } else if m.config.EncryptionEnabled() && m.config.GossipVerifyIncoming { return 0, nil, nil, fmt.Errorf("Encryption is configured but remote state is not encrypted") } // Get the msgPack decoders hd := codec.MsgpackHandle{} dec := codec.NewDecoder(bufConn, &hd) // Check if we have a compressed message if msgType == compressMsg { var c compress if err := dec.Decode(&c); err != nil { return 0, nil, nil, err } decomp, err := decompressBuffer(&c) if err != nil { return 0, nil, nil, err } // Reset the message type msgType = messageType(decomp[0]) // Create a new bufConn bufConn = bytes.NewReader(decomp[1:]) // Create a new decoder dec = codec.NewDecoder(bufConn, &hd) } return msgType, bufConn, dec, nil } // readRemoteState is used to read the remote state from a connection func (m Memberlist) readRemoteState(bufConn io.Reader, dec codec.Decoder) (bool, []pushNodeState, []byte, error) { // Read the push/pull header var header pushPullHeader if err := dec.Decode(&header); err != nil { return false, nil, nil, err } // Allocate space for the transfer remoteNodes := make([]pushNodeState, header.Nodes) // Try to decode all the states for i := 0; i < header.Nodes; i++ { if err := dec.Decode(&remoteNodes[i]); err != nil { return false, nil, nil, err } } // Read the remote user state into a buffer var userBuf []byte if header.UserStateLen > 0 { userBuf = make([]byte, header.UserStateLen) bytes, err := io.ReadAtLeast(bufConn, userBuf, header.UserStateLen) if err == nil && bytes != header.UserStateLen { err = fmt.Errorf( "Failed to read full user state (%d / %d)", bytes, header.UserStateLen) } if err != nil { return false, nil, nil, err } } // For proto versions < 2, there is no port provided. Mask old // behavior by using the configured port for idx := range remoteNodes { if m.ProtocolVersion() < 2 \|\| remoteNodes[idx].Port == 0 { remoteNodes[idx].Port = uint16(m.config.BindPort) } } return header.Join, remoteNodes, userBuf, nil } // mergeRemoteState is used to merge the remote state with our local state func (m Memberlist) mergeRemoteState(join bool, remoteNodes []pushNodeState, userBuf []byte) error { if err := m.verifyProtocol(remoteNodes); err != nil { return err } // Invoke the merge delegate if any if join && m.config.Merge != nil { nodes := make([]Node, len(remoteNodes)) for idx, n := range remoteNodes { nodes[idx] = &Node{ Name: n.Name, Addr: n.Addr, Port: n.Port, Meta: n.Meta, State: n.State, PMin: n.Vsn[0], PMax: n.Vsn[1], PCur: n.Vsn[2], DMin: n.Vsn[3], DMax: n.Vsn[4], DCur: n.Vsn[5], } } if err := m.config.Merge.NotifyMerge(nodes); err != nil { return err } } // Merge the membership state m.mergeState(remoteNodes) // Invoke the delegate for user state if userBuf != nil && m.config.Delegate != nil { m.config.Delegate.MergeRemoteState(userBuf, join) } return nil } // readUserMsg is used to decode a userMsg from a stream. func (m Memberlist) readUserMsg(bufConn io.Reader, dec codec.Decoder) error { // Read the user message header var header userMsgHeader if err := dec.Decode(&header); err != nil { return err } // Read the user message into a buffer var userBuf []byte if header.UserMsgLen > 0 { userBuf = make([]byte, header.UserMsgLen) bytes, err := io.ReadAtLeast(bufConn, userBuf, header.UserMsgLen) if err == nil && bytes != header.UserMsgLen { err = fmt.Errorf( "Failed to read full user message (%d / %d)", bytes, header.UserMsgLen) } if err != nil { return err } d := m.config.Delegate if d != nil { d.NotifyMsg(userBuf) } } return nil } // sendPingAndWaitForAck makes a stream connection to the given address, sends // a ping, and waits for an ack. All of this is done as a series of blocking // operations, given the deadline. The bool return parameter is true if we // we able to round trip a ping to the other node. func (m *Memberlist) sendPingAndWaitForAck(a Address, ping ping, deadline time.Time) (bool, error) { if a.Name == "" && m.config.RequireNodeNames { return false, errNodeNamesAreRequired } conn, err := m.transport.DialAddressTimeout(a, deadline.Sub(time.Now())) if err != nil { // If the node is actually dead we expect this to fail, so we // shouldn't spam the logs with it. After this point, errors // with the connection are real, unexpected errors and should // get propagated up. return false, nil } defer conn.Close() conn.SetDeadline(deadline) out, err := encode(pingMsg, &ping, m.config.MsgpackUseNewTimeFormat) if err != nil { return false, err } if err = m.rawSendMsgStream(conn, out.Bytes(), m.config.Label); err != nil { return false, err } msgType, _, dec, err := m.readStream(conn, m.config.Label) if err != nil { return false, err } if msgType != ackRespMsg { return false, fmt.Errorf("Unexpected msgType (%d) from ping %s", msgType, LogConn(conn)) } var ack ackResp if err = dec.Decode(&ack); err != nil { return false, err } if ack.SeqNo != ping.SeqNo { return false, fmt.Errorf("Sequence number from ack (%d) doesn't match ping (%d)", ack.SeqNo, ping.SeqNo) } return true, nil }	38.43 KB	2025-05-25 12:27
📄 net_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "encoding/binary" "fmt" "io" "log" "net" "reflect" "strconv" "strings" "testing" "time" "github.com/hashicorp/go-msgpack/v2/codec" "github.com/stretchr/testify/require" ) // As a regression we left this test very low-level and network-ey, even after // we abstracted the transport. We added some basic network-free transport tests // in transport_test.go to prove that we didn't hard code some network stuff // outside of NetTransport. func TestHandleCompoundPing(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() udpAddr := udp.LocalAddr().(net.UDPAddr) // Encode a ping ping := ping{ SeqNo: 42, SourceAddr: udpAddr.IP, SourcePort: uint16(udpAddr.Port), SourceNode: "test", } buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat) if err != nil { t.Fatalf("unexpected err %s", err) } // Make a compound message compound := makeCompoundMessage([][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()}) // Send compound version addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort} _, err = udp.WriteTo(compound.Bytes(), addr) if err != nil { t.Fatalf("unexpected err %s", err) } // Wait for responses doneCh := make(chan struct{}, 1) go func() { select { case <-doneCh: case <-time.After(2 time.Second): panic("timeout") } }() for i := 0; i < 3; i++ { in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] msgType := messageType(in[0]) if msgType != ackRespMsg { t.Fatalf("bad response %v", in) } var ack ackResp if err := decode(in[1:], &ack); err != nil { t.Fatalf("unexpected err %s", err) } if ack.SeqNo != 42 { t.Fatalf("bad sequence no") } } doneCh <- struct{}{} } func TestHandlePing(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() udpAddr := udp.LocalAddr().(net.UDPAddr) // Encode a ping ping := ping{ SeqNo: 42, SourceAddr: udpAddr.IP, SourcePort: uint16(udpAddr.Port), SourceNode: "test", } buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat) if err != nil { t.Fatalf("unexpected err %s", err) } // Send addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort} _, err = udp.WriteTo(buf.Bytes(), addr) if err != nil { t.Fatalf("unexpected err %s", err) } // Wait for response doneCh := make(chan struct{}, 1) go func() { select { case <-doneCh: case <-time.After(2 time.Second): panic("timeout") } }() in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] msgType := messageType(in[0]) if msgType != ackRespMsg { t.Fatalf("bad response %v", in) } var ack ackResp if err := decode(in[1:], &ack); err != nil { t.Fatalf("unexpected err %s", err) } if ack.SeqNo != 42 { t.Fatalf("bad sequence no") } doneCh <- struct{}{} } func TestHandlePing_WrongNode(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() udpAddr := udp.LocalAddr().(net.UDPAddr) // Encode a ping, wrong node! ping := ping{ SeqNo: 42, Node: m.config.Name + "-bad", SourceAddr: udpAddr.IP, SourcePort: uint16(udpAddr.Port), SourceNode: "test", } buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat) if err != nil { t.Fatalf("unexpected err %s", err) } // Send addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort} _, err = udp.WriteTo(buf.Bytes(), addr) if err != nil { t.Fatalf("unexpected err %s", err) } // Wait for response udp.SetDeadline(time.Now().Add(50 time.Millisecond)) in := make([]byte, 1500) _, _, err = udp.ReadFrom(in) // Should get an i/o timeout if err == nil { t.Fatalf("expected err %s", err) } } func TestHandleIndirectPing(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() udpAddr := udp.LocalAddr().(net.UDPAddr) // Encode an indirect ping ind := indirectPingReq{ SeqNo: 100, Target: net.ParseIP(m.config.BindAddr), Port: uint16(m.config.BindPort), Node: m.config.Name, SourceAddr: udpAddr.IP, SourcePort: uint16(udpAddr.Port), SourceNode: "test", } buf, err := encode(indirectPingMsg, &ind, m.config.MsgpackUseNewTimeFormat) if err != nil { t.Fatalf("unexpected err %s", err) } // Send addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort} _, err = udp.WriteTo(buf.Bytes(), addr) if err != nil { t.Fatalf("unexpected err %s", err) } // Wait for response doneCh := make(chan struct{}, 1) go func() { select { case <-doneCh: case <-time.After(2 time.Second): panic("timeout") } }() in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] msgType := messageType(in[0]) if msgType != ackRespMsg { t.Fatalf("bad response %v", in) } var ack ackResp if err := decode(in[1:], &ack); err != nil { t.Fatalf("unexpected err %s", err) } if ack.SeqNo != 100 { t.Fatalf("bad sequence no") } doneCh <- struct{}{} } func TestTCPPing(t testing.T) { var tcp net.TCPListener var tcpAddr net.TCPAddr for port := 60000; port < 61000; port++ { tcpAddr = &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: port} tcpLn, err := net.ListenTCP("tcp", tcpAddr) if err == nil { tcp = tcpLn break } } if tcp == nil { t.Fatalf("no tcp listener") } tcpAddr2 := Address{Addr: tcpAddr.String(), Name: "test"} // Note that tcp gets closed in the last test, so we avoid a deferred // Close() call here. m := GetMemberlist(t, nil) defer m.Shutdown() pingTimeout := m.config.ProbeInterval pingTimeMax := m.config.ProbeInterval + 10time.Millisecond // Do a normal round trip. pingOut := ping{SeqNo: 23, Node: "mongo"} pingErrCh := make(chan error, 1) go func() { tcp.SetDeadline(time.Now().Add(pingTimeMax)) conn, err := tcp.AcceptTCP() if err != nil { pingErrCh <- fmt.Errorf("failed to connect: %s", err) return } defer conn.Close() msgType, _, dec, err := m.readStream(conn, "") if err != nil { pingErrCh <- fmt.Errorf("failed to read ping: %s", err) return } if msgType != pingMsg { pingErrCh <- fmt.Errorf("expecting ping, got message type (%d)", msgType) return } var pingIn ping if err := dec.Decode(&pingIn); err != nil { pingErrCh <- fmt.Errorf("failed to decode ping: %s", err) return } if pingIn.SeqNo != pingOut.SeqNo { pingErrCh <- fmt.Errorf("sequence number isn't correct (%d) vs (%d)", pingIn.SeqNo, pingOut.SeqNo) return } if pingIn.Node != pingOut.Node { pingErrCh <- fmt.Errorf("node name isn't correct (%s) vs (%s)", pingIn.Node, pingOut.Node) return } ack := ackResp{pingIn.SeqNo, nil} out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat) if err != nil { pingErrCh <- fmt.Errorf("failed to encode ack: %s", err) return } err = m.rawSendMsgStream(conn, out.Bytes(), "") if err != nil { pingErrCh <- fmt.Errorf("failed to send ack: %s", err) return } pingErrCh <- nil }() deadline := time.Now().Add(pingTimeout) didContact, err := m.sendPingAndWaitForAck(tcpAddr2, pingOut, deadline) if err != nil { t.Fatalf("error trying to ping: %s", err) } if !didContact { t.Fatalf("expected successful ping") } if err = <-pingErrCh; err != nil { t.Fatal(err) } // Make sure a mis-matched sequence number is caught. go func() { tcp.SetDeadline(time.Now().Add(pingTimeMax)) conn, err := tcp.AcceptTCP() if err != nil { pingErrCh <- fmt.Errorf("failed to connect: %s", err) return } defer conn.Close() _, _, dec, err := m.readStream(conn, "") if err != nil { pingErrCh <- fmt.Errorf("failed to read ping: %s", err) return } var pingIn ping if err := dec.Decode(&pingIn); err != nil { pingErrCh <- fmt.Errorf("failed to decode ping: %s", err) return } ack := ackResp{pingIn.SeqNo + 1, nil} out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat) if err != nil { pingErrCh <- fmt.Errorf("failed to encode ack: %s", err) return } err = m.rawSendMsgStream(conn, out.Bytes(), "") if err != nil { pingErrCh <- fmt.Errorf("failed to send ack: %s", err) return } pingErrCh <- nil }() deadline = time.Now().Add(pingTimeout) didContact, err = m.sendPingAndWaitForAck(tcpAddr2, pingOut, deadline) if err == nil \|\| !strings.Contains(err.Error(), "Sequence number") { t.Fatalf("expected an error from mis-matched sequence number") } if didContact { t.Fatalf("expected failed ping") } if err = <-pingErrCh; err != nil { t.Fatal(err) } // Make sure an unexpected message type is handled gracefully. go func() { tcp.SetDeadline(time.Now().Add(pingTimeMax)) conn, err := tcp.AcceptTCP() if err != nil { pingErrCh <- fmt.Errorf("failed to connect: %s", err) return } defer conn.Close() _, _, _, err = m.readStream(conn, "") if err != nil { pingErrCh <- fmt.Errorf("failed to read ping: %s", err) return } bogus := indirectPingReq{} out, err := encode(indirectPingMsg, &bogus, m.config.MsgpackUseNewTimeFormat) if err != nil { pingErrCh <- fmt.Errorf("failed to encode bogus msg: %s", err) return } err = m.rawSendMsgStream(conn, out.Bytes(), "") if err != nil { pingErrCh <- fmt.Errorf("failed to send bogus msg: %s", err) return } pingErrCh <- nil }() deadline = time.Now().Add(pingTimeout) didContact, err = m.sendPingAndWaitForAck(tcpAddr2, pingOut, deadline) if err == nil \|\| !strings.Contains(err.Error(), "Unexpected msgType") { t.Fatalf("expected an error from bogus message") } if didContact { t.Fatalf("expected failed ping") } if err = <-pingErrCh; err != nil { t.Fatal(err) } // Make sure failed I/O respects the deadline. In this case we try the // common case of the receiving node being totally down. tcp.Close() deadline = time.Now().Add(pingTimeout) startPing := time.Now() didContact, err = m.sendPingAndWaitForAck(tcpAddr2, pingOut, deadline) pingTime := time.Now().Sub(startPing) if err != nil { t.Fatalf("expected no error during ping on closed socket, got: %s", err) } if didContact { t.Fatalf("expected failed ping") } if pingTime > pingTimeMax { t.Fatalf("took too long to fail ping, %9.6f", pingTime.Seconds()) } } func TestTCPPushPull(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() m.nodes = append(m.nodes, &nodeState{ Node: Node{ Name: "Test 0", Addr: net.ParseIP(m.config.BindAddr), Port: uint16(m.config.BindPort), }, Incarnation: 0, State: StateSuspect, StateChange: time.Now().Add(-1 time.Second), }) addr := net.JoinHostPort(m.config.BindAddr, strconv.Itoa(m.config.BindPort)) conn, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("unexpected err %s", err) } defer conn.Close() localNodes := make([]pushNodeState, 3) localNodes[0].Name = "Test 0" localNodes[0].Addr = net.ParseIP(m.config.BindAddr) localNodes[0].Port = uint16(m.config.BindPort) localNodes[0].Incarnation = 1 localNodes[0].State = StateAlive localNodes[1].Name = "Test 1" localNodes[1].Addr = net.ParseIP(m.config.BindAddr) localNodes[1].Port = uint16(m.config.BindPort) localNodes[1].Incarnation = 1 localNodes[1].State = StateAlive localNodes[2].Name = "Test 2" localNodes[2].Addr = net.ParseIP(m.config.BindAddr) localNodes[2].Port = uint16(m.config.BindPort) localNodes[2].Incarnation = 1 localNodes[2].State = StateAlive // Send our node state header := pushPullHeader{Nodes: 3} hd := codec.MsgpackHandle{ BasicHandle: codec.BasicHandle{ TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat, }, } enc := codec.NewEncoder(conn, &hd) // Send the push/pull indicator conn.Write([]byte{byte(pushPullMsg)}) if err := enc.Encode(&header); err != nil { t.Fatalf("unexpected err %s", err) } for i := 0; i < header.Nodes; i++ { if err := enc.Encode(&localNodes[i]); err != nil { t.Fatalf("unexpected err %s", err) } } // Read the message type var msgType messageType if err := binary.Read(conn, binary.BigEndian, &msgType); err != nil { t.Fatalf("unexpected err %s", err) } var bufConn io.Reader = conn msghd := codec.MsgpackHandle{ BasicHandle: codec.BasicHandle{ TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat, }, } dec := codec.NewDecoder(bufConn, &msghd) // Check if we have a compressed message if msgType == compressMsg { var c compress if err := dec.Decode(&c); err != nil { t.Fatalf("unexpected err %s", err) } decomp, err := decompressBuffer(&c) if err != nil { t.Fatalf("unexpected err %s", err) } // Reset the message type msgType = messageType(decomp[0]) // Create a new bufConn bufConn = bytes.NewReader(decomp[1:]) // Create a new decoder dec = codec.NewDecoder(bufConn, &hd) } // Quit if not push/pull if msgType != pushPullMsg { t.Fatalf("bad message type") } if err := dec.Decode(&header); err != nil { t.Fatalf("unexpected err %s", err) } // Allocate space for the transfer remoteNodes := make([]pushNodeState, header.Nodes) // Try to decode all the states for i := 0; i < header.Nodes; i++ { if err := dec.Decode(&remoteNodes[i]); err != nil { t.Fatalf("unexpected err %s", err) } } if len(remoteNodes) != 1 { t.Fatalf("bad response") } n := &remoteNodes[0] if n.Name != "Test 0" { t.Fatalf("bad name") } if bytes.Compare(n.Addr, net.ParseIP(m.config.BindAddr)) != 0 { t.Fatal("bad addr") } if n.Incarnation != 0 { t.Fatal("bad incarnation") } if n.State != StateSuspect { t.Fatal("bad state") } } func TestSendMsg_Piggyback(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() // Add a message to be broadcast a := alive{ Incarnation: 10, Node: "rand", Addr: []byte{127, 0, 0, 255}, Meta: nil, Vsn: []uint8{ ProtocolVersionMin, ProtocolVersionMax, ProtocolVersionMin, 1, 1, 1, }, } m.encodeAndBroadcast("rand", aliveMsg, &a) udp := listenUDP(t) defer udp.Close() udpAddr := udp.LocalAddr().(net.UDPAddr) // Encode a ping ping := ping{ SeqNo: 42, SourceAddr: udpAddr.IP, SourcePort: uint16(udpAddr.Port), SourceNode: "test", } buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat) if err != nil { t.Fatalf("unexpected err %s", err) } // Send addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort} _, err = udp.WriteTo(buf.Bytes(), addr) if err != nil { t.Fatalf("unexpected err %s", err) } // Wait for response doneCh := make(chan struct{}, 1) go func() { select { case <-doneCh: case <-time.After(2 * time.Second): panic("timeout") } }() in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] msgType := messageType(in[0]) if msgType != compoundMsg { t.Fatalf("bad response %v", in) } // get the parts trunc, parts, err := decodeCompoundMessage(in[1:]) if trunc != 0 { t.Fatalf("unexpected truncation") } if len(parts) != 2 { t.Fatalf("unexpected parts %v", parts) } if err != nil { t.Fatalf("unexpected err %s", err) } var ack ackResp if err := decode(parts[0][1:], &ack); err != nil { t.Fatalf("unexpected err %s", err) } if ack.SeqNo != 42 { t.Fatalf("bad sequence no") } var aliveout alive if err := decode(parts[1][1:], &aliveout); err != nil { t.Fatalf("unexpected err %s", err) } if aliveout.Node != "rand" \|\| aliveout.Incarnation != 10 { t.Fatalf("bad mesg") } doneCh <- struct{}{} } func TestEncryptDecryptState(t testing.T) { state := []byte("this is our internal state...") config := &Config{ SecretKey: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, ProtocolVersion: ProtocolVersionMax, } sink := registerInMemorySink(t) m, err := Create(config) if err != nil { t.Fatalf("err: %s", err) } defer m.Shutdown() crypt, err := m.encryptLocalState(state, "") if err != nil { t.Fatalf("err: %v", err) } // Create reader, seek past the type byte buf := bytes.NewReader(crypt) buf.Seek(1, 0) plain, err := m.decryptRemoteState(buf, "") if err != nil { t.Fatalf("err: %v", err) } verifySampleExists(t, "consul.usage.test.memberlist.size.remote", sink) if !reflect.DeepEqual(state, plain) { t.Fatalf("Decrypt failed: %v", plain) } } func TestRawSendUdp_CRC(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() a := Address{ Addr: udp.LocalAddr().String(), Name: "test", } // Pass a nil node with no nodes registered, should result in no checksum payload := []byte{3, 3, 3, 3} m.rawSendMsgPacket(a, nil, payload) in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] if len(in) != 4 { t.Fatalf("bad: %v", in) } // Pass a non-nil node with PMax >= 5, should result in a checksum m.rawSendMsgPacket(a, &Node{PMax: 5}, payload) in = make([]byte, 1500) n, _, err = udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] if len(in) != 9 { t.Fatalf("bad: %v", in) } // Register a node with PMax >= 5 to be looked up, should result in a checksum m.nodeMap["127.0.0.1"] = &nodeState{ Node: Node{PMax: 5}, } m.rawSendMsgPacket(a, nil, payload) in = make([]byte, 1500) n, _, err = udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] if len(in) != 9 { t.Fatalf("bad: %v", in) } } func TestIngestPacket_CRC(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() a := Address{ Addr: udp.LocalAddr().String(), Name: "test", } // Get a message with a checksum payload := []byte{3, 3, 3, 3} m.rawSendMsgPacket(a, &Node{PMax: 5}, payload) in := make([]byte, 1500) n, _, err := udp.ReadFrom(in) if err != nil { t.Fatalf("unexpected err %s", err) } in = in[0:n] if len(in) != 9 { t.Fatalf("bad: %v", in) } // Corrupt the checksum in[1] <<= 1 logs := &bytes.Buffer{} logger := log.New(logs, "", 0) m.logger = logger m.ingestPacket(in, udp.LocalAddr(), time.Now()) if !strings.Contains(logs.String(), "invalid checksum") { t.Fatalf("bad: %s", logs.String()) } } func TestIngestPacket_ExportedFunc_EmptyMessage(t testing.T) { m := GetMemberlist(t, func(c Config) { c.EnableCompression = false }) defer m.Shutdown() udp := listenUDP(t) defer udp.Close() emptyConn := &emptyReadNetConn{} logs := &bytes.Buffer{} logger := log.New(logs, "", 0) m.logger = logger type ingestionAwareTransport interface { IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error } err := m.transport.(ingestionAwareTransport).IngestPacket(emptyConn, udp.LocalAddr(), time.Now(), true) require.Error(t, err) require.Contains(t, err.Error(), "packet too short") } type emptyReadNetConn struct { net.Conn } func (c emptyReadNetConn) Read(b []byte) (n int, err error) { return 0, io.EOF } func (c emptyReadNetConn) Close() error { return nil } func TestGossip_MismatchedKeys(t testing.T) { // Create two agents with different gossip keys c1 := testConfig(t) c1.SecretKey = []byte("4W6DGn2VQVqDEceOdmuRTQ==") m1, err := Create(c1) require.NoError(t, err) defer m1.Shutdown() bindPort := m1.config.BindPort c2 := testConfig(t) c2.BindPort = bindPort c2.SecretKey = []byte("XhX/w702/JKKK7/7OtM9Ww==") m2, err := Create(c2) require.NoError(t, err) defer m2.Shutdown() // Make sure we get this error on the joining side _, err = m2.Join([]string{c1.Name + "/" + c1.BindAddr}) if err == nil \|\| !strings.Contains(err.Error(), "No installed keys could decrypt the message") { t.Fatalf("bad: %s", err) } } func listenUDP(t testing.T) net.UDPConn { var udp net.UDPConn for port := 60000; port < 61000; port++ { udpAddr := fmt.Sprintf("127.0.0.1:%d", port) udpLn, err := net.ListenPacket("udp", udpAddr) if err == nil { udp = udpLn.(net.UDPConn) break } } if udp == nil { t.Fatalf("no udp listener") } return udp } func TestHandleCommand(t *testing.T) { var buf bytes.Buffer m := Memberlist{ logger: log.New(&buf, "", 0), } m.handleCommand(nil, &net.TCPAddr{Port: 12345}, time.Now()) require.Contains(t, buf.String(), "missing message type byte") }	20.53 KB	2025-05-25 12:27
📄 net_transport.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "io" "log" "net" "sync" "sync/atomic" "time" "github.com/armon/go-metrics" sockaddr "github.com/hashicorp/go-sockaddr" ) const ( // udpPacketBufSize is used to buffer incoming packets during read // operations. udpPacketBufSize = 65536 // udpRecvBufSize is a large buffer size that we attempt to set UDP // sockets to in order to handle a large volume of messages. udpRecvBufSize = 2 * 1024 * 1024 ) // NetTransportConfig is used to configure a net transport. type NetTransportConfig struct { // BindAddrs is a list of addresses to bind to for both TCP and UDP // communications. BindAddrs []string // BindPort is the port to listen on, for each address above. BindPort int // Logger is a logger for operator messages. Logger log.Logger // MetricLabels is a map of optional labels to apply to all metrics // emitted by this transport. MetricLabels []metrics.Label } // NetTransport is a Transport implementation that uses connectionless UDP for // packet operations, and ad-hoc TCP connections for stream operations. type NetTransport struct { config NetTransportConfig packetCh chan Packet streamCh chan net.Conn logger log.Logger wg sync.WaitGroup tcpListeners []net.TCPListener udpListeners []net.UDPConn shutdown int32 metricLabels []metrics.Label } var _ NodeAwareTransport = (NetTransport)(nil) // NewNetTransport returns a net transport with the given configuration. On // success all the network listeners will be created and listening. func NewNetTransport(config NetTransportConfig) (NetTransport, error) { // If we reject the empty list outright we can assume that there's at // least one listener of each type later during operation. if len(config.BindAddrs) == 0 { return nil, fmt.Errorf("At least one bind address is required") } // Build out the new transport. var ok bool t := NetTransport{ config: config, packetCh: make(chan Packet), streamCh: make(chan net.Conn), logger: config.Logger, metricLabels: config.MetricLabels, } // Clean up listeners if there's an error. defer func() { if !ok { t.Shutdown() } }() // Build all the TCP and UDP listeners. port := config.BindPort for _, addr := range config.BindAddrs { ip := net.ParseIP(addr) tcpAddr := &net.TCPAddr{IP: ip, Port: port} tcpLn, err := net.ListenTCP("tcp", tcpAddr) if err != nil { return nil, fmt.Errorf("Failed to start TCP listener on %q port %d: %v", addr, port, err) } t.tcpListeners = append(t.tcpListeners, tcpLn) // If the config port given was zero, use the first TCP listener // to pick an available port and then apply that to everything // else. if port == 0 { port = tcpLn.Addr().(net.TCPAddr).Port } udpAddr := &net.UDPAddr{IP: ip, Port: port} udpLn, err := net.ListenUDP("udp", udpAddr) if err != nil { return nil, fmt.Errorf("Failed to start UDP listener on %q port %d: %v", addr, port, err) } if err := setUDPRecvBuf(udpLn); err != nil { return nil, fmt.Errorf("Failed to resize UDP buffer: %v", err) } t.udpListeners = append(t.udpListeners, udpLn) } // Fire them up now that we've been able to create them all. for i := 0; i < len(config.BindAddrs); i++ { t.wg.Add(2) go t.tcpListen(t.tcpListeners[i]) go t.udpListen(t.udpListeners[i]) } ok = true return &t, nil } // GetAutoBindPort returns the bind port that was automatically given by the // kernel, if a bind port of 0 was given. func (t NetTransport) GetAutoBindPort() int { // We made sure there's at least one TCP listener, and that one's // port was applied to all the others for the dynamic bind case. return t.tcpListeners[0].Addr().(net.TCPAddr).Port } // See Transport. func (t NetTransport) FinalAdvertiseAddr(ip string, port int) (net.IP, int, error) { var advertiseAddr net.IP var advertisePort int if ip != "" { // If they've supplied an address, use that. advertiseAddr = net.ParseIP(ip) if advertiseAddr == nil { return nil, 0, fmt.Errorf("Failed to parse advertise address %q", ip) } // Ensure IPv4 conversion if necessary. if ip4 := advertiseAddr.To4(); ip4 != nil { advertiseAddr = ip4 } advertisePort = port } else { if t.config.BindAddrs[0] == "0.0.0.0" { // Otherwise, if we're not bound to a specific IP, let's // use a suitable private IP address. var err error ip, err = sockaddr.GetPrivateIP() if err != nil { return nil, 0, fmt.Errorf("Failed to get interface addresses: %v", err) } if ip == "" { return nil, 0, fmt.Errorf("No private IP address found, and explicit IP not provided") } advertiseAddr = net.ParseIP(ip) if advertiseAddr == nil { return nil, 0, fmt.Errorf("Failed to parse advertise address: %q", ip) } } else { // Use the IP that we're bound to, based on the first // TCP listener, which we already ensure is there. advertiseAddr = t.tcpListeners[0].Addr().(net.TCPAddr).IP } // Use the port we are bound to. advertisePort = t.GetAutoBindPort() } return advertiseAddr, advertisePort, nil } // See Transport. func (t NetTransport) WriteTo(b []byte, addr string) (time.Time, error) { a := Address{Addr: addr, Name: ""} return t.WriteToAddress(b, a) } // See NodeAwareTransport. func (t NetTransport) WriteToAddress(b []byte, a Address) (time.Time, error) { addr := a.Addr udpAddr, err := net.ResolveUDPAddr("udp", addr) if err != nil { return time.Time{}, err } // We made sure there's at least one UDP listener, so just use the // packet sending interface on the first one. Take the time after the // write call comes back, which will underestimate the time a little, // but help account for any delays before the write occurs. _, err = t.udpListeners[0].WriteTo(b, udpAddr) return time.Now(), err } // See Transport. func (t NetTransport) PacketCh() <-chan Packet { return t.packetCh } // See IngestionAwareTransport. func (t NetTransport) IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error { if shouldClose { defer conn.Close() } // Copy everything from the stream into packet buffer. var buf bytes.Buffer if _, err := io.Copy(&buf, conn); err != nil { return fmt.Errorf("failed to read packet: %v", err) } // Check the length - it needs to have at least one byte to be a proper // message. This is checked elsewhere for writes coming in directly from // the UDP socket. if n := buf.Len(); n < 1 { return fmt.Errorf("packet too short (%d bytes) %s", n, LogAddress(addr)) } // Inject the packet. t.packetCh <- &Packet{ Buf: buf.Bytes(), From: addr, Timestamp: now, } return nil } // See Transport. func (t NetTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { a := Address{Addr: addr, Name: ""} return t.DialAddressTimeout(a, timeout) } // See NodeAwareTransport. func (t NetTransport) DialAddressTimeout(a Address, timeout time.Duration) (net.Conn, error) { addr := a.Addr dialer := net.Dialer{Timeout: timeout} return dialer.Dial("tcp", addr) } // See Transport. func (t NetTransport) StreamCh() <-chan net.Conn { return t.streamCh } // See IngestionAwareTransport. func (t NetTransport) IngestStream(conn net.Conn) error { t.streamCh <- conn return nil } // See Transport. func (t NetTransport) Shutdown() error { // This will avoid log spam about errors when we shut down. atomic.StoreInt32(&t.shutdown, 1) // Rip through all the connections and shut them down. for _, conn := range t.tcpListeners { conn.Close() } for _, conn := range t.udpListeners { conn.Close() } // Block until all the listener threads have died. t.wg.Wait() return nil } // tcpListen is a long running goroutine that accepts incoming TCP connections // and hands them off to the stream channel. func (t NetTransport) tcpListen(tcpLn net.TCPListener) { defer t.wg.Done() // baseDelay is the initial delay after an AcceptTCP() error before attempting again const baseDelay = 5 time.Millisecond // maxDelay is the maximum delay after an AcceptTCP() error before attempting again. // In the case that tcpListen() is error-looping, it will delay the shutdown check. // Therefore, changes to maxDelay may have an effect on the latency of shutdown. const maxDelay = 1 * time.Second var loopDelay time.Duration for { conn, err := tcpLn.AcceptTCP() if err != nil { if s := atomic.LoadInt32(&t.shutdown); s == 1 { break } if loopDelay == 0 { loopDelay = baseDelay } else { loopDelay = 2 } if loopDelay > maxDelay { loopDelay = maxDelay } t.logger.Printf("[ERR] memberlist: Error accepting TCP connection: %v", err) time.Sleep(loopDelay) continue } // No error, reset loop delay loopDelay = 0 t.streamCh <- conn } } // udpListen is a long running goroutine that accepts incoming UDP packets and // hands them off to the packet channel. func (t NetTransport) udpListen(udpLn net.UDPConn) { defer t.wg.Done() for { // Do a blocking read into a fresh buffer. Grab a time stamp as // close as possible to the I/O. buf := make([]byte, udpPacketBufSize) n, addr, err := udpLn.ReadFrom(buf) ts := time.Now() if err != nil { if s := atomic.LoadInt32(&t.shutdown); s == 1 { break } t.logger.Printf("[ERR] memberlist: Error reading UDP packet: %v", err) continue } // Check the length - it needs to have at least one byte to be a // proper message. if n < 1 { t.logger.Printf("[ERR] memberlist: UDP packet too short (%d bytes) %s", len(buf), LogAddress(addr)) continue } // Ingest the packet. metrics.IncrCounterWithLabels([]string{"memberlist", "udp", "received"}, float32(n), t.metricLabels) t.packetCh <- &Packet{ Buf: buf[:n], From: addr, Timestamp: ts, } } } // setUDPRecvBuf is used to resize the UDP receive window. The function // attempts to set the read buffer to `udpRecvBuf` but backs off until // the read buffer can be set. func setUDPRecvBuf(c net.UDPConn) error { size := udpRecvBufSize var err error for size > 0 { if err = c.SetReadBuffer(size); err == nil { return nil } size = size / 2 } return err }	10.1 KB	2025-05-25 12:27
📄 peeked_conn.go // Copyright 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Originally from: https://github.com/google/tcpproxy/blob/master/tcpproxy.go // at f5c09fbedceb69e4b238dec52cdf9f2fe9a815e2 package memberlist import "net" // peekedConn is an incoming connection that has had some bytes read from it // to determine how to route the connection. The Read method stitches // the peeked bytes and unread bytes back together. type peekedConn struct { // Peeked are the bytes that have been read from Conn for the // purposes of route matching, but have not yet been consumed // by Read calls. It set to nil by Read when fully consumed. Peeked []byte // Conn is the underlying connection. // It can be type asserted against net.TCPConn or other types // as needed. It should not be read from directly unless // Peeked is nil. net.Conn } func (c peekedConn) Read(p []byte) (n int, err error) { if len(c.Peeked) > 0 { n = copy(p, c.Peeked) c.Peeked = c.Peeked[n:] if len(c.Peeked) == 0 { c.Peeked = nil } return n, nil } return c.Conn.Read(p) }	1.56 KB	2025-05-25 12:27
📄 ping_delegate.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import "time" // PingDelegate is used to notify an observer how long it took for a ping message to // complete a round trip. It can also be used for writing arbitrary byte slices // into ack messages. Note that in order to be meaningful for RTT estimates, this // delegate does not apply to indirect pings, nor fallback pings sent over TCP. type PingDelegate interface { // AckPayload is invoked when an ack is being sent; the returned bytes will be appended to the ack AckPayload() []byte // NotifyPing is invoked when an ack for a ping is received NotifyPingComplete(other *Node, rtt time.Duration, payload []byte) }	715 B	2025-05-25 12:27
📄 queue.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "math" "sync" "github.com/google/btree" ) // TransmitLimitedQueue is used to queue messages to broadcast to // the cluster (via gossip) but limits the number of transmits per // message. It also prioritizes messages with lower transmit counts // (hence newer messages). type TransmitLimitedQueue struct { // NumNodes returns the number of nodes in the cluster. This is // used to determine the retransmit count, which is calculated // based on the log of this. NumNodes func() int // RetransmitMult is the multiplier used to determine the maximum // number of retransmissions attempted. RetransmitMult int mu sync.Mutex tq btree.BTree // stores limitedBroadcast as btree.Item tm map[string]limitedBroadcast idGen int64 } type limitedBroadcast struct { transmits int // btree-key[0]: Number of transmissions attempted. msgLen int64 // btree-key[1]: copied from len(b.Message()) id int64 // btree-key[2]: unique incrementing id stamped at submission time b Broadcast name string // set if Broadcast is a NamedBroadcast } // Less tests whether the current item is less than the given argument. // // This must provide a strict weak ordering. // If !a.Less(b) && !b.Less(a), we treat this to mean a == b (i.e. we can only // hold one of either a or b in the tree). // // default ordering is // - [transmits=0, ..., transmits=inf] // - [transmits=0:len=999, ..., transmits=0:len=2, ...] // - [transmits=0:len=999,id=999, ..., transmits=0:len=999:id=1, ...] func (b limitedBroadcast) Less(than btree.Item) bool { o := than.(limitedBroadcast) if b.transmits < o.transmits { return true } else if b.transmits > o.transmits { return false } if b.msgLen > o.msgLen { return true } else if b.msgLen < o.msgLen { return false } return b.id > o.id } // for testing; emits in transmit order if reverse=false func (q TransmitLimitedQueue) orderedView(reverse bool) []limitedBroadcast { q.mu.Lock() defer q.mu.Unlock() out := make([]limitedBroadcast, 0, q.lenLocked()) q.walkReadOnlyLocked(reverse, func(cur limitedBroadcast) bool { out = append(out, cur) return true }) return out } // walkReadOnlyLocked calls f for each item in the queue traversing it in // natural order (by Less) when reverse=false and the opposite when true. You // must hold the mutex. // // This method panics if you attempt to mutate the item during traversal. The // underlying btree should also not be mutated during traversal. func (q TransmitLimitedQueue) walkReadOnlyLocked(reverse bool, f func(limitedBroadcast) bool) { if q.lenLocked() == 0 { return } iter := func(item btree.Item) bool { cur := item.(limitedBroadcast) prevTransmits := cur.transmits prevMsgLen := cur.msgLen prevID := cur.id keepGoing := f(cur) if prevTransmits != cur.transmits \|\| prevMsgLen != cur.msgLen \|\| prevID != cur.id { panic("edited queue while walking read only") } return keepGoing } if reverse { q.tq.Descend(iter) // end with transmit 0 } else { q.tq.Ascend(iter) // start with transmit 0 } } // Broadcast is something that can be broadcasted via gossip to // the memberlist cluster. type Broadcast interface { // Invalidates checks if enqueuing the current broadcast // invalidates a previous broadcast Invalidates(b Broadcast) bool // Returns a byte form of the message Message() []byte // Finished is invoked when the message will no longer // be broadcast, either due to invalidation or to the // transmit limit being reached Finished() } // NamedBroadcast is an optional extension of the Broadcast interface that // gives each message a unique string name, and that is used to optimize // // You shoud ensure that Invalidates() checks the same uniqueness as the // example below: // // func (b foo) Invalidates(other Broadcast) bool { // nb, ok := other.(NamedBroadcast) // if !ok { // return false // } // return b.Name() == nb.Name() // } // // Invalidates() isn't currently used for NamedBroadcasts, but that may change // in the future. type NamedBroadcast interface { Broadcast // The unique identity of this broadcast message. Name() string } // UniqueBroadcast is an optional interface that indicates that each message is // intrinsically unique and there is no need to scan the broadcast queue for // duplicates. // // You should ensure that Invalidates() always returns false if implementing // this interface. Invalidates() isn't currently used for UniqueBroadcasts, but // that may change in the future. type UniqueBroadcast interface { Broadcast // UniqueBroadcast is just a marker method for this interface. UniqueBroadcast() } // QueueBroadcast is used to enqueue a broadcast func (q TransmitLimitedQueue) QueueBroadcast(b Broadcast) { q.queueBroadcast(b, 0) } // lazyInit initializes internal data structures the first time they are // needed. You must already hold the mutex. func (q TransmitLimitedQueue) lazyInit() { if q.tq == nil { q.tq = btree.New(32) } if q.tm == nil { q.tm = make(map[string]limitedBroadcast) } } // queueBroadcast is like QueueBroadcast but you can use a nonzero value for // the initial transmit tier assigned to the message. This is meant to be used // for unit testing. func (q TransmitLimitedQueue) queueBroadcast(b Broadcast, initialTransmits int) { q.mu.Lock() defer q.mu.Unlock() q.lazyInit() if q.idGen == math.MaxInt64 { // it's super duper unlikely to wrap around within the retransmit limit q.idGen = 1 } else { q.idGen++ } id := q.idGen lb := &limitedBroadcast{ transmits: initialTransmits, msgLen: int64(len(b.Message())), id: id, b: b, } unique := false if nb, ok := b.(NamedBroadcast); ok { lb.name = nb.Name() } else if _, ok := b.(UniqueBroadcast); ok { unique = true } // Check if this message invalidates another. if lb.name != "" { if old, ok := q.tm[lb.name]; ok { old.b.Finished() q.deleteItem(old) } } else if !unique { // Slow path, hopefully nothing hot hits this. var remove []limitedBroadcast q.tq.Ascend(func(item btree.Item) bool { cur := item.(limitedBroadcast) // Special Broadcasts can only invalidate each other. switch cur.b.(type) { case NamedBroadcast: // noop case UniqueBroadcast: // noop default: if b.Invalidates(cur.b) { cur.b.Finished() remove = append(remove, cur) } } return true }) for _, cur := range remove { q.deleteItem(cur) } } // Append to the relevant queue. q.addItem(lb) } // deleteItem removes the given item from the overall datastructure. You // must already hold the mutex. func (q TransmitLimitedQueue) deleteItem(cur limitedBroadcast) { _ = q.tq.Delete(cur) if cur.name != "" { delete(q.tm, cur.name) } if q.tq.Len() == 0 { // At idle there's no reason to let the id generator keep going // indefinitely. q.idGen = 0 } } // addItem adds the given item into the overall datastructure. You must already // hold the mutex. func (q TransmitLimitedQueue) addItem(cur limitedBroadcast) { _ = q.tq.ReplaceOrInsert(cur) if cur.name != "" { q.tm[cur.name] = cur } } // getTransmitRange returns a pair of min/max values for transmit values // represented by the current queue contents. Both values represent actual // transmit values on the interval [0, len). You must already hold the mutex. func (q TransmitLimitedQueue) getTransmitRange() (minTransmit, maxTransmit int) { if q.lenLocked() == 0 { return 0, 0 } minItem, maxItem := q.tq.Min(), q.tq.Max() if minItem == nil \|\| maxItem == nil { return 0, 0 } min := minItem.(limitedBroadcast).transmits max := maxItem.(limitedBroadcast).transmits return min, max } // GetBroadcasts is used to get a number of broadcasts, up to a byte limit // and applying a per-message overhead as provided. func (q TransmitLimitedQueue) GetBroadcasts(overhead, limit int) [][]byte { q.mu.Lock() defer q.mu.Unlock() // Fast path the default case if q.lenLocked() == 0 { return nil } transmitLimit := retransmitLimit(q.RetransmitMult, q.NumNodes()) var ( bytesUsed int toSend [][]byte reinsert []limitedBroadcast ) // Visit fresher items first, but only look at stuff that will fit. // We'll go tier by tier, grabbing the largest items first. minTr, maxTr := q.getTransmitRange() for transmits := minTr; transmits <= maxTr; /do not advance automatically/ { free := int64(limit - bytesUsed - overhead) if free <= 0 { break // bail out early } // Search for the least element on a given tier (by transmit count) as // defined in the limitedBroadcast.Less function that will fit into our // remaining space. greaterOrEqual := &limitedBroadcast{ transmits: transmits, msgLen: free, id: math.MaxInt64, } lessThan := &limitedBroadcast{ transmits: transmits + 1, msgLen: math.MaxInt64, id: math.MaxInt64, } var keep limitedBroadcast q.tq.AscendRange(greaterOrEqual, lessThan, func(item btree.Item) bool { cur := item.(limitedBroadcast) // Check if this is within our limits if int64(len(cur.b.Message())) > free { // If this happens it's a bug in the datastructure or // surrounding use doing something like having len(Message()) // change over time. There's enough going on here that it's // probably sane to just skip it and move on for now. return true } keep = cur return false }) if keep == nil { // No more items of an appropriate size in the tier. transmits++ continue } msg := keep.b.Message() // Add to slice to send bytesUsed += overhead + len(msg) toSend = append(toSend, msg) // Check if we should stop transmission q.deleteItem(keep) if keep.transmits+1 >= transmitLimit { keep.b.Finished() } else { // We need to bump this item down to another transmit tier, but // because it would be in the same direction that we're walking the // tiers, we will have to delay the reinsertion until we are // finished our search. Otherwise we'll possibly re-add the message // when we ascend to the next tier. keep.transmits++ reinsert = append(reinsert, keep) } } for _, cur := range reinsert { q.addItem(cur) } return toSend } // NumQueued returns the number of queued messages func (q TransmitLimitedQueue) NumQueued() int { q.mu.Lock() defer q.mu.Unlock() return q.lenLocked() } // lenLocked returns the length of the overall queue datastructure. You must // hold the mutex. func (q TransmitLimitedQueue) lenLocked() int { if q.tq == nil { return 0 } return q.tq.Len() } // Reset clears all the queued messages. Should only be used for tests. func (q TransmitLimitedQueue) Reset() { q.mu.Lock() defer q.mu.Unlock() q.walkReadOnlyLocked(false, func(cur limitedBroadcast) bool { cur.b.Finished() return true }) q.tq = nil q.tm = nil q.idGen = 0 } // Prune will retain the maxRetain latest messages, and the rest // will be discarded. This can be used to prevent unbounded queue sizes func (q TransmitLimitedQueue) Prune(maxRetain int) { q.mu.Lock() defer q.mu.Unlock() // Do nothing if queue size is less than the limit for q.tq.Len() > maxRetain { item := q.tq.Max() if item == nil { break } cur := item.(limitedBroadcast) cur.b.Finished() q.deleteItem(cur) } }	11.19 KB	2025-05-25 12:27
📄 queue_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "testing" "github.com/google/btree" "github.com/stretchr/testify/require" ) func TestLimitedBroadcastLess(t testing.T) { cases := []struct { Name string A limitedBroadcast // lesser B limitedBroadcast }{ { "diff-transmits", &limitedBroadcast{transmits: 0, msgLen: 10, id: 100}, &limitedBroadcast{transmits: 1, msgLen: 10, id: 100}, }, { "same-transmits--diff-len", &limitedBroadcast{transmits: 0, msgLen: 12, id: 100}, &limitedBroadcast{transmits: 0, msgLen: 10, id: 100}, }, { "same-transmits--same-len--diff-id", &limitedBroadcast{transmits: 0, msgLen: 12, id: 100}, &limitedBroadcast{transmits: 0, msgLen: 12, id: 90}, }, } for _, c := range cases { t.Run(c.Name, func(t testing.T) { a, b := c.A, c.B require.True(t, a.Less(b)) tree := btree.New(32) tree.ReplaceOrInsert(b) tree.ReplaceOrInsert(a) min := tree.Min().(limitedBroadcast) require.Equal(t, a.transmits, min.transmits) require.Equal(t, a.msgLen, min.msgLen) require.Equal(t, a.id, min.id) max := tree.Max().(limitedBroadcast) require.Equal(t, b.transmits, max.transmits) require.Equal(t, b.msgLen, max.msgLen) require.Equal(t, b.id, max.id) }) } } func TestTransmitLimited_Queue(t testing.T) { q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 1 }} q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil}) q.QueueBroadcast(&memberlistBroadcast{"foo", nil, nil}) q.QueueBroadcast(&memberlistBroadcast{"bar", nil, nil}) if q.NumQueued() != 3 { t.Fatalf("bad len") } dump := q.orderedView(true) if dump[0].b.(memberlistBroadcast).node != "test" { t.Fatalf("missing test") } if dump[1].b.(memberlistBroadcast).node != "foo" { t.Fatalf("missing foo") } if dump[2].b.(memberlistBroadcast).node != "bar" { t.Fatalf("missing bar") } // Should invalidate previous message q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil}) if q.NumQueued() != 3 { t.Fatalf("bad len") } dump = q.orderedView(true) if dump[0].b.(memberlistBroadcast).node != "foo" { t.Fatalf("missing foo") } if dump[1].b.(memberlistBroadcast).node != "bar" { t.Fatalf("missing bar") } if dump[2].b.(memberlistBroadcast).node != "test" { t.Fatalf("missing test") } } func TestTransmitLimited_GetBroadcasts(t testing.T) { q := &TransmitLimitedQueue{RetransmitMult: 3, NumNodes: func() int { return 10 }} // 18 bytes per message q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) // 2 byte overhead per message, should get all 4 messages all := q.GetBroadcasts(2, 80) require.Equal(t, 4, len(all), "missing messages: %v", prettyPrintMessages(all)) // 3 byte overhead, should only get 3 messages back partial := q.GetBroadcasts(3, 80) require.Equal(t, 3, len(partial), "missing messages: %v", prettyPrintMessages(partial)) } func TestTransmitLimited_GetBroadcasts_Limit(t testing.T) { q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} require.Equal(t, int64(0), q.idGen, "the id generator seed starts at zero") require.Equal(t, 2, retransmitLimit(q.RetransmitMult, q.NumNodes()), "sanity check transmit limits") // 18 bytes per message q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) require.Equal(t, int64(4), q.idGen, "we handed out 4 IDs") // 3 byte overhead, should only get 3 messages back partial1 := q.GetBroadcasts(3, 80) require.Equal(t, 3, len(partial1), "missing messages: %v", prettyPrintMessages(partial1)) require.Equal(t, int64(4), q.idGen, "id generator doesn't reset until empty") partial2 := q.GetBroadcasts(3, 80) require.Equal(t, 3, len(partial2), "missing messages: %v", prettyPrintMessages(partial2)) require.Equal(t, int64(4), q.idGen, "id generator doesn't reset until empty") // Only two not expired partial3 := q.GetBroadcasts(3, 80) require.Equal(t, 2, len(partial3), "missing messages: %v", prettyPrintMessages(partial3)) require.Equal(t, int64(0), q.idGen, "id generator resets on empty") // Should get nothing partial5 := q.GetBroadcasts(3, 80) require.Equal(t, 0, len(partial5), "missing messages: %v", prettyPrintMessages(partial5)) require.Equal(t, int64(0), q.idGen, "id generator resets on empty") } func prettyPrintMessages(msgs [][]byte) []string { var out []string for _, msg := range msgs { out = append(out, "'"+string(msg)+"'") } return out } func TestTransmitLimited_Prune(t testing.T) { q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} ch1 := make(chan struct{}, 1) ch2 := make(chan struct{}, 1) // 18 bytes per message q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), ch1}) q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), ch2}) q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) // Keep only 2 q.Prune(2) require.Equal(t, 2, q.NumQueued()) // Should notify the first two select { case <-ch1: default: t.Fatalf("expected invalidation") } select { case <-ch2: default: t.Fatalf("expected invalidation") } dump := q.orderedView(true) if dump[0].b.(memberlistBroadcast).node != "bar" { t.Fatalf("missing bar") } if dump[1].b.(memberlistBroadcast).node != "baz" { t.Fatalf("missing baz") } } func TestTransmitLimited_ordering(t testing.T) { q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} insert := func(name string, transmits int) { q.queueBroadcast(&memberlistBroadcast{name, []byte(name), make(chan struct{})}, transmits) } insert("node0", 0) insert("node1", 10) insert("node2", 3) insert("node3", 4) insert("node4", 7) dump := q.orderedView(true) if dump[0].transmits != 10 { t.Fatalf("bad val %v, %d", dump[0].b.(memberlistBroadcast).node, dump[0].transmits) } if dump[1].transmits != 7 { t.Fatalf("bad val %v, %d", dump[7].b.(memberlistBroadcast).node, dump[7].transmits) } if dump[2].transmits != 4 { t.Fatalf("bad val %v, %d", dump[2].b.(memberlistBroadcast).node, dump[2].transmits) } if dump[3].transmits != 3 { t.Fatalf("bad val %v, %d", dump[3].b.(memberlistBroadcast).node, dump[3].transmits) } if dump[4].transmits != 0 { t.Fatalf("bad val %v, %d", dump[4].b.(memberlistBroadcast).node, dump[4].transmits) } }	6.87 KB	2025-05-25 12:27
📄 security.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "crypto/aes" "crypto/cipher" "crypto/rand" "fmt" "io" ) /* Encrypted messages are prefixed with an encryptionVersion byte that is used for us to be able to properly encode/decode. We currently support the following versions: 0 - AES-GCM 128, using PKCS7 padding 1 - AES-GCM 128, no padding. Padding not needed, caused bloat. / type encryptionVersion uint8 const ( minEncryptionVersion encryptionVersion = 0 maxEncryptionVersion encryptionVersion = 1 ) const ( versionSize = 1 nonceSize = 12 tagSize = 16 maxPadOverhead = 16 blockSize = aes.BlockSize ) // pkcs7encode is used to pad a byte buffer to a specific block size using // the PKCS7 algorithm. "Ignores" some bytes to compensate for IV func pkcs7encode(buf bytes.Buffer, ignore, blockSize int) { n := buf.Len() - ignore more := blockSize - (n % blockSize) for i := 0; i < more; i++ { buf.WriteByte(byte(more)) } } // pkcs7decode is used to decode a buffer that has been padded func pkcs7decode(buf []byte, blockSize int) []byte { if len(buf) == 0 { panic("Cannot decode a PKCS7 buffer of zero length") } n := len(buf) last := buf[n-1] n -= int(last) return buf[:n] } // encryptOverhead returns the maximum possible overhead of encryption by version func encryptOverhead(vsn encryptionVersion) int { switch vsn { case 0: return 45 // Version: 1, IV: 12, Padding: 16, Tag: 16 case 1: return 29 // Version: 1, IV: 12, Tag: 16 default: panic("unsupported version") } } // encryptedLength is used to compute the buffer size needed // for a message of given length func encryptedLength(vsn encryptionVersion, inp int) int { // If we are on version 1, there is no padding if vsn >= 1 { return versionSize + nonceSize + inp + tagSize } // Determine the padding size padding := blockSize - (inp % blockSize) // Sum the extra parts to get total size return versionSize + nonceSize + inp + padding + tagSize } // encryptPayload is used to encrypt a message with a given key. // We make use of AES-128 in GCM mode. New byte buffer is the version, // nonce, ciphertext and tag func encryptPayload(vsn encryptionVersion, key []byte, msg []byte, data []byte, dst *bytes.Buffer) error { // Get the AES block cipher aesBlock, err := aes.NewCipher(key) if err != nil { return err } // Get the GCM cipher mode gcm, err := cipher.NewGCM(aesBlock) if err != nil { return err } // Grow the buffer to make room for everything offset := dst.Len() dst.Grow(encryptedLength(vsn, len(msg))) // Write the encryption version dst.WriteByte(byte(vsn)) // Add a random nonce _, err = io.CopyN(dst, rand.Reader, nonceSize) if err != nil { return err } afterNonce := dst.Len() // Ensure we are correctly padded (only version 0) if vsn == 0 { io.Copy(dst, bytes.NewReader(msg)) pkcs7encode(dst, offset+versionSize+nonceSize, aes.BlockSize) } // Encrypt message using GCM slice := dst.Bytes()[offset:] nonce := slice[versionSize : versionSize+nonceSize] // Message source depends on the encryption version. // Version 0 uses padding, version 1 does not var src []byte if vsn == 0 { src = slice[versionSize+nonceSize:] } else { src = msg } out := gcm.Seal(nil, nonce, src, data) // Truncate the plaintext, and write the cipher text dst.Truncate(afterNonce) dst.Write(out) return nil } // decryptMessage performs the actual decryption of ciphertext. This is in its // own function to allow it to be called on all keys easily. func decryptMessage(key, msg []byte, data []byte) ([]byte, error) { // Get the AES block cipher aesBlock, err := aes.NewCipher(key) if err != nil { return nil, err } // Get the GCM cipher mode gcm, err := cipher.NewGCM(aesBlock) if err != nil { return nil, err } // Decrypt the message nonce := msg[versionSize : versionSize+nonceSize] ciphertext := msg[versionSize+nonceSize:] plain, err := gcm.Open(nil, nonce, ciphertext, data) if err != nil { return nil, err } // Success! return plain, nil } // decryptPayload is used to decrypt a message with a given key, // and verify it's contents. Any padding will be removed, and a // slice to the plaintext is returned. Decryption is done IN PLACE! func decryptPayload(keys [][]byte, msg []byte, data []byte) ([]byte, error) { // Ensure we have at least one byte if len(msg) == 0 { return nil, fmt.Errorf("Cannot decrypt empty payload") } // Verify the version vsn := encryptionVersion(msg[0]) if vsn > maxEncryptionVersion { return nil, fmt.Errorf("Unsupported encryption version %d", msg[0]) } // Ensure the length is sane if len(msg) < encryptedLength(vsn, 0) { return nil, fmt.Errorf("Payload is too small to decrypt: %d", len(msg)) } for _, key := range keys { plain, err := decryptMessage(key, msg, data) if err == nil { // Remove the PKCS7 padding for vsn 0 if vsn == 0 { return pkcs7decode(plain, aes.BlockSize), nil } else { return plain, nil } } } return nil, fmt.Errorf("No installed keys could decrypt the message") } func appendBytes(first []byte, second []byte) []byte { hasFirst := len(first) > 0 hasSecond := len(second) > 0 switch { case hasFirst && hasSecond: out := make([]byte, 0, len(first)+len(second)) out = append(out, first...) out = append(out, second...) return out case hasFirst: return first case hasSecond: return second default: return nil } }	5.38 KB	2025-05-25 12:27
📄 security_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "reflect" "testing" ) func TestPKCS7(t testing.T) { for i := 0; i <= 255; i++ { // Make a buffer of size i buf := []byte{} for j := 0; j < i; j++ { buf = append(buf, byte(i)) } // Copy to bytes buffer inp := bytes.NewBuffer(nil) inp.Write(buf) // Pad this out pkcs7encode(inp, 0, 16) // Unpad dec := pkcs7decode(inp.Bytes(), 16) // Ensure equivilence if !reflect.DeepEqual(buf, dec) { t.Fatalf("mismatch: %v %v", buf, dec) } } } func TestEncryptDecrypt_V0(t testing.T) { encryptDecryptVersioned(0, t) } func TestEncryptDecrypt_V1(t testing.T) { encryptDecryptVersioned(1, t) } func encryptDecryptVersioned(vsn encryptionVersion, t testing.T) { k1 := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} plaintext := []byte("this is a plain text message") extra := []byte("random data") var buf bytes.Buffer err := encryptPayload(vsn, k1, plaintext, extra, &buf) if err != nil { t.Fatalf("err: %v", err) } expLen := encryptedLength(vsn, len(plaintext)) if buf.Len() != expLen { t.Fatalf("output length is unexpected %d %d %d", len(plaintext), buf.Len(), expLen) } msg, err := decryptPayload([][]byte{k1}, buf.Bytes(), extra) if err != nil { t.Fatalf("err: %v", err) } cmp := bytes.Compare(msg, plaintext) if cmp != 0 { t.Errorf("len %d %v", len(msg), msg) t.Errorf("len %d %v", len(plaintext), plaintext) t.Fatalf("encrypt/decrypt failed! %d '%s' '%s'", cmp, msg, plaintext) } }	1.54 KB	2025-05-25 12:27
📄 state.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "math" "math/rand" "net" "strings" "sync/atomic" "time" "github.com/armon/go-metrics" ) type NodeStateType int func (t NodeStateType) metricsString() string { switch t { case StateAlive: return "alive" case StateDead: return "dead" case StateSuspect: return "suspect" case StateLeft: return "left" default: return fmt.Sprintf("unhandled-value-%d", t) } } const ( StateAlive NodeStateType = iota StateSuspect StateDead StateLeft ) // Node represents a node in the cluster. type Node struct { Name string Addr net.IP Port uint16 Meta []byte // Metadata from the delegate for this node. State NodeStateType // State of the node. PMin uint8 // Minimum protocol version this understands PMax uint8 // Maximum protocol version this understands PCur uint8 // Current version node is speaking DMin uint8 // Min protocol version for the delegate to understand DMax uint8 // Max protocol version for the delegate to understand DCur uint8 // Current version delegate is speaking } // Address returns the host:port form of a node's address, suitable for use // with a transport. func (n Node) Address() string { return joinHostPort(n.Addr.String(), n.Port) } // FullAddress returns the node name and host:port form of a node's address, // suitable for use with a transport. func (n Node) FullAddress() Address { return Address{ Addr: joinHostPort(n.Addr.String(), n.Port), Name: n.Name, } } // String returns the node name func (n Node) String() string { return n.Name } // NodeState is used to manage our state view of another node type nodeState struct { Node Incarnation uint32 // Last known incarnation number State NodeStateType // Current state StateChange time.Time // Time last state change happened } // Address returns the host:port form of a node's address, suitable for use // with a transport. func (n nodeState) Address() string { return n.Node.Address() } // FullAddress returns the node name and host:port form of a node's address, // suitable for use with a transport. func (n nodeState) FullAddress() Address { return n.Node.FullAddress() } func (n nodeState) DeadOrLeft() bool { return n.State == StateDead \|\| n.State == StateLeft } // ackHandler is used to register handlers for incoming acks and nacks. type ackHandler struct { ackFn func([]byte, time.Time) nackFn func() timer time.Timer } // NoPingResponseError is used to indicate a 'ping' packet was // successfully issued but no response was received type NoPingResponseError struct { node string } func (f NoPingResponseError) Error() string { return fmt.Sprintf("No response from node %s", f.node) } // Schedule is used to ensure the Tick is performed periodically. This // function is safe to call multiple times. If the memberlist is already // scheduled, then it won't do anything. func (m Memberlist) schedule() { m.tickerLock.Lock() defer m.tickerLock.Unlock() // If we already have tickers, then don't do anything, since we're // scheduled if len(m.tickers) > 0 { return } // Create the stop tick channel, a blocking channel. We close this // when we should stop the tickers. stopCh := make(chan struct{}) // Create a new probeTicker if m.config.ProbeInterval > 0 { t := time.NewTicker(m.config.ProbeInterval) go m.triggerFunc(m.config.ProbeInterval, t.C, stopCh, m.probe) m.tickers = append(m.tickers, t) } // Create a push pull ticker if needed if m.config.PushPullInterval > 0 { go m.pushPullTrigger(stopCh) } // Create a gossip ticker if needed if m.config.GossipInterval > 0 && m.config.GossipNodes > 0 { t := time.NewTicker(m.config.GossipInterval) go m.triggerFunc(m.config.GossipInterval, t.C, stopCh, m.gossip) m.tickers = append(m.tickers, t) } // If we made any tickers, then record the stopTick channel for // later. if len(m.tickers) > 0 { m.stopTick = stopCh } } // triggerFunc is used to trigger a function call each time a // message is received until a stop tick arrives. func (m Memberlist) triggerFunc(stagger time.Duration, C <-chan time.Time, stop <-chan struct{}, f func()) { // Use a random stagger to avoid syncronizing randStagger := time.Duration(uint64(rand.Int63()) % uint64(stagger)) select { case <-time.After(randStagger): case <-stop: return } for { select { case <-C: f() case <-stop: return } } } // pushPullTrigger is used to periodically trigger a push/pull until // a stop tick arrives. We don't use triggerFunc since the push/pull // timer is dynamically scaled based on cluster size to avoid network // saturation func (m Memberlist) pushPullTrigger(stop <-chan struct{}) { interval := m.config.PushPullInterval // Use a random stagger to avoid syncronizing randStagger := time.Duration(uint64(rand.Int63()) % uint64(interval)) select { case <-time.After(randStagger): case <-stop: return } // Tick using a dynamic timer for { tickTime := pushPullScale(interval, m.estNumNodes()) select { case <-time.After(tickTime): m.pushPull() case <-stop: return } } } // Deschedule is used to stop the background maintenance. This is safe // to call multiple times. func (m Memberlist) deschedule() { m.tickerLock.Lock() defer m.tickerLock.Unlock() // If we have no tickers, then we aren't scheduled. if len(m.tickers) == 0 { return } // Close the stop channel so all the ticker listeners stop. close(m.stopTick) // Explicitly stop all the tickers themselves so they don't take // up any more resources, and get rid of the list. for _, t := range m.tickers { t.Stop() } m.tickers = nil } // Tick is used to perform a single round of failure detection and gossip func (m Memberlist) probe() { // Track the number of indexes we've considered probing numCheck := 0 START: m.nodeLock.RLock() // Make sure we don't wrap around infinitely if numCheck >= len(m.nodes) { m.nodeLock.RUnlock() return } // Handle the wrap around case if m.probeIndex >= len(m.nodes) { m.nodeLock.RUnlock() m.resetNodes() m.probeIndex = 0 numCheck++ goto START } // Determine if we should probe this node skip := false var node nodeState node = m.nodes[m.probeIndex] if node.Name == m.config.Name { skip = true } else if node.DeadOrLeft() { skip = true } // Potentially skip m.nodeLock.RUnlock() m.probeIndex++ if skip { numCheck++ goto START } // Probe the specific node m.probeNode(&node) } // probeNodeByAddr just safely calls probeNode given only the address of the node (for tests) func (m Memberlist) probeNodeByAddr(addr string) { m.nodeLock.RLock() n := m.nodeMap[addr] m.nodeLock.RUnlock() m.probeNode(n) } // failedRemote checks the error and decides if it indicates a failure on the // other end. func failedRemote(err error) bool { switch t := err.(type) { case net.OpError: if strings.HasPrefix(t.Net, "tcp") { switch t.Op { case "dial", "read", "write": return true } } else if strings.HasPrefix(t.Net, "udp") { switch t.Op { case "write": return true } } } return false } // probeNode handles a single round of failure checking on a node. func (m Memberlist) probeNode(node nodeState) { defer metrics.MeasureSinceWithLabels([]string{"memberlist", "probeNode"}, time.Now(), m.metricLabels) // We use our health awareness to scale the overall probe interval, so we // slow down if we detect problems. The ticker that calls us can handle // us running over the base interval, and will skip missed ticks. probeInterval := m.awareness.ScaleTimeout(m.config.ProbeInterval) if probeInterval > m.config.ProbeInterval { metrics.IncrCounterWithLabels([]string{"memberlist", "degraded", "probe"}, 1, m.metricLabels) } // Prepare a ping message and setup an ack handler. selfAddr, selfPort := m.getAdvertise() ping := ping{ SeqNo: m.nextSeqNo(), Node: node.Name, SourceAddr: selfAddr, SourcePort: selfPort, SourceNode: m.config.Name, } ackCh := make(chan ackMessage, m.config.IndirectChecks+1) nackCh := make(chan struct{}, m.config.IndirectChecks+1) m.setProbeChannels(ping.SeqNo, ackCh, nackCh, probeInterval) // Mark the sent time here, which should be after any pre-processing but // before system calls to do the actual send. This probably over-reports // a bit, but it's the best we can do. We had originally put this right // after the I/O, but that would sometimes give negative RTT measurements // which was not desirable. sent := time.Now() // Send a ping to the node. If this node looks like it's suspect or dead, // also tack on a suspect message so that it has a chance to refute as // soon as possible. deadline := sent.Add(probeInterval) addr := node.Address() // Arrange for our self-awareness to get updated. var awarenessDelta int defer func() { m.awareness.ApplyDelta(awarenessDelta) }() if node.State == StateAlive { if err := m.encodeAndSendMsg(node.FullAddress(), pingMsg, &ping); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send UDP ping: %s", err) if failedRemote(err) { goto HANDLE_REMOTE_FAILURE } else { return } } } else { var msgs [][]byte if buf, err := encode(pingMsg, &ping, m.config.MsgpackUseNewTimeFormat); err != nil { m.logger.Printf("[ERR] memberlist: Failed to encode UDP ping message: %s", err) return } else { msgs = append(msgs, buf.Bytes()) } s := suspect{Incarnation: node.Incarnation, Node: node.Name, From: m.config.Name} if buf, err := encode(suspectMsg, &s, m.config.MsgpackUseNewTimeFormat); err != nil { m.logger.Printf("[ERR] memberlist: Failed to encode suspect message: %s", err) return } else { msgs = append(msgs, buf.Bytes()) } compound := makeCompoundMessage(msgs) if err := m.rawSendMsgPacket(node.FullAddress(), &node.Node, compound.Bytes()); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send UDP compound ping and suspect message to %s: %s", addr, err) if failedRemote(err) { goto HANDLE_REMOTE_FAILURE } else { return } } } // Arrange for our self-awareness to get updated. At this point we've // sent the ping, so any return statement means the probe succeeded // which will improve our health until we get to the failure scenarios // at the end of this function, which will alter this delta variable // accordingly. awarenessDelta = -1 // Wait for response or round-trip-time. select { case v := <-ackCh: if v.Complete == true { if m.config.Ping != nil { rtt := v.Timestamp.Sub(sent) m.config.Ping.NotifyPingComplete(&node.Node, rtt, v.Payload) } return } // As an edge case, if we get a timeout, we need to re-enqueue it // here to break out of the select below. if v.Complete == false { ackCh <- v } case <-time.After(m.config.ProbeTimeout): // Note that we don't scale this timeout based on awareness and // the health score. That's because we don't really expect waiting // longer to help get UDP through. Since health does extend the // probe interval it will give the TCP fallback more time, which // is more active in dealing with lost packets, and it gives more // time to wait for indirect acks/nacks. m.logger.Printf("[DEBUG] memberlist: Failed UDP ping: %s (timeout reached)", node.Name) } HANDLE_REMOTE_FAILURE: // Get some random live nodes. m.nodeLock.RLock() kNodes := kRandomNodes(m.config.IndirectChecks, m.nodes, func(n nodeState) bool { return n.Name == m.config.Name \|\| n.Name == node.Name \|\| n.State != StateAlive }) m.nodeLock.RUnlock() // Attempt an indirect ping. expectedNacks := 0 selfAddr, selfPort = m.getAdvertise() ind := indirectPingReq{ SeqNo: ping.SeqNo, Target: node.Addr, Port: node.Port, Node: node.Name, SourceAddr: selfAddr, SourcePort: selfPort, SourceNode: m.config.Name, } for _, peer := range kNodes { // We only expect nack to be sent from peers who understand // version 4 of the protocol. if ind.Nack = peer.PMax >= 4; ind.Nack { expectedNacks++ } if err := m.encodeAndSendMsg(peer.FullAddress(), indirectPingMsg, &ind); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send indirect UDP ping: %s", err) } } // Also make an attempt to contact the node directly over TCP. This // helps prevent confused clients who get isolated from UDP traffic // but can still speak TCP (which also means they can possibly report // misinformation to other nodes via anti-entropy), avoiding flapping in // the cluster. // // This is a little unusual because we will attempt a TCP ping to any // member who understands version 3 of the protocol, regardless of // which protocol version we are speaking. That's why we've included a // config option to turn this off if desired. fallbackCh := make(chan bool, 1) disableTcpPings := m.config.DisableTcpPings \|\| (m.config.DisableTcpPingsForNode != nil && m.config.DisableTcpPingsForNode(node.Name)) if (!disableTcpPings) && (node.PMax >= 3) { go func() { defer close(fallbackCh) didContact, err := m.sendPingAndWaitForAck(node.FullAddress(), ping, deadline) if err != nil { var to string if ne, ok := err.(net.Error); ok && ne.Timeout() { to = fmt.Sprintf("timeout %s: ", probeInterval) } m.logger.Printf("[ERR] memberlist: Failed fallback TCP ping: %s%s", to, err) } else { fallbackCh <- didContact } }() } else { close(fallbackCh) } // Wait for the acks or timeout. Note that we don't check the fallback // channel here because we want to issue a warning below if that's the // only way we hear back from the peer, so we have to let this time // out first to allow the normal UDP-based acks to come in. select { case v := <-ackCh: if v.Complete == true { return } } // Finally, poll the fallback channel. The timeouts are set such that // the channel will have something or be closed without having to wait // any additional time here. for didContact := range fallbackCh { if didContact { m.logger.Printf("[WARN] memberlist: Was able to connect to %s over TCP but UDP probes failed, network may be misconfigured", node.Name) return } } // Update our self-awareness based on the results of this failed probe. // If we don't have peers who will send nacks then we penalize for any // failed probe as a simple health metric. If we do have peers to nack // verify, then we can use that as a more sophisticated measure of self- // health because we assume them to be working, and they can help us // decide if the probed node was really dead or if it was something wrong // with ourselves. awarenessDelta = 0 if expectedNacks > 0 { if nackCount := len(nackCh); nackCount < expectedNacks { awarenessDelta += (expectedNacks - nackCount) } } else { awarenessDelta += 1 } // No acks received from target, suspect it as failed. m.logger.Printf("[INFO] memberlist: Suspect %s has failed, no acks received", node.Name) s := suspect{Incarnation: node.Incarnation, Node: node.Name, From: m.config.Name} m.suspectNode(&s) } // Ping initiates a ping to the node with the specified name. func (m Memberlist) Ping(node string, addr net.Addr) (time.Duration, error) { // Prepare a ping message and setup an ack handler. selfAddr, selfPort := m.getAdvertise() ping := ping{ SeqNo: m.nextSeqNo(), Node: node, SourceAddr: selfAddr, SourcePort: selfPort, SourceNode: m.config.Name, } ackCh := make(chan ackMessage, m.config.IndirectChecks+1) m.setProbeChannels(ping.SeqNo, ackCh, nil, m.config.ProbeInterval) a := Address{Addr: addr.String(), Name: node} // Send a ping to the node. if err := m.encodeAndSendMsg(a, pingMsg, &ping); err != nil { return 0, err } // Mark the sent time here, which should be after any pre-processing and // system calls to do the actual send. This probably under-reports a bit, // but it's the best we can do. sent := time.Now() // Wait for response or timeout. select { case v := <-ackCh: if v.Complete == true { return v.Timestamp.Sub(sent), nil } case <-time.After(m.config.ProbeTimeout): // Timeout, return an error below. } m.logger.Printf("[DEBUG] memberlist: Failed UDP ping: %v (timeout reached)", node) return 0, NoPingResponseError{ping.Node} } // resetNodes is used when the tick wraps around. It will reap the // dead nodes and shuffle the node list. func (m Memberlist) resetNodes() { m.nodeLock.Lock() defer m.nodeLock.Unlock() // Move dead nodes, but respect gossip to the dead interval deadIdx := moveDeadNodes(m.nodes, m.config.GossipToTheDeadTime) // Deregister the dead nodes for i := deadIdx; i < len(m.nodes); i++ { delete(m.nodeMap, m.nodes[i].Name) m.nodes[i] = nil } // Trim the nodes to exclude the dead nodes m.nodes = m.nodes[0:deadIdx] // Update numNodes after we've trimmed the dead nodes atomic.StoreUint32(&m.numNodes, uint32(deadIdx)) // Shuffle live nodes shuffleNodes(m.nodes) } // gossip is invoked every GossipInterval period to broadcast our gossip // messages to a few random nodes. func (m Memberlist) gossip() { defer metrics.MeasureSinceWithLabels([]string{"memberlist", "gossip"}, time.Now(), m.metricLabels) // Get some random live, suspect, or recently dead nodes m.nodeLock.RLock() kNodes := kRandomNodes(m.config.GossipNodes, m.nodes, func(n nodeState) bool { if n.Name == m.config.Name { return true } switch n.State { case StateAlive, StateSuspect: return false case StateDead: return time.Since(n.StateChange) > m.config.GossipToTheDeadTime default: return true } }) m.nodeLock.RUnlock() // Compute the bytes available bytesAvail := m.config.UDPBufferSize - compoundHeaderOverhead - labelOverhead(m.config.Label) if m.config.EncryptionEnabled() { bytesAvail -= encryptOverhead(m.encryptionVersion()) } for _, node := range kNodes { // Get any pending broadcasts msgs := m.getBroadcasts(compoundOverhead, bytesAvail) if len(msgs) == 0 { return } addr := node.Address() if len(msgs) == 1 { // Send single message as is if err := m.rawSendMsgPacket(node.FullAddress(), &node, msgs[0]); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send gossip to %s: %s", addr, err) } } else { // Otherwise create and send one or more compound messages compounds := makeCompoundMessages(msgs) for _, compound := range compounds { if err := m.rawSendMsgPacket(node.FullAddress(), &node, compound.Bytes()); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send gossip to %s: %s", addr, err) } } } } } // pushPull is invoked periodically to randomly perform a complete state // exchange. Used to ensure a high level of convergence, but is also // reasonably expensive as the entire state of this node is exchanged // with the other node. func (m Memberlist) pushPull() { // Get a random live node m.nodeLock.RLock() nodes := kRandomNodes(1, m.nodes, func(n nodeState) bool { return n.Name == m.config.Name \|\| n.State != StateAlive }) m.nodeLock.RUnlock() // If no nodes, bail if len(nodes) == 0 { return } node := nodes[0] // Attempt a push pull if err := m.pushPullNode(node.FullAddress(), false); err != nil { m.logger.Printf("[ERR] memberlist: Push/Pull with %s failed: %s", node.Name, err) } } // pushPullNode does a complete state exchange with a specific node. func (m Memberlist) pushPullNode(a Address, join bool) error { defer metrics.MeasureSinceWithLabels([]string{"memberlist", "pushPullNode"}, time.Now(), m.metricLabels) // Attempt to send and receive with the node remote, userState, err := m.sendAndReceiveState(a, join) if err != nil { return err } if err := m.mergeRemoteState(join, remote, userState); err != nil { return err } return nil } // verifyProtocol verifies that all the remote nodes can speak with our // nodes and vice versa on both the core protocol as well as the // delegate protocol level. // // The verification works by finding the maximum minimum and // minimum maximum understood protocol and delegate versions. In other words, // it finds the common denominator of protocol and delegate version ranges // for the entire cluster. // // After this, it goes through the entire cluster (local and remote) and // verifies that everyone's speaking protocol versions satisfy this range. // If this passes, it means that every node can understand each other. func (m Memberlist) verifyProtocol(remote []pushNodeState) error { m.nodeLock.RLock() defer m.nodeLock.RUnlock() // Maximum minimum understood and minimum maximum understood for both // the protocol and delegate versions. We use this to verify everyone // can be understood. var maxpmin, minpmax uint8 var maxdmin, mindmax uint8 minpmax = math.MaxUint8 mindmax = math.MaxUint8 for _, rn := range remote { // If the node isn't alive, then skip it if rn.State != StateAlive { continue } // Skip nodes that don't have versions set, it just means // their version is zero. if len(rn.Vsn) == 0 { continue } if rn.Vsn[0] > maxpmin { maxpmin = rn.Vsn[0] } if rn.Vsn[1] < minpmax { minpmax = rn.Vsn[1] } if rn.Vsn[3] > maxdmin { maxdmin = rn.Vsn[3] } if rn.Vsn[4] < mindmax { mindmax = rn.Vsn[4] } } for _, n := range m.nodes { // Ignore non-alive nodes if n.State != StateAlive { continue } if n.PMin > maxpmin { maxpmin = n.PMin } if n.PMax < minpmax { minpmax = n.PMax } if n.DMin > maxdmin { maxdmin = n.DMin } if n.DMax < mindmax { mindmax = n.DMax } } // Now that we definitively know the minimum and maximum understood // version that satisfies the whole cluster, we verify that every // node in the cluster satisifies this. for _, n := range remote { var nPCur, nDCur uint8 if len(n.Vsn) > 0 { nPCur = n.Vsn[2] nDCur = n.Vsn[5] } if nPCur < maxpmin \|\| nPCur > minpmax { return fmt.Errorf( "Node '%s' protocol version (%d) is incompatible: [%d, %d]", n.Name, nPCur, maxpmin, minpmax) } if nDCur < maxdmin \|\| nDCur > mindmax { return fmt.Errorf( "Node '%s' delegate protocol version (%d) is incompatible: [%d, %d]", n.Name, nDCur, maxdmin, mindmax) } } for _, n := range m.nodes { nPCur := n.PCur nDCur := n.DCur if nPCur < maxpmin \|\| nPCur > minpmax { return fmt.Errorf( "Node '%s' protocol version (%d) is incompatible: [%d, %d]", n.Name, nPCur, maxpmin, minpmax) } if nDCur < maxdmin \|\| nDCur > mindmax { return fmt.Errorf( "Node '%s' delegate protocol version (%d) is incompatible: [%d, %d]", n.Name, nDCur, maxdmin, mindmax) } } return nil } // nextSeqNo returns a usable sequence number in a thread safe way func (m Memberlist) nextSeqNo() uint32 { return atomic.AddUint32(&m.sequenceNum, 1) } // nextIncarnation returns the next incarnation number in a thread safe way func (m Memberlist) nextIncarnation() uint32 { return atomic.AddUint32(&m.incarnation, 1) } // skipIncarnation adds the positive offset to the incarnation number. func (m Memberlist) skipIncarnation(offset uint32) uint32 { return atomic.AddUint32(&m.incarnation, offset) } // estNumNodes is used to get the current estimate of the number of nodes func (m Memberlist) estNumNodes() int { return int(atomic.LoadUint32(&m.numNodes)) } type ackMessage struct { Complete bool Payload []byte Timestamp time.Time } // setProbeChannels is used to attach the ackCh to receive a message when an ack // with a given sequence number is received. The `complete` field of the message // will be false on timeout. Any nack messages will cause an empty struct to be // passed to the nackCh, which can be nil if not needed. func (m Memberlist) setProbeChannels(seqNo uint32, ackCh chan ackMessage, nackCh chan struct{}, timeout time.Duration) { // Create handler functions for acks and nacks ackFn := func(payload []byte, timestamp time.Time) { select { case ackCh <- ackMessage{true, payload, timestamp}: default: } } nackFn := func() { select { case nackCh <- struct{}{}: default: } } // Add the handlers ah := &ackHandler{ackFn, nackFn, nil} m.ackLock.Lock() m.ackHandlers[seqNo] = ah m.ackLock.Unlock() // Setup a reaping routing ah.timer = time.AfterFunc(timeout, func() { m.ackLock.Lock() delete(m.ackHandlers, seqNo) m.ackLock.Unlock() select { case ackCh <- ackMessage{false, nil, time.Now()}: default: } }) } // setAckHandler is used to attach a handler to be invoked when an ack with a // given sequence number is received. If a timeout is reached, the handler is // deleted. This is used for indirect pings so does not configure a function // for nacks. func (m Memberlist) setAckHandler(seqNo uint32, ackFn func([]byte, time.Time), timeout time.Duration) { // Add the handler ah := &ackHandler{ackFn, nil, nil} m.ackLock.Lock() m.ackHandlers[seqNo] = ah m.ackLock.Unlock() // Setup a reaping routing ah.timer = time.AfterFunc(timeout, func() { m.ackLock.Lock() delete(m.ackHandlers, seqNo) m.ackLock.Unlock() }) } // Invokes an ack handler if any is associated, and reaps the handler immediately func (m Memberlist) invokeAckHandler(ack ackResp, timestamp time.Time) { m.ackLock.Lock() ah, ok := m.ackHandlers[ack.SeqNo] delete(m.ackHandlers, ack.SeqNo) m.ackLock.Unlock() if !ok { return } ah.timer.Stop() ah.ackFn(ack.Payload, timestamp) } // Invokes nack handler if any is associated. func (m Memberlist) invokeNackHandler(nack nackResp) { m.ackLock.Lock() ah, ok := m.ackHandlers[nack.SeqNo] m.ackLock.Unlock() if !ok \|\| ah.nackFn == nil { return } ah.nackFn() } // refute gossips an alive message in response to incoming information that we // are suspect or dead. It will make sure the incarnation number beats the given // accusedInc value, or you can supply 0 to just get the next incarnation number. // This alters the node state that's passed in so this MUST be called while the // nodeLock is held. func (m Memberlist) refute(me nodeState, accusedInc uint32) { // Make sure the incarnation number beats the accusation. inc := m.nextIncarnation() if accusedInc >= inc { inc = m.skipIncarnation(accusedInc - inc + 1) } me.Incarnation = inc // Decrease our health because we are being asked to refute a problem. m.awareness.ApplyDelta(1) // Format and broadcast an alive message. a := alive{ Incarnation: inc, Node: me.Name, Addr: me.Addr, Port: me.Port, Meta: me.Meta, Vsn: []uint8{ me.PMin, me.PMax, me.PCur, me.DMin, me.DMax, me.DCur, }, } m.encodeAndBroadcast(me.Addr.String(), aliveMsg, a) } // aliveNode is invoked by the network layer when we get a message about a // live node. func (m Memberlist) aliveNode(a alive, notify chan struct{}, bootstrap bool) { m.nodeLock.Lock() defer m.nodeLock.Unlock() state, ok := m.nodeMap[a.Node] // It is possible that during a Leave(), there is already an aliveMsg // in-queue to be processed but blocked by the locks above. If we let // that aliveMsg process, it'll cause us to re-join the cluster. This // ensures that we don't. if m.hasLeft() && a.Node == m.config.Name { return } if len(a.Vsn) >= 3 { pMin := a.Vsn[0] pMax := a.Vsn[1] pCur := a.Vsn[2] if pMin == 0 \|\| pMax == 0 \|\| pMin > pMax { m.logger.Printf("[WARN] memberlist: Ignoring an alive message for '%s' (%v:%d) because protocol version(s) are wrong: %d <= %d <= %d should be >0", a.Node, net.IP(a.Addr), a.Port, pMin, pCur, pMax) return } } // Invoke the Alive delegate if any. This can be used to filter out // alive messages based on custom logic. For example, using a cluster name. // Using a merge delegate is not enough, as it is possible for passive // cluster merging to still occur. if m.config.Alive != nil { if len(a.Vsn) < 6 { m.logger.Printf("[WARN] memberlist: ignoring alive message for '%s' (%v:%d) because Vsn is not present", a.Node, net.IP(a.Addr), a.Port) return } node := &Node{ Name: a.Node, Addr: a.Addr, Port: a.Port, Meta: a.Meta, PMin: a.Vsn[0], PMax: a.Vsn[1], PCur: a.Vsn[2], DMin: a.Vsn[3], DMax: a.Vsn[4], DCur: a.Vsn[5], } if err := m.config.Alive.NotifyAlive(node); err != nil { m.logger.Printf("[WARN] memberlist: ignoring alive message for '%s': %s", a.Node, err) return } } // Check if we've never seen this node before, and if not, then // store this node in our node map. var updatesNode bool if !ok { errCon := m.config.IPAllowed(a.Addr) if errCon != nil { m.logger.Printf("[WARN] memberlist: Rejected node %s (%v): %s", a.Node, net.IP(a.Addr), errCon) return } state = &nodeState{ Node: Node{ Name: a.Node, Addr: a.Addr, Port: a.Port, Meta: a.Meta, }, State: StateDead, } if len(a.Vsn) > 5 { state.PMin = a.Vsn[0] state.PMax = a.Vsn[1] state.PCur = a.Vsn[2] state.DMin = a.Vsn[3] state.DMax = a.Vsn[4] state.DCur = a.Vsn[5] } // Add to map m.nodeMap[a.Node] = state // Get a random offset. This is important to ensure // the failure detection bound is low on average. If all // nodes did an append, failure detection bound would be // very high. n := len(m.nodes) offset := randomOffset(n) // Add at the end and swap with the node at the offset m.nodes = append(m.nodes, state) m.nodes[offset], m.nodes[n] = m.nodes[n], m.nodes[offset] // Update numNodes after we've added a new node atomic.AddUint32(&m.numNodes, 1) } else { // Check if this address is different than the existing node unless the old node is dead. if !bytes.Equal([]byte(state.Addr), a.Addr) \|\| state.Port != a.Port { errCon := m.config.IPAllowed(a.Addr) if errCon != nil { m.logger.Printf("[WARN] memberlist: Rejected IP update from %v to %v for node %s: %s", a.Node, state.Addr, net.IP(a.Addr), errCon) return } // If DeadNodeReclaimTime is configured, check if enough time has elapsed since the node died. canReclaim := (m.config.DeadNodeReclaimTime > 0 && time.Since(state.StateChange) > m.config.DeadNodeReclaimTime) // Allow the address to be updated if a dead node is being replaced. if state.State == StateLeft \|\| (state.State == StateDead && canReclaim) { m.logger.Printf("[INFO] memberlist: Updating address for left or failed node %s from %v:%d to %v:%d", state.Name, state.Addr, state.Port, net.IP(a.Addr), a.Port) updatesNode = true } else { m.logger.Printf("[ERR] memberlist: Conflicting address for %s. Mine: %v:%d Theirs: %v:%d Old state: %v", state.Name, state.Addr, state.Port, net.IP(a.Addr), a.Port, state.State) // Inform the conflict delegate if provided if m.config.Conflict != nil { other := Node{ Name: a.Node, Addr: a.Addr, Port: a.Port, Meta: a.Meta, } m.config.Conflict.NotifyConflict(&state.Node, &other) } return } } } // Bail if the incarnation number is older, and this is not about us isLocalNode := state.Name == m.config.Name if a.Incarnation <= state.Incarnation && !isLocalNode && !updatesNode { return } // Bail if strictly less and this is about us if a.Incarnation < state.Incarnation && isLocalNode { return } // Clear out any suspicion timer that may be in effect. delete(m.nodeTimers, a.Node) // Store the old state and meta data oldState := state.State oldMeta := state.Meta // If this is us we need to refute, otherwise re-broadcast if !bootstrap && isLocalNode { // Compute the version vector versions := []uint8{ state.PMin, state.PMax, state.PCur, state.DMin, state.DMax, state.DCur, } // If the Incarnation is the same, we need special handling, since it // possible for the following situation to happen: // 1) Start with configuration C, join cluster // 2) Hard fail / Kill / Shutdown // 3) Restart with configuration C', join cluster // // In this case, other nodes and the local node see the same incarnation, // but the values may not be the same. For this reason, we always // need to do an equality check for this Incarnation. In most cases, // we just ignore, but we may need to refute. // if a.Incarnation == state.Incarnation && bytes.Equal(a.Meta, state.Meta) && bytes.Equal(a.Vsn, versions) { return } m.refute(state, a.Incarnation) m.logger.Printf("[WARN] memberlist: Refuting an alive message for '%s' (%v:%d) meta:(%v VS %v), vsn:(%v VS %v)", a.Node, net.IP(a.Addr), a.Port, a.Meta, state.Meta, a.Vsn, versions) } else { m.encodeBroadcastNotify(a.Node, aliveMsg, a, notify) // Update protocol versions if it arrived if len(a.Vsn) > 0 { state.PMin = a.Vsn[0] state.PMax = a.Vsn[1] state.PCur = a.Vsn[2] state.DMin = a.Vsn[3] state.DMax = a.Vsn[4] state.DCur = a.Vsn[5] } // Update the state and incarnation number state.Incarnation = a.Incarnation state.Meta = a.Meta state.Addr = a.Addr state.Port = a.Port if state.State != StateAlive { state.State = StateAlive state.StateChange = time.Now() } } // Update metrics metrics.IncrCounterWithLabels([]string{"memberlist", "msg", "alive"}, 1, m.metricLabels) // Notify the delegate of any relevant updates if m.config.Events != nil { if oldState == StateDead \|\| oldState == StateLeft { // if Dead/Left -> Alive, notify of join m.config.Events.NotifyJoin(&state.Node) } else if !bytes.Equal(oldMeta, state.Meta) { // if Meta changed, trigger an update notification m.config.Events.NotifyUpdate(&state.Node) } } } // suspectNode is invoked by the network layer when we get a message // about a suspect node func (m Memberlist) suspectNode(s suspect) { m.nodeLock.Lock() defer m.nodeLock.Unlock() state, ok := m.nodeMap[s.Node] // If we've never heard about this node before, ignore it if !ok { return } // Ignore old incarnation numbers if s.Incarnation < state.Incarnation { return } // See if there's a suspicion timer we can confirm. If the info is new // to us we will go ahead and re-gossip it. This allows for multiple // independent confirmations to flow even when a node probes a node // that's already suspect. if timer, ok := m.nodeTimers[s.Node]; ok { if timer.Confirm(s.From) { m.encodeAndBroadcast(s.Node, suspectMsg, s) } return } // Ignore non-alive nodes if state.State != StateAlive { return } // If this is us we need to refute, otherwise re-broadcast if state.Name == m.config.Name { m.refute(state, s.Incarnation) m.logger.Printf("[WARN] memberlist: Refuting a suspect message (from: %s)", s.From) return // Do not mark ourself suspect } else { m.encodeAndBroadcast(s.Node, suspectMsg, s) } // Update metrics metrics.IncrCounterWithLabels([]string{"memberlist", "msg", "suspect"}, 1, m.metricLabels) // Update the state state.Incarnation = s.Incarnation state.State = StateSuspect changeTime := time.Now() state.StateChange = changeTime // Setup a suspicion timer. Given that we don't have any known phase // relationship with our peers, we set up k such that we hit the nominal // timeout two probe intervals short of what we expect given the suspicion // multiplier. k := m.config.SuspicionMult - 2 // If there aren't enough nodes to give the expected confirmations, just // set k to 0 to say that we don't expect any. Note we subtract 2 from n // here to take out ourselves and the node being probed. n := m.estNumNodes() if n-2 < k { k = 0 } // Compute the timeouts based on the size of the cluster. min := suspicionTimeout(m.config.SuspicionMult, n, m.config.ProbeInterval) max := time.Duration(m.config.SuspicionMaxTimeoutMult) * min fn := func(numConfirmations int) { var d dead m.nodeLock.Lock() state, ok := m.nodeMap[s.Node] timeout := ok && state.State == StateSuspect && state.StateChange == changeTime if timeout { d = &dead{Incarnation: state.Incarnation, Node: state.Name, From: m.config.Name} } m.nodeLock.Unlock() if timeout { if k > 0 && numConfirmations < k { metrics.IncrCounterWithLabels([]string{"memberlist", "degraded", "timeout"}, 1, m.metricLabels) } m.logger.Printf("[INFO] memberlist: Marking %s as failed, suspect timeout reached (%d peer confirmations)", state.Name, numConfirmations) m.deadNode(d) } } m.nodeTimers[s.Node] = newSuspicion(s.From, k, min, max, fn) } // deadNode is invoked by the network layer when we get a message // about a dead node func (m Memberlist) deadNode(d dead) { m.nodeLock.Lock() defer m.nodeLock.Unlock() state, ok := m.nodeMap[d.Node] // If we've never heard about this node before, ignore it if !ok { return } // Ignore old incarnation numbers if d.Incarnation < state.Incarnation { return } // Clear out any suspicion timer that may be in effect. delete(m.nodeTimers, d.Node) // Ignore if node is already dead if state.DeadOrLeft() { return } // Check if this is us if state.Name == m.config.Name { // If we are not leaving we need to refute if !m.hasLeft() { m.refute(state, d.Incarnation) m.logger.Printf("[WARN] memberlist: Refuting a dead message (from: %s)", d.From) return // Do not mark ourself dead } // If we are leaving, we broadcast and wait m.encodeBroadcastNotify(d.Node, deadMsg, d, m.leaveBroadcast) } else { m.encodeAndBroadcast(d.Node, deadMsg, d) } // Update metrics metrics.IncrCounterWithLabels([]string{"memberlist", "msg", "dead"}, 1, m.metricLabels) // Update the state state.Incarnation = d.Incarnation // If the dead message was send by the node itself, mark it is left // instead of dead. if d.Node == d.From { state.State = StateLeft } else { state.State = StateDead } state.StateChange = time.Now() // Notify of death if m.config.Events != nil { m.config.Events.NotifyLeave(&state.Node) } } // mergeState is invoked by the network layer when we get a Push/Pull // state transfer func (m Memberlist) mergeState(remote []pushNodeState) { for _, r := range remote { switch r.State { case StateAlive: a := alive{ Incarnation: r.Incarnation, Node: r.Name, Addr: r.Addr, Port: r.Port, Meta: r.Meta, Vsn: r.Vsn, } m.aliveNode(&a, nil, false) case StateLeft: d := dead{Incarnation: r.Incarnation, Node: r.Name, From: r.Name} m.deadNode(&d) case StateDead: // If the remote node believes a node is dead, we prefer to // suspect that node instead of declaring it dead instantly fallthrough case StateSuspect: s := suspect{Incarnation: r.Incarnation, Node: r.Name, From: m.config.Name} m.suspectNode(&s) } } }	38.01 KB	2025-05-25 12:27
📄 state_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "fmt" "log" "net" "os" "reflect" "strconv" "sync" "sync/atomic" "testing" "time" "github.com/armon/go-metrics" iretry "github.com/hashicorp/memberlist/internal/retry" "github.com/stretchr/testify/require" ) func HostMemberlist(host string, t testing.T, f func(Config)) Memberlist { t.Helper() c := DefaultLANConfig() c.Name = host c.BindAddr = host c.BindPort = 0 // choose a free port c.Logger = log.New(os.Stderr, host, log.LstdFlags) if f != nil { f(c) } m, err := newMemberlist(c) if err != nil { t.Fatalf("failed to get memberlist: %s", err) } return m } func TestMemberList_Probe(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = time.Millisecond c.ProbeInterval = 10 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() a1 := alive{ Node: addr1.String(), Addr: []byte(addr1), Port: uint16(m1.config.BindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray(), } m1.aliveNode(&a1, nil, true) a2 := alive{ Node: addr2.String(), Addr: []byte(addr2), Port: uint16(m2.config.BindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray(), } m1.aliveNode(&a2, nil, false) // should ping addr2 m1.probe() // Should not be marked suspect n := m1.nodeMap[addr2.String()] if n.State != StateAlive { t.Fatalf("Expect node to be alive") } // Should increment seqno if m1.sequenceNum != 1 { t.Fatalf("bad seqno %v", m2.sequenceNum) } } func TestMemberList_ProbeNode_Suspect(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = time.Millisecond c.ProbeInterval = 10 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort }) defer m3.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1, Vsn: m3.config.BuildVsnArray()} m1.aliveNode(&a3, nil, false) a4 := alive{Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a4, nil, false) n := m1.nodeMap[addr4.String()] m1.probeNode(n) // Should be marked suspect. if n.State != StateSuspect { t.Fatalf("Expect node to be suspect") } time.Sleep(10 * time.Millisecond) // One of the peers should have attempted an indirect probe. if s2, s3 := atomic.LoadUint32(&m2.sequenceNum), atomic.LoadUint32(&m3.sequenceNum); s2 != 1 && s3 != 1 { t.Fatalf("bad seqnos, expected both to be 1: %v, %v", s2, s3) } } func TestMemberList_ProbeNode_Suspect_Dogpile(t testing.T) { cases := []struct { name string numPeers int confirmations int expected time.Duration }{ {"n=2, k=3 (max timeout disabled)", 1, 0, 500 time.Millisecond}, {"n=3, k=3", 2, 0, 500 * time.Millisecond}, {"n=4, k=3", 3, 0, 500 * time.Millisecond}, {"n=5, k=3 (max timeout starts to take effect)", 4, 0, 1000 * time.Millisecond}, {"n=6, k=3", 5, 0, 1000 * time.Millisecond}, {"n=6, k=3 (confirmations start to lower timeout)", 5, 1, 750 * time.Millisecond}, {"n=6, k=3", 5, 2, 604 * time.Millisecond}, {"n=6, k=3 (timeout driven to nominal value)", 5, 3, 500 * time.Millisecond}, {"n=6, k=3", 5, 4, 500 * time.Millisecond}, } for i, c := range cases { t.Run(c.name, func(t testing.T) { // Create the main memberlist under test. addr := getBindAddr() m := HostMemberlist(addr.String(), t, func(c Config) { c.ProbeTimeout = time.Millisecond c.ProbeInterval = 100 * time.Millisecond c.SuspicionMult = 5 c.SuspicionMaxTimeoutMult = 2 }) defer m.Shutdown() bindPort := m.config.BindPort a := alive{Node: addr.String(), Addr: []byte(addr), Port: uint16(bindPort), Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, true) // Make all but one peer be an real, alive instance. var peers []Memberlist for j := 0; j < c.numPeers-1; j++ { peerAddr := getBindAddr() peer := HostMemberlist(peerAddr.String(), t, func(c Config) { c.BindPort = bindPort }) defer peer.Shutdown() peers = append(peers, peer) a = alive{Node: peerAddr.String(), Addr: []byte(peerAddr), Port: uint16(bindPort), Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) } // Just use a bogus address for the last peer so it doesn't respond // to pings, but tell the memberlist it's alive. badPeerAddr := getBindAddr() a = alive{Node: badPeerAddr.String(), Addr: []byte(badPeerAddr), Port: uint16(bindPort), Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) // Force a probe, which should start us into the suspect state. m.probeNodeByAddr(badPeerAddr.String()) if m.getNodeState(badPeerAddr.String()) != StateSuspect { t.Fatalf("case %d: expected node to be suspect", i) } // Add the requested number of confirmations. for j := 0; j < c.confirmations; j++ { from := fmt.Sprintf("peer%d", j) s := suspect{Node: badPeerAddr.String(), Incarnation: 1, From: from} m.suspectNode(&s) } // Wait until right before the timeout and make sure the timer // hasn't fired. fudge := 25 * time.Millisecond time.Sleep(c.expected - fudge) if m.getNodeState(badPeerAddr.String()) != StateSuspect { t.Fatalf("case %d: expected node to still be suspect", i) } // Wait through the timeout and a little after to make sure the // timer fires. time.Sleep(2 * fudge) if m.getNodeState(badPeerAddr.String()) != StateDead { t.Fatalf("case %d: expected node to be dead", i) } }) } } /* func TestMemberList_ProbeNode_FallbackTCP(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMax time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMax = c.ProbeInterval + 20time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort }) defer m3.Shutdown() m4 := HostMemberlist(addr4.String(), t, func(c Config) { c.BindPort = bindPort }) defer m4.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a3, nil, false) // Make sure m4 is configured with the same protocol version as m1 so // the TCP fallback behavior is enabled. a4 := alive{ Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: []uint8{ ProtocolVersionMin, ProtocolVersionMax, m1.config.ProtocolVersion, m1.config.DelegateProtocolMin, m1.config.DelegateProtocolMax, m1.config.DelegateProtocolVersion, }, } m1.aliveNode(&a4, nil, false) // Isolate m4 from UDP traffic by re-opening its listener on the wrong // port. This should force the TCP fallback path to be used. var err error if err = m4.udpListener.Close(); err != nil { t.Fatalf("err: %v", err) } udpAddr := &net.UDPAddr{IP: ip4, Port: 9999} if m4.udpListener, err = net.ListenUDP("udp", udpAddr); err != nil { t.Fatalf("err: %v", err) } // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Should be marked alive because of the TCP fallback ping. if n.State != stateAlive { t.Fatalf("expect node to be alive") } // Make sure TCP activity completed in a timely manner. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. time.Sleep(probeTimeMax) if m2.sequenceNum != 1 && m3.sequenceNum != 1 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } // Now shutdown all inbound TCP traffic to make sure the TCP fallback // path properly fails when the node is really unreachable. if err = m4.tcpListener.Close(); err != nil { t.Fatalf("err: %v", err) } tcpAddr := &net.TCPAddr{IP: ip4, Port: 9999} if m4.tcpListener, err = net.ListenTCP("tcp", tcpAddr); err != nil { t.Fatalf("err: %v", err) } // Probe again, this time there should be no contact. startProbe = time.Now() m1.probeNode(n) probeTime = time.Now().Sub(startProbe) // Node should be reported suspect. if n.State != stateSuspect { t.Fatalf("expect node to be suspect") } // Make sure TCP activity didn't cause us to wait too long before // timing out. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. time.Sleep(probeTimeMax) if m2.sequenceNum != 2 && m3.sequenceNum != 2 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } } func TestMemberList_ProbeNode_FallbackTCP_Disabled(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMax time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMax = c.ProbeInterval + 20time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort }) defer m3.Shutdown() m4 := HostMemberlist(addr4.String(), t, func(c Config) { c.BindPort = bindPort }) defer m4.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a3, nil, false) // Make sure m4 is configured with the same protocol version as m1 so // the TCP fallback behavior is enabled. a4 := alive{ Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: []uint8{ ProtocolVersionMin, ProtocolVersionMax, m1.config.ProtocolVersion, m1.config.DelegateProtocolMin, m1.config.DelegateProtocolMax, m1.config.DelegateProtocolVersion, }, } m1.aliveNode(&a4, nil, false) // Isolate m4 from UDP traffic by re-opening its listener on the wrong // port. This should force the TCP fallback path to be used. var err error if err = m4.udpListener.Close(); err != nil { t.Fatalf("err: %v", err) } udpAddr := &net.UDPAddr{IP: ip4, Port: 9999} if m4.udpListener, err = net.ListenUDP("udp", udpAddr); err != nil { t.Fatalf("err: %v", err) } // Disable the TCP pings using the config mechanism. m1.config.DisableTcpPings = true // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Node should be reported suspect. if n.State != stateSuspect { t.Fatalf("expect node to be suspect") } // Make sure TCP activity didn't cause us to wait too long before // timing out. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. time.Sleep(probeTimeMax) if m2.sequenceNum != 1 && m3.sequenceNum != 1 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } } func TestMemberList_ProbeNode_FallbackTCP_OldProtocol(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMax time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMax = c.ProbeInterval + 20time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort }) defer m3.Shutdown() m4 := HostMemberlist(addr4.String(), t, func(c Config) { c.BindPort = bindPort }) defer m4.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a3, nil, false) // Set up m4 so that it doesn't understand a version of the protocol // that supports TCP pings. a4 := alive{ Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: []uint8{ ProtocolVersionMin, ProtocolVersion2Compatible, ProtocolVersion2Compatible, m1.config.DelegateProtocolMin, m1.config.DelegateProtocolMax, m1.config.DelegateProtocolVersion, }, } m1.aliveNode(&a4, nil, false) // Isolate m4 from UDP traffic by re-opening its listener on the wrong // port. This should force the TCP fallback path to be used. var err error if err = m4.udpListener.Close(); err != nil { t.Fatalf("err: %v", err) } udpAddr := &net.UDPAddr{IP: ip4, Port: 9999} if m4.udpListener, err = net.ListenUDP("udp", udpAddr); err != nil { t.Fatalf("err: %v", err) } // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Node should be reported suspect. if n.State != stateSuspect { t.Fatalf("expect node to be suspect") } // Make sure TCP activity didn't cause us to wait too long before // timing out. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. time.Sleep(probeTimeMax) if m2.sequenceNum != 1 && m3.sequenceNum != 1 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } } / func TestMemberList_ProbeNode_Awareness_Degraded(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMin time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMin = 2c.ProbeInterval - 50time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m3.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1, Vsn: m3.config.BuildVsnArray()} m1.aliveNode(&a3, nil, false) vsn4 := []uint8{ ProtocolVersionMin, ProtocolVersionMax, ProtocolVersionMin, 1, 1, 1, } // Node 4 never gets started. a4 := alive{Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: vsn4} m1.aliveNode(&a4, nil, false) // Start the health in a degraded state. m1.awareness.ApplyDelta(1) if score := m1.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Node should be reported suspect. if n.State != StateSuspect { t.Fatalf("expect node to be suspect") } // Make sure we timed out approximately on time (note that we accounted // for the slowed-down failure detector in the probeTimeMin calculation. if probeTime < probeTimeMin { t.Fatalf("probed too quickly, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. if m2.sequenceNum != 1 && m3.sequenceNum != 1 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } // We should have gotten all the nacks, so our score should remain the // same, since we didn't get a successful probe. if score := m1.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } } func TestMemberList_ProbeNode_Wrong_VSN(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m3.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1, Vsn: m3.config.BuildVsnArray()} m1.aliveNode(&a3, nil, false) vsn4 := []uint8{ 0, 0, 0, 0, 0, 0, } // Node 4 never gets started. a4 := alive{Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: vsn4} m1.aliveNode(&a4, nil, false) // Start the health in a degraded state. m1.awareness.ApplyDelta(1) if score := m1.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } // Have node m1 probe m4. n, ok := m1.nodeMap[addr4.String()] if ok \|\| n != nil { t.Fatalf("expect node a4 to be not taken into account, because of its wrong version") } } func TestMemberList_ProbeNode_Awareness_Improved(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) // Start the health in a degraded state. m1.awareness.ApplyDelta(1) if score := m1.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } // Have node m1 probe m2. n := m1.nodeMap[addr2.String()] m1.probeNode(n) // Node should be reported alive. if n.State != StateAlive { t.Fatalf("expect node to be suspect") } // Our score should have improved since we did a good probe. if score := m1.GetHealthScore(); score != 0 { t.Fatalf("bad: %d", score) } } func TestMemberList_ProbeNode_Awareness_MissedNack(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMax time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMax = c.ProbeInterval + 50time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.ProbeTimeout = 10 * time.Millisecond c.ProbeInterval = 200 * time.Millisecond }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) vsn := m1.config.BuildVsnArray() // Node 3 and node 4 never get started. a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1, Vsn: vsn} m1.aliveNode(&a3, nil, false) a4 := alive{Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1, Vsn: vsn} m1.aliveNode(&a4, nil, false) // Make sure health looks good. if score := m1.GetHealthScore(); score != 0 { t.Fatalf("bad: %d", score) } // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Node should be reported suspect. m1.nodeLock.Lock() if n.State != StateSuspect { t.Fatalf("expect node to be suspect") } m1.nodeLock.Unlock() // Make sure we timed out approximately on time. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // We should have gotten dinged for the missed nack. Note that the code under // test is waiting for probeTimeMax and then doing some other work before it // updates the awareness, so we need to wait some extra time. Rather than just // add longer and longer sleeps, we'll retry a few times. iretry.Run(t, func(r iretry.R) { if score := m1.GetHealthScore(); score != 1 { r.Fatalf("expected health score to decrement on missed nack. want %d, "+ "got: %d", 1, score) } }) } func TestMemberList_ProbeNode_Awareness_OldProtocol(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() addr4 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) ip4 := []byte(addr4) var probeTimeMax time.Duration m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 10 time.Millisecond c.ProbeInterval = 200 * time.Millisecond probeTimeMax = c.ProbeInterval + 20time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() m3 := HostMemberlist(addr3.String(), t, func(c Config) { c.BindPort = bindPort }) defer m3.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a3, nil, false) // Node 4 never gets started. a4 := alive{Node: addr4.String(), Addr: ip4, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a4, nil, false) // Make sure health looks good. if score := m1.GetHealthScore(); score != 0 { t.Fatalf("bad: %d", score) } // Have node m1 probe m4. n := m1.nodeMap[addr4.String()] startProbe := time.Now() m1.probeNode(n) probeTime := time.Now().Sub(startProbe) // Node should be reported suspect. if n.State != StateSuspect { t.Fatalf("expect node to be suspect") } // Make sure we timed out approximately on time. if probeTime > probeTimeMax { t.Fatalf("took to long to probe, %9.6f", probeTime.Seconds()) } // Confirm at least one of the peers attempted an indirect probe. time.Sleep(probeTimeMax) if m2.sequenceNum != 1 && m3.sequenceNum != 1 { t.Fatalf("bad seqnos %v, %v", m2.sequenceNum, m3.sequenceNum) } // Since we are using the old protocol here, we should have gotten dinged // for a failed health check. if score := m1.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } } func TestMemberList_ProbeNode_Buddy(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = time.Millisecond c.ProbeInterval = 10 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) m1.aliveNode(&a2, nil, false) m2.aliveNode(&a2, nil, true) // Force the state to suspect so we piggyback a suspect message with the ping. // We should see this get refuted later, and the ping will succeed. n := m1.nodeMap[addr2.String()] n.State = StateSuspect m1.probeNode(n) // Make sure a ping was sent. if m1.sequenceNum != 1 { t.Fatalf("bad seqno %v", m1.sequenceNum) } // Check a broadcast is queued. if num := m2.broadcasts.NumQueued(); num != 1 { t.Fatalf("expected only one queued message: %d", num) } // Should be alive msg. if messageType(m2.broadcasts.orderedView(true)[0].b.Message()[0]) != aliveMsg { t.Fatalf("expected queued alive msg") } } func TestMemberList_ProbeNode(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = time.Millisecond c.ProbeInterval = 10 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) n := m1.nodeMap[addr2.String()] m1.probeNode(n) // Should be marked alive if n.State != StateAlive { t.Fatalf("Expect node to be alive") } // Should increment seqno if m1.sequenceNum != 1 { t.Fatalf("bad seqno %v", m1.sequenceNum) } } func TestMemberList_Ping(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.ProbeTimeout = 1 time.Second c.ProbeInterval = 10 * time.Second }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1} m1.aliveNode(&a2, nil, false) // Do a legit ping. n := m1.nodeMap[addr2.String()] addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(addr2.String(), strconv.Itoa(bindPort))) if err != nil { t.Fatalf("err: %v", err) } rtt, err := m1.Ping(n.Name, addr) if err != nil { t.Fatalf("err: %v", err) } if !(rtt > 0) { t.Fatalf("bad: %v", rtt) } // This ping has a bad node name so should timeout. _, err = m1.Ping("bad", addr) if _, ok := err.(NoPingResponseError); !ok \|\| err == nil { t.Fatalf("bad: %v", err) } } func TestMemberList_ResetNodes(t testing.T) { m := GetMemberlist(t, func(c Config) { c.GossipToTheDeadTime = 100 time.Millisecond }) defer m.Shutdown() a1 := alive{Node: "test1", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a1, nil, false) a2 := alive{Node: "test2", Addr: []byte{127, 0, 0, 2}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a2, nil, false) a3 := alive{Node: "test3", Addr: []byte{127, 0, 0, 3}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a3, nil, false) d := dead{Node: "test2", Incarnation: 1} m.deadNode(&d) m.resetNodes() if len(m.nodes) != 3 { t.Fatalf("Bad length") } if _, ok := m.nodeMap["test2"]; !ok { t.Fatalf("test2 should not be unmapped") } time.Sleep(200 * time.Millisecond) m.resetNodes() if len(m.nodes) != 2 { t.Fatalf("Bad length") } if _, ok := m.nodeMap["test2"]; ok { t.Fatalf("test2 should be unmapped") } } func TestMemberList_NextSeq(t testing.T) { m := &Memberlist{} if m.nextSeqNo() != 1 { t.Fatalf("bad sequence no") } if m.nextSeqNo() != 2 { t.Fatalf("bad sequence no") } } func ackHandlerExists(t testing.T, m Memberlist, idx uint32) bool { t.Helper() m.ackLock.Lock() _, ok := m.ackHandlers[idx] m.ackLock.Unlock() return ok } func TestMemberList_setProbeChannels(t testing.T) { m := &Memberlist{ackHandlers: make(map[uint32]ackHandler)} ch := make(chan ackMessage, 1) m.setProbeChannels(0, ch, nil, 10time.Millisecond) require.True(t, ackHandlerExists(t, m, 0), "missing handler") time.Sleep(20 * time.Millisecond) require.False(t, ackHandlerExists(t, m, 0), "non-reaped handler") } func TestMemberList_setAckHandler(t testing.T) { m := &Memberlist{ackHandlers: make(map[uint32]ackHandler)} f := func([]byte, time.Time) {} m.setAckHandler(0, f, 10time.Millisecond) require.True(t, ackHandlerExists(t, m, 0), "missing handler") time.Sleep(20 time.Millisecond) require.False(t, ackHandlerExists(t, m, 0), "non-reaped handler") } func TestMemberList_invokeAckHandler(t testing.T) { m := &Memberlist{ackHandlers: make(map[uint32]ackHandler)} // Does nothing m.invokeAckHandler(ackResp{}, time.Now()) var b bool f := func(payload []byte, timestamp time.Time) { b = true } m.setAckHandler(0, f, 10time.Millisecond) // Should set b m.invokeAckHandler(ackResp{0, nil}, time.Now()) if !b { t.Fatalf("b not set") } require.False(t, ackHandlerExists(t, m, 0), "non-reaped handler") } func TestMemberList_invokeAckHandler_Channel_Ack(t testing.T) { m := &Memberlist{ackHandlers: make(map[uint32]ackHandler)} ack := ackResp{0, []byte{0, 0, 0}} // Does nothing m.invokeAckHandler(ack, time.Now()) ackCh := make(chan ackMessage, 1) nackCh := make(chan struct{}, 1) m.setProbeChannels(0, ackCh, nackCh, 10time.Millisecond) // Should send message m.invokeAckHandler(ack, time.Now()) select { case v := <-ackCh: if v.Complete != true { t.Fatalf("Bad value") } if bytes.Compare(v.Payload, ack.Payload) != 0 { t.Fatalf("wrong payload. expected: %v; actual: %v", ack.Payload, v.Payload) } case <-nackCh: t.Fatalf("should not get a nack") default: t.Fatalf("message not sent") } require.False(t, ackHandlerExists(t, m, 0), "non-reaped handler") } func TestMemberList_invokeAckHandler_Channel_Nack(t testing.T) { m := &Memberlist{ackHandlers: make(map[uint32]ackHandler)} nack := nackResp{0} // Does nothing. m.invokeNackHandler(nack) ackCh := make(chan ackMessage, 1) nackCh := make(chan struct{}, 1) m.setProbeChannels(0, ackCh, nackCh, 10time.Millisecond) // Should send message. m.invokeNackHandler(nack) select { case <-ackCh: t.Fatalf("should not get an ack") case <-nackCh: // Good. default: t.Fatalf("message not sent") } // Getting a nack doesn't reap the handler so that we can still forward // an ack up to the reap time, if we get one. require.True(t, ackHandlerExists(t, m, 0), "handler should not be reaped") ack := ackResp{0, []byte{0, 0, 0}} m.invokeAckHandler(ack, time.Now()) select { case v := <-ackCh: if v.Complete != true { t.Fatalf("Bad value") } if bytes.Compare(v.Payload, ack.Payload) != 0 { t.Fatalf("wrong payload. expected: %v; actual: %v", ack.Payload, v.Payload) } case <-nackCh: t.Fatalf("should not get a nack") default: t.Fatalf("message not sent") } require.False(t, ackHandlerExists(t, m, 0), "non-reaped handler") } func TestMemberList_AliveNode_NewNode(t testing.T) { ch := make(chan NodeEvent, 1) m := GetMemberlist(t, func(c Config) { c.Events = &ChannelEventDelegate{ch} }) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) if len(m.nodes) != 1 { t.Fatalf("should add node") } state, ok := m.nodeMap["test"] if !ok { t.Fatalf("should map node") } if state.Incarnation != 1 { t.Fatalf("bad incarnation") } if state.State != StateAlive { t.Fatalf("bad state") } if time.Now().Sub(state.StateChange) > time.Second { t.Fatalf("bad change delta") } // Check for a join message select { case e := <-ch: if e.Node.Name != "test" { t.Fatalf("bad node name") } default: t.Fatalf("no join message") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected queued message") } } func TestMemberList_AliveNode_SuspectNode(t testing.T) { ch := make(chan NodeEvent, 1) ted := &toggledEventDelegate{ real: &ChannelEventDelegate{ch}, } m := GetMemberlist(t, func(c Config) { c.Events = ted }) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) // Listen only after first join ted.Toggle(true) // Make suspect state := m.nodeMap["test"] state.State = StateSuspect state.StateChange = state.StateChange.Add(-time.Hour) // Old incarnation number, should not change m.aliveNode(&a, nil, false) if state.State != StateSuspect { t.Fatalf("update with old incarnation!") } // Should reset to alive now a.Incarnation = 2 m.aliveNode(&a, nil, false) if state.State != StateAlive { t.Fatalf("no update with new incarnation!") } if time.Now().Sub(state.StateChange) > time.Second { t.Fatalf("bad change delta") } // Check for a no join message select { case <-ch: t.Fatalf("got bad join message") default: } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected queued message") } } func TestMemberList_AliveNode_Idempotent(t testing.T) { ch := make(chan NodeEvent, 1) ted := &toggledEventDelegate{ real: &ChannelEventDelegate{ch}, } m := GetMemberlist(t, func(c Config) { c.Events = ted }) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) // Listen only after first join ted.Toggle(true) // Make suspect state := m.nodeMap["test"] stateTime := state.StateChange // Should reset to alive now a.Incarnation = 2 m.aliveNode(&a, nil, false) if state.State != StateAlive { t.Fatalf("non idempotent") } if stateTime != state.StateChange { t.Fatalf("should not change state") } // Check for a no join message select { case <-ch: t.Fatalf("got bad join message") default: } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } } type toggledEventDelegate struct { mu sync.Mutex real EventDelegate enabled bool } func (d toggledEventDelegate) Toggle(enabled bool) { d.mu.Lock() defer d.mu.Unlock() d.enabled = enabled } // NotifyJoin is invoked when a node is detected to have joined. // The Node argument must not be modified. func (d toggledEventDelegate) NotifyJoin(n Node) { d.mu.Lock() defer d.mu.Unlock() if d.enabled { d.real.NotifyJoin(n) } } // NotifyLeave is invoked when a node is detected to have left. // The Node argument must not be modified. func (d toggledEventDelegate) NotifyLeave(n Node) { d.mu.Lock() defer d.mu.Unlock() if d.enabled { d.real.NotifyLeave(n) } } // NotifyUpdate is invoked when a node is detected to have // updated, usually involving the meta data. The Node argument // must not be modified. func (d toggledEventDelegate) NotifyUpdate(n Node) { d.mu.Lock() defer d.mu.Unlock() if d.enabled { d.real.NotifyUpdate(n) } } // Serf Bug: GH-58, Meta data does not update func TestMemberList_AliveNode_ChangeMeta(t testing.T) { ch := make(chan NodeEvent, 1) ted := &toggledEventDelegate{ real: &ChannelEventDelegate{ch}, } m := GetMemberlist(t, func(c Config) { c.Events = ted }) defer m.Shutdown() a := alive{ Node: "test", Addr: []byte{127, 0, 0, 1}, Meta: []byte("val1"), Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) // Listen only after first join ted.Toggle(true) // Make suspect state := m.nodeMap["test"] // Should reset to alive now a.Incarnation = 2 a.Meta = []byte("val2") m.aliveNode(&a, nil, false) // Check updates if bytes.Compare(state.Meta, a.Meta) != 0 { t.Fatalf("meta did not update") } // Check for a NotifyUpdate select { case e := <-ch: if e.Event != NodeUpdate { t.Fatalf("bad event: %v", e) } if !reflect.DeepEqual(e.Node, state.Node) { t.Fatalf("expected %v, got %v", e.Node, state.Node) } if bytes.Compare(e.Node.Meta, a.Meta) != 0 { t.Fatalf("meta did not update") } default: t.Fatalf("missing event!") } } func TestMemberList_AliveNode_Refute(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: m.config.Name, Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, true) // Clear queue m.broadcasts.Reset() // Conflicting alive s := alive{ Node: m.config.Name, Addr: []byte{127, 0, 0, 1}, Incarnation: 2, Meta: []byte("foo"), Vsn: m.config.BuildVsnArray(), } m.aliveNode(&s, nil, false) state := m.nodeMap[m.config.Name] if state.State != StateAlive { t.Fatalf("should still be alive") } if state.Meta != nil { t.Fatalf("meta should still be nil") } // Check a broad cast is queued if num := m.broadcasts.NumQueued(); num != 1 { t.Fatalf("expected only one queued message: %d", num) } // Should be alive mesg if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != aliveMsg { t.Fatalf("expected queued alive msg") } } func TestMemberList_AliveNode_Conflict(t testing.T) { m := GetMemberlist(t, func(c Config) { c.DeadNodeReclaimTime = 10 time.Millisecond }) defer m.Shutdown() nodeName := "test" a := alive{Node: nodeName, Addr: []byte{127, 0, 0, 1}, Port: 8000, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, true) // Clear queue m.broadcasts.Reset() // Conflicting alive s := alive{ Node: nodeName, Addr: []byte{127, 0, 0, 2}, Port: 9000, Incarnation: 2, Meta: []byte("foo"), Vsn: m.config.BuildVsnArray(), } m.aliveNode(&s, nil, false) state := m.nodeMap[nodeName] if state.State != StateAlive { t.Fatalf("should still be alive") } if state.Meta != nil { t.Fatalf("meta should still be nil") } if bytes.Equal(state.Addr, []byte{127, 0, 0, 2}) { t.Fatalf("address should not be updated") } if state.Port == 9000 { t.Fatalf("port should not be updated") } // Check a broad cast is queued if num := m.broadcasts.NumQueued(); num != 0 { t.Fatalf("expected 0 queued messages: %d", num) } // Change the node to dead d := dead{Node: nodeName, Incarnation: 2} m.deadNode(&d) m.broadcasts.Reset() state = m.nodeMap[nodeName] if state.State != StateDead { t.Fatalf("should be dead") } time.Sleep(m.config.DeadNodeReclaimTime) // New alive node s2 := alive{ Node: nodeName, Addr: []byte{127, 0, 0, 2}, Port: 9000, Incarnation: 3, Meta: []byte("foo"), Vsn: m.config.BuildVsnArray(), } m.aliveNode(&s2, nil, false) state = m.nodeMap[nodeName] if state.State != StateAlive { t.Fatalf("should still be alive") } if !bytes.Equal(state.Meta, []byte("foo")) { t.Fatalf("meta should be updated") } if !bytes.Equal(state.Addr, []byte{127, 0, 0, 2}) { t.Fatalf("address should be updated") } if state.Port != 9000 { t.Fatalf("port should be updated") } } func TestMemberList_SuspectNode_NoNode(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() s := suspect{Node: "test", Incarnation: 1} m.suspectNode(&s) if len(m.nodes) != 0 { t.Fatalf("don't expect nodes") } } func TestMemberList_SuspectNode(t testing.T) { m := GetMemberlist(t, func(c Config) { c.ProbeInterval = time.Millisecond c.SuspicionMult = 1 }) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) m.changeNode("test", func(state nodeState) { state.StateChange = state.StateChange.Add(-time.Hour) }) s := suspect{Node: "test", Incarnation: 1} m.suspectNode(&s) if m.getNodeState("test") != StateSuspect { t.Fatalf("Bad state") } change := m.getNodeStateChange("test") if time.Now().Sub(change) > time.Second { t.Fatalf("bad change delta") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Check its a suspect message if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != suspectMsg { t.Fatalf("expected queued suspect msg") } // Wait for the timeout time.Sleep(10 * time.Millisecond) if m.getNodeState("test") != StateDead { t.Fatalf("Bad state") } newChange := m.getNodeStateChange("test") if time.Now().Sub(newChange) > time.Second { t.Fatalf("bad change delta") } if !newChange.After(change) { t.Fatalf("should increment time") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Check its a suspect message if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != deadMsg { t.Fatalf("expected queued dead msg") } } func TestMemberList_SuspectNode_DoubleSuspect(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) state := m.nodeMap["test"] state.StateChange = state.StateChange.Add(-time.Hour) s := suspect{Node: "test", Incarnation: 1} m.suspectNode(&s) if state.State != StateSuspect { t.Fatalf("Bad state") } change := state.StateChange if time.Now().Sub(change) > time.Second { t.Fatalf("bad change delta") } // clear the broadcast queue m.broadcasts.Reset() // Suspect again m.suspectNode(&s) if state.StateChange != change { t.Fatalf("unexpected state change") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 0 { t.Fatalf("expected only one queued message") } } func TestMemberList_SuspectNode_OldSuspect(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 10, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) state := m.nodeMap["test"] state.StateChange = state.StateChange.Add(-time.Hour) // Clear queue m.broadcasts.Reset() s := suspect{Node: "test", Incarnation: 1} m.suspectNode(&s) if state.State != StateAlive { t.Fatalf("Bad state") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 0 { t.Fatalf("expected only one queued message") } } func TestMemberList_SuspectNode_Refute(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: m.config.Name, Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, true) // Clear queue m.broadcasts.Reset() // Make sure health is in a good state if score := m.GetHealthScore(); score != 0 { t.Fatalf("bad: %d", score) } s := suspect{Node: m.config.Name, Incarnation: 1} m.suspectNode(&s) state := m.nodeMap[m.config.Name] if state.State != StateAlive { t.Fatalf("should still be alive") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Should be alive mesg if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != aliveMsg { t.Fatalf("expected queued alive msg") } // Health should have been dinged if score := m.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } } func TestMemberList_DeadNode_NoNode(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() d := dead{Node: "test", Incarnation: 1} m.deadNode(&d) if len(m.nodes) != 0 { t.Fatalf("don't expect nodes") } } func TestMemberList_DeadNodeLeft(t testing.T) { ch := make(chan NodeEvent, 1) m := GetMemberlist(t, func(c Config) { c.Events = &ChannelEventDelegate{ch} }) defer m.Shutdown() nodeName := "node1" s1 := alive{ Node: nodeName, Addr: []byte{127, 0, 0, 1}, Port: 8000, Incarnation: 1, Vsn: m.config.BuildVsnArray(), } m.aliveNode(&s1, nil, false) // Read the join event <-ch d := dead{Node: nodeName, From: nodeName, Incarnation: 1} m.deadNode(&d) // Read the dead event <-ch state := m.nodeMap[nodeName] if state.State != StateLeft { t.Fatalf("Bad state") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Check its a dead message if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != deadMsg { t.Fatalf("expected queued dead msg") } // Clear queue // m.broadcasts.Reset() // New alive node s2 := alive{ Node: nodeName, Addr: []byte{127, 0, 0, 2}, Port: 9000, Incarnation: 3, Meta: []byte("foo"), Vsn: m.config.BuildVsnArray(), } m.aliveNode(&s2, nil, false) // Read the join event <-ch state = m.nodeMap[nodeName] if state.State != StateAlive { t.Fatalf("should still be alive") } if !bytes.Equal(state.Meta, []byte("foo")) { t.Fatalf("meta should be updated") } if !bytes.Equal(state.Addr, []byte{127, 0, 0, 2}) { t.Fatalf("address should be updated") } if state.Port != 9000 { t.Fatalf("port should be updated") } } func TestMemberList_DeadNode(t testing.T) { ch := make(chan NodeEvent, 1) m := GetMemberlist(t, func(c Config) { c.Events = &ChannelEventDelegate{ch} }) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) // Read the join event <-ch state := m.nodeMap["test"] state.StateChange = state.StateChange.Add(-time.Hour) d := dead{Node: "test", Incarnation: 1} m.deadNode(&d) if state.State != StateDead { t.Fatalf("Bad state") } change := state.StateChange if time.Now().Sub(change) > time.Second { t.Fatalf("bad change delta") } select { case leave := <-ch: if leave.Event != NodeLeave \|\| leave.Node.Name != "test" { t.Fatalf("bad node name") } default: t.Fatalf("no leave message") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Check its a dead message if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != deadMsg { t.Fatalf("expected queued dead msg") } } func TestMemberList_DeadNode_Double(t testing.T) { ch := make(chan NodeEvent, 1) m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) state := m.nodeMap["test"] state.StateChange = state.StateChange.Add(-time.Hour) d := dead{Node: "test", Incarnation: 1} m.deadNode(&d) // Clear queue m.broadcasts.Reset() // Notify after the first dead m.config.Events = &ChannelEventDelegate{ch} // Should do nothing d.Incarnation = 2 m.deadNode(&d) select { case <-ch: t.Fatalf("should not get leave") default: } // Check a broad cast is queued if m.broadcasts.NumQueued() != 0 { t.Fatalf("expected only one queued message") } } func TestMemberList_DeadNode_OldDead(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 10, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) state := m.nodeMap["test"] state.StateChange = state.StateChange.Add(-time.Hour) d := dead{Node: "test", Incarnation: 1} m.deadNode(&d) if state.State != StateAlive { t.Fatalf("Bad state") } } func TestMemberList_DeadNode_AliveReplay(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: "test", Addr: []byte{127, 0, 0, 1}, Incarnation: 10, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, false) d := dead{Node: "test", Incarnation: 10} m.deadNode(&d) // Replay alive at same incarnation m.aliveNode(&a, nil, false) // Should remain dead state, ok := m.nodeMap["test"] if ok && state.State != StateDead { t.Fatalf("Bad state") } } func TestMemberList_DeadNode_Refute(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a := alive{Node: m.config.Name, Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a, nil, true) // Clear queue m.broadcasts.Reset() // Make sure health is in a good state if score := m.GetHealthScore(); score != 0 { t.Fatalf("bad: %d", score) } d := dead{Node: m.config.Name, Incarnation: 1} m.deadNode(&d) state := m.nodeMap[m.config.Name] if state.State != StateAlive { t.Fatalf("should still be alive") } // Check a broad cast is queued if m.broadcasts.NumQueued() != 1 { t.Fatalf("expected only one queued message") } // Should be alive mesg if messageType(m.broadcasts.orderedView(true)[0].b.Message()[0]) != aliveMsg { t.Fatalf("expected queued alive msg") } // We should have been dinged if score := m.GetHealthScore(); score != 1 { t.Fatalf("bad: %d", score) } } func TestMemberList_MergeState(t testing.T) { m := GetMemberlist(t, nil) defer m.Shutdown() a1 := alive{Node: "test1", Addr: []byte{127, 0, 0, 1}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a1, nil, false) a2 := alive{Node: "test2", Addr: []byte{127, 0, 0, 2}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a2, nil, false) a3 := alive{Node: "test3", Addr: []byte{127, 0, 0, 3}, Incarnation: 1, Vsn: m.config.BuildVsnArray()} m.aliveNode(&a3, nil, false) s := suspect{Node: "test1", Incarnation: 1} m.suspectNode(&s) remote := []pushNodeState{ pushNodeState{ Name: "test1", Addr: []byte{127, 0, 0, 1}, Incarnation: 2, State: StateAlive, }, pushNodeState{ Name: "test2", Addr: []byte{127, 0, 0, 2}, Incarnation: 1, State: StateSuspect, }, pushNodeState{ Name: "test3", Addr: []byte{127, 0, 0, 3}, Incarnation: 1, State: StateDead, }, pushNodeState{ Name: "test4", Addr: []byte{127, 0, 0, 4}, Incarnation: 2, State: StateAlive, }, } // Listen for changes eventCh := make(chan NodeEvent, 1) m.config.Events = &ChannelEventDelegate{eventCh} // Merge remote state m.mergeState(remote) // Check the states state := m.nodeMap["test1"] if state.State != StateAlive \|\| state.Incarnation != 2 { t.Fatalf("Bad state %v", state) } state = m.nodeMap["test2"] if state.State != StateSuspect \|\| state.Incarnation != 1 { t.Fatalf("Bad state %v", state) } state = m.nodeMap["test3"] if state.State != StateSuspect { t.Fatalf("Bad state %v", state) } state = m.nodeMap["test4"] if state.State != StateAlive \|\| state.Incarnation != 2 { t.Fatalf("Bad state %v", state) } // Check the channels select { case e := <-eventCh: if e.Event != NodeJoin \|\| e.Node.Name != "test4" { t.Fatalf("bad node %v", e) } default: t.Fatalf("Expect join") } select { case e := <-eventCh: t.Fatalf("Unexpect event: %v", e) default: } } func TestMemberlist_Gossip(t testing.T) { ch := make(chan NodeEvent, 3) addr1 := getBindAddr() addr2 := getBindAddr() addr3 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) ip3 := []byte(addr3) m1 := HostMemberlist(addr1.String(), t, func(c Config) { // Set the gossip interval fast enough to get a reasonable test, // but slow enough to avoid "sendto: operation not permitted" c.GossipInterval = 10 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.Events = &ChannelEventDelegate{ch} // Set the gossip interval fast enough to get a reasonable test, // but slow enough to avoid "sendto: operation not permitted" c.GossipInterval = 10 time.Millisecond }) defer m2.Shutdown() m3 := HostMemberlist(addr2.String(), t, func(c Config) { }) defer m3.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) a3 := alive{Node: addr3.String(), Addr: ip3, Port: uint16(bindPort), Incarnation: 1, Vsn: m3.config.BuildVsnArray()} m1.aliveNode(&a3, nil, false) // Gossip should send all this to m2. Retry a few times because it's UDP and // timing and stuff makes this flaky without. retry(t, 15, 250time.Millisecond, func(failf func(string, ...interface{})) { m1.gossip() time.Sleep(3 * time.Millisecond) if len(ch) < 3 { failf("expected 3 messages from gossip but only got %d", len(ch)) } }) } func retry(t testing.T, n int, w time.Duration, fn func(func(string, ...interface{}))) { t.Helper() for try := 1; try <= n; try++ { failed := false failFormat := "" failArgs := []interface{}{} failf := func(format string, args ...interface{}) { failed = true failFormat = format failArgs = args } fn(failf) if !failed { return } if try == n { t.Fatalf(failFormat, failArgs...) } time.Sleep(w) } } func TestMemberlist_GossipToDead(t testing.T) { ch := make(chan NodeEvent, 2) addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.GossipInterval = time.Millisecond c.GossipToTheDeadTime = 100 time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.Events = &ChannelEventDelegate{ch} }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) // Shouldn't send anything to m2 here, node has been dead for 2x the GossipToTheDeadTime m1.nodeMap[addr2.String()].State = StateDead m1.nodeMap[addr2.String()].StateChange = time.Now().Add(-200 time.Millisecond) m1.gossip() select { case <-ch: t.Fatalf("shouldn't get gossip") case <-time.After(50 * time.Millisecond): } // Should gossip to m2 because its state has changed within GossipToTheDeadTime m1.nodeMap[addr2.String()].StateChange = time.Now().Add(-20 * time.Millisecond) retry(t, 5, 10time.Millisecond, func(failf func(string, ...interface{})) { m1.gossip() time.Sleep(3 time.Millisecond) if len(ch) < 2 { failf("expected 2 messages from gossip") } }) } func TestMemberlist_FailedRemote(t testing.T) { type test struct { name string err error expected bool } tests := []test{ {"nil error", nil, false}, {"normal error", fmt.Errorf(""), false}, {"net.OpError for file", &net.OpError{Net: "file"}, false}, {"net.OpError for udp", &net.OpError{Net: "udp"}, false}, {"net.OpError for udp4", &net.OpError{Net: "udp4"}, false}, {"net.OpError for udp6", &net.OpError{Net: "udp6"}, false}, {"net.OpError for tcp", &net.OpError{Net: "tcp"}, false}, {"net.OpError for tcp4", &net.OpError{Net: "tcp4"}, false}, {"net.OpError for tcp6", &net.OpError{Net: "tcp6"}, false}, {"net.OpError for tcp with dial", &net.OpError{Net: "tcp", Op: "dial"}, true}, {"net.OpError for tcp with write", &net.OpError{Net: "tcp", Op: "write"}, true}, {"net.OpError for tcp with read", &net.OpError{Net: "tcp", Op: "read"}, true}, {"net.OpError for udp with write", &net.OpError{Net: "udp", Op: "write"}, true}, {"net.OpError for udp with read", &net.OpError{Net: "udp", Op: "read"}, false}, } for _, test := range tests { t.Run(test.name, func(t testing.T) { actual := failedRemote(test.err) if actual != test.expected { t.Fatalf("expected %t, got %t", test.expected, actual) } }) } } func TestMemberlist_PushPull(t testing.T) { addr1 := getBindAddr() addr2 := getBindAddr() ip1 := []byte(addr1) ip2 := []byte(addr2) sink := registerInMemorySink(t) ch := make(chan NodeEvent, 3) m1 := HostMemberlist(addr1.String(), t, func(c Config) { c.GossipInterval = 10 * time.Second c.PushPullInterval = time.Millisecond }) defer m1.Shutdown() bindPort := m1.config.BindPort m2 := HostMemberlist(addr2.String(), t, func(c Config) { c.BindPort = bindPort c.GossipInterval = 10 time.Second c.Events = &ChannelEventDelegate{ch} }) defer m2.Shutdown() a1 := alive{Node: addr1.String(), Addr: ip1, Port: uint16(bindPort), Incarnation: 1, Vsn: m1.config.BuildVsnArray()} m1.aliveNode(&a1, nil, true) a2 := alive{Node: addr2.String(), Addr: ip2, Port: uint16(bindPort), Incarnation: 1, Vsn: m2.config.BuildVsnArray()} m1.aliveNode(&a2, nil, false) // Gossip should send all this to m2. It's UDP though so retry a few times retry(t, 5, 10time.Millisecond, func(failf func(string, ...interface{})) { m1.pushPull() time.Sleep(3 time.Millisecond) if len(ch) < 2 { failf("expected 2 messages from pushPull") } instancesMetricName := "consul.usage.test.memberlist.node.instances" verifyGaugeExists(t, "consul.usage.test.memberlist.size.local", sink) verifyGaugeExists(t, fmt.Sprintf("%s;node_state=%s", instancesMetricName, StateAlive.metricsString()), sink) verifyGaugeExists(t, fmt.Sprintf("%s;node_state=%s", instancesMetricName, StateDead.metricsString()), sink) verifyGaugeExists(t, fmt.Sprintf("%s;node_state=%s", instancesMetricName, StateLeft.metricsString()), sink) verifyGaugeExists(t, fmt.Sprintf("%s;node_state=%s", instancesMetricName, StateSuspect.metricsString()), sink) }) } func TestVerifyProtocol(t testing.T) { cases := []struct { Anodes [][3]uint8 Bnodes [][3]uint8 expected bool }{ // Both running identical everything { Anodes: [][3]uint8{ {0, 0, 0}, }, Bnodes: [][3]uint8{ {0, 0, 0}, }, expected: true, }, // One can understand newer, but speaking same protocol { Anodes: [][3]uint8{ {0, 0, 0}, }, Bnodes: [][3]uint8{ {0, 1, 0}, }, expected: true, }, // One is speaking outside the range { Anodes: [][3]uint8{ {0, 0, 0}, }, Bnodes: [][3]uint8{ {1, 1, 1}, }, expected: false, }, // Transitively outside the range { Anodes: [][3]uint8{ {0, 1, 0}, {0, 2, 1}, }, Bnodes: [][3]uint8{ {1, 3, 1}, }, expected: false, }, // Multi-node { Anodes: [][3]uint8{ {0, 3, 2}, {0, 2, 0}, }, Bnodes: [][3]uint8{ {0, 2, 1}, {0, 5, 0}, }, expected: true, }, } for _, tc := range cases { aCore := make([][6]uint8, len(tc.Anodes)) aApp := make([][6]uint8, len(tc.Anodes)) for i, n := range tc.Anodes { aCore[i] = [6]uint8{n[0], n[1], n[2], 0, 0, 0} aApp[i] = [6]uint8{0, 0, 0, n[0], n[1], n[2]} } bCore := make([][6]uint8, len(tc.Bnodes)) bApp := make([][6]uint8, len(tc.Bnodes)) for i, n := range tc.Bnodes { bCore[i] = [6]uint8{n[0], n[1], n[2], 0, 0, 0} bApp[i] = [6]uint8{0, 0, 0, n[0], n[1], n[2]} } // Test core protocol verification testVerifyProtocolSingle(t, aCore, bCore, tc.expected) testVerifyProtocolSingle(t, bCore, aCore, tc.expected) // Test app protocol verification testVerifyProtocolSingle(t, aApp, bApp, tc.expected) testVerifyProtocolSingle(t, bApp, aApp, tc.expected) } } func testVerifyProtocolSingle(t testing.T, A [][6]uint8, B [][6]uint8, expect bool) { m := GetMemberlist(t, nil) defer m.Shutdown() m.nodes = make([]nodeState, len(A)) for i, n := range A { m.nodes[i] = &nodeState{ Node: Node{ PMin: n[0], PMax: n[1], PCur: n[2], DMin: n[3], DMax: n[4], DCur: n[5], }, } } remote := make([]pushNodeState, len(B)) for i, n := range B { remote[i] = pushNodeState{ Name: fmt.Sprintf("node %d", i), Vsn: []uint8{n[0], n[1], n[2], n[3], n[4], n[5]}, } } err := m.verifyProtocol(remote) if (err == nil) != expect { t.Fatalf("bad:\nA: %v\nB: %v\nErr: %s", A, B, err) } } func registerInMemorySink(t testing.T) metrics.InmemSink { t.Helper() // Only have a single interval for the test sink := metrics.NewInmemSink(1time.Minute, 1time.Minute) cfg := metrics.DefaultConfig("consul.usage.test") cfg.EnableHostname = false metrics.NewGlobal(cfg, sink) return sink } func getIntervalMetrics(t testing.T, sink metrics.InmemSink) metrics.IntervalMetrics { t.Helper() intervals := sink.Data() require.Len(t, intervals, 1) intv := intervals[0] return intv } func verifyGaugeExists(t testing.T, name string, sink metrics.InmemSink) { t.Helper() interval := getIntervalMetrics(t, sink) interval.RLock() defer interval.RUnlock() if _, ok := interval.Gauges[name]; !ok { t.Fatalf("%s gauge not emmited", name) } } func verifySampleExists(t testing.T, name string, sink metrics.InmemSink) { t.Helper() interval := getIntervalMetrics(t, sink) interval.RLock() defer interval.RUnlock() if _, ok := interval.Samples[name]; !ok { t.Fatalf("%s sample not emmited", name) } }	62.49 KB	2025-05-25 12:27
📄 suspicion.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "math" "sync/atomic" "time" ) // suspicion manages the suspect timer for a node and provides an interface // to accelerate the timeout as we get more independent confirmations that // a node is suspect. type suspicion struct { // n is the number of independent confirmations we've seen. This must // be updated using atomic instructions to prevent contention with the // timer callback. n int32 // k is the number of independent confirmations we'd like to see in // order to drive the timer to its minimum value. k int32 // min is the minimum timer value. min time.Duration // max is the maximum timer value. max time.Duration // start captures the timestamp when we began the timer. This is used // so we can calculate durations to feed the timer during updates in // a way the achieves the overall time we'd like. start time.Time // timer is the underlying timer that implements the timeout. timer time.Timer // f is the function to call when the timer expires. We hold on to this // because there are cases where we call it directly. timeoutFn func() // confirmations is a map of "from" nodes that have confirmed a given // node is suspect. This prevents double counting. confirmations map[string]struct{} } // newSuspicion returns a timer started with the max time, and that will drive // to the min time after seeing k or more confirmations. The from node will be // excluded from confirmations since we might get our own suspicion message // gossiped back to us. The minimum time will be used if no confirmations are // called for (k <= 0). func newSuspicion(from string, k int, min time.Duration, max time.Duration, fn func(int)) suspicion { s := &suspicion{ k: int32(k), min: min, max: max, confirmations: make(map[string]struct{}), } // Exclude the from node from any confirmations. s.confirmations[from] = struct{}{} // Pass the number of confirmations into the timeout function for // easy telemetry. s.timeoutFn = func() { fn(int(atomic.LoadInt32(&s.n))) } // If there aren't any confirmations to be made then take the min // time from the start. timeout := max if k < 1 { timeout = min } s.timer = time.AfterFunc(timeout, s.timeoutFn) // Capture the start time right after starting the timer above so // we should always err on the side of a little longer timeout if // there's any preemption that separates this and the step above. s.start = time.Now() return s } // remainingSuspicionTime takes the state variables of the suspicion timer and // calculates the remaining time to wait before considering a node dead. The // return value can be negative, so be prepared to fire the timer immediately in // that case. func remainingSuspicionTime(n, k int32, elapsed time.Duration, min, max time.Duration) time.Duration { frac := math.Log(float64(n)+1.0) / math.Log(float64(k)+1.0) raw := max.Seconds() - frac(max.Seconds()-min.Seconds()) timeout := time.Duration(math.Floor(1000.0raw)) * time.Millisecond if timeout < min { timeout = min } // We have to take into account the amount of time that has passed so // far, so we get the right overall timeout. return timeout - elapsed } // Confirm registers that a possibly new peer has also determined the given // node is suspect. This returns true if this was new information, and false // if it was a duplicate confirmation, or if we've got enough confirmations to // hit the minimum. func (s suspicion) Confirm(from string) bool { // If we've got enough confirmations then stop accepting them. if atomic.LoadInt32(&s.n) >= s.k { return false } // Only allow one confirmation from each possible peer. if _, ok := s.confirmations[from]; ok { return false } s.confirmations[from] = struct{}{} // Compute the new timeout given the current number of confirmations and // adjust the timer. If the timeout becomes negative and* we can cleanly // stop the timer then we will call the timeout function directly from // here. n := atomic.AddInt32(&s.n, 1) elapsed := time.Since(s.start) remaining := remainingSuspicionTime(n, s.k, elapsed, s.min, s.max) if s.timer.Stop() { if remaining > 0 { s.timer.Reset(remaining) } else { go s.timeoutFn() } } return true }	4.27 KB	2025-05-25 12:27
📄 suspicion_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "testing" "time" ) func TestSuspicion_remainingSuspicionTime(t testing.T) { cases := []struct { n int32 k int32 elapsed time.Duration min time.Duration max time.Duration expected time.Duration }{ {0, 3, 0, 2 time.Second, 30 * time.Second, 30 * time.Second}, {1, 3, 2 * time.Second, 2 * time.Second, 30 * time.Second, 14 * time.Second}, {2, 3, 3 * time.Second, 2 * time.Second, 30 * time.Second, 4810 * time.Millisecond}, {3, 3, 4 * time.Second, 2 * time.Second, 30 * time.Second, -2 * time.Second}, {4, 3, 5 * time.Second, 2 * time.Second, 30 * time.Second, -3 * time.Second}, {5, 3, 10 * time.Second, 2 * time.Second, 30 * time.Second, -8 * time.Second}, } for i, c := range cases { remaining := remainingSuspicionTime(c.n, c.k, c.elapsed, c.min, c.max) if remaining != c.expected { t.Errorf("case %d: remaining %9.6f != expected %9.6f", i, remaining.Seconds(), c.expected.Seconds()) } } } func TestSuspicion_Timer(t testing.T) { const k = 3 const min = 500 time.Millisecond const max = 2 * time.Second type pair struct { from string newInfo bool } cases := []struct { numConfirmations int from string confirmations []pair expected time.Duration }{ { 0, "me", []pair{}, max, }, { 1, "me", []pair{ pair{"me", false}, pair{"foo", true}, }, 1250 * time.Millisecond, }, { 1, "me", []pair{ pair{"me", false}, pair{"foo", true}, pair{"foo", false}, pair{"foo", false}, }, 1250 * time.Millisecond, }, { 2, "me", []pair{ pair{"me", false}, pair{"foo", true}, pair{"bar", true}, }, 810 * time.Millisecond, }, { 3, "me", []pair{ pair{"me", false}, pair{"foo", true}, pair{"bar", true}, pair{"baz", true}, }, min, }, { 3, "me", []pair{ pair{"me", false}, pair{"foo", true}, pair{"bar", true}, pair{"baz", true}, pair{"zoo", false}, }, min, }, } for i, c := range cases { ch := make(chan time.Duration, 1) start := time.Now() f := func(numConfirmations int) { if numConfirmations != c.numConfirmations { t.Errorf("case %d: bad %d != %d", i, numConfirmations, c.numConfirmations) } ch <- time.Now().Sub(start) } // Create the timer and add the requested confirmations. Wait // the fudge amount to help make sure we calculate the timeout // overall, and don't accumulate extra time. s := newSuspicion(c.from, k, min, max, f) fudge := 25 * time.Millisecond for _, p := range c.confirmations { time.Sleep(fudge) if s.Confirm(p.from) != p.newInfo { t.Fatalf("case %d: newInfo mismatch for %s", i, p.from) } } // Wait until right before the timeout and make sure the // timer hasn't fired. already := time.Duration(len(c.confirmations)) * fudge time.Sleep(c.expected - already - fudge) select { case d := <-ch: t.Fatalf("case %d: should not have fired (%9.6f)", i, d.Seconds()) default: } // Wait through the timeout and a little after and make sure it // fires. time.Sleep(2 * fudge) select { case <-ch: default: t.Fatalf("case %d: should have fired", i) } // Confirm after to make sure it handles a negative remaining // time correctly and doesn't fire again. s.Confirm("late") time.Sleep(c.expected + 2fudge) select { case d := <-ch: t.Fatalf("case %d: should not have fired (%9.6f)", i, d.Seconds()) default: } } } func TestSuspicion_Timer_ZeroK(t testing.T) { ch := make(chan struct{}, 1) f := func(int) { ch <- struct{}{} } // This should select the min time since there are no expected // confirmations to accelerate the timer. s := newSuspicion("me", 0, 25time.Millisecond, 30time.Second, f) if s.Confirm("foo") { t.Fatalf("should not provide new information") } select { case <-ch: case <-time.After(50 * time.Millisecond): t.Fatalf("should have fired") } } func TestSuspicion_Timer_Immediate(t testing.T) { ch := make(chan struct{}, 1) f := func(int) { ch <- struct{}{} } // This should underflow the timeout and fire immediately. s := newSuspicion("me", 1, 100time.Millisecond, 30time.Second, f) time.Sleep(200 time.Millisecond) s.Confirm("foo") // Wait a little while since the function gets called in a goroutine. select { case <-ch: case <-time.After(25 * time.Millisecond): t.Fatalf("should have fired") } }	4.46 KB	2025-05-25 12:27
📄 tag.sh #!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: MPL-2.0 set -e # The version must be supplied from the environment. Do not include the # leading "v". if [ -z $VERSION ]; then echo "Please specify a version." exit 1 fi # Generate the tag. echo "==> Tagging version $VERSION..." git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION" git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master exit 0	467 B	2025-05-25 12:27
📁 test	-	2025-05-25 12:27
📄 todo.md # TODO * Dynamic RTT discovery * Compute 99th percentile for ping/ack * Better lower bound for ping/ack, faster failure detection * Dynamic MTU discovery * Prevent lost updates, increases efficiency	211 B	2025-05-25 12:27
📄 transport.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "net" "time" ) // Packet is used to provide some metadata about incoming packets from peers // over a packet connection, as well as the packet payload. type Packet struct { // Buf has the raw contents of the packet. Buf []byte // From has the address of the peer. This is an actual net.Addr so we // can expose some concrete details about incoming packets. From net.Addr // Timestamp is the time when the packet was received. This should be // taken as close as possible to the actual receipt time to help make an // accurate RTT measurement during probes. Timestamp time.Time } // Transport is used to abstract over communicating with other peers. The packet // interface is assumed to be best-effort and the stream interface is assumed to // be reliable. type Transport interface { // FinalAdvertiseAddr is given the user's configured values (which // might be empty) and returns the desired IP and port to advertise to // the rest of the cluster. FinalAdvertiseAddr(ip string, port int) (net.IP, int, error) // WriteTo is a packet-oriented interface that fires off the given // payload to the given address in a connectionless fashion. This should // return a time stamp that's as close as possible to when the packet // was transmitted to help make accurate RTT measurements during probes. // // This is similar to net.PacketConn, though we didn't want to expose // that full set of required methods to keep assumptions about the // underlying plumbing to a minimum. We also treat the address here as a // string, similar to Dial, so it's network neutral, so this usually is // in the form of "host:port". WriteTo(b []byte, addr string) (time.Time, error) // PacketCh returns a channel that can be read to receive incoming // packets from other peers. How this is set up for listening is left as // an exercise for the concrete transport implementations. PacketCh() <-chan Packet // DialTimeout is used to create a connection that allows us to perform // two-way communication with a peer. This is generally more expensive // than packet connections so is used for more infrequent operations // such as anti-entropy or fallback probes if the packet-oriented probe // failed. DialTimeout(addr string, timeout time.Duration) (net.Conn, error) // StreamCh returns a channel that can be read to handle incoming stream // connections from other peers. How this is set up for listening is // left as an exercise for the concrete transport implementations. StreamCh() <-chan net.Conn // Shutdown is called when memberlist is shutting down; this gives the // transport a chance to clean up any listeners. Shutdown() error } type Address struct { // Addr is a network address as a string, similar to Dial. This usually is // in the form of "host:port". This is required. Addr string // Name is the name of the node being addressed. This is optional but // transports may require it. Name string } func (a Address) String() string { if a.Name != "" { return fmt.Sprintf("%s (%s)", a.Name, a.Addr) } return a.Addr } // IngestionAwareTransport is not used. // // Deprecated: IngestionAwareTransport is not used and may be removed in a future // version. Define the interface locally instead of referencing this exported // interface. type IngestionAwareTransport interface { IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error IngestStream(conn net.Conn) error } type NodeAwareTransport interface { Transport WriteToAddress(b []byte, addr Address) (time.Time, error) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) } type shimNodeAwareTransport struct { Transport } var _ NodeAwareTransport = (shimNodeAwareTransport)(nil) func (t shimNodeAwareTransport) WriteToAddress(b []byte, addr Address) (time.Time, error) { return t.WriteTo(b, addr.Addr) } func (t shimNodeAwareTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) { return t.DialTimeout(addr.Addr, timeout) } type labelWrappedTransport struct { label string NodeAwareTransport } var _ NodeAwareTransport = (labelWrappedTransport)(nil) func (t labelWrappedTransport) WriteToAddress(buf []byte, addr Address) (time.Time, error) { var err error buf, err = AddLabelHeaderToPacket(buf, t.label) if err != nil { return time.Time{}, fmt.Errorf("failed to add label header to packet: %w", err) } return t.NodeAwareTransport.WriteToAddress(buf, addr) } func (t labelWrappedTransport) WriteTo(buf []byte, addr string) (time.Time, error) { var err error buf, err = AddLabelHeaderToPacket(buf, t.label) if err != nil { return time.Time{}, err } return t.NodeAwareTransport.WriteTo(buf, addr) } func (t labelWrappedTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) { conn, err := t.NodeAwareTransport.DialAddressTimeout(addr, timeout) if err != nil { return nil, err } if err := AddLabelHeaderToStream(conn, t.label); err != nil { return nil, fmt.Errorf("failed to add label header to stream: %w", err) } return conn, nil } func (t labelWrappedTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { conn, err := t.NodeAwareTransport.DialTimeout(addr, timeout) if err != nil { return nil, err } if err := AddLabelHeaderToStream(conn, t.label); err != nil { return nil, fmt.Errorf("failed to add label header to stream: %w", err) } return conn, nil }	5.44 KB	2025-05-25 12:27
📄 transport_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "log" "net" "strings" "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" ) func TestTransport_Join(t testing.T) { net := &MockNetwork{} t1 := net.NewTransport("node1") c1 := DefaultLANConfig() c1.Name = "node1" c1.Transport = t1 m1, err := Create(c1) if err != nil { t.Fatalf("err: %v", err) } m1.setAlive() m1.schedule() defer m1.Shutdown() c2 := DefaultLANConfig() c2.Name = "node2" c2.Transport = net.NewTransport("node2") m2, err := Create(c2) if err != nil { t.Fatalf("err: %v", err) } m2.setAlive() m2.schedule() defer m2.Shutdown() num, err := m2.Join([]string{c1.Name + "/" + t1.addr.String()}) if num != 1 { t.Fatalf("bad: %d", num) } if err != nil { t.Fatalf("err: %v", err) } if len(m2.Members()) != 2 { t.Fatalf("bad: %v", m2.Members()) } if m2.estNumNodes() != 2 { t.Fatalf("bad: %v", m2.Members()) } } func TestTransport_Send(t testing.T) { net := &MockNetwork{} t1 := net.NewTransport("node1") d1 := &MockDelegate{} c1 := DefaultLANConfig() c1.Name = "node1" c1.Transport = t1 c1.Delegate = d1 m1, err := Create(c1) if err != nil { t.Fatalf("err: %v", err) } m1.setAlive() m1.schedule() defer m1.Shutdown() c2 := DefaultLANConfig() c2.Name = "node2" c2.Transport = net.NewTransport("node2") m2, err := Create(c2) if err != nil { t.Fatalf("err: %v", err) } m2.setAlive() m2.schedule() defer m2.Shutdown() num, err := m2.Join([]string{c1.Name + "/" + t1.addr.String()}) if num != 1 { t.Fatalf("bad: %d", num) } if err != nil { t.Fatalf("err: %v", err) } if err := m2.SendTo(t1.addr, []byte("SendTo")); err != nil { t.Fatalf("err: %v", err) } var n1 Node for _, n := range m2.Members() { if n.Name == c1.Name { n1 = n break } } if n1 == nil { t.Fatalf("bad") } if err := m2.SendToUDP(n1, []byte("SendToUDP")); err != nil { t.Fatalf("err: %v", err) } if err := m2.SendToTCP(n1, []byte("SendToTCP")); err != nil { t.Fatalf("err: %v", err) } if err := m2.SendBestEffort(n1, []byte("SendBestEffort")); err != nil { t.Fatalf("err: %v", err) } if err := m2.SendReliable(n1, []byte("SendReliable")); err != nil { t.Fatalf("err: %v", err) } time.Sleep(100 time.Millisecond) expected := []string{"SendTo", "SendToUDP", "SendToTCP", "SendBestEffort", "SendReliable"} msgs1 := d1.getMessages() received := make([]string, len(msgs1)) for i, bs := range msgs1 { received[i] = string(bs) } // Some of these are UDP so often get re-ordered making the test flaky if we // assert send ordering. Sort both slices to be tolerant of re-ordering. require.ElementsMatch(t, expected, received) } type testCountingWriter struct { t testing.T numCalls int32 } func (tw testCountingWriter) Write(p []byte) (n int, err error) { atomic.AddInt32(tw.numCalls, 1) if !strings.Contains(string(p), "memberlist: Error accepting TCP connection") { tw.t.Error("did not receive expected log message") } tw.t.Log("countingWriter:", string(p)) return len(p), nil } // TestTransport_TcpListenBackoff tests that AcceptTCP() errors in NetTransport#tcpListen() // do not result in a tight loop and spam the log. We verify this here by counting the number // of entries logged in a given time period. func TestTransport_TcpListenBackoff(t testing.T) { // testTime is the amount of time we will allow NetTransport#tcpListen() to run // This needs to be long enough that to verify that maxDelay is in force, // but not so long as to be obnoxious when running the test suite. const testTime = 4 time.Second var numCalls int32 countingWriter := testCountingWriter{t, &numCalls} countingLogger := log.New(countingWriter, "test", log.LstdFlags) transport := NetTransport{ streamCh: make(chan net.Conn), logger: countingLogger, } transport.wg.Add(1) // create a listener that will cause AcceptTCP calls to fail listener, _ := net.ListenTCP("tcp", nil) listener.Close() go transport.tcpListen(listener) // sleep (+yield) for testTime seconds before asking the accept loop to shut down time.Sleep(testTime) atomic.StoreInt32(&transport.shutdown, 1) // Verify that the wg was completed on exit (but without blocking this test) // maxDelay == 1s, so we will give the routine 1.25s to loop around and shut down. c := make(chan struct{}) go func() { defer close(c) transport.wg.Wait() }() select { case <-c: case <-time.After(1250 * time.Millisecond): t.Error("timed out waiting for transport waitgroup to be done after flagging shutdown") } // In testTime==4s, we expect to loop approximately 12 times (and log approximately 11 errors), // with the following delays (in ms): // 0+5+10+20+40+80+160+320+640+1000+1000+1000 == 4275 ms // Too few calls suggests that the minDelay is not in force; too many calls suggests that the // maxDelay is not in force or that the back-off isn't working at all. // We'll leave a little flex; the important thing here is the asymptotic behavior. // If the minDelay or maxDelay in NetTransport#tcpListen() are modified, this test may fail // and need to be adjusted. require.True(t, numCalls > 8) require.True(t, numCalls < 14) // no connections should have been accepted and sent to the channel require.Equal(t, len(transport.streamCh), 0) }	5.26 KB	2025-05-25 12:27
📄 util.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "bytes" "compress/lzw" "encoding/binary" "fmt" "io" "math" "math/rand" "net" "strconv" "strings" "time" "github.com/hashicorp/go-msgpack/v2/codec" "github.com/sean-/seed" ) // pushPullScale is the minimum number of nodes // before we start scaling the push/pull timing. The scale // effect is the log2(Nodes) - log2(pushPullScale). This means // that the 33rd node will cause us to double the interval, // while the 65th will triple it. const pushPullScaleThreshold = 32 const ( // Constant litWidth 2-8 lzwLitWidth = 8 ) func init() { seed.Init() } // Decode reverses the encode operation on a byte slice input func decode(buf []byte, out interface{}) error { r := bytes.NewReader(buf) hd := codec.MsgpackHandle{} dec := codec.NewDecoder(r, &hd) return dec.Decode(out) } // Encode writes an encoded object to a new bytes buffer func encode(msgType messageType, in interface{}, msgpackUseNewTimeFormat bool) (bytes.Buffer, error) { buf := bytes.NewBuffer(nil) buf.WriteByte(uint8(msgType)) hd := codec.MsgpackHandle{ BasicHandle: codec.BasicHandle{ TimeNotBuiltin: !msgpackUseNewTimeFormat, }, } enc := codec.NewEncoder(buf, &hd) err := enc.Encode(in) return buf, err } // Returns a random offset between 0 and n func randomOffset(n int) int { if n == 0 { return 0 } return int(rand.Uint32() % uint32(n)) } // suspicionTimeout computes the timeout that should be used when // a node is suspected func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration { nodeScale := math.Max(1.0, math.Log10(math.Max(1.0, float64(n)))) // multiply by 1000 to keep some precision because time.Duration is an int64 type timeout := time.Duration(suspicionMult) time.Duration(nodeScale1000) interval / 1000 return timeout } // retransmitLimit computes the limit of retransmissions func retransmitLimit(retransmitMult, n int) int { nodeScale := math.Ceil(math.Log10(float64(n + 1))) limit := retransmitMult * int(nodeScale) return limit } // shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle func shuffleNodes(nodes []nodeState) { n := len(nodes) rand.Shuffle(n, func(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] }) } // pushPushScale is used to scale the time interval at which push/pull // syncs take place. It is used to prevent network saturation as the // cluster size grows func pushPullScale(interval time.Duration, n int) time.Duration { // Don't scale until we cross the threshold if n <= pushPullScaleThreshold { return interval } multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0 return time.Duration(multiplier) interval } // moveDeadNodes moves dead and left nodes that that have not changed during the gossipToTheDeadTime interval // to the end of the slice and returns the index of the first moved node. func moveDeadNodes(nodes []nodeState, gossipToTheDeadTime time.Duration) int { numDead := 0 n := len(nodes) for i := 0; i < n-numDead; i++ { if !nodes[i].DeadOrLeft() { continue } // Respect the gossip to the dead interval if time.Since(nodes[i].StateChange) <= gossipToTheDeadTime { continue } // Move this node to the end nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i] numDead++ i-- } return n - numDead } // kRandomNodes is used to select up to k random Nodes, excluding any nodes where // the exclude function returns true. It is possible that less than k nodes are // returned. func kRandomNodes(k int, nodes []nodeState, exclude func(nodeState) bool) []Node { n := len(nodes) kNodes := make([]Node, 0, k) OUTER: // Probe up to 3n times, with large n this is not necessary // since k << n, but with small n we want search to be // exhaustive for i := 0; i < 3n && len(kNodes) < k; i++ { // Get random nodeState idx := randomOffset(n) state := nodes[idx] // Give the filter a shot at it. if exclude != nil && exclude(state) { continue OUTER } // Check if we have this node already for j := 0; j < len(kNodes); j++ { if state.Node.Name == kNodes[j].Name { continue OUTER } } // Append the node kNodes = append(kNodes, state.Node) } return kNodes } // makeCompoundMessages takes a list of messages and packs // them into one or multiple messages based on the limitations // of compound messages (255 messages each). func makeCompoundMessages(msgs [][]byte) []bytes.Buffer { const maxMsgs = 255 bufs := make([]bytes.Buffer, 0, (len(msgs)+(maxMsgs-1))/maxMsgs) for ; len(msgs) > maxMsgs; msgs = msgs[maxMsgs:] { bufs = append(bufs, makeCompoundMessage(msgs[:maxMsgs])) } if len(msgs) > 0 { bufs = append(bufs, makeCompoundMessage(msgs)) } return bufs } // makeCompoundMessage takes a list of messages and generates // a single compound message containing all of them func makeCompoundMessage(msgs [][]byte) bytes.Buffer { // Create a local buffer buf := bytes.NewBuffer(nil) // Write out the type buf.WriteByte(uint8(compoundMsg)) // Write out the number of message buf.WriteByte(uint8(len(msgs))) // Add the message lengths for _, m := range msgs { binary.Write(buf, binary.BigEndian, uint16(len(m))) } // Append the messages for _, m := range msgs { buf.Write(m) } return buf } // decodeCompoundMessage splits a compound message and returns // the slices of individual messages. Also returns the number // of truncated messages and any potential error func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) { if len(buf) < 1 { err = fmt.Errorf("missing compound length byte") return } numParts := int(buf[0]) buf = buf[1:] // Check we have enough bytes if len(buf) < numParts2 { err = fmt.Errorf("truncated len slice") return } // Decode the lengths lengths := make([]uint16, numParts) for i := 0; i < numParts; i++ { lengths[i] = binary.BigEndian.Uint16(buf[i2 : i2+2]) } buf = buf[numParts2:] // Split each message for idx, msgLen := range lengths { if len(buf) < int(msgLen) { trunc = numParts - idx return } // Extract the slice, seek past on the buffer slice := buf[:msgLen] buf = buf[msgLen:] parts = append(parts, slice) } return } // compressPayload takes an opaque input buffer, compresses it // and wraps it in a compress{} message that is encoded. func compressPayload(inp []byte, msgpackUseNewTimeFormat bool) (bytes.Buffer, error) { var buf bytes.Buffer compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth) _, err := compressor.Write(inp) if err != nil { return nil, err } // Ensure we flush everything out if err := compressor.Close(); err != nil { return nil, err } // Create a compressed message c := compress{ Algo: lzwAlgo, Buf: buf.Bytes(), } return encode(compressMsg, &c, msgpackUseNewTimeFormat) } // decompressPayload is used to unpack an encoded compress{} // message and return its payload uncompressed func decompressPayload(msg []byte) ([]byte, error) { // Decode the message var c compress if err := decode(msg, &c); err != nil { return nil, err } return decompressBuffer(&c) } // decompressBuffer is used to decompress the buffer of // a single compress message, handling multiple algorithms func decompressBuffer(c compress) ([]byte, error) { // Verify the algorithm if c.Algo != lzwAlgo { return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo) } // Create a uncompressor uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth) defer uncomp.Close() // Read all the data var b bytes.Buffer _, err := io.Copy(&b, uncomp) if err != nil { return nil, err } // Return the uncompressed bytes return b.Bytes(), nil } // joinHostPort returns the host:port form of an address, for use with a // transport. func joinHostPort(host string, port uint16) string { return net.JoinHostPort(host, strconv.Itoa(int(port))) } // hasPort is given a string of the form "host", "host:port", "ipv6::address", // or "[ipv6::address]:port", and returns true if the string includes a port. func hasPort(s string) bool { // IPv6 address in brackets. if strings.LastIndex(s, "[") == 0 { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } // Otherwise the presence of a single colon determines if there's a port // since IPv6 addresses outside of brackets (count > 1) can't have a // port. return strings.Count(s, ":") == 1 } // ensurePort makes sure the given string has a port number on it, otherwise it // appends the given port as a default. func ensurePort(s string, port int) string { if hasPort(s) { return s } // If this is an IPv6 address, the join call will add another set of // brackets, so we have to trim before we add the default port. s = strings.Trim(s, "[]") s = net.JoinHostPort(s, strconv.Itoa(port)) return s }	8.74 KB	2025-05-25 12:27
📄 util_test.go // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package memberlist import ( "fmt" "reflect" "testing" "time" "github.com/stretchr/testify/require" ) func TestUtil_PortFunctions(t testing.T) { tests := []struct { addr string hasPort bool ensurePort string }{ {"1.2.3.4", false, "1.2.3.4:8301"}, {"1.2.3.4:1234", true, "1.2.3.4:1234"}, {"2600:1f14:e22:1501:f9a:2e0c:a167:67e8", false, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:8301"}, {"[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]", false, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:8301"}, {"[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:1234", true, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:1234"}, {"localhost", false, "localhost:8301"}, {"localhost:1234", true, "localhost:1234"}, {"hashicorp.com", false, "hashicorp.com:8301"}, {"hashicorp.com:1234", true, "hashicorp.com:1234"}, } for _, tt := range tests { t.Run(tt.addr, func(t testing.T) { if got, want := hasPort(tt.addr), tt.hasPort; got != want { t.Fatalf("got %v want %v", got, want) } if got, want := ensurePort(tt.addr, 8301), tt.ensurePort; got != want { t.Fatalf("got %v want %v", got, want) } }) } } func TestEncodeDecode(t testing.T) { msg := &ping{SeqNo: 100} buf, err := encode(pingMsg, msg, false) if err != nil { t.Fatalf("unexpected err: %s", err) } var out ping if err := decode(buf.Bytes()[1:], &out); err != nil { t.Fatalf("unexpected err: %s", err) } if msg.SeqNo != out.SeqNo { t.Fatalf("bad sequence no") } } func TestRandomOffset(t testing.T) { vals := make(map[int]struct{}) for i := 0; i < 100; i++ { offset := randomOffset(2 << 30) if _, ok := vals[offset]; ok { t.Fatalf("got collision") } vals[offset] = struct{}{} } } func TestRandomOffset_Zero(t testing.T) { offset := randomOffset(0) if offset != 0 { t.Fatalf("bad offset") } } func TestSuspicionTimeout(t testing.T) { timeouts := map[int]time.Duration{ 5: 1000 * time.Millisecond, 10: 1000 * time.Millisecond, 50: 1698 * time.Millisecond, 100: 2000 * time.Millisecond, 500: 2698 * time.Millisecond, 1000: 3000 * time.Millisecond, } for n, expected := range timeouts { timeout := suspicionTimeout(3, n, time.Second) / 3 if timeout != expected { t.Fatalf("bad: %v, %v", expected, timeout) } } } func TestRetransmitLimit(t testing.T) { lim := retransmitLimit(3, 0) if lim != 0 { t.Fatalf("bad val %v", lim) } lim = retransmitLimit(3, 1) if lim != 3 { t.Fatalf("bad val %v", lim) } lim = retransmitLimit(3, 99) if lim != 6 { t.Fatalf("bad val %v", lim) } } func TestShuffleNodes(t testing.T) { orig := []nodeState{ &nodeState{ State: StateDead, }, &nodeState{ State: StateAlive, }, &nodeState{ State: StateAlive, }, &nodeState{ State: StateDead, }, &nodeState{ State: StateAlive, }, &nodeState{ State: StateAlive, }, &nodeState{ State: StateDead, }, &nodeState{ State: StateAlive, }, } nodes := make([]nodeState, len(orig)) copy(nodes[:], orig[:]) if !reflect.DeepEqual(nodes, orig) { t.Fatalf("should match") } shuffleNodes(nodes) if reflect.DeepEqual(nodes, orig) { t.Fatalf("should not match") } } func TestPushPullScale(t testing.T) { sec := time.Second for i := 0; i <= 32; i++ { if s := pushPullScale(sec, i); s != sec { t.Fatalf("Bad time scale: %v", s) } } for i := 33; i <= 64; i++ { if s := pushPullScale(sec, i); s != 2sec { t.Fatalf("Bad time scale: %v", s) } } for i := 65; i <= 128; i++ { if s := pushPullScale(sec, i); s != 3sec { t.Fatalf("Bad time scale: %v", s) } } } func TestMoveDeadNodes(t testing.T) { nodes := []nodeState{ &nodeState{ State: StateDead, StateChange: time.Now().Add(-20 time.Second), }, &nodeState{ State: StateAlive, StateChange: time.Now().Add(-20 * time.Second), }, // This dead node should not be moved, as its state changed // less than the specified GossipToTheDead time ago &nodeState{ State: StateDead, StateChange: time.Now().Add(-10 * time.Second), }, // This left node should not be moved, as its state changed // less than the specified GossipToTheDead time ago &nodeState{ State: StateLeft, StateChange: time.Now().Add(-10 * time.Second), }, &nodeState{ State: StateLeft, StateChange: time.Now().Add(-20 * time.Second), }, &nodeState{ State: StateAlive, StateChange: time.Now().Add(-20 * time.Second), }, &nodeState{ State: StateDead, StateChange: time.Now().Add(-20 * time.Second), }, &nodeState{ State: StateAlive, StateChange: time.Now().Add(-20 * time.Second), }, &nodeState{ State: StateLeft, StateChange: time.Now().Add(-20 * time.Second), }, } idx := moveDeadNodes(nodes, (15 * time.Second)) if idx != 5 { t.Fatalf("bad index") } for i := 0; i < idx; i++ { switch i { case 2: // Recently dead node remains at index 2, // since nodes are swapped out to move to end. if nodes[i].State != StateDead { t.Fatalf("Bad state %d", i) } case 3: //Recently left node should remain at 3 if nodes[i].State != StateLeft { t.Fatalf("Bad State %d", i) } default: if nodes[i].State != StateAlive { t.Fatalf("Bad state %d", i) } } } for i := idx; i < len(nodes); i++ { if !nodes[i].DeadOrLeft() { t.Fatalf("Bad state %d", i) } } } func TestKRandomNodes(t testing.T) { nodes := []nodeState{} for i := 0; i < 90; i++ { // Half the nodes are in a bad state state := StateAlive switch i % 3 { case 0: state = StateAlive case 1: state = StateSuspect case 2: state = StateDead } nodes = append(nodes, &nodeState{ Node: Node{ Name: fmt.Sprintf("test%d", i), }, State: state, }) } filterFunc := func(n nodeState) bool { if n.Name == "test0" \|\| n.State != StateAlive { return true } return false } s1 := kRandomNodes(3, nodes, filterFunc) s2 := kRandomNodes(3, nodes, filterFunc) s3 := kRandomNodes(3, nodes, filterFunc) if reflect.DeepEqual(s1, s2) { t.Fatalf("unexpected equal") } if reflect.DeepEqual(s1, s3) { t.Fatalf("unexpected equal") } if reflect.DeepEqual(s2, s3) { t.Fatalf("unexpected equal") } for _, s := range [][]Node{s1, s2, s3} { if len(s) != 3 { t.Fatalf("bad len") } for _, n := range s { if n.Name == "test0" { t.Fatalf("Bad name") } if n.State != StateAlive { t.Fatalf("Bad state") } } } } func TestMakeCompoundMessage(t testing.T) { msg := &ping{SeqNo: 100} buf, err := encode(pingMsg, msg, false) if err != nil { t.Fatalf("unexpected err: %s", err) } msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} compound := makeCompoundMessage(msgs) if compound.Len() != 3buf.Len()+3compoundOverhead+compoundHeaderOverhead { t.Fatalf("bad len") } } func TestDecodeCompoundMessage(t testing.T) { msg := &ping{SeqNo: 100} buf, err := encode(pingMsg, msg, false) if err != nil { t.Fatalf("unexpected err: %s", err) } msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} compound := makeCompoundMessage(msgs) trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:]) if err != nil { t.Fatalf("unexpected err: %s", err) } if trunc != 0 { t.Fatalf("should not truncate") } if len(parts) != 3 { t.Fatalf("bad parts") } for _, p := range parts { if len(p) != buf.Len() { t.Fatalf("bad part len") } } } func TestDecodeCompoundMessage_NumberOfPartsOverflow(t testing.T) { buf := []byte{0x80} _, _, err := decodeCompoundMessage(buf) require.Error(t, err) require.Equal(t, err.Error(), "truncated len slice") } func TestDecodeCompoundMessage_Trunc(t testing.T) { msg := &ping{SeqNo: 100} buf, err := encode(pingMsg, msg, false) if err != nil { t.Fatalf("unexpected err: %s", err) } msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} compound := makeCompoundMessage(msgs) trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:38]) if err != nil { t.Fatalf("unexpected err: %s", err) } if trunc != 1 { t.Fatalf("truncate: %d", trunc) } if len(parts) != 2 { t.Fatalf("bad parts") } for _, p := range parts { if len(p) != buf.Len() { t.Fatalf("bad part len") } } } func TestCompressDecompressPayload(t testing.T) { buf, err := compressPayload([]byte("testing"), false) if err != nil { t.Fatalf("unexpected err: %s", err) } decomp, err := decompressPayload(buf.Bytes()[1:]) if err != nil { t.Fatalf("unexpected err: %s", err) } if !reflect.DeepEqual(decomp, []byte("testing")) { t.Fatalf("bad payload: %v", decomp) } }	8.48 KB	2025-05-25 12:27

Edit File:

Terminal