Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2025-05-25 12:27
📄 silence.go // Copyright 2016 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package silence provides a storage for silences, which can share its // state over a mesh network and snapshot it. package silence import ( "bytes" "errors" "fmt" "io" "log/slog" "math/rand" "os" "reflect" "regexp" "sort" "sync" "time" "github.com/coder/quartz" uuid "github.com/gofrs/uuid" "github.com/matttproud/golang_protobuf_extensions/pbutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/alertmanager/cluster" "github.com/prometheus/alertmanager/matcher/compat" "github.com/prometheus/alertmanager/pkg/labels" pb "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/alertmanager/types" ) // ErrNotFound is returned if a silence was not found. var ErrNotFound = errors.New("silence not found") // ErrInvalidState is returned if the state isn't valid. var ErrInvalidState = errors.New("invalid state") type matcherCache map[string]labels.Matchers // Get retrieves the matchers for a given silence. If it is a missed cache // access, it compiles and adds the matchers of the requested silence to the // cache. func (c matcherCache) Get(s pb.Silence) (labels.Matchers, error) { if m, ok := c[s.Id]; ok { return m, nil } return c.add(s) } // add compiles a silences' matchers and adds them to the cache. // It returns the compiled matchers. func (c matcherCache) add(s pb.Silence) (labels.Matchers, error) { ms := make(labels.Matchers, len(s.Matchers)) for i, m := range s.Matchers { var mt labels.MatchType switch m.Type { case pb.Matcher_EQUAL: mt = labels.MatchEqual case pb.Matcher_NOT_EQUAL: mt = labels.MatchNotEqual case pb.Matcher_REGEXP: mt = labels.MatchRegexp case pb.Matcher_NOT_REGEXP: mt = labels.MatchNotRegexp default: return nil, fmt.Errorf("unknown matcher type %q", m.Type) } matcher, err := labels.NewMatcher(mt, m.Name, m.Pattern) if err != nil { return nil, err } ms[i] = matcher } c[s.Id] = ms return ms, nil } // Silencer binds together a AlertMarker and a Silences to implement the Muter // interface. type Silencer struct { silences Silences marker types.AlertMarker logger slog.Logger } // NewSilencer returns a new Silencer. func NewSilencer(s Silences, m types.AlertMarker, l slog.Logger) Silencer { return &Silencer{ silences: s, marker: m, logger: l, } } // Mutes implements the Muter interface. func (s Silencer) Mutes(lset model.LabelSet) bool { fp := lset.Fingerprint() activeIDs, pendingIDs, markerVersion, _ := s.marker.Silenced(fp) var ( err error allSils []pb.Silence newVersion = markerVersion ) if markerVersion == s.silences.Version() { totalSilences := len(activeIDs) + len(pendingIDs) // No new silences added, just need to check which of the old // silences are still relevant and which of the pending ones // have become active. if totalSilences == 0 { // Super fast path: No silences ever applied to this // alert, none have been added. We are done. return false } // This is still a quite fast path: No silences have been added, // we only need to check which of the applicable silences are // currently active. Note that newVersion is left at // markerVersion because the Query call might already return a // newer version, which is not the version our old list of // applicable silences is based on. allIDs := append(append(make([]string, 0, totalSilences), activeIDs...), pendingIDs...) allSils, _, err = s.silences.Query( QIDs(allIDs...), QState(types.SilenceStateActive, types.SilenceStatePending), ) } else { // New silences have been added, do a full query. allSils, newVersion, err = s.silences.Query( QState(types.SilenceStateActive, types.SilenceStatePending), QMatches(lset), ) } if err != nil { s.logger.Error("Querying silences failed, alerts might not get silenced correctly", "err", err) } if len(allSils) == 0 { // Easy case, neither active nor pending silences anymore. s.marker.SetActiveOrSilenced(fp, newVersion, nil, nil) return false } // It is still possible that nothing has changed, but finding out is not // much less effort than just recreating the IDs from the query // result. So let's do it in any case. Note that we cannot reuse the // current ID slices for concurrency reasons. activeIDs, pendingIDs = nil, nil now := s.silences.nowUTC() for _, sil := range allSils { switch getState(sil, now) { case types.SilenceStatePending: pendingIDs = append(pendingIDs, sil.Id) case types.SilenceStateActive: activeIDs = append(activeIDs, sil.Id) default: // Do nothing, silence has expired in the meantime. } } s.logger.Debug( "determined current silences state", "now", now, "total", len(allSils), "active", len(activeIDs), "pending", len(pendingIDs), ) sort.Strings(activeIDs) sort.Strings(pendingIDs) s.marker.SetActiveOrSilenced(fp, newVersion, activeIDs, pendingIDs) return len(activeIDs) > 0 } // Silences holds a silence state that can be modified, queried, and snapshot. type Silences struct { clock quartz.Clock logger slog.Logger metrics metrics retention time.Duration limits Limits mtx sync.RWMutex st state version int // Increments whenever silences are added. broadcast func([]byte) mc matcherCache } // Limits contains the limits for silences. type Limits struct { // MaxSilences limits the maximum number of silences, including expired // silences. MaxSilences func() int // MaxSilenceSizeBytes is the maximum size of an individual silence as // stored on disk. MaxSilenceSizeBytes func() int } // MaintenanceFunc represents the function to run as part of the periodic maintenance for silences. // It returns the size of the snapshot taken or an error if it failed. type MaintenanceFunc func() (int64, error) type metrics struct { gcDuration prometheus.Summary snapshotDuration prometheus.Summary snapshotSize prometheus.Gauge queriesTotal prometheus.Counter queryErrorsTotal prometheus.Counter queryDuration prometheus.Histogram silencesActive prometheus.GaugeFunc silencesPending prometheus.GaugeFunc silencesExpired prometheus.GaugeFunc propagatedMessagesTotal prometheus.Counter maintenanceTotal prometheus.Counter maintenanceErrorsTotal prometheus.Counter } func newSilenceMetricByState(s Silences, st types.SilenceState) prometheus.GaugeFunc { return prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Name: "alertmanager_silences", Help: "How many silences by state.", ConstLabels: prometheus.Labels{"state": string(st)}, }, func() float64 { count, err := s.CountState(st) if err != nil { s.logger.Error("Counting silences failed", "err", err) } return float64(count) }, ) } func newMetrics(r prometheus.Registerer, s Silences) metrics { m := &metrics{} m.gcDuration = prometheus.NewSummary(prometheus.SummaryOpts{ Name: "alertmanager_silences_gc_duration_seconds", Help: "Duration of the last silence garbage collection cycle.", Objectives: map[float64]float64{}, }) m.snapshotDuration = prometheus.NewSummary(prometheus.SummaryOpts{ Name: "alertmanager_silences_snapshot_duration_seconds", Help: "Duration of the last silence snapshot.", Objectives: map[float64]float64{}, }) m.snapshotSize = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "alertmanager_silences_snapshot_size_bytes", Help: "Size of the last silence snapshot in bytes.", }) m.maintenanceTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "alertmanager_silences_maintenance_total", Help: "How many maintenances were executed for silences.", }) m.maintenanceErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "alertmanager_silences_maintenance_errors_total", Help: "How many maintenances were executed for silences that failed.", }) m.queriesTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "alertmanager_silences_queries_total", Help: "How many silence queries were received.", }) m.queryErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "alertmanager_silences_query_errors_total", Help: "How many silence received queries did not succeed.", }) m.queryDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "alertmanager_silences_query_duration_seconds", Help: "Duration of silence query evaluation.", Buckets: prometheus.DefBuckets, NativeHistogramBucketFactor: 1.1, NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, }) m.propagatedMessagesTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "alertmanager_silences_gossip_messages_propagated_total", Help: "Number of received gossip messages that have been further gossiped.", }) if s != nil { m.silencesActive = newSilenceMetricByState(s, types.SilenceStateActive) m.silencesPending = newSilenceMetricByState(s, types.SilenceStatePending) m.silencesExpired = newSilenceMetricByState(s, types.SilenceStateExpired) } if r != nil { r.MustRegister( m.gcDuration, m.snapshotDuration, m.snapshotSize, m.queriesTotal, m.queryErrorsTotal, m.queryDuration, m.silencesActive, m.silencesPending, m.silencesExpired, m.propagatedMessagesTotal, m.maintenanceTotal, m.maintenanceErrorsTotal, ) } return m } // Options exposes configuration options for creating a new Silences object. // Its zero value is a safe default. type Options struct { // A snapshot file or reader from which the initial state is loaded. // None or only one of them must be set. SnapshotFile string SnapshotReader io.Reader // Retention time for newly created Silences. Silences may be // garbage collected after the given duration after they ended. Retention time.Duration Limits Limits // A logger used by background processing. Logger slog.Logger Metrics prometheus.Registerer } func (o Options) validate() error { if o.SnapshotFile != "" && o.SnapshotReader != nil { return errors.New("only one of SnapshotFile and SnapshotReader must be set") } return nil } // New returns a new Silences object with the given configuration. func New(o Options) (Silences, error) { if err := o.validate(); err != nil { return nil, err } s := &Silences{ clock: quartz.NewReal(), mc: matcherCache{}, logger: promslog.NewNopLogger(), retention: o.Retention, limits: o.Limits, broadcast: func([]byte) {}, st: state{}, } s.metrics = newMetrics(o.Metrics, s) if o.Logger != nil { s.logger = o.Logger } if o.SnapshotFile != "" { if r, err := os.Open(o.SnapshotFile); err != nil { if !os.IsNotExist(err) { return nil, err } s.logger.Debug("silences snapshot file doesn't exist", "err", err) } else { o.SnapshotReader = r defer r.Close() } } if o.SnapshotReader != nil { if err := s.loadSnapshot(o.SnapshotReader); err != nil { return s, err } } return s, nil } func (s Silences) nowUTC() time.Time { return s.clock.Now().UTC() } // Maintenance garbage collects the silence state at the given interval. If the snapshot // file is set, a snapshot is written to it afterwards. // Terminates on receiving from stopc. // If not nil, the last argument is an override for what to do as part of the maintenance - for advanced usage. func (s Silences) Maintenance(interval time.Duration, snapf string, stopc <-chan struct{}, override MaintenanceFunc) { if interval == 0 \|\| stopc == nil { s.logger.Error("interval or stop signal are missing - not running maintenance") return } t := s.clock.NewTicker(interval) defer t.Stop() var doMaintenance MaintenanceFunc doMaintenance = func() (int64, error) { var size int64 if _, err := s.GC(); err != nil { return size, err } if snapf == "" { return size, nil } f, err := openReplace(snapf) if err != nil { return size, err } if size, err = s.Snapshot(f); err != nil { f.Close() return size, err } return size, f.Close() } if override != nil { doMaintenance = override } runMaintenance := func(do MaintenanceFunc) error { s.metrics.maintenanceTotal.Inc() s.logger.Debug("Running maintenance") start := s.nowUTC() size, err := do() s.metrics.snapshotSize.Set(float64(size)) if err != nil { s.metrics.maintenanceErrorsTotal.Inc() return err } s.logger.Debug("Maintenance done", "duration", s.clock.Since(start), "size", size) return nil } Loop: for { select { case <-stopc: break Loop case <-t.C: if err := runMaintenance(doMaintenance); err != nil { // @tjhop: this should probably log at error level s.logger.Info("Running maintenance failed", "err", err) } } } // No need for final maintenance if we don't want to snapshot. if snapf == "" { return } if err := runMaintenance(doMaintenance); err != nil { // @tjhop: this should probably log at error level s.logger.Info("Creating shutdown snapshot failed", "err", err) } } // GC runs a garbage collection that removes silences that have ended longer // than the configured retention time ago. func (s Silences) GC() (int, error) { start := time.Now() defer func() { s.metrics.gcDuration.Observe(time.Since(start).Seconds()) }() now := s.nowUTC() var n int s.mtx.Lock() defer s.mtx.Unlock() for id, sil := range s.st { if sil.ExpiresAt.IsZero() { return n, errors.New("unexpected zero expiration timestamp") } if !sil.ExpiresAt.After(now) { delete(s.st, id) delete(s.mc, sil.Silence.Id) n++ } } return n, nil } func validateMatcher(m pb.Matcher) error { if !compat.IsValidLabelName(model.LabelName(m.Name)) { return fmt.Errorf("invalid label name %q", m.Name) } switch m.Type { case pb.Matcher_EQUAL, pb.Matcher_NOT_EQUAL: if !model.LabelValue(m.Pattern).IsValid() { return fmt.Errorf("invalid label value %q", m.Pattern) } case pb.Matcher_REGEXP, pb.Matcher_NOT_REGEXP: if _, err := regexp.Compile(m.Pattern); err != nil { return fmt.Errorf("invalid regular expression %q: %w", m.Pattern, err) } default: return fmt.Errorf("unknown matcher type %q", m.Type) } return nil } func matchesEmpty(m pb.Matcher) bool { switch m.Type { case pb.Matcher_EQUAL: return m.Pattern == "" case pb.Matcher_REGEXP: matched, _ := regexp.MatchString(m.Pattern, "") return matched default: return false } } func validateSilence(s pb.Silence) error { if len(s.Matchers) == 0 { return errors.New("at least one matcher required") } allMatchEmpty := true for i, m := range s.Matchers { if err := validateMatcher(m); err != nil { return fmt.Errorf("invalid label matcher %d: %w", i, err) } allMatchEmpty = allMatchEmpty && matchesEmpty(m) } if allMatchEmpty { return errors.New("at least one matcher must not match the empty string") } if s.StartsAt.IsZero() { return errors.New("invalid zero start timestamp") } if s.EndsAt.IsZero() { return errors.New("invalid zero end timestamp") } if s.EndsAt.Before(s.StartsAt) { return errors.New("end time must not be before start time") } return nil } // cloneSilence returns a shallow copy of a silence. func cloneSilence(sil pb.Silence) pb.Silence { s := sil return &s } func (s Silences) checkSizeLimits(msil pb.MeshSilence) error { if s.limits.MaxSilenceSizeBytes != nil { n := msil.Size() if m := s.limits.MaxSilenceSizeBytes(); m > 0 && n > m { return fmt.Errorf("silence exceeded maximum size: %d bytes (limit: %d bytes)", n, m) } } return nil } func (s Silences) getSilence(id string) (pb.Silence, bool) { msil, ok := s.st[id] if !ok { return nil, false } return msil.Silence, true } func (s Silences) toMeshSilence(sil pb.Silence) pb.MeshSilence { return &pb.MeshSilence{ Silence: sil, ExpiresAt: sil.EndsAt.Add(s.retention), } } func (s Silences) setSilence(msil pb.MeshSilence, now time.Time) error { b, err := marshalMeshSilence(msil) if err != nil { return err } _, added := s.st.merge(msil, now) if added { s.version++ } s.broadcast(b) return nil } // Set the specified silence. If a silence with the ID already exists and the modification // modifies history, the old silence gets expired and a new one is created. func (s Silences) Set(sil pb.Silence) error { s.mtx.Lock() defer s.mtx.Unlock() now := s.nowUTC() if sil.StartsAt.IsZero() { sil.StartsAt = now } if err := validateSilence(sil); err != nil { return fmt.Errorf("invalid silence: %w", err) } prev, ok := s.getSilence(sil.Id) if sil.Id != "" && !ok { return ErrNotFound } if ok && canUpdate(prev, sil, now) { sil.UpdatedAt = now msil := s.toMeshSilence(sil) if err := s.checkSizeLimits(msil); err != nil { return err } return s.setSilence(msil, now) } // If we got here it's either a new silence or a replacing one (which would // also create a new silence) so we need to make sure we have capacity for // the new silence. if s.limits.MaxSilences != nil { if m := s.limits.MaxSilences(); m > 0 && len(s.st)+1 > m { return fmt.Errorf("exceeded maximum number of silences: %d (limit: %d)", len(s.st), m) } } uid, err := uuid.NewV4() if err != nil { return fmt.Errorf("generate uuid: %w", err) } sil.Id = uid.String() if sil.StartsAt.Before(now) { sil.StartsAt = now } sil.UpdatedAt = now msil := s.toMeshSilence(sil) if err := s.checkSizeLimits(msil); err != nil { return err } if ok && getState(prev, s.nowUTC()) != types.SilenceStateExpired { // We cannot update the silence, expire the old one to leave a history of // the silence before modification. if err := s.expire(prev.Id); err != nil { return fmt.Errorf("expire previous silence: %w", err) } } return s.setSilence(msil, now) } // canUpdate returns true if silence a can be updated to b without // affecting the historic view of silencing. func canUpdate(a, b pb.Silence, now time.Time) bool { if !reflect.DeepEqual(a.Matchers, b.Matchers) { return false } // Allowed timestamp modifications depend on the current time. switch st := getState(a, now); st { case types.SilenceStateActive: if b.StartsAt.Unix() != a.StartsAt.Unix() { return false } if b.EndsAt.Before(now) { return false } case types.SilenceStatePending: if b.StartsAt.Before(now) { return false } case types.SilenceStateExpired: return false default: panic("unknown silence state") } return true } // Expire the silence with the given ID immediately. func (s Silences) Expire(id string) error { s.mtx.Lock() defer s.mtx.Unlock() return s.expire(id) } // Expire the silence with the given ID immediately. // It is idempotent, nil is returned if the silence already expired before it is GC'd. // If the silence is not found an error is returned. func (s Silences) expire(id string) error { sil, ok := s.getSilence(id) if !ok { return ErrNotFound } sil = cloneSilence(sil) now := s.nowUTC() switch getState(sil, now) { case types.SilenceStateExpired: return nil case types.SilenceStateActive: sil.EndsAt = now case types.SilenceStatePending: // Set both to now to make Silence move to "expired" state sil.StartsAt = now sil.EndsAt = now } sil.UpdatedAt = now return s.setSilence(s.toMeshSilence(sil), now) } // QueryParam expresses parameters along which silences are queried. type QueryParam func(query) error type query struct { ids []string filters []silenceFilter } // silenceFilter is a function that returns true if a silence // should be dropped from a result set for a given time. type silenceFilter func(pb.Silence, Silences, time.Time) (bool, error) // QIDs configures a query to select the given silence IDs. func QIDs(ids ...string) QueryParam { return func(q query) error { q.ids = append(q.ids, ids...) return nil } } // QMatches returns silences that match the given label set. func QMatches(set model.LabelSet) QueryParam { return func(q query) error { f := func(sil pb.Silence, s Silences, _ time.Time) (bool, error) { m, err := s.mc.Get(sil) if err != nil { return true, err } return m.Matches(set), nil } q.filters = append(q.filters, f) return nil } } // getState returns a silence's SilenceState at the given timestamp. func getState(sil pb.Silence, ts time.Time) types.SilenceState { if ts.Before(sil.StartsAt) { return types.SilenceStatePending } if ts.After(sil.EndsAt) { return types.SilenceStateExpired } return types.SilenceStateActive } // QState filters queried silences by the given states. func QState(states ...types.SilenceState) QueryParam { return func(q query) error { f := func(sil pb.Silence, _ Silences, now time.Time) (bool, error) { s := getState(sil, now) for _, ps := range states { if s == ps { return true, nil } } return false, nil } q.filters = append(q.filters, f) return nil } } // QueryOne queries with the given parameters and returns the first result. // Returns ErrNotFound if the query result is empty. func (s Silences) QueryOne(params ...QueryParam) (pb.Silence, error) { res, _, err := s.Query(params...) if err != nil { return nil, err } if len(res) == 0 { return nil, ErrNotFound } return res[0], nil } // Query for silences based on the given query parameters. It returns the // resulting silences and the state version the result is based on. func (s Silences) Query(params ...QueryParam) ([]pb.Silence, int, error) { s.mtx.Lock() defer s.mtx.Unlock() s.metrics.queriesTotal.Inc() defer prometheus.NewTimer(s.metrics.queryDuration).ObserveDuration() q := &query{} for _, p := range params { if err := p(q); err != nil { s.metrics.queryErrorsTotal.Inc() return nil, s.Version(), err } } sils, version, err := s.query(q, s.nowUTC()) if err != nil { s.metrics.queryErrorsTotal.Inc() } return sils, version, err } // Version of the silence state. func (s Silences) Version() int { s.mtx.RLock() defer s.mtx.RUnlock() return s.version } // CountState counts silences by state. func (s Silences) CountState(states ...types.SilenceState) (int, error) { // This could probably be optimized. sils, _, err := s.Query(QState(states...)) if err != nil { return -1, err } return len(sils), nil } func (s Silences) query(q query, now time.Time) ([]pb.Silence, int, error) { // If we have no ID constraint, all silences are our base set. This and // the use of post-filter functions is the trivial solution for now. var res []pb.Silence if q.ids != nil { for _, id := range q.ids { if s, ok := s.st[id]; ok { res = append(res, s.Silence) } } } else { for _, sil := range s.st { res = append(res, sil.Silence) } } var resf []pb.Silence for _, sil := range res { remove := false for _, f := range q.filters { ok, err := f(sil, s, now) if err != nil { return nil, s.version, err } if !ok { remove = true break } } if !remove { resf = append(resf, cloneSilence(sil)) } } return resf, s.version, nil } // loadSnapshot loads a snapshot generated by Snapshot() into the state. // Any previous state is wiped. func (s Silences) loadSnapshot(r io.Reader) error { st, err := decodeState(r) if err != nil { return err } for _, e := range st { // Comments list was moved to a single comment. Upgrade on loading the snapshot. if len(e.Silence.Comments) > 0 { e.Silence.Comment = e.Silence.Comments[0].Comment e.Silence.CreatedBy = e.Silence.Comments[0].Author e.Silence.Comments = nil } st[e.Silence.Id] = e } s.mtx.Lock() s.st = st s.version++ s.mtx.Unlock() return nil } // Snapshot writes the full internal state into the writer and returns the number of bytes // written. func (s Silences) Snapshot(w io.Writer) (int64, error) { start := time.Now() defer func() { s.metrics.snapshotDuration.Observe(time.Since(start).Seconds()) }() s.mtx.RLock() defer s.mtx.RUnlock() b, err := s.st.MarshalBinary() if err != nil { return 0, err } return io.Copy(w, bytes.NewReader(b)) } // MarshalBinary serializes all silences. func (s Silences) MarshalBinary() ([]byte, error) { s.mtx.Lock() defer s.mtx.Unlock() return s.st.MarshalBinary() } // Merge merges silence state received from the cluster with the local state. func (s Silences) Merge(b []byte) error { st, err := decodeState(bytes.NewReader(b)) if err != nil { return err } s.mtx.Lock() defer s.mtx.Unlock() now := s.nowUTC() for _, e := range st { merged, added := s.st.merge(e, now) if merged { if added { s.version++ } if !cluster.OversizedMessage(b) { // If this is the first we've seen the message and it's // not oversized, gossip it to other nodes. We don't // propagate oversized messages because they're sent to // all nodes already. s.broadcast(b) s.metrics.propagatedMessagesTotal.Inc() s.logger.Debug("Gossiping new silence", "silence", e) } } } return nil } // SetBroadcast sets the provided function as the one creating data to be // broadcast. func (s Silences) SetBroadcast(f func([]byte)) { s.mtx.Lock() s.broadcast = f s.mtx.Unlock() } type state map[string]pb.MeshSilence // merge returns two bools: the first is true when merge caused a state change. The second // is true if that state change added a new silence. In other words, the second return is // true whenever a silence with a new ID has been added to the state as a result of merge. func (s state) merge(e pb.MeshSilence, now time.Time) (bool, bool) { id := e.Silence.Id if e.ExpiresAt.Before(now) { return false, false } // Comments list was moved to a single comment. Apply upgrade // on silences received from peers. if len(e.Silence.Comments) > 0 { e.Silence.Comment = e.Silence.Comments[0].Comment e.Silence.CreatedBy = e.Silence.Comments[0].Author e.Silence.Comments = nil } prev, ok := s[id] if !ok \|\| prev.Silence.UpdatedAt.Before(e.Silence.UpdatedAt) { s[id] = e return true, !ok } return false, false } func (s state) MarshalBinary() ([]byte, error) { var buf bytes.Buffer for _, e := range s { if _, err := pbutil.WriteDelimited(&buf, e); err != nil { return nil, err } } return buf.Bytes(), nil } func decodeState(r io.Reader) (state, error) { st := state{} for { var s pb.MeshSilence _, err := pbutil.ReadDelimited(r, &s) if err == nil { if s.Silence == nil { return nil, ErrInvalidState } st[s.Silence.Id] = &s continue } if errors.Is(err, io.EOF) { break } return nil, err } return st, nil } func marshalMeshSilence(e pb.MeshSilence) ([]byte, error) { var buf bytes.Buffer if _, err := pbutil.WriteDelimited(&buf, e); err != nil { return nil, err } return buf.Bytes(), nil } // replaceFile wraps a file that is moved to another filename on closing. type replaceFile struct { os.File filename string } func (f replaceFile) Close() error { if err := f.File.Sync(); err != nil { return err } if err := f.File.Close(); err != nil { return err } return os.Rename(f.File.Name(), f.filename) } // openReplace opens a new temporary file that is moved to filename on closing. func openReplace(filename string) (*replaceFile, error) { tmpFilename := fmt.Sprintf("%s.%x", filename, uint64(rand.Int63())) f, err := os.Create(tmpFilename) if err != nil { return nil, err } rf := &replaceFile{ File: f, filename: filename, } return rf, nil }	27.66 KB	2025-05-25 12:27
📄 silence_bench_test.go // Copyright 2024 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package silence import ( "strconv" "testing" "time" "github.com/coder/quartz" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/alertmanager/types" ) // BenchmarkMutes benchmarks the Mutes method for the Muter interface for // different numbers of silences, where all silences match the alert. func BenchmarkMutes(b testing.B) { b.Run("1 silence mutes alert", func(b testing.B) { benchmarkMutes(b, 1) }) b.Run("10 silences mute alert", func(b testing.B) { benchmarkMutes(b, 10) }) b.Run("100 silences mute alert", func(b testing.B) { benchmarkMutes(b, 100) }) b.Run("1000 silences mute alert", func(b testing.B) { benchmarkMutes(b, 1000) }) b.Run("10000 silences mute alert", func(b testing.B) { benchmarkMutes(b, 10000) }) } func benchmarkMutes(b testing.B, n int) { silences, err := New(Options{}) require.NoError(b, err) clock := quartz.NewMock(b) silences.clock = clock now := clock.Now() var silenceIDs []string for i := 0; i < n; i++ { s := &silencepb.Silence{ Matchers: []silencepb.Matcher{{ Type: silencepb.Matcher_EQUAL, Name: "foo", Pattern: "bar", }}, StartsAt: now, EndsAt: now.Add(time.Minute), } require.NoError(b, silences.Set(s)) require.NoError(b, err) silenceIDs = append(silenceIDs, s.Id) } require.Len(b, silenceIDs, n) m := types.NewMarker(prometheus.NewRegistry()) s := NewSilencer(silences, m, promslog.NewNopLogger()) b.ResetTimer() for i := 0; i < b.N; i++ { s.Mutes(model.LabelSet{"foo": "bar"}) } b.StopTimer() // The alert should be marked as silenced for each silence. activeIDs, pendingIDs, _, silenced := m.Silenced(model.LabelSet{"foo": "bar"}.Fingerprint()) require.True(b, silenced) require.Empty(b, pendingIDs) require.Len(b, activeIDs, n) } // BenchmarkQuery benchmarks the Query method for the Silences struct // for different numbers of silences. Not all silences match the query // to prevent compiler and runtime optimizations from affecting the benchmarks. func BenchmarkQuery(b testing.B) { b.Run("100 silences", func(b testing.B) { benchmarkQuery(b, 100) }) b.Run("1000 silences", func(b testing.B) { benchmarkQuery(b, 1000) }) b.Run("10000 silences", func(b testing.B) { benchmarkQuery(b, 10000) }) } func benchmarkQuery(b testing.B, numSilences int) { s, err := New(Options{}) require.NoError(b, err) clock := quartz.NewMock(b) s.clock = clock now := clock.Now() lset := model.LabelSet{"aaaa": "AAAA", "bbbb": "BBBB", "cccc": "CCCC"} s.st = state{} for i := 0; i < numSilences; i++ { id := strconv.Itoa(i) // Include an offset to avoid optimizations. patA := "A{4}\|" + id patB := id // Does not match. if i%10 == 0 { // Every 10th time, have an actually matching pattern. patB = "B(B\|C)B.\|" + id } s.st[id] = &silencepb.MeshSilence{Silence: &silencepb.Silence{ Id: id, Matchers: []silencepb.Matcher{ {Type: silencepb.Matcher_REGEXP, Name: "aaaa", Pattern: patA}, {Type: silencepb.Matcher_REGEXP, Name: "bbbb", Pattern: patB}, }, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), }} } // Run things once to populate the matcherCache. sils, _, err := s.Query( QState(types.SilenceStateActive), QMatches(lset), ) require.NoError(b, err) require.Len(b, sils, numSilences/10) b.ResetTimer() for i := 0; i < b.N; i++ { sils, _, err := s.Query( QState(types.SilenceStateActive), QMatches(lset), ) require.NoError(b, err) require.Len(b, sils, numSilences/10) } }	4.25 KB	2025-05-25 12:27
📄 silence_test.go // Copyright 2016 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package silence import ( "bytes" "fmt" "os" "runtime" "sort" "strings" "sync" "testing" "time" "github.com/coder/quartz" "github.com/matttproud/golang_protobuf_extensions/pbutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "go.uber.org/atomic" "github.com/prometheus/alertmanager/featurecontrol" "github.com/prometheus/alertmanager/matcher/compat" pb "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/alertmanager/types" ) func checkErr(t testing.T, expected string, got error) { t.Helper() if expected == "" { require.NoError(t, got) return } if got == nil { t.Errorf("expected error containing %q but got none", expected) return } require.Contains(t, got.Error(), expected) } func TestOptionsValidate(t testing.T) { cases := []struct { options Options err string }{ { options: &Options{ SnapshotReader: &bytes.Buffer{}, }, }, { options: &Options{ SnapshotFile: "test.bkp", }, }, { options: &Options{ SnapshotFile: "test bkp", SnapshotReader: &bytes.Buffer{}, }, err: "only one of SnapshotFile and SnapshotReader must be set", }, } for _, c := range cases { checkErr(t, c.err, c.options.validate()) } } func TestSilenceGCOverTime(t testing.T) { t.Run("GC does not remove active silences", func(t testing.T) { s, err := New(Options{}) require.NoError(t, err) s.clock = quartz.NewMock(t) now := s.nowUTC() s.st = state{ "1": &pb.MeshSilence{Silence: &pb.Silence{Id: "1"}, ExpiresAt: now}, "2": &pb.MeshSilence{Silence: &pb.Silence{Id: "2"}, ExpiresAt: now.Add(-time.Second)}, "3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}, ExpiresAt: now.Add(time.Second)}, } want := state{ "3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}, ExpiresAt: now.Add(time.Second)}, } n, err := s.GC() require.NoError(t, err) require.Equal(t, 2, n) require.Equal(t, want, s.st) }) t.Run("GC does not leak cache entries", func(t testing.T) { s, err := New(Options{}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock sil1 := &pb.Silence{ Matchers: []pb.Matcher{{ Type: pb.Matcher_EQUAL, Name: "foo", Pattern: "bar", }}, StartsAt: clock.Now(), EndsAt: clock.Now().Add(time.Minute), } require.NoError(t, s.Set(sil1)) // Need to query the silence to populate the matcher cache. s.Query(QMatches(model.LabelSet{"foo": "bar"})) require.Len(t, s.st, 1) require.Len(t, s.mc, 1) // Move time forward and both silence and cache entry should be garbage // collected. clock.Advance(time.Minute) n, err := s.GC() require.NoError(t, err) require.Equal(t, 1, n) require.Empty(t, s.st) require.Empty(t, s.mc) }) t.Run("replacing a silences does not leak cache entries", func(t testing.T) { s, err := New(Options{}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock sil1 := &pb.Silence{ Matchers: []pb.Matcher{{ Type: pb.Matcher_EQUAL, Name: "foo", Pattern: "bar", }}, StartsAt: clock.Now(), EndsAt: clock.Now().Add(time.Minute), } require.NoError(t, s.Set(sil1)) // Need to query the silence to populate the matcher cache. s.Query(QMatches(model.LabelSet{"foo": "bar"})) require.Len(t, s.st, 1) require.Len(t, s.mc, 1) // must clone sil1 before replacing it. sil2 := cloneSilence(sil1) sil2.Matchers = []pb.Matcher{{ Type: pb.Matcher_EQUAL, Name: "bar", Pattern: "baz", }} require.NoError(t, s.Set(sil2)) // Need to query the silence to populate the matcher cache. s.Query(QMatches(model.LabelSet{"bar": "baz"})) require.Len(t, s.st, 2) require.Len(t, s.mc, 2) // Move time forward and both silence and cache entry should be garbage // collected. clock.Advance(time.Minute) n, err := s.GC() require.NoError(t, err) require.Equal(t, 2, n) require.Empty(t, s.st) require.Empty(t, s.mc) }) // This test checks for a memory leak that occurred in the matcher cache when // updating an existing silence. t.Run("updating a silences does not leak cache entries", func(t testing.T) { s, err := New(Options{}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock sil1 := &pb.Silence{ Id: "1", Matchers: []pb.Matcher{{ Type: pb.Matcher_EQUAL, Name: "foo", Pattern: "bar", }}, StartsAt: clock.Now(), EndsAt: clock.Now().Add(time.Minute), } s.st["1"] = &pb.MeshSilence{Silence: sil1, ExpiresAt: clock.Now().Add(time.Minute)} // Need to query the silence to populate the matcher cache. s.Query(QMatches(model.LabelSet{"foo": "bar"})) require.Len(t, s.mc, 1) // must clone sil1 before updating it. sil2 := cloneSilence(sil1) require.NoError(t, s.Set(sil2)) // The memory leak occurred because updating a silence would add a new // entry in the matcher cache even though no new silence was created. // This check asserts that this no longer happens. s.Query(QMatches(model.LabelSet{"foo": "bar"})) require.Len(t, s.st, 1) require.Len(t, s.mc, 1) // Move time forward and both silence and cache entry should be garbage // collected. clock.Advance(time.Minute) n, err := s.GC() require.NoError(t, err) require.Equal(t, 1, n) require.Empty(t, s.st) require.Empty(t, s.mc) }) } func TestSilencesSnapshot(t testing.T) { // Check whether storing and loading the snapshot is symmetric. now := quartz.NewMock(t).Now().UTC() cases := []struct { entries []pb.MeshSilence }{ { entries: []pb.MeshSilence{ { Silence: &pb.Silence{ Id: "3be80475-e219-4ee7-b6fc-4b65114e362f", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL}, {Name: "label2", Pattern: "val.+", Type: pb.Matcher_REGEXP}, }, StartsAt: now, EndsAt: now, UpdatedAt: now, }, ExpiresAt: now, }, { Silence: &pb.Silence{ Id: "3dfb2528-59ce-41eb-b465-f875a4e744a4", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_NOT_EQUAL}, {Name: "label2", Pattern: "val.+", Type: pb.Matcher_NOT_REGEXP}, }, StartsAt: now, EndsAt: now, UpdatedAt: now, }, ExpiresAt: now, }, { Silence: &pb.Silence{ Id: "4b1e760d-182c-4980-b873-c1a6827c9817", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL}, }, StartsAt: now.Add(time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now, }, ExpiresAt: now.Add(24 * time.Hour), }, }, }, } for _, c := range cases { f, err := os.CreateTemp("", "snapshot") require.NoError(t, err, "creating temp file failed") s1 := &Silences{st: state{}, metrics: newMetrics(nil, nil)} // Setup internal state manually. for _, e := range c.entries { s1.st[e.Silence.Id] = e } _, err = s1.Snapshot(f) require.NoError(t, err, "creating snapshot failed") require.NoError(t, f.Close(), "closing snapshot file failed") f, err = os.Open(f.Name()) require.NoError(t, err, "opening snapshot file failed") // Check again against new nlog instance. s2 := &Silences{mc: matcherCache{}, st: state{}} err = s2.loadSnapshot(f) require.NoError(t, err, "error loading snapshot") require.Equal(t, s1.st, s2.st, "state after loading snapshot did not match snapshotted state") require.NoError(t, f.Close(), "closing snapshot file failed") } } // This tests a regression introduced by https://github.com/prometheus/alertmanager/pull/2689. func TestSilences_Maintenance_DefaultMaintenanceFuncDoesntCrash(t testing.T) { f, err := os.CreateTemp("", "snapshot") require.NoError(t, err, "creating temp file failed") clock := quartz.NewMock(t) s := &Silences{st: state{}, logger: promslog.NewNopLogger(), clock: clock, metrics: newMetrics(nil, nil)} stopc := make(chan struct{}) done := make(chan struct{}) go func() { s.Maintenance(100time.Millisecond, f.Name(), stopc, nil) close(done) }() runtime.Gosched() clock.Advance(100 * time.Millisecond) close(stopc) <-done } func TestSilences_Maintenance_SupportsCustomCallback(t testing.T) { f, err := os.CreateTemp("", "snapshot") require.NoError(t, err, "creating temp file failed") clock := quartz.NewMock(t) reg := prometheus.NewRegistry() s := &Silences{st: state{}, logger: promslog.NewNopLogger(), clock: clock} s.metrics = newMetrics(reg, s) stopc := make(chan struct{}) var calls atomic.Int32 var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() s.Maintenance(10time.Second, f.Name(), stopc, func() (int64, error) { calls.Add(1) return 0, nil }) }() gosched() // Before the first tick, no maintenance executed. clock.Advance(9 * time.Second) require.EqualValues(t, 0, calls.Load()) // Tick once. clock.Advance(1 * time.Second) require.Eventually(t, func() bool { return calls.Load() == 1 }, 5time.Second, time.Second) // Stop the maintenance loop. We should get exactly one more execution of the maintenance func. close(stopc) wg.Wait() require.EqualValues(t, 2, calls.Load()) // Check the maintenance metrics. require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP alertmanager_silences_maintenance_errors_total How many maintenances were executed for silences that failed. # TYPE alertmanager_silences_maintenance_errors_total counter alertmanager_silences_maintenance_errors_total 0 # HELP alertmanager_silences_maintenance_total How many maintenances were executed for silences. # TYPE alertmanager_silences_maintenance_total counter alertmanager_silences_maintenance_total 2 `), "alertmanager_silences_maintenance_total", "alertmanager_silences_maintenance_errors_total")) } func TestSilencesSetSilence(t testing.T) { s, err := New(Options{ Retention: time.Minute, }) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock nowpb := s.nowUTC() sil := &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{{Name: "abc", Pattern: "def"}}, StartsAt: nowpb, EndsAt: nowpb, } want := state{ "some_id": &pb.MeshSilence{ Silence: sil, ExpiresAt: nowpb.Add(time.Minute), }, } done := make(chan struct{}) s.broadcast = func(b []byte) { var e pb.MeshSilence r := bytes.NewReader(b) _, err := pbutil.ReadDelimited(r, &e) require.NoError(t, err) require.Equal(t, &e, want["some_id"]) close(done) } // setSilence() is always called with s.mtx locked() in the application code func() { s.mtx.Lock() defer s.mtx.Unlock() require.NoError(t, s.setSilence(s.toMeshSilence(sil), nowpb)) }() // Ensure broadcast was called. if _, isOpen := <-done; isOpen { t.Fatal("broadcast was not called") } require.Equal(t, want, s.st, "Unexpected silence state") } func TestSilenceSet(t testing.T) { s, err := New(Options{ Retention: time.Hour, }) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock start1 := s.nowUTC() // Insert silence with fixed start time. sil1 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start1.Add(2 time.Minute), EndsAt: start1.Add(5 * time.Minute), } versionBeforeOp := s.Version() require.NoError(t, s.Set(sil1)) require.NotEqual(t, "", sil1.Id) require.NotEqual(t, versionBeforeOp, s.Version()) want := state{ sil1.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil1.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start1.Add(2 time.Minute), EndsAt: start1.Add(5 * time.Minute), UpdatedAt: start1, }, ExpiresAt: start1.Add(5time.Minute + s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") // Insert silence with unset start time. Must be set to now. clock.Advance(time.Minute) start2 := s.nowUTC() sil2 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, EndsAt: start2.Add(1 * time.Minute), } versionBeforeOp = s.Version() require.NoError(t, s.Set(sil2)) require.NotEqual(t, "", sil2.Id) require.NotEqual(t, versionBeforeOp, s.Version()) want = state{ sil1.Id: want[sil1.Id], sil2.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil2.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start2, EndsAt: start2.Add(1 time.Minute), UpdatedAt: start2, }, ExpiresAt: start2.Add(1time.Minute + s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") // Should be able to update silence without modifications. It is expected to // keep the same ID. sil3 := cloneSilence(sil2) versionBeforeOp = s.Version() require.NoError(t, s.Set(sil3)) require.Equal(t, sil2.Id, sil3.Id) require.Equal(t, versionBeforeOp, s.Version()) // Should be able to update silence with comment. It is also expected to // keep the same ID. sil4 := cloneSilence(sil3) sil4.Comment = "c" versionBeforeOp = s.Version() require.NoError(t, s.Set(sil4)) require.Equal(t, sil3.Id, sil4.Id) require.Equal(t, versionBeforeOp, s.Version()) // Extend sil4 to expire at a later time. This should not expire the // existing silence, and so should also keep the same ID. clock.Advance(time.Minute) start5 := s.nowUTC() sil5 := cloneSilence(sil4) sil5.EndsAt = start5.Add(100 time.Minute) versionBeforeOp = s.Version() require.NoError(t, s.Set(sil5)) require.Equal(t, sil4.Id, sil5.Id) want = state{ sil1.Id: want[sil1.Id], sil2.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil2.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start2, EndsAt: start5.Add(100 time.Minute), UpdatedAt: start5, Comment: "c", }, ExpiresAt: start5.Add(100time.Minute + s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") require.Equal(t, versionBeforeOp, s.Version()) // Replace the silence sil5 with another silence with different matchers. // Unlike previous updates, changing the matchers for an existing silence // will expire the existing silence and create a new silence. The new // silence is expected to have a different ID to preserve the history of // the previous silence. clock.Advance(time.Minute) start6 := s.nowUTC() sil6 := cloneSilence(sil5) sil6.Matchers = []pb.Matcher{{Name: "a", Pattern: "c"}} versionBeforeOp = s.Version() require.NoError(t, s.Set(sil6)) require.NotEqual(t, sil5.Id, sil6.Id) want = state{ sil1.Id: want[sil1.Id], sil2.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil2.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start2, EndsAt: start6, // Expired UpdatedAt: start6, Comment: "c", }, ExpiresAt: start6.Add(s.retention), }, sil6.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil6.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "c"}}, StartsAt: start6, EndsAt: start5.Add(100 * time.Minute), UpdatedAt: start6, Comment: "c", }, ExpiresAt: start5.Add(100time.Minute + s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") require.NotEqual(t, versionBeforeOp, s.Version()) // Re-create the silence that we just replaced. Changing the start time, // just like changing the matchers, creates a new silence with a different // ID. This is again to preserve the history of the original silence. clock.Advance(time.Minute) start7 := s.nowUTC() sil7 := cloneSilence(sil5) sil7.StartsAt = start1 sil7.EndsAt = start1.Add(5 time.Minute) versionBeforeOp = s.Version() require.NoError(t, s.Set(sil7)) require.NotEqual(t, sil2.Id, sil7.Id) want = state{ sil1.Id: want[sil1.Id], sil2.Id: want[sil2.Id], sil6.Id: want[sil6.Id], sil7.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil7.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: start7, // New silences have their start time set to "now" when created. EndsAt: start1.Add(5 time.Minute), UpdatedAt: start7, Comment: "c", }, ExpiresAt: start1.Add(5time.Minute + s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") require.NotEqual(t, versionBeforeOp, s.Version()) // Updating an existing silence with an invalid silence should not expire // the original silence. clock.Advance(time.Millisecond) sil8 := cloneSilence(sil7) sil8.EndsAt = time.Time{} versionBeforeOp = s.Version() require.EqualError(t, s.Set(sil8), "invalid silence: invalid zero end timestamp") // sil7 should not be expired because the update failed. clock.Advance(time.Millisecond) sil7, err = s.QueryOne(QIDs(sil7.Id)) require.NoError(t, err) require.Equal(t, types.SilenceStateActive, getState(sil7, s.nowUTC())) require.Equal(t, versionBeforeOp, s.Version()) } func TestSilenceLimits(t testing.T) { s, err := New(Options{ Limits: Limits{ MaxSilences: func() int { return 1 }, MaxSilenceSizeBytes: func() int { return 2 << 11 }, // 4KB }, }) require.NoError(t, err) // Insert sil1 should succeed without error. sil1 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: time.Now(), EndsAt: time.Now().Add(5 time.Minute), } require.NoError(t, s.Set(sil1)) // Insert sil2 should fail because maximum number of silences has been // exceeded. sil2 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "c", Pattern: "d"}}, StartsAt: time.Now(), EndsAt: time.Now().Add(5 time.Minute), } require.EqualError(t, s.Set(sil2), "exceeded maximum number of silences: 1 (limit: 1)") // Expire sil1 and run the GC. This should allow sil2 to be inserted. require.NoError(t, s.Expire(sil1.Id)) n, err := s.GC() require.NoError(t, err) require.Equal(t, 1, n) require.NoError(t, s.Set(sil2)) // Expire sil2 and run the GC. require.NoError(t, s.Expire(sil2.Id)) n, err = s.GC() require.NoError(t, err) require.Equal(t, 1, n) // Insert sil3 should fail because it exceeds maximum size. sil3 := &pb.Silence{ Matchers: []pb.Matcher{ { Name: strings.Repeat("e", 2<<9), Pattern: strings.Repeat("f", 2<<9), }, { Name: strings.Repeat("g", 2<<9), Pattern: strings.Repeat("h", 2<<9), }, }, CreatedBy: strings.Repeat("i", 2<<9), Comment: strings.Repeat("j", 2<<9), StartsAt: time.Now(), EndsAt: time.Now().Add(5 time.Minute), } require.EqualError(t, s.Set(sil3), fmt.Sprintf("silence exceeded maximum size: %d bytes (limit: 4096 bytes)", s.toMeshSilence(sil3).Size())) // Should be able to insert sil4. sil4 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "k", Pattern: "l"}}, StartsAt: time.Now(), EndsAt: time.Now().Add(5 time.Minute), } require.NoError(t, s.Set(sil4)) // Should be able to update sil4 without modifications. It is expected to // keep the same ID. sil5 := cloneSilence(sil4) require.NoError(t, s.Set(sil5)) require.Equal(t, sil4.Id, sil5.Id) // Should be able to update the comment. It is also expected to keep the // same ID. sil6 := cloneSilence(sil5) sil6.Comment = "m" require.NoError(t, s.Set(sil6)) require.Equal(t, sil5.Id, sil6.Id) // Should not be able to update the start and end time as this requires // sil6 to be expired and a new silence to be created. However, this would // exceed the maximum number of silences, which counts both active and // expired silences. sil7 := cloneSilence(sil6) sil7.StartsAt = time.Now().Add(5 * time.Minute) sil7.EndsAt = time.Now().Add(10 * time.Minute) require.EqualError(t, s.Set(sil7), "exceeded maximum number of silences: 1 (limit: 1)") // sil6 should not be expired because the update failed. sil6, err = s.QueryOne(QIDs(sil6.Id)) require.NoError(t, err) require.Equal(t, types.SilenceStateActive, getState(sil6, s.nowUTC())) // Should not be able to update with a comment that exceeds maximum size. // Need to increase the maximum number of silences to test this. s.limits.MaxSilences = func() int { return 2 } sil8 := cloneSilence(sil6) sil8.Comment = strings.Repeat("m", 2<<11) require.EqualError(t, s.Set(sil8), fmt.Sprintf("silence exceeded maximum size: %d bytes (limit: 4096 bytes)", s.toMeshSilence(sil8).Size())) // sil6 should not be expired because the update failed. sil6, err = s.QueryOne(QIDs(sil6.Id)) require.NoError(t, err) require.Equal(t, types.SilenceStateActive, getState(sil6, s.nowUTC())) // Should not be able to replace with a silence that exceeds maximum size. // This is different from the previous assertion as unlike when adding or // updating a comment, changing the matchers for a silence should expire // the existing silence, unless the silence that is replacing it exceeds // limits, in which case the operation should fail and the existing silence // should still be active. sil9 := cloneSilence(sil8) sil9.Matchers = []pb.Matcher{{Name: "n", Pattern: "o"}} require.EqualError(t, s.Set(sil9), fmt.Sprintf("silence exceeded maximum size: %d bytes (limit: 4096 bytes)", s.toMeshSilence(sil9).Size())) // sil6 should not be expired because the update failed. sil6, err = s.QueryOne(QIDs(sil6.Id)) require.NoError(t, err) require.Equal(t, types.SilenceStateActive, getState(sil6, s.nowUTC())) } func TestSilenceNoLimits(t testing.T) { s, err := New(Options{ Limits: Limits{}, }) require.NoError(t, err) // Insert sil should succeed without error. sil := &pb.Silence{ Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: time.Now(), EndsAt: time.Now().Add(5 time.Minute), Comment: strings.Repeat("c", 2<<9), } require.NoError(t, s.Set(sil)) require.NotEqual(t, "", sil.Id) } func TestSetActiveSilence(t testing.T) { s, err := New(Options{ Retention: time.Hour, }) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock now := clock.Now() startsAt := now.Add(-1 time.Minute) endsAt := now.Add(5 * time.Minute) // Insert silence with fixed start time. sil1 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: startsAt, EndsAt: endsAt, } require.NoError(t, s.Set(sil1)) // Update silence with 2 extra nanoseconds so the "seconds" part should not change newStartsAt := now.Add(2 time.Nanosecond) newEndsAt := endsAt.Add(2 * time.Minute) sil2 := cloneSilence(sil1) sil2.Id = sil1.Id sil2.StartsAt = newStartsAt sil2.EndsAt = newEndsAt clock.Advance(time.Minute) now = s.nowUTC() require.NoError(t, s.Set(sil2)) require.Equal(t, sil1.Id, sil2.Id) want := state{ sil2.Id: &pb.MeshSilence{ Silence: &pb.Silence{ Id: sil1.Id, Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, StartsAt: newStartsAt, EndsAt: newEndsAt, UpdatedAt: now, }, ExpiresAt: newEndsAt.Add(s.retention), }, } require.Equal(t, want, s.st, "unexpected state after silence creation") } func TestSilencesSetFail(t testing.T) { s, err := New(Options{}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock cases := []struct { s pb.Silence err string }{ { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{{Name: "a", Pattern: "b"}}, EndsAt: clock.Now().Add(5 * time.Minute), }, err: ErrNotFound.Error(), }, { s: &pb.Silence{}, // Silence without matcher. err: "invalid silence", }, } for _, c := range cases { checkErr(t, c.err, s.Set(c.s)) } } func TestQState(t testing.T) { now := time.Now().UTC() cases := []struct { sil pb.Silence states []types.SilenceState keep bool }{ { sil: &pb.Silence{ StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Hour), }, states: []types.SilenceState{types.SilenceStateActive, types.SilenceStateExpired}, keep: false, }, { sil: &pb.Silence{ StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Hour), }, states: []types.SilenceState{types.SilenceStatePending}, keep: true, }, { sil: &pb.Silence{ StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Hour), }, states: []types.SilenceState{types.SilenceStateExpired, types.SilenceStatePending}, keep: true, }, } for i, c := range cases { q := &query{} QState(c.states...)(q) f := q.filters[0] keep, err := f(c.sil, nil, now) require.NoError(t, err) require.Equal(t, c.keep, keep, "unexpected filter result for case %d", i) } } func TestQMatches(t testing.T) { qp := QMatches(model.LabelSet{ "job": "test", "instance": "web-1", "path": "/user/profile", "method": "GET", }) q := &query{} qp(q) f := q.filters[0] cases := []struct { sil pb.Silence drop bool }{ { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL}, }, }, drop: true, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "job", Pattern: "test", Type: pb.Matcher_NOT_EQUAL}, }, }, drop: false, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL}, {Name: "method", Pattern: "POST", Type: pb.Matcher_EQUAL}, }, }, drop: false, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL}, {Name: "method", Pattern: "POST", Type: pb.Matcher_NOT_EQUAL}, }, }, drop: true, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "path", Pattern: "/user/.+", Type: pb.Matcher_REGEXP}, }, }, drop: true, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "path", Pattern: "/user/.+", Type: pb.Matcher_NOT_REGEXP}, }, }, drop: false, }, { sil: &pb.Silence{ Matchers: []pb.Matcher{ {Name: "path", Pattern: "/user/.+", Type: pb.Matcher_REGEXP}, {Name: "path", Pattern: "/nothing/.+", Type: pb.Matcher_REGEXP}, }, }, drop: false, }, } for _, c := range cases { drop, err := f(c.sil, &Silences{mc: matcherCache{}, st: state{}}, time.Time{}) require.NoError(t, err) require.Equal(t, c.drop, drop, "unexpected filter result") } } func TestSilencesQuery(t testing.T) { s, err := New(Options{}) require.NoError(t, err) s.st = state{ "1": &pb.MeshSilence{Silence: &pb.Silence{Id: "1"}}, "2": &pb.MeshSilence{Silence: &pb.Silence{Id: "2"}}, "3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}}, "4": &pb.MeshSilence{Silence: &pb.Silence{Id: "4"}}, "5": &pb.MeshSilence{Silence: &pb.Silence{Id: "5"}}, } cases := []struct { q query exp []pb.Silence }{ { // Default query of retrieving all silences. q: &query{}, exp: []pb.Silence{ {Id: "1"}, {Id: "2"}, {Id: "3"}, {Id: "4"}, {Id: "5"}, }, }, { // Retrieve by IDs. q: &query{ ids: []string{"2", "5"}, }, exp: []pb.Silence{ {Id: "2"}, {Id: "5"}, }, }, { // Retrieve all and filter q: &query{ filters: []silenceFilter{ func(sil pb.Silence, _ Silences, _ time.Time) (bool, error) { return sil.Id == "1" \|\| sil.Id == "2", nil }, }, }, exp: []pb.Silence{ {Id: "1"}, {Id: "2"}, }, }, { // Retrieve by IDs and filter q: &query{ ids: []string{"2", "5"}, filters: []silenceFilter{ func(sil pb.Silence, _ Silences, _ time.Time) (bool, error) { return sil.Id == "1" \|\| sil.Id == "2", nil }, }, }, exp: []pb.Silence{ {Id: "2"}, }, }, } for _, c := range cases { // Run default query of retrieving all silences. res, _, err := s.query(c.q, time.Time{}) require.NoError(t, err, "unexpected error on querying") // Currently there are no sorting guarantees in the querying API. sort.Sort(silencesByID(c.exp)) sort.Sort(silencesByID(res)) require.Equal(t, c.exp, res, "unexpected silences in result") } } type silencesByID []pb.Silence func (s silencesByID) Len() int { return len(s) } func (s silencesByID) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s silencesByID) Less(i, j int) bool { return s[i].Id < s[j].Id } func TestSilenceCanUpdate(t testing.T) { now := time.Now().UTC() cases := []struct { a, b pb.Silence ok bool }{ // Bad arguments. { a: &pb.Silence{}, b: &pb.Silence{ StartsAt: now, EndsAt: now.Add(-time.Minute), }, ok: false, }, // Expired silence. { a: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(-time.Second), }, b: &pb.Silence{ StartsAt: now, EndsAt: now, }, ok: false, }, // Pending silences. { a: &pb.Silence{ StartsAt: now.Add(time.Hour), EndsAt: now.Add(2 time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour), }, ok: false, }, { a: &pb.Silence{ StartsAt: now.Add(time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Minute), }, ok: true, }, { a: &pb.Silence{ StartsAt: now.Add(time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now, // set to exactly start now. EndsAt: now.Add(2 * time.Hour), }, ok: true, }, // Active silences. { a: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(-time.Minute), EndsAt: now.Add(2 * time.Hour), }, ok: false, }, { a: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(-time.Second), }, ok: false, }, { a: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now, }, ok: true, }, { a: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(2 * time.Hour), UpdatedAt: now.Add(-time.Hour), }, b: &pb.Silence{ StartsAt: now.Add(-time.Hour), EndsAt: now.Add(3 * time.Hour), }, ok: true, }, } for _, c := range cases { ok := canUpdate(c.a, c.b, now) if ok && !c.ok { t.Errorf("expected not-updateable but was: %v, %v", c.a, c.b) } if ok && !c.ok { t.Errorf("expected updateable but was not: %v, %v", c.a, c.b) } } } func TestSilenceExpire(t testing.T) { s, err := New(Options{Retention: time.Hour}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock now := s.nowUTC() m := &pb.Matcher{Type: pb.Matcher_EQUAL, Name: "a", Pattern: "b"} s.st = state{ "pending": &pb.MeshSilence{Silence: &pb.Silence{ Id: "pending", Matchers: []pb.Matcher{m}, StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), }}, "active": &pb.MeshSilence{Silence: &pb.Silence{ Id: "active", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), }}, "expired": &pb.MeshSilence{Silence: &pb.Silence{ Id: "expired", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Hour), EndsAt: now.Add(-time.Minute), UpdatedAt: now.Add(-time.Hour), }}, } count, err := s.CountState(types.SilenceStatePending) require.NoError(t, err) require.Equal(t, 1, count) count, err = s.CountState(types.SilenceStateExpired) require.NoError(t, err) require.Equal(t, 1, count) require.NoError(t, s.Expire("pending")) require.NoError(t, s.Expire("active")) require.NoError(t, s.Expire("expired")) sil, err := s.QueryOne(QIDs("pending")) require.NoError(t, err) require.Equal(t, &pb.Silence{ Id: "pending", Matchers: []pb.Matcher{m}, StartsAt: now, EndsAt: now, UpdatedAt: now, }, sil) // Let time pass... clock.Advance(time.Second) count, err = s.CountState(types.SilenceStatePending) require.NoError(t, err) require.Equal(t, 0, count) count, err = s.CountState(types.SilenceStateExpired) require.NoError(t, err) require.Equal(t, 3, count) // Expiring a pending Silence should make the API return the // SilenceStateExpired Silence state. silenceState := types.CalcSilenceState(sil.StartsAt, sil.EndsAt) require.Equal(t, types.SilenceStateExpired, silenceState) sil, err = s.QueryOne(QIDs("active")) require.NoError(t, err) require.Equal(t, &pb.Silence{ Id: "active", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Minute), EndsAt: now, UpdatedAt: now, }, sil) sil, err = s.QueryOne(QIDs("expired")) require.NoError(t, err) require.Equal(t, &pb.Silence{ Id: "expired", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Hour), EndsAt: now.Add(-time.Minute), UpdatedAt: now.Add(-time.Hour), }, sil) } // TestSilenceExpireWithZeroRetention covers the problem that, with zero // retention time, a silence explicitly set to expired will also immediately // expire from the silence storage. func TestSilenceExpireWithZeroRetention(t testing.T) { s, err := New(Options{Retention: 0}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock now := s.nowUTC() m := &pb.Matcher{Type: pb.Matcher_EQUAL, Name: "a", Pattern: "b"} s.st = state{ "pending": &pb.MeshSilence{Silence: &pb.Silence{ Id: "pending", Matchers: []pb.Matcher{m}, StartsAt: now.Add(time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), }}, "active": &pb.MeshSilence{Silence: &pb.Silence{ Id: "active", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), }}, "expired": &pb.MeshSilence{Silence: &pb.Silence{ Id: "expired", Matchers: []pb.Matcher{m}, StartsAt: now.Add(-time.Hour), EndsAt: now.Add(-time.Minute), UpdatedAt: now.Add(-time.Hour), }}, } count, err := s.CountState(types.SilenceStatePending) require.NoError(t, err) require.Equal(t, 1, count) count, err = s.CountState(types.SilenceStateActive) require.NoError(t, err) require.Equal(t, 1, count) count, err = s.CountState(types.SilenceStateExpired) require.NoError(t, err) require.Equal(t, 1, count) // Advance time. The silence state management code uses update time when // merging, and the logic is "first write wins". So we must advance the clock // one tick for updates to take effect. clock.Advance(1 time.Millisecond) require.NoError(t, s.Expire("pending")) require.NoError(t, s.Expire("active")) require.NoError(t, s.Expire("expired")) // Advance time again. Despite what the function name says, s.Expire() does // not expire a silence. It sets the silence to EndAt the current time. This // means that the silence is active immediately after calling Expire. clock.Advance(1 * time.Millisecond) // Verify all silences have expired. count, err = s.CountState(types.SilenceStatePending) require.NoError(t, err) require.Equal(t, 0, count) count, err = s.CountState(types.SilenceStateActive) require.NoError(t, err) require.Equal(t, 0, count) count, err = s.CountState(types.SilenceStateExpired) require.NoError(t, err) require.Equal(t, 3, count) } // This test checks that invalid silences can be expired. func TestSilenceExpireInvalid(t testing.T) { s, err := New(Options{Retention: time.Hour}) require.NoError(t, err) clock := quartz.NewMock(t) s.clock = clock now := s.nowUTC() // In this test the matcher has an invalid type. silence := pb.Silence{ Id: "active", Matchers: []pb.Matcher{{Type: -1, Name: "a", Pattern: "b"}}, StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour), UpdatedAt: now.Add(-time.Hour), } // Assert that this silence is invalid. require.EqualError(t, validateSilence(&silence), "invalid label matcher 0: unknown matcher type \"-1\"") s.st = state{"active": &pb.MeshSilence{Silence: &silence}} // The silence should be active. count, err := s.CountState(types.SilenceStateActive) require.NoError(t, err) require.Equal(t, 1, count) clock.Advance(time.Millisecond) require.NoError(t, s.Expire("active")) clock.Advance(time.Millisecond) // The silence should be expired. count, err = s.CountState(types.SilenceStateActive) require.NoError(t, err) require.Equal(t, 0, count) count, err = s.CountState(types.SilenceStateExpired) require.NoError(t, err) require.Equal(t, 1, count) } func TestSilencer(t testing.T) { ss, err := New(Options{Retention: time.Hour}) require.NoError(t, err) clock := quartz.NewMock(t) ss.clock = clock now := ss.nowUTC() m := types.NewMarker(prometheus.NewRegistry()) s := NewSilencer(ss, m, promslog.NewNopLogger()) require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced without any silences") sil1 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "foo", Pattern: "baz"}}, StartsAt: now.Add(-time.Hour), EndsAt: now.Add(5 * time.Minute), } require.NoError(t, ss.Set(sil1)) require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by non-matching silence") sil2 := &pb.Silence{ Matchers: []pb.Matcher{{Name: "foo", Pattern: "bar"}}, StartsAt: now.Add(-time.Hour), EndsAt: now.Add(5 time.Minute), } require.NoError(t, ss.Set(sil2)) require.NotEmpty(t, sil2.Id) require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by matching silence") // One hour passes, silence expires. clock.Advance(time.Hour) now = ss.nowUTC() require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by expired silence") // Update silence to start in the future. err = ss.Set(&pb.Silence{ Id: sil2.Id, Matchers: []pb.Matcher{{Name: "foo", Pattern: "bar"}}, StartsAt: now.Add(time.Hour), EndsAt: now.Add(3 time.Hour), }) require.NoError(t, err) require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by future silence") // Two hours pass, silence becomes active. clock.Advance(2 * time.Hour) now = ss.nowUTC() // Exposes issue #2426. require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by activated silence") err = ss.Set(&pb.Silence{ Matchers: []pb.Matcher{{Name: "foo", Pattern: "b..", Type: pb.Matcher_REGEXP}}, StartsAt: now.Add(time.Hour), EndsAt: now.Add(3 time.Hour), }) require.NoError(t, err) // Note that issue #2426 doesn't apply anymore because we added a new silence. require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert still silenced by activated silence") // Two hours pass, first silence expires, overlapping second silence becomes active. clock.Advance(2 * time.Hour) // Another variant of issue #2426 (overlapping silences). require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by activated second silence") } func TestValidateClassicMatcher(t testing.T) { cases := []struct { m pb.Matcher err string }{ { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_NOT_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_REGEXP, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_NOT_REGEXP, }, err: "", }, { m: &pb.Matcher{ Name: "00", Pattern: "a", Type: pb.Matcher_EQUAL, }, err: "invalid label name", }, { m: &pb.Matcher{ Name: "\xf0\x9f\x99\x82", // U+1F642 Pattern: "a", Type: pb.Matcher_EQUAL, }, err: "invalid label name", }, { m: &pb.Matcher{ Name: "a", Pattern: "((", Type: pb.Matcher_REGEXP, }, err: "invalid regular expression", }, { m: &pb.Matcher{ Name: "a", Pattern: "))", Type: pb.Matcher_NOT_REGEXP, }, err: "invalid regular expression", }, { m: &pb.Matcher{ Name: "a", Pattern: "\xff", Type: pb.Matcher_EQUAL, }, err: "invalid label value", }, { m: &pb.Matcher{ Name: "a", Pattern: "\xf0\x9f\x99\x82", // U+1F642 Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: 333, }, err: "unknown matcher type", }, } for _, c := range cases { checkErr(t, c.err, validateMatcher(c.m)) } } func TestValidateUTF8Matcher(t testing.T) { cases := []struct { m pb.Matcher err string }{ { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_NOT_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_REGEXP, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: pb.Matcher_NOT_REGEXP, }, err: "", }, { m: &pb.Matcher{ Name: "00", Pattern: "a", Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "\xf0\x9f\x99\x82", // U+1F642 Pattern: "a", Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "((", Type: pb.Matcher_REGEXP, }, err: "invalid regular expression", }, { m: &pb.Matcher{ Name: "a", Pattern: "))", Type: pb.Matcher_NOT_REGEXP, }, err: "invalid regular expression", }, { m: &pb.Matcher{ Name: "a", Pattern: "\xff", Type: pb.Matcher_EQUAL, }, err: "invalid label value", }, { m: &pb.Matcher{ Name: "a", Pattern: "\xf0\x9f\x99\x82", // U+1F642 Type: pb.Matcher_EQUAL, }, err: "", }, { m: &pb.Matcher{ Name: "a", Pattern: "b", Type: 333, }, err: "unknown matcher type", }, } // Change the mode to UTF-8 mode. ff, err := featurecontrol.NewFlags(promslog.NewNopLogger(), featurecontrol.FeatureUTF8StrictMode) require.NoError(t, err) compat.InitFromFlags(promslog.NewNopLogger(), ff) // Restore the mode to classic at the end of the test. ff, err = featurecontrol.NewFlags(promslog.NewNopLogger(), featurecontrol.FeatureClassicMode) require.NoError(t, err) defer compat.InitFromFlags(promslog.NewNopLogger(), ff) for _, c := range cases { checkErr(t, c.err, validateMatcher(c.m)) } } func TestValidateSilence(t testing.T) { var ( now = time.Now().UTC() zeroTimestamp = time.Time{} validTimestamp = now ) cases := []struct { s pb.Silence err string }{ { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: "b"}, }, StartsAt: validTimestamp, EndsAt: validTimestamp, UpdatedAt: validTimestamp, }, err: "", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{}, StartsAt: validTimestamp, EndsAt: validTimestamp, UpdatedAt: validTimestamp, }, err: "at least one matcher required", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: "b"}, {Name: "00", Pattern: "b"}, }, StartsAt: validTimestamp, EndsAt: validTimestamp, UpdatedAt: validTimestamp, }, err: "invalid label matcher", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: ""}, {Name: "b", Pattern: ".", Type: pb.Matcher_REGEXP}, }, StartsAt: validTimestamp, EndsAt: validTimestamp, UpdatedAt: validTimestamp, }, err: "at least one matcher must not match the empty string", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: "b"}, }, StartsAt: now, EndsAt: now.Add(-time.Second), UpdatedAt: validTimestamp, }, err: "end time must not be before start time", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: "b"}, }, StartsAt: zeroTimestamp, EndsAt: validTimestamp, UpdatedAt: validTimestamp, }, err: "invalid zero start timestamp", }, { s: &pb.Silence{ Id: "some_id", Matchers: []pb.Matcher{ {Name: "a", Pattern: "b"}, }, StartsAt: validTimestamp, EndsAt: zeroTimestamp, UpdatedAt: validTimestamp, }, err: "invalid zero end timestamp", }, } for _, c := range cases { checkErr(t, c.err, validateSilence(c.s)) } } func TestStateMerge(t testing.T) { now := time.Now().UTC() // We only care about key names and timestamps for the // merging logic. newSilence := func(id string, ts, exp time.Time) pb.MeshSilence { return &pb.MeshSilence{ Silence: &pb.Silence{Id: id, UpdatedAt: ts}, ExpiresAt: exp, } } exp := now.Add(time.Minute) cases := []struct { a, b state final state }{ { a: state{ "a1": newSilence("a1", now, exp), "a2": newSilence("a2", now, exp), "a3": newSilence("a3", now, exp), }, b: state{ "b1": newSilence("b1", now, exp), // new key, should be added "a2": newSilence("a2", now.Add(-time.Minute), exp), // older timestamp, should be dropped "a3": newSilence("a3", now.Add(time.Minute), exp), // newer timestamp, should overwrite "a4": newSilence("a4", now.Add(-time.Minute), now.Add(-time.Millisecond)), // new key, expired, should not be added }, final: state{ "a1": newSilence("a1", now, exp), "a2": newSilence("a2", now, exp), "a3": newSilence("a3", now.Add(time.Minute), exp), "b1": newSilence("b1", now, exp), }, }, } for _, c := range cases { for _, e := range c.b { c.a.merge(e, now) } require.Equal(t, c.final, c.a, "Merge result should match expectation") } } func TestStateCoding(t testing.T) { // Check whether encoding and decoding the data is symmetric. now := time.Now().UTC() cases := []struct { entries []pb.MeshSilence }{ { entries: []pb.MeshSilence{ { Silence: &pb.Silence{ Id: "3be80475-e219-4ee7-b6fc-4b65114e362f", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL}, {Name: "label2", Pattern: "val.+", Type: pb.Matcher_REGEXP}, }, StartsAt: now, EndsAt: now, UpdatedAt: now, }, ExpiresAt: now, }, { Silence: &pb.Silence{ Id: "4b1e760d-182c-4980-b873-c1a6827c9817", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL}, }, StartsAt: now.Add(time.Hour), EndsAt: now.Add(2 time.Hour), UpdatedAt: now, }, ExpiresAt: now.Add(24 * time.Hour), }, { Silence: &pb.Silence{ Id: "3dfb2528-59ce-41eb-b465-f875a4e744a4", Matchers: []pb.Matcher{ {Name: "label1", Pattern: "val1", Type: pb.Matcher_NOT_EQUAL}, {Name: "label2", Pattern: "val.+", Type: pb.Matcher_NOT_REGEXP}, }, StartsAt: now, EndsAt: now, UpdatedAt: now, }, ExpiresAt: now, }, }, }, } for _, c := range cases { // Create gossip data from input. in := state{} for _, e := range c.entries { in[e.Silence.Id] = e } msg, err := in.MarshalBinary() require.NoError(t, err) out, err := decodeState(bytes.NewReader(msg)) require.NoError(t, err, "decoding message failed") require.Equal(t, in, out, "decoded data doesn't match encoded data") } } func TestStateDecodingError(t testing.T) { // Check whether decoding copes with erroneous data. s := state{"": &pb.MeshSilence{}} msg, err := s.MarshalBinary() require.NoError(t, err) _, err = decodeState(bytes.NewReader(msg)) require.Equal(t, ErrInvalidState, err) } // runtime.Gosched() does not "suspend" the current goroutine so there's no guarantee that the main goroutine won't // be able to continue. For more see https://pkg.go.dev/runtime#Gosched. func gosched() { time.Sleep(1 * time.Millisecond) }	48.22 KB	2025-05-25 12:27
📁 silencepb	-	2025-05-25 12:27

Terminal