diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 640b22abe..104471e3d 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -107,7 +107,7 @@ func NewDispatcher( ap provider.Alerts, r *Route, s notify.Stage, - mk types.Marker, + mk types.AlertMarker, to func(time.Duration) time.Duration, lim Limits, l log.Logger, diff --git a/inhibit/inhibit.go b/inhibit/inhibit.go index a8f96f28a..42e20242e 100644 --- a/inhibit/inhibit.go +++ b/inhibit/inhibit.go @@ -36,7 +36,7 @@ import ( type Inhibitor struct { alerts provider.Alerts rules []*InhibitRule - marker types.Marker + marker types.AlertMarker logger log.Logger mtx sync.RWMutex @@ -44,7 +44,7 @@ type Inhibitor struct { } // NewInhibitor returns a new Inhibitor. -func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.Marker, logger log.Logger) *Inhibitor { +func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger log.Logger) *Inhibitor { ih := &Inhibitor{ alerts: ap, marker: mk, diff --git a/provider/mem/mem.go b/provider/mem/mem.go index 23bde9ba7..cfc3bfc36 100644 --- a/provider/mem/mem.go +++ b/provider/mem/mem.go @@ -35,7 +35,7 @@ type Alerts struct { cancel context.CancelFunc alerts *store.Alerts - marker types.Marker + marker types.AlertMarker mtx sync.Mutex listeners map[int]listeningAlerts @@ -85,7 +85,7 @@ func (a *Alerts) registerMetrics(r prometheus.Registerer) { } // NewAlerts returns a new alert provider. -func NewAlerts(ctx context.Context, m types.Marker, intervalGC time.Duration, alertCallback AlertStoreCallback, l log.Logger, r prometheus.Registerer) (*Alerts, error) { +func NewAlerts(ctx context.Context, m types.AlertMarker, intervalGC time.Duration, alertCallback AlertStoreCallback, l log.Logger, r prometheus.Registerer) (*Alerts, error) { if alertCallback == nil { alertCallback = noopCallback{} } diff --git a/silence/silence.go b/silence/silence.go index c87ab76e4..bab409f7a 100644 --- a/silence/silence.go +++ b/silence/silence.go @@ -92,16 +92,16 @@ func (c matcherCache) add(s *pb.Silence) (labels.Matchers, error) { return ms, nil } -// Silencer binds together a Marker and a Silences to implement the Muter +// Silencer binds together a AlertMarker and a Silences to implement the Muter // interface. type Silencer struct { silences *Silences - marker types.Marker + marker types.AlertMarker logger log.Logger } // NewSilencer returns a new Silencer. -func NewSilencer(s *Silences, m types.Marker, l log.Logger) *Silencer { +func NewSilencer(s *Silences, m types.AlertMarker, l log.Logger) *Silencer { return &Silencer{ silences: s, marker: m, diff --git a/types/types.go b/types/types.go index 6dbc01750..648f0a7e0 100644 --- a/types/types.go +++ b/types/types.go @@ -52,9 +52,17 @@ type AlertStatus struct { silencesVersion int } -// Marker helps to mark alerts as silenced and/or inhibited. +// groupStatus stores the state of the group, and, as applicable, the names +// of all active and mute time intervals that are muting it. +type groupStatus struct { + // mutedBy contains the names of all active and mute time intervals that + // are muting it. + mutedBy []string +} + +// AlertMarker helps to mark alerts as silenced and/or inhibited. // All methods are goroutine-safe. -type Marker interface { +type AlertMarker interface { // SetActiveOrSilenced replaces the previous SilencedBy by the provided IDs of // active and pending silences, including the version number of the // silences state. The set of provided IDs is supposed to represent the @@ -92,24 +100,65 @@ type Marker interface { Inhibited(model.Fingerprint) ([]string, bool) } -// NewMarker returns an instance of a Marker implementation. -func NewMarker(r prometheus.Registerer) Marker { - m := &memMarker{ - m: map[model.Fingerprint]*AlertStatus{}, +// GroupMarker helps to mark groups as active or muted. +// All methods are goroutine-safe. +// +// TODO(grobinson): routeID is used in Muted and SetMuted because groupKey +// is not unique (see #3817). Once groupKey uniqueness is fixed routeID can +// be removed from the GroupMarker interface. +type GroupMarker interface { + // Muted returns true if the group is muted, otherwise false. If the group + // is muted then it also returns the names of the time intervals that muted + // it. + Muted(routeID, groupKey string) ([]string, bool) + + // SetMuted marks the group as muted, and sets the names of the time + // intervals that mute it. If the list of names is nil or the empty slice + // then the muted marker is removed. + SetMuted(routeID, groupKey string, timeIntervalNames []string) +} + +// NewMarker returns an instance of a AlertMarker implementation. +func NewMarker(r prometheus.Registerer) *MemMarker { + m := &MemMarker{ + alerts: map[model.Fingerprint]*AlertStatus{}, + groups: map[string]*groupStatus{}, } - m.registerMetrics(r) - return m } -type memMarker struct { - m map[model.Fingerprint]*AlertStatus +type MemMarker struct { + alerts map[model.Fingerprint]*AlertStatus + groups map[string]*groupStatus mtx sync.RWMutex } -func (m *memMarker) registerMetrics(r prometheus.Registerer) { +// Muted implements GroupMarker. +func (m *MemMarker) Muted(routeID, groupKey string) ([]string, bool) { + m.mtx.Lock() + defer m.mtx.Unlock() + status, ok := m.groups[routeID+groupKey] + if !ok { + return nil, false + } + return status.mutedBy, len(status.mutedBy) > 0 +} + +// SetMuted implements GroupMarker. +func (m *MemMarker) SetMuted(routeID, groupKey string, timeIntervalNames []string) { + m.mtx.Lock() + defer m.mtx.Unlock() + status, ok := m.groups[routeID+groupKey] + if !ok { + status = &groupStatus{} + m.groups[routeID+groupKey] = status + } + status.mutedBy = timeIntervalNames +} + +func (m *MemMarker) registerMetrics(r prometheus.Registerer) { newMarkedAlertMetricByState := func(st AlertState) prometheus.GaugeFunc { return prometheus.NewGaugeFunc( prometheus.GaugeOpts{ @@ -132,17 +181,17 @@ func (m *memMarker) registerMetrics(r prometheus.Registerer) { r.MustRegister(alertStateUnprocessed) } -// Count implements Marker. -func (m *memMarker) Count(states ...AlertState) int { +// Count implements AlertMarker. +func (m *MemMarker) Count(states ...AlertState) int { m.mtx.RLock() defer m.mtx.RUnlock() if len(states) == 0 { - return len(m.m) + return len(m.alerts) } var count int - for _, status := range m.m { + for _, status := range m.alerts { for _, state := range states { if status.State == state { count++ @@ -152,15 +201,15 @@ func (m *memMarker) Count(states ...AlertState) int { return count } -// SetActiveOrSilenced implements Marker. -func (m *memMarker) SetActiveOrSilenced(alert model.Fingerprint, version int, activeIDs, pendingIDs []string) { +// SetActiveOrSilenced implements AlertMarker. +func (m *MemMarker) SetActiveOrSilenced(alert model.Fingerprint, version int, activeIDs, pendingIDs []string) { m.mtx.Lock() defer m.mtx.Unlock() - s, found := m.m[alert] + s, found := m.alerts[alert] if !found { s = &AlertStatus{} - m.m[alert] = s + m.alerts[alert] = s } s.SilencedBy = activeIDs s.pendingSilences = pendingIDs @@ -177,15 +226,15 @@ func (m *memMarker) SetActiveOrSilenced(alert model.Fingerprint, version int, ac s.State = AlertStateSuppressed } -// SetInhibited implements Marker. -func (m *memMarker) SetInhibited(alert model.Fingerprint, ids ...string) { +// SetInhibited implements AlertMarker. +func (m *MemMarker) SetInhibited(alert model.Fingerprint, ids ...string) { m.mtx.Lock() defer m.mtx.Unlock() - s, found := m.m[alert] + s, found := m.alerts[alert] if !found { s = &AlertStatus{} - m.m[alert] = s + m.alerts[alert] = s } s.InhibitedBy = ids @@ -200,12 +249,12 @@ func (m *memMarker) SetInhibited(alert model.Fingerprint, ids ...string) { s.State = AlertStateSuppressed } -// Status implements Marker. -func (m *memMarker) Status(alert model.Fingerprint) AlertStatus { +// Status implements AlertMarker. +func (m *MemMarker) Status(alert model.Fingerprint) AlertStatus { m.mtx.RLock() defer m.mtx.RUnlock() - if s, found := m.m[alert]; found { + if s, found := m.alerts[alert]; found { return *s } return AlertStatus{ @@ -215,26 +264,26 @@ func (m *memMarker) Status(alert model.Fingerprint) AlertStatus { } } -// Delete implements Marker. -func (m *memMarker) Delete(alert model.Fingerprint) { +// Delete implements AlertMarker. +func (m *MemMarker) Delete(alert model.Fingerprint) { m.mtx.Lock() defer m.mtx.Unlock() - delete(m.m, alert) + delete(m.alerts, alert) } -// Unprocessed implements Marker. -func (m *memMarker) Unprocessed(alert model.Fingerprint) bool { +// Unprocessed implements AlertMarker. +func (m *MemMarker) Unprocessed(alert model.Fingerprint) bool { return m.Status(alert).State == AlertStateUnprocessed } -// Active implements Marker. -func (m *memMarker) Active(alert model.Fingerprint) bool { +// Active implements AlertMarker. +func (m *MemMarker) Active(alert model.Fingerprint) bool { return m.Status(alert).State == AlertStateActive } -// Inhibited implements Marker. -func (m *memMarker) Inhibited(alert model.Fingerprint) ([]string, bool) { +// Inhibited implements AlertMarker. +func (m *MemMarker) Inhibited(alert model.Fingerprint) ([]string, bool) { s := m.Status(alert) return s.InhibitedBy, s.State == AlertStateSuppressed && len(s.InhibitedBy) > 0 @@ -243,7 +292,7 @@ func (m *memMarker) Inhibited(alert model.Fingerprint) ([]string, bool) { // Silenced returns whether the alert for the given Fingerprint is in the // Silenced state, any associated silence IDs, and the silences state version // the result is based on. -func (m *memMarker) Silenced(alert model.Fingerprint) (activeIDs, pendingIDs []string, version int, silenced bool) { +func (m *MemMarker) Silenced(alert model.Fingerprint) (activeIDs, pendingIDs []string, version int, silenced bool) { s := m.Status(alert) return s.SilencedBy, s.pendingSilences, s.silencesVersion, s.State == AlertStateSuppressed && len(s.SilencedBy) > 0 @@ -410,7 +459,7 @@ func (a *Alert) Merge(o *Alert) *Alert { } // A Muter determines whether a given label set is muted. Implementers that -// maintain an underlying Marker are expected to update it during a call of +// maintain an underlying AlertMarker are expected to update it during a call of // Mutes. type Muter interface { Mutes(model.LabelSet) bool diff --git a/types/types_test.go b/types/types_test.go index ece6fb5c5..45bc59e2f 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -29,6 +29,38 @@ import ( "github.com/prometheus/alertmanager/matchers/compat" ) +func TestMemMarker_Muted(t *testing.T) { + r := prometheus.NewRegistry() + marker := NewMarker(r) + + // No groups should be muted. + timeIntervalNames, isMuted := marker.Muted("route1", "group1") + require.False(t, isMuted) + require.Empty(t, timeIntervalNames) + + // Mark the group as muted because it's the weekend. + marker.SetMuted("route1", "group1", []string{"weekends"}) + timeIntervalNames, isMuted = marker.Muted("route1", "group1") + require.True(t, isMuted) + require.Equal(t, []string{"weekends"}, timeIntervalNames) + + // Other groups should not be marked as muted. + timeIntervalNames, isMuted = marker.Muted("route1", "group2") + require.False(t, isMuted) + require.Empty(t, timeIntervalNames) + + // Other routes should not be marked as muted either. + timeIntervalNames, isMuted = marker.Muted("route2", "group1") + require.False(t, isMuted) + require.Empty(t, timeIntervalNames) + + // The group is no longer muted. + marker.SetMuted("route1", "group1", nil) + timeIntervalNames, isMuted = marker.Muted("route1", "group1") + require.False(t, isMuted) + require.Empty(t, timeIntervalNames) +} + func TestMemMarker_Count(t *testing.T) { r := prometheus.NewRegistry() marker := NewMarker(r)