Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Limits should include expired silences #3862

Merged
merged 2 commits into from
Jun 3, 2024
Merged

Limits should include expired silences #3862

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0f9fe2
Limits should include expired silences
grobinson-grafana Jun 3, 2024
01938d1
Fix docs
grobinson-grafana Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/alertmanager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ func run() int {
dataDir = kingpin.Flag("storage.path", "Base path for data storage.").Default("data/").String()
retention = kingpin.Flag("data.retention", "How long to keep data for.").Default("120h").Duration()
maintenanceInterval = kingpin.Flag("data.maintenance-interval", "Interval between garbage collection and snapshotting to disk of the silences and the notification logs.").Default("15m").Duration()
maxSilences = kingpin.Flag("silences.max-silences", "Maximum number of active and pending silences, excluding expired silences. If negative or zero, no limit is set.").Default("0").Int()
maxSilences = kingpin.Flag("silences.max-silences", "Maximum number of silences, including expired silences. If negative or zero, no limit is set.").Default("0").Int()
maxPerSilenceBytes = kingpin.Flag("silences.max-per-silence-bytes", "Maximum per silence size in bytes. If negative or zero, no limit is set.").Default("0").Int()
alertGCInterval = kingpin.Flag("alerts.gc-interval", "Interval between alert GC.").Default("30m").Duration()

Expand Down
18 changes: 4 additions & 14 deletions silence/silence.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,8 +204,8 @@ type Silences struct {

// Limits contains the limits for silences.
type Limits struct {
// MaxSilences limits the maximum number active and pending silences.
// It does not include expired silences.
// MaxSilences limits the maximum number of silences, including expired
// silences.
MaxSilences int
// MaxPerSilenceBytes is the maximum size of an individual silence as
// stored on disk.
Expand Down Expand Up @@ -623,18 +623,8 @@ func (s *Silences) Set(sil *pb.Silence) (string, error) {

// If we got here it's either a new silence or a replacing one.
if s.limits.MaxSilences > 0 {
// Get the number of active and pending silences to enforce limits.
q := &query{}
err := QState(types.SilenceStateActive, types.SilenceStatePending)(q)
if err != nil {
return "", fmt.Errorf("unable to query silences while checking limits: %w", err)
}
sils, _, err := s.query(q, s.nowUTC())
if err != nil {
return "", fmt.Errorf("unable to query silences while checking limits: %w", err)
}
if len(sils)+1 > s.limits.MaxSilences {
return "", fmt.Errorf("exceeded maximum number of silences: %d (limit: %d)", len(sils), s.limits.MaxSilences)
if len(s.st)+1 > s.limits.MaxSilences {
return "", fmt.Errorf("exceeded maximum number of silences: %d (limit: %d)", len(s.st), s.limits.MaxSilences)
}
}

Expand Down
9 changes: 8 additions & 1 deletion silence/silence_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,8 +465,12 @@ func TestSilenceLimits(t *testing.T) {
MaxSilences: 1,
MaxPerSilenceBytes: 2 << 11, // 4KB
},
Retention: 100 * time.Millisecond,
})
require.NoError(t, err)
stopCh := make(chan struct{})
defer close(stopCh)
go s.Maintenance(100*time.Millisecond, "", stopCh, nil)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We cannot force a maintenance without running the goroutine, hence the calls to time.Sleep. I think this is fine, and I can improve the tests later.


// Insert sil1 should succeed without error.
sil1 := &pb.Silence{
Expand All @@ -489,8 +493,10 @@ func TestSilenceLimits(t *testing.T) {
require.EqualError(t, err, "exceeded maximum number of silences: 1 (limit: 1)")
require.Equal(t, "", id2)

// Expire sil1. This should allow sil2 to be inserted.
// Expire sil1 and wait for the maintenance to finish.
// This should allow sil2 to be inserted.
require.NoError(t, s.Expire(id1))
time.Sleep(150 * time.Millisecond)
id2, err = s.Set(sil2)
require.NoError(t, err)
require.NotEqual(t, "", id2)
Expand All @@ -501,6 +507,7 @@ func TestSilenceLimits(t *testing.T) {

// Expire sil2.
require.NoError(t, s.Expire(id2))
time.Sleep(150 * time.Millisecond)

// Insert sil3 should fail because it exceeds maximum size.
sil3 := &pb.Silence{
Expand Down
Loading