Ruler: Add alert source template

CHANGELOG.md

@@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 ### Added
 
 - [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified.
+- [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message.
 
 ### Changed
 

cmd/thanos/config.go

@@ -220,6 +220,7 @@ type alertMgrConfig struct {
 	alertExcludeLabels     []string
 	alertQueryURL          *string
 	alertRelabelConfigPath *extflag.PathOrContent
+	alertSourceTemplate    *string
 }
 
 func (ac *alertMgrConfig) registerFlag(cmd extflag.FlagClause) *alertMgrConfig {
@@ -234,5 +235,7 @@ func (ac *alertMgrConfig) registerFlag(cmd extflag.FlagClause) *alertMgrConfig {
 	cmd.Flag("alert.label-drop", "Labels by name to drop before sending to alertmanager. This allows alert to be deduplicated on replica label (repeated). Similar Prometheus alert relabelling").
 		StringsVar(&ac.alertExcludeLabels)
 	ac.alertRelabelConfigPath = extflag.RegisterPathOrContent(cmd, "alert.relabel-config", "YAML file that contains alert relabelling configuration.", extflag.WithEnvSubstitution())
+	ac.alertSourceTemplate = cmd.Flag("alert.query-template", "Template to use in alerts source field. Need only include {{.Expr}} parameter").Default("/graph?g0.expr={{.Expr}}&g0.tab=1").String()
+
 	return ac
 }

cmd/thanos/rule.go

@@ -4,7 +4,10 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"html/template"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -38,7 +41,6 @@ import (
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/agent"
 	"github.com/prometheus/prometheus/tsdb/wlog"
-	"github.com/prometheus/prometheus/util/strutil"
 
 	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/objstore/client"
@@ -101,6 +103,10 @@ type ruleConfig struct {
 	storeRateLimits   store.SeriesSelectLimits
 }
 
+type Expression struct {
+	Expr string
+}
+
 func (rc *ruleConfig) registerFlag(cmd extkingpin.FlagClause) {
 	rc.http.registerFlag(cmd)
 	rc.grpc.registerFlag(cmd)
@@ -329,6 +335,10 @@ func runRule(
 		}
 	}
 
+	if err := validateTemplate(*conf.alertmgr.alertSourceTemplate); err != nil {
+		return errors.Wrap(err, "invalid alert source template")
+	}
+
 	queryProvider := dns.NewProvider(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_rule_query_apis_", reg),
@@ -492,11 +502,15 @@ func runRule(
 				if alrt.State == rules.StatePending {
 					continue
 				}
+				expressionURL, err := tableLinkForExpression(*conf.alertmgr.alertSourceTemplate, expr)
+				if err != nil {
+					level.Warn(logger).Log("msg", "failed to generate link for expression", "expr", expr, "err", err)
+				}
 				a := &notifier.Alert{
 					StartsAt:     alrt.FiredAt,
 					Labels:       alrt.Labels,
 					Annotations:  alrt.Annotations,
-					GeneratorURL: conf.alertQueryURL.String() + strutil.TableLinkForExpression(expr),
+					GeneratorURL: conf.alertQueryURL.String() + expressionURL,
 				}
 				if !alrt.ResolvedAt.IsZero() {
 					a.EndsAt = alrt.ResolvedAt
@@ -934,3 +948,33 @@ func reloadRules(logger log.Logger,
 	}
 	return errs.Err()
 }
+
+func tableLinkForExpression(tmpl string, expr string) (string, error) {
+	// template example: "/graph?g0.expr={{.Expr}}&g0.tab=1"
+	escapedExpression := url.QueryEscape(expr)
+
+	escapedExpr := Expression{Expr: escapedExpression}
+	t, err := template.New("url").Parse(tmpl)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse template")
+	}
+
+	var buf bytes.Buffer
+	if err := t.Execute(&buf, escapedExpr); err != nil {
+		return "", errors.Wrap(err, "failed to execute template")
+	}
+	return buf.String(), nil
+}
+
+func validateTemplate(tmplStr string) error {
+	tmpl, err := template.New("test").Parse(tmplStr)
+	if err != nil {
+		return fmt.Errorf("failed to parse the template: %w", err)
+	}
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, Expression{Expr: "test_expr"})
+	if err != nil {
+		return fmt.Errorf("failed to execute the template: %w", err)
+	}
+	return nil
+}

cmd/thanos/rule_test.go

@@ -48,3 +48,59 @@ func Test_parseFlagLabels(t *testing.T) {
 		testutil.Equals(t, err != nil, td.expectErr)
 	}
 }
+
+func Test_validateTemplate(t *testing.T) {
+	tData := []struct {
+		template  string
+		expectErr bool
+	}{
+		{
+			template:  `/graph?g0.expr={{.Expr}}&g0.tab=1`,
+			expectErr: false,
+		},
+		{
+			template:  `/graph?g0.expr={{.Expression}}&g0.tab=1`,
+			expectErr: true,
+		},
+		{
+			template:  `another template includes {{.Expr}}`,
+			expectErr: false,
+		},
+	}
+	for _, td := range tData {
+		err := validateTemplate(td.template)
+		testutil.Equals(t, err != nil, td.expectErr)
+	}
+}
+
+func Test_tableLinkForExpression(t *testing.T) {
+	tData := []struct {
+		template  string
+		expr      string
+		expectStr string
+		expectErr bool
+	}{
+		{
+			template:  `/graph?g0.expr={{.Expr}}&g0.tab=1`,
+			expr:      `up{app="foo"}`,
+			expectStr: `/graph?g0.expr=up%7Bapp%3D%22foo%22%7D&g0.tab=1`,
+			expectErr: false,
+		},
+		{
+			template:  `/graph?g0.expr={{.Expression}}&g0.tab=1`,
+			expr:      "test_expr",
+			expectErr: true,
+		},
+		{
+			template:  `another template includes {{.Expr}}`,
+			expr:      "test_expr",
+			expectStr: `another template includes test_expr`,
+			expectErr: false,
+		},
+	}
+	for _, td := range tData {
+		resStr, err := tableLinkForExpression(td.template, td.expr)
+		testutil.Equals(t, err != nil, td.expectErr)
+		testutil.Equals(t, resStr, td.expectStr)
+	}
+}

docs/components/rule.md

@@ -267,6 +267,9 @@ Flags:
                                  to alertmanager. This allows alert to be
                                  deduplicated on replica label (repeated).
                                  Similar Prometheus alert relabelling
+      --alert.query-template="/graph?g0.expr={{.Expr}}&g0.tab=1"
+                                 Template to use in alerts source field.
+                                 Need only include {{.Expr}} parameter
       --alert.query-url=ALERT.QUERY-URL
                                  The external Thanos Query URL that would be set
                                  in all alerts 'Source' field