LogQL: add __line__ function to | line_format template

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+* [4879](https://github.com/grafana/loki/pull/4879) **cyriltovena**: LogQL: add __line__ function to | line_format template.
 * [4860](https://github.com/grafana/loki/pull/4860) **cyriltovena**: Add rate limiting and metrics to hedging
 * [4865](https://github.com/grafana/loki/pull/4865) **taisho6339**: Fix duplicate registry.MustRegister call in Promtail Kafka
 * [4845](https://github.com/grafana/loki/pull/4845) **chaudum** Return error responses consistently as JSON

docs/sources/logql/template_functions.md

@@ -5,7 +5,6 @@ weight: 30
 
 # Template functions
 
-
 The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support the usage of functions.
 
 All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`). For example the following template will output the value of the path label:
@@ -14,6 +13,8 @@ All labels are added as variables in the template engine. They can be referenced
 {{ .path }}
 ```
 
+Additionally you can also access the log line using the [`__line__`](#__line__) function.
+
 You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to join together multiple functions.
 In a chained pipeline, the result of each command is passed as the last argument of the following command.
 
@@ -23,6 +24,23 @@ Example:
 {{ .path | replace " " "_" | trunc 5 | upper }}
 ```
 
+## __line__
+
+This function return the current log line.
+
+> Added in Loki 2.5
+Signature:
+
+`line() string`
+
+Examples:
+
+```template
+"{{ __line__ | lower }}"
+`{{ __line__ }}`
+```
+
+
 ## ToLower and ToUpper
 
 This function converts the entire string to lowercase or uppercase.
@@ -631,7 +649,7 @@ Example of a query to print a newline per queries stored as a json array in the
 
 ## date
 
-`date` returns a textual representation of the time value formatted according to the provided [golang datetime layout](https://pkg.go.dev/time#pkg-constants). 
+`date` returns a textual representation of the time value formatted according to the provided [golang datetime layout](https://pkg.go.dev/time#pkg-constants).
 
 ```template
 { date "2006-01-02" now }}

pkg/logql/log/fmt.go

@@ -13,6 +13,10 @@ import (
 	"github.com/grafana/loki/pkg/logqlmodel"
 )
 
+const (
+	functionLineName = "__line__"
+)
+
 var (
 	_ Stage = &LineFormatter{}
 	_ Stage = &LabelsFormatter{}
@@ -95,22 +99,29 @@ func init() {
 type LineFormatter struct {
 	*template.Template
 	buf *bytes.Buffer
+
+	currentLine []byte
 }
 
 // NewFormatter creates a new log line formatter from a given text template.
 func NewFormatter(tmpl string) (*LineFormatter, error) {
+	lf := &LineFormatter{
+		buf: bytes.NewBuffer(make([]byte, 4096)),
+	}
+	functionMap[functionLineName] = func() string {
+		return unsafeGetString(lf.currentLine)
+	}
 	t, err := template.New("line").Option("missingkey=zero").Funcs(functionMap).Parse(tmpl)
 	if err != nil {
 		return nil, fmt.Errorf("invalid line template: %w", err)
 	}
-	return &LineFormatter{
-		Template: t,
-		buf:      bytes.NewBuffer(make([]byte, 4096)),
-	}, nil
+	lf.Template = t
+	return lf, nil
 }
 
 func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) {
 	lf.buf.Reset()
+	lf.currentLine = line
 
 	if err := lf.Template.Execute(lf.buf, lbs.Labels().Map()); err != nil {
 		lbs.SetErr(errTemplateFormat)

pkg/logql/log/fmt_test.go

@@ -19,34 +19,39 @@ func Test_lineFormatter_Format(t *testing.T) {
 
 		want    []byte
 		wantLbs labels.Labels
+		in      []byte
 	}{
 		{
 			"combining",
 			newMustLineFormatter("foo{{.foo}}buzz{{  .bar  }}"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("fooblipbuzzblop"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"Replace",
 			newMustLineFormatter(`foo{{.foo}}buzz{{ Replace .bar "blop" "bar" -1 }}`),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("fooblipbuzzbar"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"replace",
 			newMustLineFormatter(`foo{{.foo}}buzz{{ .bar | replace "blop" "bar" }}`),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("fooblipbuzzbar"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"title",
 			newMustLineFormatter(`{{.foo | title }}`),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("Blip"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"substr and trunc",
@@ -56,6 +61,7 @@ func Test_lineFormatter_Format(t *testing.T) {
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("li b blo"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"trim",
@@ -65,167 +71,199 @@ func Test_lineFormatter_Format(t *testing.T) {
 			labels.Labels{{Name: "foo", Value: "  blip "}, {Name: "bar", Value: "blop"}},
 			[]byte("blip bl lop blo"),
 			labels.Labels{{Name: "foo", Value: "  blip "}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"lower and upper",
 			newMustLineFormatter(`{{.foo | lower }} {{ .bar  | upper }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("blip BLOP"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"repeat",
 			newMustLineFormatter(`{{ "foo" | repeat 3 }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("foofoofoo"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"indent",
 			newMustLineFormatter(`{{ "foo\n bar" | indent 4 }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("    foo\n     bar"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"nindent",
 			newMustLineFormatter(`{{ "foo" | nindent 2 }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("\n  foo"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"contains",
 			newMustLineFormatter(`{{ if  .foo | contains "p"}}yes{{end}}-{{ if  .foo | contains "z"}}no{{end}}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("yes-"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"hasPrefix",
 			newMustLineFormatter(`{{ if  .foo | hasPrefix "BL" }}yes{{end}}-{{ if  .foo | hasPrefix "p"}}no{{end}}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("yes-"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"hasSuffix",
 			newMustLineFormatter(`{{ if  .foo | hasSuffix "Ip" }}yes{{end}}-{{ if  .foo | hasSuffix "pw"}}no{{end}}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("yes-"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"regexReplaceAll",
 			newMustLineFormatter(`{{ regexReplaceAll "(p)" .foo "t" }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("BLIt"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"regexReplaceAllLiteral",
 			newMustLineFormatter(`{{ regexReplaceAllLiteral "(p)" .foo "${1}" }}`),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
 			[]byte("BLI${1}"),
 			labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"err",
 			newMustLineFormatter(`{{.foo Replace "foo"}}`),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			nil,
 			labels.Labels{{Name: logqlmodel.ErrorLabel, Value: errTemplateFormat}, {Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"missing",
 			newMustLineFormatter("foo {{.foo}}buzz{{  .bar  }}"),
 			labels.Labels{{Name: "bar", Value: "blop"}},
 			[]byte("foo buzzblop"),
 			labels.Labels{{Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"function",
 			newMustLineFormatter("foo {{.foo | ToUpper }} buzz{{  .bar  }}"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
 			[]byte("foo BLIP buzzblop"),
 			labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
+			nil,
 		},
 		{
 			"mathint",
 			newMustLineFormatter("{{ add .foo 1 | sub .bar | mul .baz | div .bazz}}"),
 			labels.Labels{{Name: "foo", Value: "1"}, {Name: "bar", Value: "3"}, {Name: "baz", Value: "10"}, {Name: "bazz", Value: "20"}},
 			[]byte("2"),
 			labels.Labels{{Name: "foo", Value: "1"}, {Name: "bar", Value: "3"}, {Name: "baz", Value: "10"}, {Name: "bazz", Value: "20"}},
+			nil,
 		},
 		{
 			"mathfloat",
 			newMustLineFormatter("{{ addf .foo 1.5 | subf .bar 1.5 | mulf .baz | divf .bazz }}"),
 			labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.2"}},
 			[]byte("3.8476190476190477"),
 			labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.2"}},
+			nil,
 		},
 		{
 			"mathfloatround",
 			newMustLineFormatter("{{ round (addf .foo 1.5 | subf .bar | mulf .baz | divf .bazz) 5 .2}}"),
 			labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "3.5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.4"}},
 			[]byte("3.88572"),
 			labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "3.5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.4"}},
+			nil,
 		},
 		{
 			"min",
 			newMustLineFormatter("min is {{ min .foo .bar .baz }} and max is {{ max .foo .bar .baz }}"),
 			labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "10"}, {Name: "baz", Value: "15"}},
 			[]byte("min is 5 and max is 15"),
 			labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "10"}, {Name: "baz", Value: "15"}},
+			nil,
 		},
 		{
 			"max",
 			newMustLineFormatter("minf is {{ minf .foo .bar .baz }} and maxf is {{maxf .foo .bar .baz}}"),
 			labels.Labels{{Name: "foo", Value: "5.3"}, {Name: "bar", Value: "10.5"}, {Name: "baz", Value: "15.2"}},
 			[]byte("minf is 5.3 and maxf is 15.2"),
 			labels.Labels{{Name: "foo", Value: "5.3"}, {Name: "bar", Value: "10.5"}, {Name: "baz", Value: "15.2"}},
+			nil,
 		},
 		{
 			"ceilfloor",
 			newMustLineFormatter("ceil is {{ ceil .foo }} and floor is {{floor .foo }}"),
 			labels.Labels{{Name: "foo", Value: "5.3"}},
 			[]byte("ceil is 6 and floor is 5"),
 			labels.Labels{{Name: "foo", Value: "5.3"}},
+			nil,
 		},
 		{
 			"mod",
 			newMustLineFormatter("mod is {{ mod .foo 3 }}"),
 			labels.Labels{{Name: "foo", Value: "20"}},
 			[]byte("mod is 2"),
 			labels.Labels{{Name: "foo", Value: "20"}},
+			nil,
 		},
 		{
 			"float64int",
 			newMustLineFormatter("{{ \"2.5\" | float64 | int | add 10}}"),
 			labels.Labels{{Name: "foo", Value: "2.5"}},
 			[]byte("12"),
 			labels.Labels{{Name: "foo", Value: "2.5"}},
+			nil,
 		},
 		{
 			"datetime",
 			newMustLineFormatter("{{ sub (unixEpoch (toDate \"2006-01-02\" \"2021-11-02\")) (unixEpoch (toDate \"2006-01-02\" \"2021-11-01\")) }}"),
 			labels.Labels{},
 			[]byte("86400"),
 			labels.Labels{},
+			nil,
 		},
 		{
 			"dateformat",
 			newMustLineFormatter("{{ date \"2006-01-02\" (toDate \"2006-01-02\" \"2021-11-02\") }}"),
 			labels.Labels{},
 			[]byte("2021-11-02"),
 			labels.Labels{},
+			nil,
 		},
 		{
 			"now",
 			newMustLineFormatter("{{ div (unixEpoch now) (unixEpoch now) }}"),
 			labels.Labels{},
 			[]byte("1"),
 			labels.Labels{},
+			nil,
+		},
+		{
+			"line",
+			newMustLineFormatter("{{ __line__ }} bar {{ .bar }}"),
+			labels.Labels{{Name: "bar", Value: "2"}},
+			[]byte("1 bar 2"),
+			labels.Labels{{Name: "bar", Value: "2"}},
+			[]byte("1"),
 		},
 	}
 	for _, tt := range tests {
@@ -234,7 +272,7 @@ func Test_lineFormatter_Format(t *testing.T) {
 			sort.Sort(tt.wantLbs)
 			builder := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash())
 			builder.Reset()
-			outLine, _ := tt.fmter.Process(nil, builder)
+			outLine, _ := tt.fmter.Process(tt.in, builder)
 			require.Equal(t, tt.want, outLine)
 			require.Equal(t, tt.wantLbs, builder.Labels())
 		})