Port promtail changes part 1 (grafana#6559)

* Port promtail changes part 1

* changelog

CHANGELOG.md

@@ -33,6 +33,8 @@ v0.40.1 (2024-02-27)
 
 - Fix an issue where flow mode panics if the `logging` config block is given a `null` Loki receiver to write log entries to. (@rfratto)
 
+- Fix an issue where Loki could reject a batch of logs when structured metadata feature is used. (@thampiotr) 
+
 v0.40.0 (2024-02-27)
 --------------------
 

internal/component/common/loki/client/batch.go

@@ -61,7 +61,7 @@ func newBatch(maxStreams int, entries ...loki.Entry) *batch {
 
 // add an entry to the batch
 func (b *batch) add(entry loki.Entry) error {
-	b.totalBytes += len(entry.Line)
+	b.totalBytes += entrySize(entry.Entry)
 
 	// Append the entry to an already existing stream (if any)
 	labels := labelsMapToString(entry.Labels, ReservedLabelTenantID)
@@ -150,8 +150,8 @@ func (b *batch) sizeBytes() int {
 
 // sizeBytesAfter returns the size of the batch after the input entry
 // will be added to the batch itself
-func (b *batch) sizeBytesAfter(line string) int {
-	return b.totalBytes + len(line)
+func (b *batch) sizeBytesAfter(entry logproto.Entry) int {
+	return b.totalBytes + entrySize(entry)
 }
 
 // age of the batch since its creation
@@ -201,3 +201,11 @@ func (b *batch) reportAsSentData(h SentDataMarkerHandler) {
 		h.UpdateSentData(seg, data)
 	}
 }
+
+func entrySize(entry logproto.Entry) int {
+	structuredMetadataSize := 0
+	for _, label := range entry.StructuredMetadata {
+		structuredMetadataSize += label.Size()
+	}
+	return len(entry.Line) + structuredMetadataSize
+}

internal/component/common/loki/client/batch_test.go

@@ -57,8 +57,9 @@ func TestBatch_add(t *testing.T) {
 			inputEntries: []loki.Entry{
 				{Labels: model.LabelSet{}, Entry: logEntries[0].Entry},
 				{Labels: model.LabelSet{}, Entry: logEntries[1].Entry},
+				{Labels: model.LabelSet{}, Entry: logEntries[7].Entry},
 			},
-			expectedSizeBytes: len(logEntries[0].Entry.Line) + len(logEntries[1].Entry.Line),
+			expectedSizeBytes: entrySize(logEntries[0].Entry) + entrySize(logEntries[0].Entry) + entrySize(logEntries[7].Entry),
 		},
 		"multiple streams with multiple log entries": {
 			inputEntries: []loki.Entry{

internal/component/common/loki/client/client.go

@@ -308,7 +308,7 @@ func (c *client) run() {
 
 			// If adding the entry to the batch will increase the size over the max
 			// size allowed, we do send the current batch and then create a new one
-			if batch.sizeBytesAfter(e.Line) > c.cfg.BatchSize {
+			if batch.sizeBytesAfter(e.Entry) > c.cfg.BatchSize {
 				c.sendBatch(tenantID, batch)
 
 				batches[tenantID] = newBatch(c.maxStreams, e)

internal/component/common/loki/client/client_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/loki/pkg/push"
+
 	"github.com/go-kit/log"
 	"github.com/grafana/dskit/backoff"
 	"github.com/grafana/dskit/flagext"
@@ -34,6 +36,16 @@ var logEntries = []loki.Entry{
 	{Labels: model.LabelSet{"__tenant_id__": "tenant-1"}, Entry: logproto.Entry{Timestamp: time.Unix(5, 0).UTC(), Line: "line5"}},
 	{Labels: model.LabelSet{"__tenant_id__": "tenant-2"}, Entry: logproto.Entry{Timestamp: time.Unix(6, 0).UTC(), Line: "line6"}},
 	{Labels: model.LabelSet{}, Entry: logproto.Entry{Timestamp: time.Unix(6, 0).UTC(), Line: "line0123456789"}},
+	{
+		Labels: model.LabelSet{},
+		Entry: logproto.Entry{
+			Timestamp: time.Unix(7, 0).UTC(),
+			Line:      "line7",
+			StructuredMetadata: push.LabelsAdapter{
+				{Name: "trace_id", Value: "12345"},
+			},
+		},
+	},
 }
 
 func TestClient_Handle(t *testing.T) {

internal/component/common/loki/client/queue_client.go

@@ -344,7 +344,7 @@ func (c *queueClient) appendSingleEntry(segmentNum int, lbs model.LabelSet, e lo
 
 	// If adding the entry to the batch will increase the size over the max
 	// size allowed, we do send the current batch and then create a new one
-	if batch.sizeBytesAfter(e.Line) > c.cfg.BatchSize {
+	if batch.sizeBytesAfter(e) > c.cfg.BatchSize {
 		c.sendQueue.enqueue(queuedBatch{
 			TenantID: tenantID,
 			Batch:    batch,

web/ui/build/asset-manifest.json

@@ -0,0 +1,26 @@
+{
+  "files": {
+    "main.css": "./public/static/css/main.d3ff49f0.css",
+    "main.js": "./public/static/js/main.180d839c.js",
+    "static/media/fira-code-v21-latin-500.woff2": "./public/static/media/fira-code-v21-latin-500.3452777c99809dd85b2a.woff2",
+    "static/media/fira-code-v21-latin-regular.woff2": "./public/static/media/fira-code-v21-latin-regular.e662d57ce51fba1f46e3.woff2",
+    "static/media/roboto-v30-latin-900italic.woff2": "./public/static/media/roboto-v30-latin-900italic.5b387ea565e67898ca3a.woff2",
+    "static/media/roboto-v30-latin-300italic.woff2": "./public/static/media/roboto-v30-latin-300italic.1128daa312ec555266d5.woff2",
+    "static/media/roboto-v30-latin-italic.woff2": "./public/static/media/roboto-v30-latin-italic.e10742dbb1d4a0864ba8.woff2",
+    "static/media/roboto-v30-latin-500italic.woff2": "./public/static/media/roboto-v30-latin-500italic.3a43b67e5bbdfb3ab0a6.woff2",
+    "static/media/roboto-v30-latin-100italic.woff2": "./public/static/media/roboto-v30-latin-100italic.d3935eb61aa8422388e2.woff2",
+    "static/media/roboto-v30-latin-700italic.woff2": "./public/static/media/roboto-v30-latin-700italic.d92a5d1451f249359639.woff2",
+    "static/media/roboto-v30-latin-500.woff2": "./public/static/media/roboto-v30-latin-500.f25d774ecfe0996f8eb5.woff2",
+    "static/media/roboto-v30-latin-700.woff2": "./public/static/media/roboto-v30-latin-700.227c93190fe7f82de3f8.woff2",
+    "static/media/roboto-v30-latin-100.woff2": "./public/static/media/roboto-v30-latin-100.30fb0679553d179c3343.woff2",
+    "static/media/roboto-v30-latin-900.woff2": "./public/static/media/roboto-v30-latin-900.2e8becfcae330421664b.woff2",
+    "static/media/roboto-v30-latin-regular.woff2": "./public/static/media/roboto-v30-latin-regular.b009a76ad6afe4ebd301.woff2",
+    "static/media/roboto-v30-latin-300.woff2": "./public/static/media/roboto-v30-latin-300.c48fb6765a9fcb00b330.woff2",
+    "static/media/logo.svg": "./public/static/media/logo.c578039fa12bc65fb40a9b7ee1391e66.svg",
+    "index.html": "./public/index.html"
+  },
+  "entrypoints": [
+    "static/css/main.d3ff49f0.css",
+    "static/js/main.180d839c.js"
+  ]
+}

web/ui/build/index.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><meta http-equiv="Content-Security-Policy" content="default-src 'self'; style-src 'self' 'unsafe-inline'"><base href="{{! .PublicURL !}}/" target="_top"/><meta charset="utf-8"/><link rel="icon" href="./public/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/><meta name="theme-color" content="#000000"/><link rel="manifest" href="./public/manifest.json"/><title>Grafana Agent</title><script defer="defer" src="./public/static/js/main.180d839c.js"></script><link href="./public/static/css/main.d3ff49f0.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

web/ui/build/manifest.json

@@ -0,0 +1,15 @@
+{
+  "short_name": "Grafana Agent",
+  "name": "Grafana Agent",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

web/ui/build/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow: