TEST: mask secret values only via hash

.cspell.json

@@ -74,6 +74,7 @@
     "GOPATH",
     "Gource",
     "handlebargh",
+    "hashvalue",
     "HEALTHCHECK",
     "healthz",
     "Hetzner",

agent/logger.go

@@ -15,9 +15,11 @@
 package agent
 
 import (
+	"crypto/sha256"
 	"io"
 	"sync"
 
+	hashvalue_replacer "github.com/6543/go-hashvalue-replacer"
 	"github.com/rs/zerolog"
 
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline"
@@ -35,22 +37,31 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w
 			Logger()
 
 		uploads.Add(1)
-
-		var secrets []string
-		for _, secret := range workflow.Config.Secrets {
-			secrets = append(secrets, secret.Value)
-		}
+		defer uploads.Done()
 
 		logger.Debug().Msg("log stream opened")
 
-		logStream := log.NewLineWriter(r.client, step.UUID, secrets...)
-		if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil {
+		// mask secrets from reader
+		maskedReader, err := hashvalue_replacer.NewReader(rc, workflow.Config.SecretMask.Salt, workflow.Config.SecretMask.Hashes, workflow.Config.SecretMask.Lengths, hashvalue_replacer.Options{
+			Mask: "********",
+			Hash: func(salt, data []byte) []byte {
+				h := sha256.New()
+				h.Write(salt)
+				h.Write([]byte(data))
+				return h.Sum(nil)
+			},
+		})
+		if err != nil {
+			logger.Error().Err(err).Msg("could not create masked reader")
+			return nil
+		}
+
+		logStream := log.NewLineWriter(r.client, step.UUID)
+		if err := log.CopyLineByLine(logStream, maskedReader, pipeline.MaxLogLineLength); err != nil {
 			logger.Error().Err(err).Msg("copy limited logStream part")
 		}
 
 		logger.Debug().Msg("log stream copied, close ...")
-		uploads.Done()
-
 		return nil
 	}
 }

go.mod

@@ -1,15 +1,14 @@
 module go.woodpecker-ci.org/woodpecker/v2
 
-go 1.22.0
-
-toolchain go1.23.2
+go 1.23.2
 
 require (
 	al.essio.dev/pkg/shellescape v1.5.1
 	code.gitea.io/sdk/gitea v0.19.0
 	codeberg.org/6543/go-yaml2json v1.0.0
 	codeberg.org/6543/xyaml v1.1.0
 	codeberg.org/mvdkleijn/forgejo-sdk/forgejo v1.2.0
+	github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109
 	github.com/6543/logfile-open v1.2.1
 	github.com/adrg/xdg v0.5.2
 	github.com/bmatcuk/doublestar/v4 v4.7.1

go.sum

@@ -15,6 +15,8 @@ gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0p
 gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE=
 github.com/42wim/httpsig v1.2.2 h1:ofAYoHUNs/MJOLqQ8hIxeyz2QxOz8qdSVvp3PX/oPgA=
 github.com/42wim/httpsig v1.2.2/go.mod h1:P/UYo7ytNBFwc+dg35IubuAUIs8zj5zzFIgUCEl55WY=
+github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109 h1:5DPvI79163nINaU7cj3/6XGDYtnh49hOIOJbPB97/Lk=
+github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109/go.mod h1:+fCz/+h1AIDEtSgeZG6wVPXOwag1WgfosmIz7KaeHDM=
 github.com/6543/logfile-open v1.2.1 h1:az+TtNHclTAKaHfFCTSbuduMllANox1gM9qLQr7LV5I=
 github.com/6543/logfile-open v1.2.1/go.mod h1:ZoEy7pW2mexmQxiZIqPCeh8vUxVuiHYXmSZNbvEb51g=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=

pipeline/backend/types/config.go

@@ -16,10 +16,10 @@ package types
 
 // Config defines the runtime configuration of a workflow.
 type Config struct {
-	Stages   []*Stage   `json:"pipeline"` // workflow stages
-	Networks []*Network `json:"networks"` // network definitions
-	Volumes  []*Volume  `json:"volumes"`  // volume definitions
-	Secrets  []*Secret  `json:"secrets"`  // secret definitions
+	Stages     []*Stage   `json:"pipeline"` // workflow stages
+	Networks   []*Network `json:"networks"` // network definitions
+	Volumes    []*Volume  `json:"volumes"`  // volume definitions
+	SecretMask SecretMask `json:"secret_mask"`
 }
 
 // CliCommand is the context key to pass cli context to backends if needed.

pipeline/backend/types/secret.go

@@ -15,7 +15,8 @@
 package types
 
 // Secret defines a runtime secret.
-type Secret struct {
-	Name  string `json:"name,omitempty"`
-	Value string `json:"value,omitempty"`
+type SecretMask struct {
+	Salt    []byte   `json:"salt,omitempty"`
+	Hashes  [][]byte `json:"value,omitempty"`
+	Lengths []int    `json:"lengths,omitempty"`
 }

pipeline/frontend/yaml/compiler/compiler.go

@@ -15,9 +15,13 @@
 package compiler
 
 import (
+	"crypto/rand"
+	"crypto/sha256"
 	"fmt"
 	"path"
 
+	hashvalue_replacer "github.com/6543/go-hashvalue-replacer"
+
 	backend_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
 	yaml_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/types"
@@ -139,13 +143,23 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		Name: fmt.Sprintf("%s_default", c.prefix),
 	})
 
-	// create secrets for mask
+	// create mask for secrets
+	secretValues := make([]string, len(c.secrets))
 	for _, sec := range c.secrets {
-		config.Secrets = append(config.Secrets, &backend_types.Secret{
-			Name:  sec.Name,
-			Value: sec.Value,
-		})
+		secretValues = append(secretValues, sec.Value)
+	}
+	salt := make([]byte, 64)
+	_, err := rand.Read(salt)
+	if err != nil {
+		return nil, fmt.Errorf("could not generate salt for secret masker: %w", err)
 	}
+	config.SecretMask.Salt = salt
+	config.SecretMask.Hashes, config.SecretMask.Lengths = hashvalue_replacer.ValuesToArgs(func(salt, data []byte) []byte {
+		h := sha256.New()
+		h.Write(salt)
+		h.Write([]byte(data))
+		return h.Sum(nil)
+	}, salt, secretValues)
 
 	// overrides the default workspace paths when specified
 	// in the YAML file.

pipeline/log/line_writer.go

@@ -24,7 +24,6 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
-	"go.woodpecker-ci.org/woodpecker/v2/pipeline/shared"
 )
 
 // LineWriter sends logs to the client.
@@ -35,25 +34,20 @@ type LineWriter struct {
 	stepUUID  string
 	num       int
 	startTime time.Time
-	replacer  *strings.Replacer
 }
 
 // NewLineWriter returns a new line reader.
-func NewLineWriter(peer rpc.Peer, stepUUID string, secret ...string) io.Writer {
+func NewLineWriter(peer rpc.Peer, stepUUID string) io.Writer {
 	lw := &LineWriter{
 		peer:      peer,
 		stepUUID:  stepUUID,
 		startTime: time.Now().UTC(),
-		replacer:  shared.NewSecretsReplacer(secret),
 	}
 	return lw
 }
 
 func (w *LineWriter) Write(p []byte) (n int, err error) {
 	data := string(p)
-	if w.replacer != nil {
-		data = w.replacer.Replace(data)
-	}
 	log.Trace().Str("step-uuid", w.stepUUID).Msgf("grpc write line: %s", data)
 
 	line := &rpc.LogEntry{

pipeline/log/line_writer_test.go

@@ -29,8 +29,7 @@ func TestLineWriter(t *testing.T) {
 	peer := mocks.NewPeer(t)
 	peer.On("EnqueueLog", mock.Anything)
 
-	secrets := []string{"world"}
-	lw := log.NewLineWriter(peer, "e9ea76a5-44a1-4059-9c4a-6956c478b26d", secrets...)
+	lw := log.NewLineWriter(peer, "e9ea76a5-44a1-4059-9c4a-6956c478b26d")
 
 	_, err := lw.Write([]byte("hello world\n"))
 	assert.NoError(t, err)
@@ -42,7 +41,7 @@ func TestLineWriter(t *testing.T) {
 		Time:     0,
 		Type:     rpc.LogEntryStdout,
 		Line:     0,
-		Data:     []byte("hello ********"),
+		Data:     []byte("hello world"),
 	})
 
 	peer.AssertCalled(t, "EnqueueLog", &rpc.LogEntry{

pipeline/rpc/proto/version.go

@@ -16,4 +16,4 @@ package proto
 
 // Version is the version of the woodpecker.proto file,
 // IMPORTANT: increased by 1 each time it get changed.
-const Version int32 = 11
+const Version int32 = 12

pipeline/shared/replace_secrets_test.go

@@ -15,6 +15,7 @@
 package shared
 
 import (
+	"bytes"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,7 +28,7 @@ func TestNewSecretsReplacer(t *testing.T) {
 		secrets []string
 		expect  string
 	}{{
-		name:    "dont replace secrets with less than 3 chars",
+		name:    "dont replace secrets with less than 4 chars",
 		log:     "start log\ndone",
 		secrets: []string{"", "d", "art"},
 		expect:  "start log\ndone",
@@ -61,3 +62,78 @@ func TestNewSecretsReplacer(t *testing.T) {
 		})
 	}
 }
+
+func BenchmarkReader(b *testing.B) {
+	testCases := []struct {
+		name    string
+		log     string
+		secrets []string
+	}{
+		{
+			name:    "single line",
+			log:     "this is a log with secret password and more text",
+			secrets: []string{"password"},
+		},
+		{
+			name:    "multi line",
+			log:     "log start\nthis is a multi\nline secret\nlog end",
+			secrets: []string{"multi\nline secret"},
+		},
+		{
+			name:    "many secrets",
+			log:     "log with many secrets: secret1 secret2 secret3 secret4 secret5",
+			secrets: []string{"secret1", "secret2", "secret3", "secret4", "secret5"},
+		},
+		{
+			name:    "large log",
+			log:     "start " + string(bytes.Repeat([]byte("test secret test "), 1000)) + " end",
+			secrets: []string{"secret"},
+		},
+		{
+			name:    "large log no match",
+			log:     "start " + string(bytes.Repeat([]byte("test secret test "), 1000)) + " end",
+			secrets: []string{"XXXXXXX"},
+		},
+	}
+
+	for _, tc := range testCases {
+		b.Run(tc.name, func(b *testing.B) {
+			rep := NewSecretsReplacer(tc.secrets)
+			b.ResetTimer()
+			b.SetBytes(int64(len(tc.log)))
+			for i := 0; i < b.N; i++ {
+				_ = rep.Replace(tc.log)
+			}
+		})
+	}
+}
+
+// go test -benchmem -run='^$' -tags test -bench '^BenchmarkReader$' -benchtime=100000x go.woodpecker-ci.org/woodpecker/v2/pipeline/shared
+//
+// cpu: AMD Ryzen 9 7940HS (16-Core)
+// BenchmarkReader/single_line-16            100000             55.13 ns/op         870.70 MB/s          48 B/op          1 allocs/op
+// BenchmarkReader/multi_line-16             100000             149.0 ns/op         302.06 MB/s         120 B/op          3 allocs/op
+// BenchmarkReader/many_secrets-16           100000             273.0 ns/op         227.10 MB/s         296 B/op          4 allocs/op
+// BenchmarkReader/large_log-16              100000             19544 ns/op         870.33 MB/s       40520 B/op          9 allocs/op
+// BenchmarkReader/large_log_no_match-16     100000              5080 ns/op        3348.63 MB/s           0 B/op          0 allocs/op
+//
+// cpu: AMD Ryzen 9 3900XT (12-Core)
+// BenchmarkReader/single_line-24            100000                90.87 ns/op      528.23 MB/s          48 B/op          1 allocs/op
+// BenchmarkReader/multi_line-24             100000               276.2 ns/op       162.94 MB/s         120 B/op          3 allocs/op
+// BenchmarkReader/many_secrets-24           100000               433.7 ns/op       142.97 MB/s         296 B/op          4 allocs/op
+// BenchmarkReader/large_log-24              100000             26542 ns/op         640.88 MB/s       40520 B/op          9 allocs/op
+// BenchmarkReader/large_log_no_match-24     100000              6212 ns/op        2738.45 MB/s           0 B/op          0 allocs/op
+//
+// cpu: Ampere Altra (2 vCPUs)
+// BenchmarkReader/single_line-2             100000             105.1 ns/op         456.89 MB/s          48 B/op          1 allocs/op
+// BenchmarkReader/multi_line-2              100000             441.7 ns/op         101.88 MB/s         120 B/op          3 allocs/op
+// BenchmarkReader/many_secrets-2            100000             868.7 ns/op          71.37 MB/s         296 B/op          4 allocs/op
+// BenchmarkReader/large_log-2               100000             48947 ns/op         347.52 MB/s       40520 B/op          9 allocs/op
+// BenchmarkReader/large_log_no_match-2      100000              9156 ns/op        1857.79 MB/s           0 B/op          0 allocs/op
+//
+// cpu: Intel Xeon Processor (Skylake, IBRS, no TSX) (2 vCPUs)
+// BenchmarkReader/single_line-2             100000               167.7 ns/op       286.25 MB/s          48 B/op          1 allocs/op
+// BenchmarkReader/multi_line-2              100000               640.7 ns/op        70.24 MB/s         120 B/op          3 allocs/op
+// BenchmarkReader/many_secrets-2            100000              1044 ns/op          59.38 MB/s         296 B/op          4 allocs/op
+// BenchmarkReader/large_log-2               100000             45271 ns/op         375.73 MB/s       40520 B/op          9 allocs/op
+// BenchmarkReader/large_log_no_match-2      100000             11240 ns/op        1513.37 MB/s           0 B/op          0 allocs/op