br: redact ak/sk in logging (#55622) (#55779)

close #55273
pingcap · Sep 5, 2024 · 390215e · 390215e
1 parent 0ad9365
commit 390215e
Show file tree

Hide file tree

Showing 5 changed files with 73 additions and 3 deletions.
diff --git a/br/pkg/redact/BUILD.bazel b/br/pkg/redact/BUILD.bazel
@@ -5,7 +5,10 @@ go_library(
     srcs = ["redact.go"],
     importpath = "github.com/pingcap/tidb/br/pkg/redact",
     visibility = ["//visibility:public"],
-    deps = ["@com_github_pingcap_errors//:errors"],
+    deps = [
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_kvproto//pkg/brpb",
+    ],
 )
 
 go_test(

diff --git a/br/pkg/redact/redact.go b/br/pkg/redact/redact.go
@@ -4,9 +4,20 @@ package redact
 
 import (
 	"encoding/hex"
+	"regexp"
 	"strings"
 
 	"github.com/pingcap/errors"
+	backup "github.com/pingcap/kvproto/pkg/brpb"
+)
+
+var (
+	reAccessKey       = regexp.MustCompile(`access_key:\"[^\"]*\"`)
+	reSecretAccessKey = regexp.MustCompile(`secret_access_key:\"[^\"]*\"`)
+	reSharedKey       = regexp.MustCompile(`shared_key:\"[^\"]*\"`)
+	reCredentialsBlob = regexp.MustCompile(`credentials_blob:\"[^\"]*\"`)
+	reAccessSig       = regexp.MustCompile(`access_sig:\"[^\"]*\"`)
+	reEncryptKey      = regexp.MustCompile(`encryption_key:<.*?>`)
 )
 
 // InitRedact inits the enableRedactLog
@@ -34,3 +45,25 @@ func Key(key []byte) string {
 	}
 	return strings.ToUpper(hex.EncodeToString(key))
 }
+
+// TaskInfoRedacted is a wrapper of backup.StreamBackupTaskInfo to redact sensitive information
+type TaskInfoRedacted struct {
+	Info *backup.StreamBackupTaskInfo
+}
+
+func (TaskInfoRedacted) redact(input string) string {
+	// Replace the matched fields with redacted versions
+	output := reAccessKey.ReplaceAllString(input, `access_key:"[REDACTED]"`)
+	output = reSecretAccessKey.ReplaceAllString(output, `secret_access_key:"[REDACTED]"`)
+	output = reSharedKey.ReplaceAllString(output, `shared_key:"[REDACTED]"`)
+	output = reCredentialsBlob.ReplaceAllString(output, `CredentialsBlob:"[REDACTED]"`)
+	output = reAccessSig.ReplaceAllString(output, `access_sig:"[REDACTED]"`)
+	output = reEncryptKey.ReplaceAllString(output, `encryption_key:<[REDACTED]>`)
+
+	return output
+}
+
+// String returns the redacted string of the task info
+func (t TaskInfoRedacted) String() string {
+	return t.redact(t.Info.String())
+}
diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel
@@ -68,7 +68,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 28,
+    shard_count = 29,
     deps = [
         ":streamhelper",
         "//br/pkg/errors",

diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go
@@ -17,6 +17,7 @@ import (
 	backuppb "github.com/pingcap/kvproto/pkg/brpb"
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/br/pkg/logutil"
+	"github.com/pingcap/tidb/br/pkg/redact"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/config"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/spans"
 	"github.com/pingcap/tidb/br/pkg/utils"
@@ -438,7 +439,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error
 		}
 		log.Info("get global checkpoint", zap.Uint64("checkpoint", globalCheckpointTs))
 		c.lastCheckpoint = newCheckpointWithTS(globalCheckpointTs)
-		log.Info("added event", zap.Stringer("task", e.Info),
+		log.Info("added event", zap.Stringer("task", redact.TaskInfoRedacted{Info: e.Info}),
 			zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange)))
 	case EventDel:
 		utils.LogBackupTaskCountDec()

diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go
@@ -15,6 +15,7 @@ import (
 	backup "github.com/pingcap/kvproto/pkg/brpb"
 	logbackup "github.com/pingcap/kvproto/pkg/logbackuppb"
 	"github.com/pingcap/log"
+	"github.com/pingcap/tidb/br/pkg/redact"
 	"github.com/pingcap/tidb/br/pkg/streamhelper"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/config"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/spans"
@@ -801,3 +802,35 @@ func TestSubscriptionPanic(t *testing.T) {
 	cancel()
 	wg.Wait()
 }
+
+func TestRedactBackend(t *testing.T) {
+	info := new(backup.StreamBackupTaskInfo)
+	info.Name = "test"
+	info.Storage = &backup.StorageBackend{
+		Backend: &backup.StorageBackend_S3{
+			S3: &backup.S3{
+				Endpoint:        "http://",
+				Bucket:          "test",
+				Prefix:          "test",
+				AccessKey:       "12abCD!@#[]{}?/\\",
+				SecretAccessKey: "12abCD!@#[]{}?/\\",
+			},
+		},
+	}
+
+	redacted := redact.TaskInfoRedacted{Info: info}
+	require.Equal(t, "storage:<s3:<endpoint:\"http://\" bucket:\"test\" prefix:\"test\" access_key:\"[REDACTED]\" secret_access_key:\"[REDACTED]\" > > name:\"test\" ", redacted.String())
+
+	info.Storage = &backup.StorageBackend{
+		Backend: &backup.StorageBackend_Gcs{
+			Gcs: &backup.GCS{
+				Endpoint:        "http://",
+				Bucket:          "test",
+				Prefix:          "test",
+				CredentialsBlob: "12abCD!@#[]{}?/\\",
+			},
+		},
+	}
+	redacted = redact.TaskInfoRedacted{Info: info}
+	require.Equal(t, "storage:<gcs:<endpoint:\"http://\" bucket:\"test\" prefix:\"test\" CredentialsBlob:\"[REDACTED]\" > > name:\"test\" ", redacted.String())
+}