-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
137646: changefeedccl: Fix Kafka v2 Sink GZIP Compression Level Issue r=asg0451 a=yaothao Previously, the Kafka v2 sink could not properly handle negative compression levels due to differences in the underlying compression libraries used between the v1 and v2 sinks. The Kafka v1 sink implementation, which relies on the Sarama library, uses the klauspost/compress library that supports a compression level of -3. However, our v2 sink has transitioned to using franz-go, which utilizes the standard library's compression/gzip, and does not support the -3 level. In this update, the validation function now checks the GZIP compression range between HuffmanOnly (-2) and BestCompression (9). Fixes: #136492 Epic: none Release note (bug fix): We have resolved an issue in the Kafka v2 sink configuration within CockroachDB, where users were previously unable to set negative GZIP compression levels. Now, users can configure the CompressionLevel for the Kafka sink in the range of [-2, 9]. Please update the user guide to include the new valid GZIP compression level range of [-2, 9], where -2 enables Huffman encoding and -1 sets the default compression. 137842: testutils: add a helper to capture a side-eye snapshot on demand r=srosenberg a=asg0451 Add a helper to capture a side-eye snapshot on demand. This can be used to investigate test failures that you can reproduce locally but only by running the test many times. Epic: None Release note: None Co-authored-by: yaothao <[email protected]> Co-authored-by: Miles Frankel <[email protected]>
- Loading branch information
Showing
11 changed files
with
114 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2405,10 +2405,10 @@ def go_deps(): | |
| name = "com_github_dataexmachina_dev_side_eye_go", | ||
| build_file_proto_mode = "disable_global", | ||
| importpath = "github.com/DataExMachina-dev/side-eye-go", | ||
| sha256 = "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f", | ||
| strip_prefix = "github.com/DataExMachina-dev/[email protected]20240528211710-5eb9c7a69e1d", | ||
| sha256 = "ea5676af6abb1965ba70c24a9090932c9e45a002411e9c86298f60f60f16a10a", | ||
| strip_prefix = "github.com/DataExMachina-dev/[email protected]20250102012104-645b45402adf", | ||
| urls = [ | ||
| "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip", | ||
| "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20250102012104-645b45402adf.zip", | ||
| ], | ||
| ) | ||
| go_repository( | ||
|
|
@@ -6505,6 +6505,16 @@ def go_deps(): | |
| "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip", | ||
| ], | ||
| ) | ||
| go_repository( | ||
| name = "com_github_minio_highwayhash", | ||
| build_file_proto_mode = "disable_global", | ||
| importpath = "github.com/minio/highwayhash", | ||
| sha256 = "3ab23da1595a6b8543edf3de80e31afacfba2b1bc9e9f4cf60c6f54ce3f66fa9", | ||
| strip_prefix = "github.com/minio/[email protected]", | ||
| urls = [ | ||
| "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/highwayhash/com_github_minio_highwayhash-v1.0.2.zip", | ||
| ], | ||
| ) | ||
| go_repository( | ||
| name = "com_github_minio_md5_simd", | ||
| build_file_proto_mode = "disable_global", | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // Copyright 2024 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the CockroachDB Software License | ||
| // included in the /LICENSE file. | ||
|
|
||
| package testutils | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/DataExMachina-dev/side-eye-go/sideeye" | ||
| "github.com/cockroachdb/errors" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| type testingT interface { | ||
| require.TestingT | ||
| TestFatalerLogger | ||
| } | ||
|
|
||
| // CaptureSideEyeSnapshot captures a Side-Eye snapshot if the | ||
| // SIDE_EYE_TOKEN env var is set. If the snapshot is captured, the snapshot's | ||
| // URL is logged. Snapshots are captured with a 90s timeout. | ||
| func CaptureSideEyeSnapshot(ctx context.Context, t testingT) { | ||
| t.Helper() | ||
|
|
||
| if sideEyeToken := os.Getenv("SIDE_EYE_TOKEN"); sideEyeToken == "" { | ||
| t.Logf("not capturing Side-Eye snapshot; SIDE_EYE_TOKEN env var not set. You can find it in slack or confluence " + | ||
| "or on your profile page in the Side-Eye app. If using ./dev, make sure you pass it in the environment: " + | ||
| "`./dev test mytest -- --test_env SIDE_EYE_TOKEN=xxx --strip=never`") | ||
| return | ||
| } | ||
|
|
||
| username := os.Getenv("USER") | ||
| hostname, err := os.Hostname() | ||
| require.NoError(t, err) | ||
|
|
||
| var name string | ||
| if t, ok := t.(TestNamedFatalerLogger); ok { | ||
| name = t.Name() | ||
| } else { | ||
| name = "unknown test" | ||
| } | ||
| name = fmt.Sprintf("%s@%s: %s", username, hostname, name) | ||
|
|
||
| snapshotCtx, cancel := context.WithTimeoutCause( | ||
| ctx, 90*time.Second, errors.New("timed out waiting for Side-Eye snapshot"), | ||
| ) | ||
| defer cancel() | ||
| snapshotURL, err := sideeye.CaptureSelfSnapshot(snapshotCtx, name, sideeye.WithEnvironment("unit tests")) | ||
| if err != nil { | ||
| if errors.As(err, &sideeye.BinaryStrippedError{}) { | ||
| t.Logf("failed to capture Side-Eye snapshot because the binary is stripped of debug info; " + | ||
| "if running with `go test` instead of bazel, use `go test -o test.out` " + | ||
| "for creating a non-stripped binary. If running inside bazel, " + | ||
| "add `build --strip=never` to your .bazelrc.user file, or pass `--strip=never` to " + | ||
| "bazel test, or with `dev`: `./dev test mytest -- --strip=never`") | ||
| } | ||
| t.Logf("failed to capture Side-Eye snapshot: %s", err) | ||
| return | ||
| } | ||
| t.Logf("captured Side-Eye snapshot: %s", snapshotURL) | ||
|
|
||
| } |