Skip to content

Commit

Permalink
*: Add fast gzip compression library.
Browse files Browse the repository at this point in the history
Add https://github.com/klauspost/pgzip fast gzip
compression library.

Addresses #88585

Release notes: None
  • Loading branch information
Yevgeniy Miretskiy committed Sep 26, 2022
1 parent c432bd7 commit 688430b
Show file tree
Hide file tree
Showing 7 changed files with 14 additions and 9 deletions.
6 changes: 3 additions & 3 deletions DEPS.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -5043,10 +5043,10 @@ def go_deps():
name = "com_github_klauspost_compress",
build_file_proto_mode = "disable_global",
importpath = "github.com/klauspost/compress",
sha256 = "25990a3b573b4568fa9c98c880e82abe5fe4dc6a3c784935923643ef28791acd",
strip_prefix = "github.com/klauspost/compress@v1.14.2",
sha256 = "5f85779b0a96cf9a66f6cee4a91382e03a71919121ebe8f6a90936300eb683c1",
strip_prefix = "github.com/klauspost/compress@v1.15.11",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/compress/com_github_klauspost_compress-v1.14.2.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/compress/com_github_klauspost_compress-v1.15.11.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion build/bazelutil/distdir_files.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/kevinburke/go-bindata/com_github_kevinburke_go_bindata-v3.13.0+incompatible.zip": "f087b3a77624a113883bac519ebd1a4de07b70ab2ebe73e61e52325ac30777e0",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/kisielk/errcheck/com_github_kisielk_errcheck-v1.6.1-0.20210625163953-8ddee489636a.zip": "99d3220891162cb684f8e05d54f3d0dc58abdd496a2f0cfda7fd4a28917a719e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/kisielk/gotool/com_github_kisielk_gotool-v1.0.0.zip": "089dbba6e3aa09944fdb40d72acc86694e8bdde01cfc0f40fe0248309eb80a3f",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/compress/com_github_klauspost_compress-v1.14.2.zip": "25990a3b573b4568fa9c98c880e82abe5fe4dc6a3c784935923643ef28791acd",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/compress/com_github_klauspost_compress-v1.15.11.zip": "5f85779b0a96cf9a66f6cee4a91382e03a71919121ebe8f6a90936300eb683c1",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/cpuid/com_github_klauspost_cpuid-v1.3.1.zip": "f61266e43d5c247fdb55d843e2d93974717c1052cba9f331b181f60c4cf687d9",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.0.9.zip": "52c716413296dce2b1698c6cdbc4c53927ce4aee2a60980daf9672e6b6a3b4cb",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/crc32/com_github_klauspost_crc32-v0.0.0-20161016154125-cb6bfca970f6.zip": "6b632853a19f039138f251f94dbbdfdb72809adc3a02da08e4301d3d48275b06",
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ require (
github.com/kevinburke/go-bindata v3.13.0+incompatible
github.com/kisielk/errcheck v1.6.1-0.20210625163953-8ddee489636a
github.com/kisielk/gotool v1.0.0
github.com/klauspost/compress v1.15.11
github.com/klauspost/pgzip v1.2.5
github.com/knz/go-libedit v1.10.1
github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9
github.com/kr/pretty v0.3.0
Expand Down Expand Up @@ -274,9 +276,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1488,8 +1488,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
Expand Down
2 changes: 2 additions & 0 deletions pkg/ccl/changefeedccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ go_library(
"@com_github_cockroachdb_redact//:redact",
"@com_github_gogo_protobuf//jsonpb",
"@com_github_google_btree//:btree",
"@com_github_klauspost_compress//zstd",
"@com_github_klauspost_pgzip//:pgzip",
"@com_github_linkedin_goavro_v2//:goavro",
"@com_github_shopify_sarama//:sarama",
"@com_github_xdg_go_scram//:scram",
Expand Down
3 changes: 3 additions & 0 deletions pkg/ccl/changefeedccl/sink_cloudstorage.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ import (
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/errors"
"github.com/google/btree"
// Placeholder for pgzip and zdstd.
_ "github.com/klauspost/compress/zstd"
_ "github.com/klauspost/pgzip"
)

func isCloudStorageSink(u *url.URL) bool {
Expand Down
2 changes: 1 addition & 1 deletion vendor
Submodule vendor updated 78 files
+7 −0 github.com/klauspost/compress/.gitignore
+133 −7 github.com/klauspost/compress/README.md
+22 −23 github.com/klauspost/compress/flate/deflate.go
+12 −12 github.com/klauspost/compress/flate/dict_decoder.go
+22 −39 github.com/klauspost/compress/flate/fast_encoder.go
+151 −97 github.com/klauspost/compress/flate/huffman_bit_writer.go
+82 −37 github.com/klauspost/compress/flate/huffman_code.go
+7 −215 github.com/klauspost/compress/flate/inflate.go
+505 −224 github.com/klauspost/compress/flate/inflate_gen.go
+79 −17 github.com/klauspost/compress/flate/level1.go
+27 −18 github.com/klauspost/compress/flate/level2.go
+46 −34 github.com/klauspost/compress/flate/level3.go
+15 −6 github.com/klauspost/compress/flate/level4.go
+27 −11 github.com/klauspost/compress/flate/level5.go
+29 −11 github.com/klauspost/compress/flate/level6.go
+11 −3 github.com/klauspost/compress/flate/stateless.go
+16 −14 github.com/klauspost/compress/flate/token.go
+13 −109 github.com/klauspost/compress/huff0/bitreader.go
+0 −115 github.com/klauspost/compress/huff0/bitwriter.go
+0 −10 github.com/klauspost/compress/huff0/bytereader.go
+10 −0 github.com/klauspost/compress/huff0/compress.go
+200 −420 github.com/klauspost/compress/huff0/decompress.go
+226 −0 github.com/klauspost/compress/huff0/decompress_amd64.go
+846 −0 github.com/klauspost/compress/huff0/decompress_amd64.s
+299 −0 github.com/klauspost/compress/huff0/decompress_generic.go
+2 −0 github.com/klauspost/compress/huff0/huff0.go
+34 −0 github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go
+11 −0 github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go
+36 −0 github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s
+5 −1 github.com/klauspost/compress/internal/snapref/encode_other.go
+156 −157 github.com/klauspost/compress/s2/README.md
+295 −11 github.com/klauspost/compress/s2/decode.go
+23 −11 github.com/klauspost/compress/s2/decode_other.go
+0 −6 github.com/klauspost/compress/s2/encode.go
+2 −1 github.com/klauspost/compress/s2/encode_all.go
+8 −4 github.com/klauspost/compress/s2/encode_amd64.go
+55 −8 github.com/klauspost/compress/s2/encode_best.go
+76 −29 github.com/klauspost/compress/s2/encode_better.go
+24 −8 github.com/klauspost/compress/s2/encode_go.go
+16 −11 github.com/klauspost/compress/s2/encodeblock_amd64.go
+3,113 −943 github.com/klauspost/compress/s2/encodeblock_amd64.s
+73 −0 github.com/klauspost/compress/s2/index.go
+119 −100 github.com/klauspost/compress/zip/reader.go
+0 −626 github.com/klauspost/compress/zip/reader_leg.go
+1 −2 github.com/klauspost/compress/zip/register.go
+0 −11 github.com/klauspost/compress/zip/struct.go
+0 −390 github.com/klauspost/compress/zip/struct_leg.go
+2 −5 github.com/klauspost/compress/zip/writer.go
+0 −752 github.com/klauspost/compress/zip/writer_leg.go
+86 −86 github.com/klauspost/compress/zstd/README.md
+4 −7 github.com/klauspost/compress/zstd/bitreader.go
+0 −76 github.com/klauspost/compress/zstd/bitwriter.go
+247 −251 github.com/klauspost/compress/zstd/blockdec.go
+12 −11 github.com/klauspost/compress/zstd/bytebuf.go
+0 −6 github.com/klauspost/compress/zstd/bytereader.go
+532 −137 github.com/klauspost/compress/zstd/decoder.go
+63 −16 github.com/klauspost/compress/zstd/decoder_options.go
+1 −0 github.com/klauspost/compress/zstd/enc_best.go
+20 −11 github.com/klauspost/compress/zstd/enc_better.go
+10 −7 github.com/klauspost/compress/zstd/enc_dfast.go
+6 −4 github.com/klauspost/compress/zstd/enc_fast.go
+57 −15 github.com/klauspost/compress/zstd/encoder.go
+2 −1 github.com/klauspost/compress/zstd/encoder_options.go
+108 −193 github.com/klauspost/compress/zstd/framedec.go
+25 −103 github.com/klauspost/compress/zstd/fse_decoder.go
+65 −0 github.com/klauspost/compress/zstd/fse_decoder_amd64.go
+126 −0 github.com/klauspost/compress/zstd/fse_decoder_amd64.s
+72 −0 github.com/klauspost/compress/zstd/fse_decoder_generic.go
+0 −23 github.com/klauspost/compress/zstd/fse_encoder.go
+0 −6 github.com/klauspost/compress/zstd/hash.go
+47 −20 github.com/klauspost/compress/zstd/history.go
+171 −154 github.com/klauspost/compress/zstd/seqdec.go
+379 −0 github.com/klauspost/compress/zstd/seqdec_amd64.go
+4,099 −0 github.com/klauspost/compress/zstd/seqdec_amd64.s
+237 −0 github.com/klauspost/compress/zstd/seqdec_generic.go
+44 −25 github.com/klauspost/compress/zstd/zip.go
+11 −11 github.com/klauspost/compress/zstd/zstd.go
+3 −2 modules.txt

0 comments on commit 688430b

Please sign in to comment.