Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Revert "backup: Use pargzip instead of pgzip for compression." #8174

Merged
merged 1 commit into from
May 24, 2021
Merged

Revert "backup: Use pargzip instead of pgzip for compression." #8174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ require (
github.com/philhofer/fwd v1.0.0 // indirect
github.com/pires/go-proxyproto v0.0.0-20191211124218-517ecdf5bb2b
github.com/pkg/errors v0.9.1
github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a
github.com/prometheus/client_golang v1.4.1
github.com/prometheus/common v0.9.1
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
Expand Down
2 changes: 0 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a h1:y0OpQ4+5tKxeh9+H+2cVgASl9yMZYV9CILinKOiKafA=
github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a/go.mod h1:GJFUzQuXIoB2Kjn1ZfDhJr/42D5nWOqRcIQVgCxTuIE=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down
12 changes: 6 additions & 6 deletions go/vt/mysqlctl/builtinbackupengine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"time"

"github.com/klauspost/pgzip"
"github.com/planetscale/pargzip"

"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/sync2"
Expand Down Expand Up @@ -409,12 +408,13 @@ func (be *BuiltinBackupEngine) backupFile(ctx context.Context, params BackupPara
}

// Create the gzip compression pipe, if necessary.
var gzip *pargzip.Writer
var gzip *pgzip.Writer
if *backupStorageCompress {
gzip = pargzip.NewWriter(writer)
gzip.ChunkSize = *backupCompressBlockSize
gzip.Parallel = *backupCompressBlocks
gzip.CompressionLevel = pargzip.BestSpeed
gzip, err = pgzip.NewWriterLevel(writer, pgzip.BestSpeed)
if err != nil {
return vterrors.Wrap(err, "cannot create gziper")
}
gzip.SetConcurrency(*backupCompressBlockSize, *backupCompressBlocks)
writer = gzip
}

Expand Down
16 changes: 8 additions & 8 deletions go/vt/mysqlctl/xtrabackupengine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"time"

"github.com/klauspost/pgzip"
"github.com/planetscale/pargzip"

"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/vt/logutil"
Expand Down Expand Up @@ -271,18 +270,19 @@ func (be *XtrabackupEngine) backupFiles(ctx context.Context, params BackupParams

destWriters := []io.Writer{}
destBuffers := []*bufio.Writer{}
destCompressors := []io.WriteCloser{}
destCompressors := []*pgzip.Writer{}
for _, file := range destFiles {
buffer := bufio.NewWriterSize(file, writerBufferSize)
destBuffers = append(destBuffers, buffer)
writer := io.Writer(buffer)

// Create the gzip compression pipe, if necessary.
if *backupStorageCompress {
compressor := pargzip.NewWriter(writer)
compressor.ChunkSize = *backupCompressBlockSize
compressor.Parallel = *backupCompressBlocks
compressor.CompressionLevel = pargzip.BestSpeed
compressor, err := pgzip.NewWriterLevel(writer, pgzip.BestSpeed)
if err != nil {
return replicationPosition, vterrors.Wrap(err, "cannot create gzip compressor")
}
compressor.SetConcurrency(*backupCompressBlockSize, *backupCompressBlocks)
writer = compressor
destCompressors = append(destCompressors, compressor)
}
Expand Down Expand Up @@ -520,7 +520,7 @@ func (be *XtrabackupEngine) extractFiles(ctx context.Context, logger logutil.Log
}()

srcReaders := []io.Reader{}
srcDecompressors := []io.ReadCloser{}
srcDecompressors := []*pgzip.Reader{}
for _, file := range srcFiles {
reader := io.Reader(file)

Expand Down Expand Up @@ -734,7 +734,7 @@ func copyToStripes(writers []io.Writer, reader io.Reader, blockSize int64) (writ
}

// Read blocks from source and round-robin them to destination writers.
// Since we put a buffer in front of the destination file, and pargzip has its
// Since we put a buffer in front of the destination file, and pgzip has its
// own buffer as well, we are writing into a buffer either way (whether a
// compressor is in the chain or not). That means these writes should not
// block often, so we shouldn't need separate goroutines here.
Expand Down