Skip to content

Commit

Permalink
storage: ingest columnar-block sstables when enabled
Browse files Browse the repository at this point in the history
When the active cluster verison is sufficiently high and the columnar blocks
cluster setting is enabled, build sstables for ingestion in table format
TableFormatPebblev5 with columnar blocks.

Epic: none
Release note: none
  • Loading branch information
jbowens committed Nov 12, 2024
1 parent deb2445 commit bfd1ad3
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 6 deletions.
5 changes: 3 additions & 2 deletions pkg/storage/mvcc_history_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ func TestMVCCHistories(t *testing.T) {
// SST iterator in order to accurately represent the raw SST data.
reportSSTEntries := func(buf *redact.StringBuilder, name string, sst []byte) error {
r, err := sstable.NewMemReader(sst, sstable.ReaderOptions{
Comparer: storage.EngineComparer,
Comparer: storage.EngineComparer,
KeySchemas: sstable.MakeKeySchemas(storage.KeySchemas...),
})
if err != nil {
return err
Expand All @@ -275,7 +276,7 @@ func TestMVCCHistories(t *testing.T) {
return err
}
defer func() { _ = iter.Close() }()
for kv := iter.SeekGE(nil, 0); kv != nil; kv = iter.Next() {
for kv := iter.First(); kv != nil; kv = iter.Next() {
if err := iter.Error(); err != nil {
return err
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/storage/pebble_key_schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,9 @@ func (ks *cockroachKeySeeker) IsLowerBound(k []byte, syntheticSuffix []byte) boo
func (ks *cockroachKeySeeker) SeekGE(
key []byte, boundRow int, searchDir int8,
) (row int, equalPrefix bool) {
if buildutil.CrdbTestBuild && len(key) == 0 {
panic(errors.AssertionFailedf("seeking to empty key"))
}
// TODO(jackson): Inline EngineKeySplit.
si := EngineKeySplit(key)
row, eq := ks.roachKeys.Search(key[:si-1])
Expand Down
12 changes: 8 additions & 4 deletions pkg/storage/sst_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"context"
"io"

"github.com/cockroachdb/cockroach/pkg/clusterversion"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/settings"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
Expand Down Expand Up @@ -80,11 +81,14 @@ func (*noopFinishAbort) Abort() {}
// MakeIngestionWriterOptions returns writer options suitable for writing SSTs
// that will subsequently be ingested (e.g. with AddSSTable).
func MakeIngestionWriterOptions(ctx context.Context, cs *cluster.Settings) sstable.WriterOptions {
// By default, take a conservative approach and assume we don't have newer
// table features available. Upgrade to an appropriate version only if the
// cluster supports it. Currently, all supported versions understand
// TableFormatPebblev4.
// All supported versions understand TableFormatPebblev4. If columnar blocks
// are enabled and the active cluster version is at least 24.3, use
// TableFormatPebblev5.
format := sstable.TableFormatPebblev4
if cs.Version.IsActive(ctx, clusterversion.V24_3) && columnarBlocksEnabled.Get(&cs.SV) {
format = sstable.TableFormatPebblev5
}

opts := DefaultPebbleOptions().MakeWriterOptions(0, format)
// By default, compress with the algorithm used for storage in a Pebble store.
// There are other, more specific, use cases that may call for a different
Expand Down

0 comments on commit bfd1ad3

Please sign in to comment.