From cc9da6e7f6fa0a840ffb8e80e8b8dfbeb80c2582 Mon Sep 17 00:00:00 2001 From: Andrew Werner Date: Mon, 15 Aug 2022 10:06:41 -0400 Subject: [PATCH 1/8] sql: use DelRange with tombstone in `force_delete_table_data` Fixes #85754 Release note: None --- pkg/sql/repair.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/sql/repair.go b/pkg/sql/repair.go index cf49330f4673..2a6c59d5b4fc 100644 --- a/pkg/sql/repair.go +++ b/pkg/sql/repair.go @@ -17,6 +17,7 @@ import ( "sort" "github.com/cockroachdb/cockroach/pkg/cloud" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -780,17 +781,22 @@ func (p *planner) ForceDeleteTableData(ctx context.Context, descID int64) error } prefix := p.extendedEvalCtx.Codec.TablePrefix(uint32(id)) - tableSpans := roachpb.Span{Key: prefix, EndKey: prefix.PrefixEnd()} + tableSpan := roachpb.Span{Key: prefix, EndKey: prefix.PrefixEnd()} + requestHeader := roachpb.RequestHeader{ + Key: tableSpan.Key, EndKey: tableSpan.EndKey, + } b := &kv.Batch{} - b.AddRawRequest(&roachpb.ClearRangeRequest{ - RequestHeader: roachpb.RequestHeader{ - Key: tableSpans.Key, - EndKey: tableSpans.EndKey, - }, - }) - - err = p.txn.DB().Run(ctx, b) - if err != nil { + if p.execCfg.Settings.Version.IsActive(ctx, clusterversion.UseDelRangeInGCJob) { + b.AddRawRequest(&roachpb.DeleteRangeRequest{ + RequestHeader: requestHeader, + UseRangeTombstone: true, + }) + } else { + b.AddRawRequest(&roachpb.ClearRangeRequest{ + RequestHeader: requestHeader, + }) + } + if err := p.txn.DB().Run(ctx, b); err != nil { return err } From f17a6cd46fb3baed9fb328d0a827342ab8a97b7a Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Mon, 15 Aug 2022 10:48:55 -0700 Subject: [PATCH 2/8] colexecerror: do not annotate the context canceled error This commit makes it so that the context canceled error doesn't get annotated with an assertion failure when it doesn't have a valid PG code. This makes sure that the sentry issues don't get filed for the context canceled errors - they are expected to occur. Release note: None --- pkg/sql/colexecerror/error.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/sql/colexecerror/error.go b/pkg/sql/colexecerror/error.go index 5658b16112ad..1034a00e743d 100644 --- a/pkg/sql/colexecerror/error.go +++ b/pkg/sql/colexecerror/error.go @@ -75,10 +75,11 @@ func CatchVectorizedRuntimeError(operation func()) (retErr error) { annotateErrorWithoutCode := true var nie *notInternalError - if errors.As(err, &nie) { - // A notInternalError was not caused by the vectorized engine and - // represents an error that we don't want to annotate in case it - // doesn't have a valid PG code. + if errors.Is(err, context.Canceled) || errors.As(err, &nie) { + // We don't want to annotate the context cancellation and + // notInternalError errors in case they don't have a valid PG code + // so that the sentry report is not sent (errors with failed + // assertions get sentry reports). annotateErrorWithoutCode = false } if code := pgerror.GetPGCode(err); annotateErrorWithoutCode && code == pgcode.Uncategorized { From 9d1a0d433dfab01e8f0b1824571e807d9b98e2ff Mon Sep 17 00:00:00 2001 From: richardjcai Date: Mon, 15 Aug 2022 14:51:52 -0400 Subject: [PATCH 3/8] sql: deflake TestRoleOptionsMigration15000User Previously it was flakey because we always assumed the first user created had ID 100, however this is not the case due to transaction failures. Release note: None Release justification: test only --- pkg/upgrade/upgrades/role_options_migration_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/upgrade/upgrades/role_options_migration_test.go b/pkg/upgrade/upgrades/role_options_migration_test.go index ccdb36241746..ca65d0d1d8dd 100644 --- a/pkg/upgrade/upgrades/role_options_migration_test.go +++ b/pkg/upgrade/upgrades/role_options_migration_test.go @@ -102,7 +102,11 @@ func runTestRoleOptionsMigration(t *testing.T, numUsers int) { } if numUsers > 100 { - id := uint32(100) + var id uint32 + // We're not guaranteed that the ID of the first created + // user is 100. + idRow := tdb.QueryRow(t, `SELECT last_value - 1 FROM system.role_id_seq`) + idRow.Scan(&id) var wg sync.WaitGroup wg.Add(100) // Parallelize user creation. From 6b452bce4e6459f620ad6f2a7b01349e4e319347 Mon Sep 17 00:00:00 2001 From: Rebecca Taft Date: Mon, 15 Aug 2022 14:56:52 -0500 Subject: [PATCH 4/8] opt: fix error due to unsupported comparison for partitioned secondary index This commit fixes a bug where we were attempting to find the locality of the partitions in a secondary index, but we passed the incorrect index ordinal to the function IndexPartitionLocality. Fixes #86168 Release note (bug fix): Fixed a bug that existed on v22.1.0-v22.1.5, where attempting to select data from a table that had different partitioning columns used for the primary and secondary indexes could cause an error. This occured if the primary index had zone configurations applied to the index partitions with different regions for different partitions, and the secondary index had a different column type than the primary index for its partitioning column(s). --- pkg/sql/opt/xform/select_funcs.go | 2 +- pkg/sql/opt/xform/testdata/rules/select | 57 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 72d725a4fb76..68d75f7fbd8b 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -432,7 +432,7 @@ func (c *CustomFuncs) GenerateConstrainedScans( // Create a prefix sorter that describes which index partitions are // local to the gateway region. - prefixSorter, _ := tabMeta.IndexPartitionLocality(scanPrivate.Index, index, c.e.evalCtx) + prefixSorter, _ := tabMeta.IndexPartitionLocality(index.Ordinal(), index, c.e.evalCtx) // Build Constraints to scan a subset of the table Spans. if partitionFilters, remainingFilters, combinedConstraint, ok = diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 016f2d842902..c40c116ee199 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -2181,6 +2181,63 @@ index-join e ├── key: (1) └── fd: (1)-->(2,4) +# Regression test for #86168. Selecting from a table with a partitioned +# primary and secondary index, where the primary index is partitioned by +# region and the secondary index is partitioned by another column, should +# not cause an error. +exec-ddl +CREATE TYPE region_enum AS ENUM ('AP_SOUTHEAST', 'CA_CENTRAL', 'US_EAST'); +---- + +exec-ddl +CREATE TABLE "user" ( + region region_enum NOT NULL, + id uuid NOT NULL DEFAULT uuid_generate_v4(), + col1 int2 AS (col2::int2) VIRTUAL, + col2 varchar NOT NULL, + PRIMARY KEY (region, id), + UNIQUE (col1, col2) PARTITION BY LIST (col1) ( + PARTITION user_col1_col2_key_ap_southeast VALUES IN (43,32), + PARTITION user_col1_col2_key_ca_central VALUES IN (1), + PARTITION DEFAULT VALUES IN (default) + ) +) PARTITION BY LIST (region) ( + PARTITION user_ap_southeast VALUES IN ('AP_SOUTHEAST'), + PARTITION user_us_east VALUES IN ('US_EAST'), + PARTITION user_ca_central VALUES IN ('CA_CENTRAL'), + PARTITION DEFAULT VALUES IN (default) +); +---- + +exec-ddl +ALTER PARTITION user_ap_southeast OF TABLE "user" CONFIGURE ZONE USING + num_replicas = 5, + num_voters = 3, + lease_preferences = '[[+region=ap-southeast-2]]', + voter_constraints = '[+region=ap-southeast-2]'; +---- + +opt locality=(region=ap-southeast-2) +SELECT * +FROM "user" +WHERE region = 'AP_SOUTHEAST'; +---- +project + ├── columns: region:1!null id:2!null col1:3!null col2:4!null + ├── immutable + ├── key: (2) + ├── fd: ()-->(1), (2)-->(4), (4)-->(3) + ├── distribution: ap-southeast-2 + ├── scan user + │ ├── columns: region:1!null id:2!null col2:4!null + │ ├── constraint: /1/2: [/'AP_SOUTHEAST' - /'AP_SOUTHEAST'] + │ ├── immutable + │ ├── key: (2) + │ ├── fd: ()-->(1), (2)-->(4) + │ └── distribution: ap-southeast-2 + └── projections + └── col2:4::INT2 [as=col1:3, outer=(4), immutable] + # -------------------------------------------------- # GenerateInvertedIndexScans # -------------------------------------------------- From 32f38c54f82faa424d643dbbaa6e325dc286be7c Mon Sep 17 00:00:00 2001 From: Michael Erickson Date: Fri, 12 Aug 2022 16:14:22 -0700 Subject: [PATCH 5/8] sql/stats: use nil *eval.Context as CompareContext when forecasting When forecasting table statistics, we don't need a full *eval.Context. We can simply use a nil *eval.Context as a tree.CompareContext. This means we don't have to plumb an eval.Context into the stats cache. Assists: #79872 Release note: None --- pkg/sql/rowexec/sample_aggregator.go | 2 ++ pkg/sql/sem/eval/context.go | 3 +++ pkg/sql/show_stats.go | 2 +- pkg/sql/stats/forecast.go | 32 +++++++++------------------- pkg/sql/stats/forecast_test.go | 5 +---- pkg/sql/stats/quantile.go | 9 +++++--- pkg/sql/stats/quantile_test.go | 28 ++++++++++-------------- 7 files changed, 34 insertions(+), 47 deletions(-) diff --git a/pkg/sql/rowexec/sample_aggregator.go b/pkg/sql/rowexec/sample_aggregator.go index 46d09424d877..cba01aacd3dc 100644 --- a/pkg/sql/rowexec/sample_aggregator.go +++ b/pkg/sql/rowexec/sample_aggregator.go @@ -616,6 +616,8 @@ func (s *sampleAggregator) generateHistogram( prevCapacity, sr.Cap(), ) } + // TODO(michae2): Instead of using the flowCtx's evalCtx, investigate + // whether this can use a nil *eval.Context. h, _, err := stats.EquiDepthHistogram(evalCtx, colType, values, numRows, distinctCount, maxBuckets) return h, err } diff --git a/pkg/sql/sem/eval/context.go b/pkg/sql/sem/eval/context.go index 666328012ec0..8d92cf6973c7 100644 --- a/pkg/sql/sem/eval/context.go +++ b/pkg/sql/sem/eval/context.go @@ -566,6 +566,9 @@ func (ec *Context) SetStmtTimestamp(ts time.Time) { // GetLocation returns the session timezone. func (ec *Context) GetLocation() *time.Location { + if ec == nil { + return time.UTC + } return ec.SessionData().GetLocation() } diff --git a/pkg/sql/show_stats.go b/pkg/sql/show_stats.go index d683b246b6b7..86e847d2e3fe 100644 --- a/pkg/sql/show_stats.go +++ b/pkg/sql/show_stats.go @@ -179,7 +179,7 @@ func (p *planner) ShowTableStats(ctx context.Context, n *tree.ShowTableStats) (p observed[i], observed[j] = observed[j], observed[i] } - forecasts := stats.ForecastTableStatistics(ctx, p.EvalContext(), observed) + forecasts := stats.ForecastTableStatistics(ctx, observed) // Iterate in reverse order to match the ORDER BY "columnIDs". for i := len(forecasts) - 1; i >= 0; i-- { diff --git a/pkg/sql/stats/forecast.go b/pkg/sql/stats/forecast.go index e1ed47595bb0..08670e78ab23 100644 --- a/pkg/sql/stats/forecast.go +++ b/pkg/sql/stats/forecast.go @@ -17,7 +17,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" - "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" @@ -57,12 +57,7 @@ const maxForecastDistance = time.Hour * 24 * 7 // // ForecastTableStatistics is deterministic: given the same observations it will // return the same forecasts. -// -// TODO(michae2): Use nil *eval.Context or custom tree.CompareContext instead of -// taking an evalCtx. -func ForecastTableStatistics( - ctx context.Context, evalCtx tree.CompareContext, observed []*TableStatistic, -) []*TableStatistic { +func ForecastTableStatistics(ctx context.Context, observed []*TableStatistic) []*TableStatistic { // Early sanity check. We'll check this again in forecastColumnStatistics. if len(observed) < minObservationsForForecast { return nil @@ -102,9 +97,7 @@ func ForecastTableStatistics( forecasts := make([]*TableStatistic, 0, len(forecastCols)) for _, colKey := range forecastCols { - forecast, err := forecastColumnStatistics( - ctx, evalCtx, observedByCols[colKey], at, minGoodnessOfFit, - ) + forecast, err := forecastColumnStatistics(ctx, observedByCols[colKey], at, minGoodnessOfFit) if err != nil { log.VEventf( ctx, 2, "could not forecast statistics for table %v columns %s: %v", @@ -135,11 +128,7 @@ func ForecastTableStatistics( // forecastColumnStatistics is deterministic: given the same observations and // forecast time, it will return the same forecast. func forecastColumnStatistics( - ctx context.Context, - evalCtx tree.CompareContext, - observed []*TableStatistic, - at time.Time, - minRequiredFit float64, + ctx context.Context, observed []*TableStatistic, at time.Time, minRequiredFit float64, ) (forecast *TableStatistic, err error) { if len(observed) < minObservationsForForecast { return nil, errors.New("not enough observations to forecast statistics") @@ -263,9 +252,7 @@ func forecastColumnStatistics( // histogram. NOTE: If any of the observed histograms were for inverted // indexes this will produce an incorrect histogram. if observed[0].HistogramData != nil { - hist, err := predictHistogram( - ctx, evalCtx, observed, forecastAt, minRequiredFit, nonNullRowCount, - ) + hist, err := predictHistogram(ctx, observed, forecastAt, minRequiredFit, nonNullRowCount) if err != nil { // If we did not successfully predict a histogram then copy the latest // histogram so we can adjust it. @@ -276,8 +263,10 @@ func forecastColumnStatistics( hist.buckets = append([]cat.HistogramBucket{}, observed[0].nonNullHistogram().buckets...) } - // Now adjust for consistency. - hist.adjustCounts(evalCtx, nonNullRowCount, nonNullDistinctCount) + // Now adjust for consistency. We don't use any session data for operations + // on upper bounds, so a nil *eval.Context works as our tree.CompareContext. + var compareCtx *eval.Context + hist.adjustCounts(compareCtx, nonNullRowCount, nonNullDistinctCount) // Finally, convert back to HistogramData. histData, err := hist.toHistogramData(observed[0].HistogramData.ColumnType) @@ -294,7 +283,6 @@ func forecastColumnStatistics( // predictHistogram tries to predict the histogram at forecast time. func predictHistogram( ctx context.Context, - evalCtx tree.CompareContext, observed []*TableStatistic, forecastAt float64, minRequiredFit float64, @@ -359,5 +347,5 @@ func predictHistogram( } // Finally, convert the predicted quantile function back to a histogram. - return yₙ.toHistogram(evalCtx, colType, nonNullRowCount) + return yₙ.toHistogram(colType, nonNullRowCount) } diff --git a/pkg/sql/stats/forecast_test.go b/pkg/sql/stats/forecast_test.go index b4a4ad43dc5a..65d384711964 100644 --- a/pkg/sql/stats/forecast_test.go +++ b/pkg/sql/stats/forecast_test.go @@ -18,10 +18,8 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/sem/catid" - "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/timeutil" ) @@ -581,7 +579,6 @@ func TestForecastColumnStatistics(t *testing.T) { }, } ctx := context.Background() - evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { @@ -593,7 +590,7 @@ func TestForecastColumnStatistics(t *testing.T) { expected := tc.forecast.toTableStatistic(jobspb.ForecastStatsName, i) at := testStatTime(tc.at) - forecast, err := forecastColumnStatistics(ctx, evalCtx, observed, at, 1) + forecast, err := forecastColumnStatistics(ctx, observed, at, 1) if err != nil { if !tc.err { t.Errorf("test case %d unexpected forecastColumnStatistics err: %v", i, err) diff --git a/pkg/sql/stats/quantile.go b/pkg/sql/stats/quantile.go index e48130f36109..2e69d6cd154c 100644 --- a/pkg/sql/stats/quantile.go +++ b/pkg/sql/stats/quantile.go @@ -16,6 +16,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" + "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -237,9 +238,7 @@ func makeQuantile(hist histogram, rowCount float64) (quantile, error) { // toHistogram converts a quantile into a histogram, using the provided type and // row count. It returns an error if the conversion fails. The quantile must be // well-formed before calling toHistogram. -func (q quantile) toHistogram( - compareCtx tree.CompareContext, colType *types.T, rowCount float64, -) (histogram, error) { +func (q quantile) toHistogram(colType *types.T, rowCount float64) (histogram, error) { if len(q) < 2 || q[0].p != 0 || q[len(q)-1].p != 1 { return histogram{}, errors.AssertionFailedf("invalid quantile: %v", q) } @@ -249,6 +248,10 @@ func (q quantile) toHistogram( return histogram{buckets: make([]cat.HistogramBucket, 0)}, nil } + // We don't use any session data for conversions or operations on upper + // bounds, so a nil *eval.Context works as our tree.CompareContext. + var compareCtx *eval.Context + hist := histogram{buckets: make([]cat.HistogramBucket, 0, len(q)-1)} var i quantileIndex diff --git a/pkg/sql/stats/quantile_test.go b/pkg/sql/stats/quantile_test.go index ea2818a07ae2..48381cd1898f 100644 --- a/pkg/sql/stats/quantile_test.go +++ b/pkg/sql/stats/quantile_test.go @@ -20,7 +20,6 @@ import ( "strconv" "testing" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -41,19 +40,18 @@ func TestRandomQuantileRoundTrip(t *testing.T) { types.Float4, } colTypes = append(colTypes, types.Scalar...) - evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) rng, seed := randutil.NewTestRand() for _, colType := range colTypes { if canMakeQuantile(histVersion, colType) { for i := 0; i < 5; i++ { t.Run(fmt.Sprintf("%v/%v", colType.Name(), i), func(t *testing.T) { - hist, rowCount := randHist(evalCtx, colType, rng) + hist, rowCount := randHist(colType, rng) qfun, err := makeQuantile(hist, rowCount) if err != nil { t.Errorf("seed: %v unexpected makeQuantile error: %v", seed, err) return } - hist2, err := qfun.toHistogram(evalCtx, colType, rowCount) + hist2, err := qfun.toHistogram(colType, rowCount) if err != nil { t.Errorf("seed: %v unexpected quantile.toHistogram error: %v", seed, err) return @@ -70,12 +68,10 @@ func TestRandomQuantileRoundTrip(t *testing.T) { // randHist makes a random histogram of the specified type, with [1, 200] // buckets. Not all types are supported. Every bucket will have NumEq > 0 but // could have NumRange == 0. -func randHist( - compareCtx tree.CompareContext, colType *types.T, rng *rand.Rand, -) (histogram, float64) { +func randHist(colType *types.T, rng *rand.Rand) (histogram, float64) { numBuckets := rng.Intn(200) + 1 buckets := make([]cat.HistogramBucket, numBuckets) - bounds := randBounds(compareCtx, colType, rng, numBuckets) + bounds := randBounds(colType, rng, numBuckets) buckets[0].NumEq = float64(rng.Intn(100) + 1) buckets[0].UpperBound = bounds[0] rowCount := buckets[0].NumEq @@ -98,6 +94,7 @@ func randHist( rowCount += rows } // Set DistinctRange in all buckets. + var compareCtx *eval.Context for i := 1; i < len(buckets); i++ { lowerBound := getNextLowerBound(compareCtx, buckets[i-1].UpperBound) buckets[i].DistinctRange = estimatedDistinctValuesInRange( @@ -111,9 +108,7 @@ func randHist( // type. Not all types are supported. This differs from randgen.RandDatum in // that it generates no "interesting" Datums, and differs from // randgen.RandDatumSimple in that it generates distinct Datums without repeats. -func randBounds( - compareCtx tree.CompareContext, colType *types.T, rng *rand.Rand, num int, -) tree.Datums { +func randBounds(colType *types.T, rng *rand.Rand, num int) tree.Datums { datums := make(tree.Datums, num) // randInts creates an ordered slice of num distinct random ints in the closed @@ -566,10 +561,9 @@ func TestQuantileToHistogram(t *testing.T) { err: true, }, } - evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { - hist, err := tc.qfun.toHistogram(evalCtx, types.Float, tc.rows) + hist, err := tc.qfun.toHistogram(types.Float, tc.rows) if err != nil { if !tc.err { t.Errorf("test case %d unexpected quantile.toHistogram err: %v", i, err) @@ -843,7 +837,7 @@ func TestQuantileValueRoundTrip(t *testing.T) { err: true, }, } - evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) + var compareCtx *eval.Context for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { val, err := toQuantileValue(tc.dat) @@ -867,7 +861,7 @@ func TestQuantileValueRoundTrip(t *testing.T) { t.Errorf("test case %d (%v) unexpected fromQuantileValue err: %v", i, tc.typ.Name(), err) return } - cmp, err := res.CompareError(evalCtx, tc.dat) + cmp, err := res.CompareError(compareCtx, tc.dat) if err != nil { t.Errorf("test case %d (%v) unexpected CompareError err: %v", i, tc.typ.Name(), err) return @@ -1120,7 +1114,7 @@ func TestQuantileValueRoundTripOverflow(t *testing.T) { res: quantileMaxTimestampSec, }, } - evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) + var compareCtx *eval.Context for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { d, err := fromQuantileValue(tc.typ, tc.val) @@ -1134,7 +1128,7 @@ func TestQuantileValueRoundTripOverflow(t *testing.T) { t.Errorf("test case %d (%v) expected fromQuantileValue err", i, tc.typ.Name()) return } - cmp, err := d.CompareError(evalCtx, tc.dat) + cmp, err := d.CompareError(compareCtx, tc.dat) if err != nil { t.Errorf("test case %d (%v) unexpected CompareError err: %v", i, tc.typ.Name(), err) return From b4420e56b006bb98d932df1a1260fe2f996c2d60 Mon Sep 17 00:00:00 2001 From: Michael Erickson Date: Fri, 12 Aug 2022 21:02:41 -0700 Subject: [PATCH 6/8] sql/stats: generate statistics forecasts in the stats cache As of this commit, we now try to generate statistics forecasts for every column of every table. This happens whenever statistics are loaded into or refreshed in the stats cache. We use only the forecasts that fit the historical collected statistics very well, meaning we have high confidence in their accuracy. Fixes: #79872 Release note (performance improvement): Enable table statistics forecasts, which predict future statistics based on historical collected statistics. Forecasts help the optimizer produce better plans for queries that read data modified after the latest statistics collection. We use only the forecasts that fit the historical collected statistics very well, meaning we have high confidence in their accuracy. Forecasts can be viewed using `SHOW STATISTICS FOR TABLE ... WITH FORECAST`. --- pkg/sql/stats/automatic_stats_test.go | 10 ++++++++-- pkg/sql/stats/stats_cache.go | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/sql/stats/automatic_stats_test.go b/pkg/sql/stats/automatic_stats_test.go index efb2ca0e9053..8b3d861abc58 100644 --- a/pkg/sql/stats/automatic_stats_test.go +++ b/pkg/sql/stats/automatic_stats_test.go @@ -773,8 +773,14 @@ func checkStatsCount( if err != nil { return err } - if len(stats) != expected { - return fmt.Errorf("expected %d stat(s) but found %d", expected, len(stats)) + var count int + for i := range stats { + if stats[i].Name != jobspb.ForecastStatsName { + count++ + } + } + if count != expected { + return fmt.Errorf("expected %d stat(s) but found %d", expected, count) } return nil }) diff --git a/pkg/sql/stats/stats_cache.go b/pkg/sql/stats/stats_cache.go index 1e9e1cc252d4..abe779ef7a1f 100644 --- a/pkg/sql/stats/stats_cache.go +++ b/pkg/sql/stats/stats_cache.go @@ -720,5 +720,8 @@ ORDER BY "createdAt" DESC, "columnIDs" DESC, "statisticID" DESC return nil, err } + forecasts := ForecastTableStatistics(ctx, statsList) + statsList = append(forecasts, statsList...) + return statsList, nil } From 430c82407fa12d49fcf0825c585a214003d2ec4e Mon Sep 17 00:00:00 2001 From: Michael Erickson Date: Fri, 12 Aug 2022 22:31:17 -0700 Subject: [PATCH 7/8] sql: show forecasted stats time in EXPLAIN When using statistics forecasts, add the forecast time (which could be in the future) to EXPLAIN output. This both indicates that forecasts are in use, and gives us an idea of how up-to-date / ahead they are. Assists: #79872 Release note: None --- pkg/sql/opt/cat/table.go | 3 +++ pkg/sql/opt/exec/execbuilder/relational.go | 12 +++++++++ pkg/sql/opt/exec/explain/emit.go | 25 +++++++++++++++++-- pkg/sql/opt/exec/factory.go | 6 +++++ pkg/sql/opt/testutils/testcat/BUILD.bazel | 1 + pkg/sql/opt/testutils/testcat/test_catalog.go | 6 +++++ pkg/sql/opt_catalog.go | 6 +++++ 7 files changed, 57 insertions(+), 2 deletions(-) diff --git a/pkg/sql/opt/cat/table.go b/pkg/sql/opt/cat/table.go index 2f7e0b674290..ef24aa2ef8d7 100644 --- a/pkg/sql/opt/cat/table.go +++ b/pkg/sql/opt/cat/table.go @@ -201,6 +201,9 @@ type TableStatistic interface { // HistogramType returns the type that the histogram was created on. For // inverted index histograms, this will always return types.Bytes. HistogramType() *types.T + + // IsForecast returns true if this statistic is a forecast. + IsForecast() bool } // HistogramBucket contains the data for a single histogram bucket. Note diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index b0c07e34b673..d0d40c179cd6 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -400,6 +400,18 @@ func (b *Builder) maybeAnnotateWithEstimates(node exec.Node, e memo.RelExpr) { } val.TableStatsCreatedAt = stat.CreatedAt() val.LimitHint = scan.RequiredPhysical().LimitHint + val.Forecast = stat.IsForecast() + if val.Forecast { + val.ForecastAt = stat.CreatedAt() + // Find the first non-forecast stat. + for i := 0; i < tab.StatisticCount(); i++ { + nextStat := tab.Statistic(i) + if !nextStat.IsForecast() { + val.TableStatsCreatedAt = nextStat.CreatedAt() + break + } + } + } } } ef.AnnotateNode(node, exec.EstimatedStatsID, &val) diff --git a/pkg/sql/opt/exec/explain/emit.go b/pkg/sql/opt/exec/explain/emit.go index cb0c9e24143d..e16d2384192c 100644 --- a/pkg/sql/opt/exec/explain/emit.go +++ b/pkg/sql/opt/exec/explain/emit.go @@ -475,10 +475,31 @@ func (e *emitter) emitNodeAttributes(n *Node) error { } duration = string(humanizeutil.LongDuration(timeSinceStats)) } + + var forecastStr string + if s.Forecast { + if e.ob.flags.Redact.Has(RedactVolatile) { + forecastStr = "; using stats forecast" + } else { + timeSinceStats := timeutil.Since(s.ForecastAt) + if timeSinceStats >= 0 { + forecastStr = fmt.Sprintf( + "; using stats forecast for %s ago", humanizeutil.LongDuration(timeSinceStats), + ) + } else { + timeSinceStats *= -1 + forecastStr = fmt.Sprintf( + "; using stats forecast for %s in the future", + humanizeutil.LongDuration(timeSinceStats), + ) + } + } + } + e.ob.AddField("estimated row count", fmt.Sprintf( - "%s (%s%% of the table; stats collected %s ago)", + "%s (%s%% of the table; stats collected %s ago%s)", estimatedRowCountString, percentageStr, - duration, + duration, forecastStr, )) } else { e.ob.AddField("estimated row count", estimatedRowCountString) diff --git a/pkg/sql/opt/exec/factory.go b/pkg/sql/opt/exec/factory.go index b3acbaa4abad..5fa4d4435bca 100644 --- a/pkg/sql/opt/exec/factory.go +++ b/pkg/sql/opt/exec/factory.go @@ -310,6 +310,12 @@ type EstimatedStats struct { // LimitHint is the "soft limit" of the number of result rows that may be // required. See physical.Required for details. LimitHint float64 + // Forecast is set only for scans; it is true if the stats for the scan were + // forecasted rather than collected. + Forecast bool + // ForecastAt is set only for scans with forecasted stats; it is the time the + // forecast was for (which could be in the past, present, or future). + ForecastAt time.Time } // ExecutionStats contain statistics about a given operator gathered from the diff --git a/pkg/sql/opt/testutils/testcat/BUILD.bazel b/pkg/sql/opt/testutils/testcat/BUILD.bazel index 8a6a47a7c08a..8df7a6e10b7e 100644 --- a/pkg/sql/opt/testutils/testcat/BUILD.bazel +++ b/pkg/sql/opt/testutils/testcat/BUILD.bazel @@ -23,6 +23,7 @@ go_library( deps = [ "//pkg/config/zonepb", "//pkg/geo/geoindex", + "//pkg/jobs/jobspb", "//pkg/roachpb", "//pkg/security/username", "//pkg/settings/cluster", diff --git a/pkg/sql/opt/testutils/testcat/test_catalog.go b/pkg/sql/opt/testutils/testcat/test_catalog.go index 285a4151a7d1..f337fe857927 100644 --- a/pkg/sql/opt/testutils/testcat/test_catalog.go +++ b/pkg/sql/opt/testutils/testcat/test_catalog.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/geo/geoindex" + "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -1230,6 +1231,11 @@ func (ts *TableStat) HistogramType() *types.T { return tree.MustBeStaticallyKnownType(colTypeRef) } +// IsForecast is part of the cat.TableStatistic interface. +func (ts *TableStat) IsForecast() bool { + return ts.js.Name == jobspb.ForecastStatsName +} + // TableStats is a slice of TableStat pointers. type TableStats []*TableStat diff --git a/pkg/sql/opt_catalog.go b/pkg/sql/opt_catalog.go index c274ee4b45e8..351bfe334e6e 100644 --- a/pkg/sql/opt_catalog.go +++ b/pkg/sql/opt_catalog.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/geo/geoindex" + "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -1685,6 +1686,11 @@ func (os *optTableStat) HistogramType() *types.T { return os.stat.HistogramData.ColumnType } +// IsForecast is part of the cat.TableStatistic interface. +func (os *optTableStat) IsForecast() bool { + return os.stat.Name == jobspb.ForecastStatsName +} + // optFamily is a wrapper around descpb.ColumnFamilyDescriptor that keeps a // reference to the table wrapper. type optFamily struct { From 1c21c3a3ab396ac24b7ecac1573c5903b99b4451 Mon Sep 17 00:00:00 2001 From: Michael Erickson Date: Sat, 13 Aug 2022 22:05:55 -0700 Subject: [PATCH 8/8] sql/opt: add tests for statistics forecasts Add a few simple testcases for usage of statistics forecasts by the optimizer. Assists: #79872 Release note: None --- .../opt/exec/execbuilder/testdata/forecast | 599 ++++++++++++++++++ .../execbuilder/tests/local/generated_test.go | 7 + 2 files changed, 606 insertions(+) create mode 100644 pkg/sql/opt/exec/execbuilder/testdata/forecast diff --git a/pkg/sql/opt/exec/execbuilder/testdata/forecast b/pkg/sql/opt/exec/execbuilder/testdata/forecast new file mode 100644 index 000000000000..17d009ac4d28 --- /dev/null +++ b/pkg/sql/opt/exec/execbuilder/testdata/forecast @@ -0,0 +1,599 @@ +# LogicTest: local + +# Tests that verify we create and use table statistics forecasts correctly. + +# Verify that we create and use statistics forecasts for a simple table that +# grows at a constant rate. + +statement ok +CREATE TABLE g (a INT PRIMARY KEY) WITH (sql_stats_automatic_collection_enabled = false) + +statement ok +ALTER TABLE g INJECT STATISTICS '[ + { + "avg_size": 1, + "columns": [ + "a" + ], + "created_at": "1988-08-05 00:00:00.000000", + "distinct_count": 3, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 3 + }, + { + "avg_size": 1, + "columns": [ + "a" + ], + "created_at": "1988-08-06 00:00:00.000000", + "distinct_count": 6, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "3" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "4" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "5" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 6 + }, + { + "avg_size": 1, + "columns": [ + "a" + ], + "created_at": "1988-08-07 00:00:00.000000", + "distinct_count": 9, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "3" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "4" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "5" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "6" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "7" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "8" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 9 + } +]' + +query TTTIIII +SELECT statistics_name, column_names, created, row_count, distinct_count, null_count, avg_size +FROM [SHOW STATISTICS FOR TABLE g WITH FORECAST] +ORDER BY created +---- +__auto__ {a} 1988-08-05 00:00:00 +0000 +0000 3 3 0 1 +__auto__ {a} 1988-08-06 00:00:00 +0000 +0000 6 6 0 1 +__auto__ {a} 1988-08-07 00:00:00 +0000 +0000 9 9 0 1 +__forecast__ {a} 1988-08-08 00:00:00 +0000 +0000 12 12 0 1 + +query T +EXPLAIN SELECT * FROM g WHERE a >= 9 AND a < 12 +---- +distribution: local +vectorized: true +· +• scan + estimated row count: 3 (25% of the table; stats collected ago; using stats forecast) + table: g@g_pkey + spans: [/9 - /11] + +query T +EXPLAIN (OPT, VERBOSE) SELECT * FROM g WHERE a >= 0 AND a < 100 +---- +scan g + ├── columns: a:1 + ├── constraint: /1: [/0 - /99] + ├── cardinality: [0 - 100] + ├── stats: [rows=12, distinct(1)=10, null(1)=0, avgsize(1)=1] + │ histogram(1)= 0 1.3333 0 0.66667 0 0.66667 0 1.3333 0 1.3333 0 0.66667 0 0.66667 0 1.3333 0 1.3333 0 0.66667 0 0.66667 0 1.3333 + │ <---- 0 ------- 1 ------- 2 ------ 3 ------ 4 ------- 5 ------- 6 ------ 7 ------ 8 ------- 9 ------ 10 ------ 11 - + ├── cost: 21.13 + ├── key: (1) + └── distribution: test + +# Verify that we create and use statistics forecasts for a simple table that +# shrinks at a constant rate. + +statement ok +CREATE TABLE s (b INT PRIMARY KEY) WITH (sql_stats_automatic_collection_enabled = false) + +statement ok +ALTER TABLE s INJECT STATISTICS '[ + { + "avg_size": 1, + "columns": [ + "b" + ], + "created_at": "1988-08-05 00:00:00.000000", + "distinct_count": 9, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "3" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "4" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "5" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "6" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "7" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "8" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 9 + }, + { + "avg_size": 1, + "columns": [ + "b" + ], + "created_at": "1988-08-06 00:00:00.000000", + "distinct_count": 6, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "3" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "4" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "5" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 6 + }, + { + "avg_size": 1, + "columns": [ + "b" + ], + "created_at": "1988-08-07 00:00:00.000000", + "distinct_count": 3, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "0" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1" + }, + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "2" + } + ], + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 3 + } +]' + +query TTTIIII +SELECT statistics_name, column_names, created, row_count, distinct_count, null_count, avg_size +FROM [SHOW STATISTICS FOR TABLE s WITH FORECAST] +ORDER BY created +---- +__auto__ {b} 1988-08-05 00:00:00 +0000 +0000 9 9 0 1 +__auto__ {b} 1988-08-06 00:00:00 +0000 +0000 6 6 0 1 +__auto__ {b} 1988-08-07 00:00:00 +0000 +0000 3 3 0 1 +__forecast__ {b} 1988-08-08 00:00:00 +0000 +0000 0 0 0 1 + +query T +SELECT jsonb_pretty(stat) +FROM ( +SELECT jsonb_array_elements(statistics) AS stat +FROM [SHOW STATISTICS USING JSON FOR TABLE s WITH FORECAST] +) +WHERE stat->>'name' = '__forecast__' +---- +{ + "avg_size": 1, + "columns": [ + "b" + ], + "created_at": "1988-08-08 00:00:00", + "distinct_count": 0, + "histo_col_type": "INT8", + "histo_version": 2, + "name": "__forecast__", + "null_count": 0, + "row_count": 0 +} + +query T +EXPLAIN SELECT * FROM s WHERE b >= 0 AND b < 12 +---- +distribution: local +vectorized: true +· +• scan + estimated row count: 1 (100% of the table; stats collected ago; using stats forecast) + table: s@s_pkey + spans: [/0 - /11] + +query T +EXPLAIN (OPT, VERBOSE) SELECT * FROM s WHERE b >= 0 AND b < 100 +---- +scan s + ├── columns: b:1 + ├── constraint: /1: [/0 - /99] + ├── cardinality: [0 - 100] + ├── stats: [rows=1, distinct(1)=1, null(1)=0, avgsize(1)=1] + │ histogram(1)= + ├── cost: 10.02 + ├── key: (1) + └── distribution: test + +# Verify that we create and use statistics forecasts for a simple table that +# changes at a constant rate. + +statement ok +CREATE TABLE c (h TIMESTAMPTZ PRIMARY KEY) WITH (sql_stats_automatic_collection_enabled = false) + +statement ok +ALTER TABLE c INJECT STATISTICS '[ + { + "avg_size": 7, + "columns": [ + "h" + ], + "created_at": "1988-08-05 00:00:00.000000", + "distinct_count": 24, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1988-08-04 00:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-04 06:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-04 12:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-04 18:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 0, + "num_range": 5, + "upper_bound": "1988-08-05 00:00:00+00:00" + } + ], + "histo_col_type": "TIMESTAMPTZ", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 24 + }, + { + "avg_size": 7, + "columns": [ + "h" + ], + "created_at": "1988-08-06 00:00:00.000000", + "distinct_count": 24, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1988-08-05 00:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-05 06:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-05 12:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-05 18:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 0, + "num_range": 5, + "upper_bound": "1988-08-06 00:00:00+00:00" + } + ], + "histo_col_type": "TIMESTAMPTZ", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 24 + }, + { + "avg_size": 7, + "columns": [ + "h" + ], + "created_at": "1988-08-07 00:00:00.000000", + "distinct_count": 24, + "histo_buckets": [ + { + "distinct_range": 0, + "num_eq": 1, + "num_range": 0, + "upper_bound": "1988-08-06 00:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-06 06:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-06 12:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 1, + "num_range": 5, + "upper_bound": "1988-08-06 18:00:00+00:00" + }, + { + "distinct_range": 5, + "num_eq": 0, + "num_range": 5, + "upper_bound": "1988-08-07 00:00:00+00:00" + } + ], + "histo_col_type": "TIMESTAMPTZ", + "histo_version": 2, + "name": "__auto__", + "null_count": 0, + "row_count": 24 + } +]' + +query TTTIIII +SELECT statistics_name, column_names, created, row_count, distinct_count, null_count, avg_size +FROM [SHOW STATISTICS FOR TABLE c WITH FORECAST] +ORDER BY created +---- +__auto__ {h} 1988-08-05 00:00:00 +0000 +0000 24 24 0 7 +__auto__ {h} 1988-08-06 00:00:00 +0000 +0000 24 24 0 7 +__auto__ {h} 1988-08-07 00:00:00 +0000 +0000 24 24 0 7 +__forecast__ {h} 1988-08-08 00:00:00 +0000 +0000 24 24 0 7 + +query T +EXPLAIN SELECT * FROM c WHERE h >= '1988-08-07' +---- +distribution: local +vectorized: true +· +• scan + estimated row count: 24 (100% of the table; stats collected ago; using stats forecast) + table: c@c_pkey + spans: [/'1988-08-07 00:00:00+00:00' - ] + +query T +EXPLAIN (OPT, VERBOSE) SELECT * FROM c WHERE h >= '1988-08-07' +---- +scan c + ├── columns: h:1 + ├── constraint: /1: [/'1988-08-07 00:00:00+00:00' - ] + ├── stats: [rows=24, distinct(1)=24, null(1)=0, avgsize(1)=7] + │ histogram(1)= 0 1 5 1 5 1 5 1 4 1 + │ <--- '1988-08-07 00:00:00+00:00' --- '1988-08-07 06:00:00+00:00' --- '1988-08-07 12:00:00+00:00' --- '1988-08-07 18:00:00+00:00' --- '1988-08-08 00:00:00+00:00' + ├── cost: 39.7 + ├── key: (1) + └── distribution: test diff --git a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go index 99f94c36dc2c..349ee1d25a79 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go @@ -207,6 +207,13 @@ func TestExecBuild_fk( runExecBuildLogicTest(t, "fk") } +func TestExecBuild_forecast( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runExecBuildLogicTest(t, "forecast") +} + func TestExecBuild_geospatial( t *testing.T, ) {