From 28f56270676fc62d1f4545d33b040d5137e9c8c1 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Tue, 5 Jul 2022 18:04:38 +0000 Subject: [PATCH 1/7] asim: update range size split threshold, more keys This patch updates the range size split threshold to 512mb by default. Additionally it adds support for initializing a testing replica distribution, where the ranges contain an equal span of the keyspace. Release note: None --- pkg/kv/kvserver/asim/asim_test.go | 8 ++++++-- pkg/kv/kvserver/asim/settings.go | 2 +- pkg/kv/kvserver/asim/state/helpers.go | 10 ++++++++-- pkg/kv/kvserver/asim/state/impl.go | 12 ++++++------ pkg/kv/kvserver/asim/state/state.go | 8 ++++---- pkg/kv/kvserver/asim/state/state_test.go | 12 ++++++------ 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/pkg/kv/kvserver/asim/asim_test.go b/pkg/kv/kvserver/asim/asim_test.go index e1e434f11927..f33c8c7de393 100644 --- a/pkg/kv/kvserver/asim/asim_test.go +++ b/pkg/kv/kvserver/asim/asim_test.go @@ -100,10 +100,14 @@ func TestAllocatorSimulatorSpeed(t *testing.T) { // NB: We want 1000 replicas per store, so the number of ranges required // will be 1/3 of the total replicas. ranges := (replicasPerStore * stores) / replsPerRange + // NB: In this test we are using a uniform workload and expect to see at + // most 3 splits occur due to range size, therefore the keyspace need not + // be larger than 3 keys per range. + keyspace := 3 * ranges sample := func() int64 { rwg := make([]workload.Generator, 1) - rwg[0] = testCreateWorkloadGenerator(start, stores, int64(ranges)) + rwg[0] = testCreateWorkloadGenerator(start, stores, int64(keyspace)) exchange := state.NewFixedDelayExhange(preGossipStart, settings.StateExchangeInterval, settings.StateExchangeDelay) changer := state.NewReplicaChanger() m := asim.NewMetricsTracker() // no output @@ -120,7 +124,7 @@ func TestAllocatorSimulatorSpeed(t *testing.T) { replicaDistribution[i] = 0 } - s := state.NewTestStateReplDistribution(ranges, replicaDistribution, replsPerRange) + s := state.NewTestStateReplDistribution(replicaDistribution, ranges, replsPerRange, keyspace) testPreGossipStores(s, exchange, preGossipStart) sim := asim.NewSimulator(start, end, interval, rwg, s, exchange, changer, settings, m) diff --git a/pkg/kv/kvserver/asim/settings.go b/pkg/kv/kvserver/asim/settings.go index 510b3768cab3..c29220ecb54d 100644 --- a/pkg/kv/kvserver/asim/settings.go +++ b/pkg/kv/kvserver/asim/settings.go @@ -16,7 +16,7 @@ const ( defaultReplicaChangeBaseDelay = 100 * time.Millisecond defaultReplicaAddDelayFactor = 16 defaultSplitQueueDelay = 100 * time.Millisecond - defaultRangeSizeSplitThreshold = 512 + defaultRangeSizeSplitThreshold = 512 * 1024 * 1024 // 512mb defaultRangeRebalanceThreshold = 0.05 defaultPacerLoopInterval = 10 * time.Minute defaultPacerMinIterInterval = 10 * time.Millisecond diff --git a/pkg/kv/kvserver/asim/state/helpers.go b/pkg/kv/kvserver/asim/state/helpers.go index c810f9fcdaa9..66f35356c644 100644 --- a/pkg/kv/kvserver/asim/state/helpers.go +++ b/pkg/kv/kvserver/asim/state/helpers.go @@ -196,7 +196,7 @@ func (s storeRangeCounts) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // replication factor of 3. A best effort distribution is applied in these // cases. func NewTestStateReplDistribution( - ranges int, percentOfReplicas []float64, replicationFactor int, + percentOfReplicas []float64, ranges, replicationFactor, keyspace int, ) State { targetRangeCount := make(storeRangeCounts, len(percentOfReplicas)) for i, percent := range percentOfReplicas { @@ -204,10 +204,16 @@ func NewTestStateReplDistribution( targetRangeCount[i] = storeRangeCount{requestedReplicas: requiredRanges, storeID: StoreID(i + 1)} } + // There cannot be less keys than there are ranges. + if ranges > keyspace { + keyspace = ranges + } + rangeInterval := keyspace / ranges + startKeys := make([]Key, ranges) replicas := make(map[Key][]StoreID) for i := 0; i < ranges; i++ { - key := Key(i) + key := Key(i * rangeInterval) startKeys[i] = key replicas[key] = make([]StoreID, replicationFactor) diff --git a/pkg/kv/kvserver/asim/state/impl.go b/pkg/kv/kvserver/asim/state/impl.go index e292de9d25fe..363c20bbb4ff 100644 --- a/pkg/kv/kvserver/asim/state/impl.go +++ b/pkg/kv/kvserver/asim/state/impl.go @@ -83,20 +83,20 @@ func (r *rng) Less(than btree.Item) bool { } // initFirstRange initializes the first range within the rangemap, with -// [minKey, maxKey) start and end key. All other ranges are split from this. +// [MinKey, MaxKey) start and end key. All other ranges are split from this. func (rm *rmap) initFirstRange() { rm.rangeSeqGen++ rangeID := rm.rangeSeqGen desc := roachpb.RangeDescriptor{ RangeID: roachpb.RangeID(rangeID), - StartKey: minKey.ToRKey(), - EndKey: maxKey.ToRKey(), + StartKey: MinKey.ToRKey(), + EndKey: MaxKey.ToRKey(), NextReplicaID: 1, } rng := &rng{ rangeID: rangeID, - startKey: minKey, - endKey: maxKey, + startKey: MinKey, + endKey: MaxKey, desc: desc, config: defaultSpanConfig, replicas: make(map[StoreID]*replica), @@ -116,7 +116,7 @@ func (s *state) String() string { s.ranges.rangeTree.Ascend(func(i btree.Item) bool { r := i.(*rng) orderedRanges = append(orderedRanges, r) - return !r.desc.EndKey.Equal(maxKey.ToRKey()) + return !r.desc.EndKey.Equal(MaxKey.ToRKey()) }) nStores := len(s.stores) diff --git a/pkg/kv/kvserver/asim/state/state.go b/pkg/kv/kvserver/asim/state/state.go index 132d0181685e..3ff03560e532 100644 --- a/pkg/kv/kvserver/asim/state/state.go +++ b/pkg/kv/kvserver/asim/state/state.go @@ -239,11 +239,11 @@ func (m *ManualSimClock) Set(tsNanos int64) { // Key is a single slot in the keyspace. type Key int64 -// minKey is the minimum key in the keyspace. -const minKey Key = -1 +// MinKey is the minimum key in the keyspace. +const MinKey Key = -1 -// maxKey is the maximum key in the keyspace. -const maxKey Key = 9999999999 +// MaxKey is the maximum key in the keyspace. +const MaxKey Key = 9999999999 // InvalidKey is a placeholder key that does not exist in the keyspace. const InvalidKey Key = -2 diff --git a/pkg/kv/kvserver/asim/state/state_test.go b/pkg/kv/kvserver/asim/state/state_test.go index 9238b65856d5..066d3961e83c 100644 --- a/pkg/kv/kvserver/asim/state/state_test.go +++ b/pkg/kv/kvserver/asim/state/state_test.go @@ -31,7 +31,7 @@ func TestStateUpdates(t *testing.T) { // the post-split keys are correct. func TestRangeSplit(t *testing.T) { s := newState() - k1 := minKey + k1 := MinKey r1 := s.rangeFor(k1) n1 := s.AddNode() @@ -64,10 +64,10 @@ func TestRangeMap(t *testing.T) { require.Len(t, s.ranges.rangeMap, 1) require.Equal(t, s.ranges.rangeTree.Max(), s.ranges.rangeTree.Min()) firstRange := s.ranges.rangeMap[1] - require.Equal(t, s.rangeFor(minKey), firstRange) - require.Equal(t, firstRange.startKey, minKey) - require.Equal(t, firstRange.desc.StartKey, minKey.ToRKey()) - require.Equal(t, firstRange.desc.EndKey, maxKey.ToRKey()) + require.Equal(t, s.rangeFor(MinKey), firstRange) + require.Equal(t, firstRange.startKey, MinKey) + require.Equal(t, firstRange.desc.StartKey, MinKey.ToRKey()) + require.Equal(t, firstRange.desc.EndKey, MaxKey.ToRKey()) require.Equal(t, defaultSpanConfig, firstRange.SpanConfig()) k2 := Key(1) @@ -81,7 +81,7 @@ func TestRangeMap(t *testing.T) { _, r4, ok := s.SplitRange(k4) require.True(t, ok) - // Assert that the range is segmented into [minKey, EndKey) intervals. + // Assert that the range is segmented into [MinKey, EndKey) intervals. require.Equal(t, k2.ToRKey(), r1.Descriptor().EndKey) require.Equal(t, k3.ToRKey(), r2.Descriptor().EndKey) require.Equal(t, k4.ToRKey(), r3.Descriptor().EndKey) From e96d23c158866606421b5fc1b79997ad5313cb63 Mon Sep 17 00:00:00 2001 From: Rafi Shamim Date: Wed, 6 Jul 2022 15:23:35 -0400 Subject: [PATCH 2/7] pgwire: use better error for invalid Describe message Release note: None --- pkg/sql/conn_executor_prepare.go | 6 ++++-- pkg/sql/pgwire/testdata/pgtest/portals | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/sql/conn_executor_prepare.go b/pkg/sql/conn_executor_prepare.go index 2da1e78528cd..f6bb0e0c5104 100644 --- a/pkg/sql/conn_executor_prepare.go +++ b/pkg/sql/conn_executor_prepare.go @@ -599,8 +599,10 @@ func (ex *connExecutor) execDescribe( res.SetPortalOutput(ctx, portal.Stmt.Columns, portal.OutFormats) } default: - return retErr(errors.AssertionFailedf( - "unknown describe type: %s", errors.Safe(descCmd.Type))) + return retErr(pgerror.Newf( + pgcode.ProtocolViolation, + "invalid DESCRIBE message subtype %d", errors.Safe(byte(descCmd.Type)), + )) } return nil, nil } diff --git a/pkg/sql/pgwire/testdata/pgtest/portals b/pkg/sql/pgwire/testdata/pgtest/portals index def589e3c10c..8a9a5123650f 100644 --- a/pkg/sql/pgwire/testdata/pgtest/portals +++ b/pkg/sql/pgwire/testdata/pgtest/portals @@ -1394,3 +1394,29 @@ ReadyForQuery {"Type":"DataRow","Values":[{"text":"2"}]} {"Type":"CommandComplete","CommandTag":"FETCH 2"} {"Type":"ReadyForQuery","TxStatus":"T"} + +send +Query {"String": "ROLLBACK"} +---- + +until +ReadyForQuery +---- +{"Type":"CommandComplete","CommandTag":"ROLLBACK"} +{"Type":"ReadyForQuery","TxStatus":"I"} + +send +Parse {"Query": "SELECT * FROM generate_series(1, 4)"} +Describe +Bind +Execute +Sync +---- + +until keepErrMessage +ErrorResponse +ReadyForQuery +---- +{"Type":"ParseComplete"} +{"Type":"ErrorResponse","Code":"08P01","Message":"invalid DESCRIBE message subtype 0"} +{"Type":"ReadyForQuery","TxStatus":"I"} From 2904f8937912c5a6e6a8e8fab4bdf853b443ae1f Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Thu, 30 Dec 2021 15:43:50 -0500 Subject: [PATCH 3/7] storage: short-circuit in recordIteratorStats if not recording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a short-circuit fast-path in recordIteratorStats that avoids a call to `tracing.Span.RecordStructured` when the tracing span is not recording. This avoids a heap allocation in the common case where tracing is not enabled. ``` name old time/op new time/op delta KV/Scan/Native/rows=1-10 18.8µs ± 2% 18.5µs ± 3% -1.14% (p=0.003 n=20+19) KV/Scan/SQL/rows=1-10 95.4µs ± 3% 95.7µs ± 5% ~ (p=0.602 n=20+20) name old alloc/op new alloc/op delta KV/Scan/Native/rows=1-10 7.42kB ± 1% 7.35kB ± 1% -1.01% (p=0.000 n=20+20) KV/Scan/SQL/rows=1-10 23.5kB ± 0% 23.4kB ± 0% -0.31% (p=0.000 n=19+20) name old allocs/op new allocs/op delta KV/Scan/Native/rows=1-10 58.0 ± 0% 57.0 ± 0% -1.72% (p=0.000 n=19+20) KV/Scan/SQL/rows=1-10 279 ± 0% 278 ± 0% -0.36% (p=0.000 n=18+19) ``` --- pkg/storage/BUILD.bazel | 1 + pkg/storage/mvcc.go | 49 +++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel index 85831e80cc84..4981bf29aefd 100644 --- a/pkg/storage/BUILD.bazel +++ b/pkg/storage/BUILD.bazel @@ -74,6 +74,7 @@ go_library( "//pkg/util/sysutil", "//pkg/util/timeutil", "//pkg/util/tracing", + "//pkg/util/tracing/tracingpb", "//pkg/util/uuid", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_errors//oserror", diff --git a/pkg/storage/mvcc.go b/pkg/storage/mvcc.go index 5e3af3068456..1d67cd2f062a 100644 --- a/pkg/storage/mvcc.go +++ b/pkg/storage/mvcc.go @@ -36,6 +36,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/tracing" + "github.com/cockroachdb/cockroach/pkg/util/tracing/tracingpb" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" ) @@ -852,8 +853,7 @@ func mvccGet( mvccScanner.get(ctx) // If we have a trace, emit the scan stats that we produced. - traceSpan := tracing.SpanFromContext(ctx) - recordIteratorStats(traceSpan, mvccScanner.stats()) + recordIteratorStats(ctx, mvccScanner.stats()) if mvccScanner.err != nil { return optionalValue{}, nil, mvccScanner.err @@ -2777,26 +2777,29 @@ func MVCCDeleteRangeUsingTombstone( return nil } -func recordIteratorStats(traceSpan *tracing.Span, iteratorStats IteratorStats) { - stats := iteratorStats.Stats - if traceSpan != nil { - steps := stats.ReverseStepCount[pebble.InterfaceCall] + stats.ForwardStepCount[pebble.InterfaceCall] - seeks := stats.ReverseSeekCount[pebble.InterfaceCall] + stats.ForwardSeekCount[pebble.InterfaceCall] - internalSteps := stats.ReverseStepCount[pebble.InternalIterCall] + stats.ForwardStepCount[pebble.InternalIterCall] - internalSeeks := stats.ReverseSeekCount[pebble.InternalIterCall] + stats.ForwardSeekCount[pebble.InternalIterCall] - traceSpan.RecordStructured(&roachpb.ScanStats{ - NumInterfaceSeeks: uint64(seeks), - NumInternalSeeks: uint64(internalSeeks), - NumInterfaceSteps: uint64(steps), - NumInternalSteps: uint64(internalSteps), - BlockBytes: stats.InternalStats.BlockBytes, - BlockBytesInCache: stats.InternalStats.BlockBytesInCache, - KeyBytes: stats.InternalStats.KeyBytes, - ValueBytes: stats.InternalStats.ValueBytes, - PointCount: stats.InternalStats.PointCount, - PointsCoveredByRangeTombstones: stats.InternalStats.PointsCoveredByRangeTombstones, - }) +func recordIteratorStats(ctx context.Context, iteratorStats IteratorStats) { + sp := tracing.SpanFromContext(ctx) + if sp.RecordingType() == tracingpb.RecordingOff { + // Short-circuit before allocating ScanStats object. + return } + stats := &iteratorStats.Stats + steps := stats.ReverseStepCount[pebble.InterfaceCall] + stats.ForwardStepCount[pebble.InterfaceCall] + seeks := stats.ReverseSeekCount[pebble.InterfaceCall] + stats.ForwardSeekCount[pebble.InterfaceCall] + internalSteps := stats.ReverseStepCount[pebble.InternalIterCall] + stats.ForwardStepCount[pebble.InternalIterCall] + internalSeeks := stats.ReverseSeekCount[pebble.InternalIterCall] + stats.ForwardSeekCount[pebble.InternalIterCall] + sp.RecordStructured(&roachpb.ScanStats{ + NumInterfaceSeeks: uint64(seeks), + NumInternalSeeks: uint64(internalSeeks), + NumInterfaceSteps: uint64(steps), + NumInternalSteps: uint64(internalSteps), + BlockBytes: stats.InternalStats.BlockBytes, + BlockBytesInCache: stats.InternalStats.BlockBytesInCache, + KeyBytes: stats.InternalStats.KeyBytes, + ValueBytes: stats.InternalStats.ValueBytes, + PointCount: stats.InternalStats.PointCount, + PointsCoveredByRangeTombstones: stats.InternalStats.PointsCoveredByRangeTombstones, + }) } func mvccScanToBytes( @@ -2868,9 +2871,7 @@ func mvccScanToBytes( res.NumBytes = mvccScanner.results.bytes // If we have a trace, emit the scan stats that we produced. - traceSpan := tracing.SpanFromContext(ctx) - - recordIteratorStats(traceSpan, mvccScanner.stats()) + recordIteratorStats(ctx, mvccScanner.stats()) res.Intents, err = buildScanIntents(mvccScanner.intentsRepr()) if err != nil { From bf86973471d270da6034409e3a0d229e1b440bf6 Mon Sep 17 00:00:00 2001 From: Oleg Afanasyev Date: Mon, 11 Jul 2022 11:39:36 +0100 Subject: [PATCH 4/7] gc: fix NumKeysAffected counting more than collected Previously if key is not collected after a GC batch is sent out it would still be included as affected in GC stats. Those stats are mostly used for logging and tests, the unfortunate effect is that randomized test could fail. This commit fixes the bug. Release note: None --- pkg/kv/kvserver/gc/gc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/kv/kvserver/gc/gc.go b/pkg/kv/kvserver/gc/gc.go index 5fc168809869..fc9b42d63291 100644 --- a/pkg/kv/kvserver/gc/gc.go +++ b/pkg/kv/kvserver/gc/gc.go @@ -379,6 +379,10 @@ func processReplicatedKeyRange( } if affected := isNewest && (sentBatchForThisKey || haveGarbageForThisKey); affected { info.NumKeysAffected++ + // If we reached newest timestamp for the key then we should reset sent + // batch to ensure subsequent keys are not included in affected keys if + // they don't have garbage. + sentBatchForThisKey = false } shouldSendBatch := batchGCKeysBytes >= KeyVersionChunkBytes if shouldSendBatch || isNewest && haveGarbageForThisKey { From ee5c0803c35d18d067184ec981ee65042cc641ca Mon Sep 17 00:00:00 2001 From: Chengxiong Ruan Date: Wed, 6 Jul 2022 10:03:38 -0400 Subject: [PATCH 5/7] sql: sql parser for `CREATE FUNCTION` statement This commit added parser support for `CREATE FUNCTION` sql statement. Scanner was extended so that it can recognize the `BEGIN ATOMIC` context so that it doesnot return early when it sees `;` charater which normally indicates the end of a statement. Release note (sql change): `CREATE FUNCTION` statement now can be parsed by crdb, but an unimplemented error would be thrown since the statement processing is not done yet. --- docs/generated/sql/bnf/BUILD.bazel | 7 + docs/generated/sql/bnf/begin_stmt.bnf | 18 - docs/generated/sql/bnf/commit_transaction.bnf | 2 - docs/generated/sql/bnf/create_ddl_stmt.bnf | 1 + docs/generated/sql/bnf/create_func_stmt.bnf | 2 + docs/generated/sql/bnf/legacy_begin_stmt.bnf | 2 + docs/generated/sql/bnf/legacy_end_stmt.bnf | 2 + .../sql/bnf/legacy_transaction_stmt.bnf | 3 + docs/generated/sql/bnf/routine_body_stmt.bnf | 3 + .../generated/sql/bnf/routine_return_stmt.bnf | 2 + docs/generated/sql/bnf/stmt.bnf | 23 +- docs/generated/sql/bnf/stmt_block.bnf | 168 ++++++- .../bnf/stmt_without_legacy_transaction.bnf | 22 + pkg/gen/docs.bzl | 7 + pkg/sql/delegate/show_completions_test.go | 4 +- .../testdata/logic_test/show_completions | 2 + pkg/sql/logictest/testdata/logic_test/udf | 2 + pkg/sql/opt/optbuilder/builder.go | 4 + pkg/sql/parser/parse.go | 17 +- pkg/sql/parser/parse_test.go | 2 - pkg/sql/parser/sql.y | 437 ++++++++++++++++-- pkg/sql/parser/testdata/create_function | 388 ++++++++++++++++ pkg/sql/scanner/scan.go | 16 +- pkg/sql/sem/tree/BUILD.bazel | 1 + pkg/sql/sem/tree/object_name.go | 18 +- pkg/sql/sem/tree/stmt.go | 23 + pkg/sql/sem/tree/udf.go | 291 ++++++++++++ 27 files changed, 1359 insertions(+), 108 deletions(-) create mode 100644 docs/generated/sql/bnf/create_func_stmt.bnf create mode 100644 docs/generated/sql/bnf/legacy_begin_stmt.bnf create mode 100644 docs/generated/sql/bnf/legacy_end_stmt.bnf create mode 100644 docs/generated/sql/bnf/legacy_transaction_stmt.bnf create mode 100644 docs/generated/sql/bnf/routine_body_stmt.bnf create mode 100644 docs/generated/sql/bnf/routine_return_stmt.bnf create mode 100644 docs/generated/sql/bnf/stmt_without_legacy_transaction.bnf create mode 100644 pkg/sql/logictest/testdata/logic_test/udf create mode 100644 pkg/sql/parser/testdata/create_function create mode 100644 pkg/sql/sem/tree/udf.go diff --git a/docs/generated/sql/bnf/BUILD.bazel b/docs/generated/sql/bnf/BUILD.bazel index 2f0d3379d811..6233c1748466 100644 --- a/docs/generated/sql/bnf/BUILD.bazel +++ b/docs/generated/sql/bnf/BUILD.bazel @@ -76,6 +76,7 @@ FILES = [ "create_database_stmt", "create_ddl_stmt", "create_extension_stmt", + "create_func_stmt", "create_index_stmt", "create_index_with_storage_param", "create_inverted_index_stmt", @@ -130,6 +131,9 @@ FILES = [ "insert_stmt", "iso_level", "joined_table", + "legacy_begin_stmt", + "legacy_end_stmt", + "legacy_transaction_stmt", "like_table_option_list", "limit_clause", "move_cursor_stmt", @@ -167,6 +171,8 @@ FILES = [ "resume_stmt", "revoke_stmt", "rollback_transaction", + "routine_body_stmt", + "routine_return_stmt", "row_source_extension_stmt", "savepoint_stmt", "scrub_database_stmt", @@ -230,6 +236,7 @@ FILES = [ "split_table_at", "stmt", "stmt_block", + "stmt_without_legacy_transaction", "table_clause", "table_constraint", "table_ref", diff --git a/docs/generated/sql/bnf/begin_stmt.bnf b/docs/generated/sql/bnf/begin_stmt.bnf index 16c09600aae2..1059df25a0ad 100644 --- a/docs/generated/sql/bnf/begin_stmt.bnf +++ b/docs/generated/sql/bnf/begin_stmt.bnf @@ -1,19 +1 @@ begin_stmt ::= - 'BEGIN' 'TRANSACTION' 'PRIORITY' 'LOW' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'PRIORITY' 'NORMAL' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'PRIORITY' 'HIGH' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'READ' 'ONLY' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'READ' 'WRITE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'DEFERRABLE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' 'NOT' 'DEFERRABLE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'TRANSACTION' - | 'BEGIN' 'PRIORITY' 'LOW' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'PRIORITY' 'NORMAL' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'PRIORITY' 'HIGH' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'READ' 'ONLY' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'READ' 'WRITE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'DEFERRABLE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' 'NOT' 'DEFERRABLE' ( ( ( ',' | ) ( ( 'PRIORITY' ( 'LOW' | 'NORMAL' | 'HIGH' ) ) | ( 'READ' 'ONLY' | 'READ' 'WRITE' ) | ( 'AS' 'OF' 'SYSTEM' 'TIME' a_expr ) | ( 'DEFERRABLE' | 'NOT' 'DEFERRABLE' ) ) ) )* - | 'BEGIN' diff --git a/docs/generated/sql/bnf/commit_transaction.bnf b/docs/generated/sql/bnf/commit_transaction.bnf index 852a7e983367..f4d95a9e6f30 100644 --- a/docs/generated/sql/bnf/commit_transaction.bnf +++ b/docs/generated/sql/bnf/commit_transaction.bnf @@ -1,5 +1,3 @@ commit_stmt ::= 'COMMIT' 'TRANSACTION' | 'COMMIT' - | 'END' 'TRANSACTION' - | 'END' diff --git a/docs/generated/sql/bnf/create_ddl_stmt.bnf b/docs/generated/sql/bnf/create_ddl_stmt.bnf index c14860b51768..308db39242eb 100644 --- a/docs/generated/sql/bnf/create_ddl_stmt.bnf +++ b/docs/generated/sql/bnf/create_ddl_stmt.bnf @@ -7,3 +7,4 @@ create_ddl_stmt ::= | create_type_stmt | create_view_stmt | create_sequence_stmt + | create_func_stmt diff --git a/docs/generated/sql/bnf/create_func_stmt.bnf b/docs/generated/sql/bnf/create_func_stmt.bnf new file mode 100644 index 000000000000..6261dca3715a --- /dev/null +++ b/docs/generated/sql/bnf/create_func_stmt.bnf @@ -0,0 +1,2 @@ +create_func_stmt ::= + 'CREATE' opt_or_replace 'FUNCTION' func_create_name '(' opt_func_arg_with_default_list ')' 'RETURNS' opt_return_set func_return_type opt_create_func_opt_list opt_routine_body diff --git a/docs/generated/sql/bnf/legacy_begin_stmt.bnf b/docs/generated/sql/bnf/legacy_begin_stmt.bnf new file mode 100644 index 000000000000..3974be960389 --- /dev/null +++ b/docs/generated/sql/bnf/legacy_begin_stmt.bnf @@ -0,0 +1,2 @@ +legacy_begin_stmt ::= + 'BEGIN' opt_transaction begin_transaction diff --git a/docs/generated/sql/bnf/legacy_end_stmt.bnf b/docs/generated/sql/bnf/legacy_end_stmt.bnf new file mode 100644 index 000000000000..51e539de45ac --- /dev/null +++ b/docs/generated/sql/bnf/legacy_end_stmt.bnf @@ -0,0 +1,2 @@ +legacy_end_stmt ::= + 'END' opt_transaction diff --git a/docs/generated/sql/bnf/legacy_transaction_stmt.bnf b/docs/generated/sql/bnf/legacy_transaction_stmt.bnf new file mode 100644 index 000000000000..86e69e3eaa9c --- /dev/null +++ b/docs/generated/sql/bnf/legacy_transaction_stmt.bnf @@ -0,0 +1,3 @@ +legacy_transaction_stmt ::= + legacy_begin_stmt + | legacy_end_stmt diff --git a/docs/generated/sql/bnf/routine_body_stmt.bnf b/docs/generated/sql/bnf/routine_body_stmt.bnf new file mode 100644 index 000000000000..f6143b545995 --- /dev/null +++ b/docs/generated/sql/bnf/routine_body_stmt.bnf @@ -0,0 +1,3 @@ +routine_body_stmt ::= + stmt_without_legacy_transaction + | routine_return_stmt diff --git a/docs/generated/sql/bnf/routine_return_stmt.bnf b/docs/generated/sql/bnf/routine_return_stmt.bnf new file mode 100644 index 000000000000..ee632ac76569 --- /dev/null +++ b/docs/generated/sql/bnf/routine_return_stmt.bnf @@ -0,0 +1,2 @@ +routine_return_stmt ::= + 'RETURN' a_expr diff --git a/docs/generated/sql/bnf/stmt.bnf b/docs/generated/sql/bnf/stmt.bnf index e8e556311237..7ccaafdc0f5b 100644 --- a/docs/generated/sql/bnf/stmt.bnf +++ b/docs/generated/sql/bnf/stmt.bnf @@ -1,23 +1,4 @@ stmt ::= 'HELPTOKEN' - | preparable_stmt - | analyze_stmt - | copy_from_stmt - | comment_stmt - | execute_stmt - | deallocate_stmt - | discard_stmt - | grant_stmt - | prepare_stmt - | revoke_stmt - | savepoint_stmt - | reassign_owned_by_stmt - | drop_owned_by_stmt - | release_stmt - | refresh_stmt - | nonpreparable_set_stmt - | transaction_stmt - | close_cursor_stmt - | declare_cursor_stmt - | fetch_cursor_stmt - | move_cursor_stmt + | stmt_without_legacy_transaction + | legacy_transaction_stmt diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf index 6841b2ea7e01..064b7e4a7dff 100644 --- a/docs/generated/sql/bnf/stmt_block.bnf +++ b/docs/generated/sql/bnf/stmt_block.bnf @@ -3,7 +3,12 @@ stmt_block ::= stmt ::= 'HELPTOKEN' - | preparable_stmt + | stmt_without_legacy_transaction + | legacy_transaction_stmt + | + +stmt_without_legacy_transaction ::= + preparable_stmt | analyze_stmt | copy_from_stmt | comment_stmt @@ -24,7 +29,10 @@ stmt ::= | declare_cursor_stmt | fetch_cursor_stmt | move_cursor_stmt - | + +legacy_transaction_stmt ::= + legacy_begin_stmt + | legacy_end_stmt preparable_stmt ::= alter_stmt @@ -140,6 +148,12 @@ fetch_cursor_stmt ::= move_cursor_stmt ::= 'MOVE' cursor_movement_specifier +legacy_begin_stmt ::= + 'BEGIN' opt_transaction begin_transaction + +legacy_end_stmt ::= + 'END' opt_transaction + alter_stmt ::= alter_ddl_stmt | alter_role_stmt @@ -397,12 +411,10 @@ set_transaction_stmt ::= | 'SET' 'SESSION' 'TRANSACTION' transaction_mode_list begin_stmt ::= - 'BEGIN' opt_transaction begin_transaction - | 'START' 'TRANSACTION' begin_transaction + 'START' 'TRANSACTION' begin_transaction commit_stmt ::= 'COMMIT' opt_transaction - | 'END' opt_transaction rollback_stmt ::= 'ROLLBACK' opt_transaction @@ -445,6 +457,14 @@ cursor_movement_specifier ::= | 'FIRST' opt_from_or_in cursor_name | 'LAST' opt_from_or_in cursor_name +opt_transaction ::= + 'TRANSACTION' + | + +begin_transaction ::= + transaction_mode_list + | + alter_ddl_stmt ::= alter_table_stmt | alter_index_stmt @@ -528,6 +548,7 @@ create_ddl_stmt ::= | create_type_stmt | create_view_stmt | create_sequence_stmt + | create_func_stmt create_stats_stmt ::= 'CREATE' 'STATISTICS' statistics_name opt_stats_columns 'FROM' create_stats_target opt_create_stats_options @@ -927,6 +948,7 @@ unreserved_keyword ::= | 'ALWAYS' | 'ASENSITIVE' | 'AT' + | 'ATOMIC' | 'ATTRIBUTE' | 'AUTOMATIC' | 'AVAILABILITY' @@ -940,6 +962,7 @@ unreserved_keyword ::= | 'BUNDLE' | 'BY' | 'CACHE' + | 'CALLED' | 'CANCEL' | 'CANCELQUERY' | 'CASCADE' @@ -965,6 +988,7 @@ unreserved_keyword ::= | 'CONVERSION' | 'CONVERT' | 'COPY' + | 'COST' | 'COVERING' | 'CREATEDB' | 'CREATELOGIN' @@ -984,6 +1008,7 @@ unreserved_keyword ::= | 'DELETE' | 'DEFAULTS' | 'DEFERRED' + | 'DEFINER' | 'DELIMITER' | 'DESTINATION' | 'DETACHED' @@ -1010,6 +1035,7 @@ unreserved_keyword ::= | 'EXPLAIN' | 'EXPORT' | 'EXTENSION' + | 'EXTERNAL' | 'FAILURE' | 'FILES' | 'FILTER' @@ -1042,6 +1068,7 @@ unreserved_keyword ::= | 'HOUR' | 'IDENTITY' | 'IMMEDIATE' + | 'IMMUTABLE' | 'IMPORT' | 'INCLUDE' | 'INCLUDING' @@ -1051,10 +1078,12 @@ unreserved_keyword ::= | 'INDEXES' | 'INHERITS' | 'INJECT' + | 'INPUT' | 'INSERT' | 'INTO_DB' | 'INVERTED' | 'ISOLATION' + | 'INVOKER' | 'JOB' | 'JOBS' | 'JSON' @@ -1067,6 +1096,7 @@ unreserved_keyword ::= | 'LATEST' | 'LC_COLLATE' | 'LC_CTYPE' + | 'LEAKPROOF' | 'LEASE' | 'LESS' | 'LEVEL' @@ -1144,6 +1174,7 @@ unreserved_keyword ::= | 'OVER' | 'OWNED' | 'OWNER' + | 'PARALLEL' | 'PARENT' | 'PARTIAL' | 'PARTITION' @@ -1199,6 +1230,8 @@ unreserved_keyword ::= | 'RESTRICTED' | 'RESUME' | 'RETRY' + | 'RETURN' + | 'RETURNS' | 'REVISION_HISTORY' | 'REVOKE' | 'ROLE' @@ -1223,6 +1256,7 @@ unreserved_keyword ::= | 'SCRUB' | 'SEARCH' | 'SECOND' + | 'SECURITY' | 'SERIALIZABLE' | 'SEQUENCE' | 'SEQUENCES' @@ -1244,6 +1278,7 @@ unreserved_keyword ::= | 'SPLIT' | 'SQL' | 'SQLLOGIN' + | 'STABLE' | 'START' | 'STATE' | 'STATEMENTS' @@ -1257,6 +1292,7 @@ unreserved_keyword ::= | 'STRICT' | 'SUBSCRIPTION' | 'SUPER' + | 'SUPPORT' | 'SURVIVE' | 'SURVIVAL' | 'SYNTAX' @@ -1275,6 +1311,7 @@ unreserved_keyword ::= | 'TRANSACTION' | 'TRANSACTIONS' | 'TRANSFER' + | 'TRANSFORM' | 'TRIGGER' | 'TRUNCATE' | 'TRUSTED' @@ -1301,6 +1338,7 @@ unreserved_keyword ::= | 'VIEWACTIVITYREDACTED' | 'VIEWCLUSTERSETTING' | 'VISIBLE' + | 'VOLATILE' | 'VOTERS' | 'WITHIN' | 'WITHOUT' @@ -1332,6 +1370,7 @@ col_name_keyword ::= | 'IF' | 'IFERROR' | 'IFNULL' + | 'INOUT' | 'INT' | 'INTEGER' | 'INTERVAL' @@ -1347,6 +1386,7 @@ col_name_keyword ::= | 'PRECISION' | 'REAL' | 'ROW' + | 'SETOF' | 'SMALLINT' | 'STRING' | 'SUBSTRING' @@ -1397,14 +1437,6 @@ type_list ::= transaction_mode_list ::= ( transaction_mode ) ( ( opt_comma transaction_mode ) )* -opt_transaction ::= - 'TRANSACTION' - | - -begin_transaction ::= - transaction_mode_list - | - opt_abort_mod ::= 'TRANSACTION' | 'WORK' @@ -1607,6 +1639,9 @@ create_sequence_stmt ::= 'CREATE' opt_temp 'SEQUENCE' sequence_name opt_sequence_option_list | 'CREATE' opt_temp 'SEQUENCE' 'IF' 'NOT' 'EXISTS' sequence_name opt_sequence_option_list +create_func_stmt ::= + 'CREATE' opt_or_replace 'FUNCTION' func_create_name '(' opt_func_arg_with_default_list ')' 'RETURNS' opt_return_set func_return_type opt_create_func_opt_list opt_routine_body + statistics_name ::= name @@ -2266,12 +2301,39 @@ opt_sequence_option_list ::= sequence_option_list | +opt_or_replace ::= + 'OR' 'REPLACE' + | + +func_create_name ::= + db_object_name + +opt_func_arg_with_default_list ::= + func_arg_with_default_list + | + +opt_return_set ::= + 'SETOF' + | + +func_return_type ::= + func_arg_type + +opt_create_func_opt_list ::= + create_func_opt_list + | + +opt_routine_body ::= + routine_return_stmt + | 'BEGIN' 'ATOMIC' routine_body_stmt_list 'END' + | + changefeed_target ::= opt_table_prefix table_name opt_changefeed_family target_elem ::= a_expr 'AS' target_name - | a_expr 'identifier' + | a_expr bare_col_label | a_expr | '*' @@ -2788,6 +2850,21 @@ create_as_table_defs ::= enum_val_list ::= ( 'SCONST' ) ( ( ',' 'SCONST' ) )* +func_arg_with_default_list ::= + ( func_arg_with_default ) ( ( ',' func_arg_with_default ) )* + +func_arg_type ::= + typename + +create_func_opt_list ::= + ( create_func_opt_item ) ( ( create_func_opt_item ) )* + +routine_return_stmt ::= + 'RETURN' a_expr + +routine_body_stmt_list ::= + ( ) ( ( routine_body_stmt ';' ) )* + opt_table_prefix ::= 'TABLE' | @@ -2799,6 +2876,10 @@ opt_changefeed_family ::= target_name ::= unrestricted_name +bare_col_label ::= + 'identifier' + | bare_label_keywords + common_table_expr ::= table_alias_name opt_column_list 'AS' '(' preparable_stmt ')' | table_alias_name opt_column_list 'AS' materialize_clause '(' preparable_stmt ')' @@ -3157,9 +3238,42 @@ family_def ::= create_as_constraint_def ::= create_as_constraint_elem +func_arg_with_default ::= + func_arg + | func_arg 'DEFAULT' a_expr + | func_arg '=' a_expr + +create_func_opt_item ::= + 'AS' func_as + | 'LANGUAGE' non_reserved_word_or_sconst + | common_func_opt_item + +routine_body_stmt ::= + stmt_without_legacy_transaction + | routine_return_stmt + family_name ::= name +bare_label_keywords ::= + 'ATOMIC' + | 'CALLED' + | 'DEFINER' + | 'EXTERNAL' + | 'IMMUTABLE' + | 'INPUT' + | 'INVOKER' + | 'LEAKPROOF' + | 'PARALLEL' + | 'RETURN' + | 'RETURNS' + | 'SECURITY' + | 'STABLE' + | 'SUPPORT' + | 'TRANSFORM' + | 'VOLATILE' + | 'SETOF' + materialize_clause ::= 'MATERIALIZED' | 'NOT' 'MATERIALIZED' @@ -3424,6 +3538,26 @@ opt_family_name ::= create_as_constraint_elem ::= 'PRIMARY' 'KEY' '(' create_as_params ')' opt_with_storage_parameter_list +func_arg ::= + func_arg_class param_name func_arg_type + | param_name func_arg_class func_arg_type + | param_name func_arg_type + | func_arg_class func_arg_type + | func_arg_type + +func_as ::= + 'SCONST' + +common_func_opt_item ::= + 'CALLED' 'ON' 'NULL' 'INPUT' + | 'RETURNS' 'NULL' 'ON' 'NULL' 'INPUT' + | 'STRICT' + | 'IMMUTABLE' + | 'STABLE' + | 'VOLATILE' + | 'LEAKPROOF' + | 'NOT' 'LEAKPROOF' + group_by_item ::= a_expr @@ -3505,6 +3639,12 @@ create_as_col_qualification_elem ::= create_as_params ::= ( create_as_param ) ( ( ',' create_as_param ) )* +func_arg_class ::= + 'IN' + +param_name ::= + type_function_name + col_qualification ::= 'CONSTRAINT' constraint_name col_qualification_elem | col_qualification_elem diff --git a/docs/generated/sql/bnf/stmt_without_legacy_transaction.bnf b/docs/generated/sql/bnf/stmt_without_legacy_transaction.bnf new file mode 100644 index 000000000000..efb2571bd1a6 --- /dev/null +++ b/docs/generated/sql/bnf/stmt_without_legacy_transaction.bnf @@ -0,0 +1,22 @@ +stmt_without_legacy_transaction ::= + preparable_stmt + | analyze_stmt + | copy_from_stmt + | comment_stmt + | execute_stmt + | deallocate_stmt + | discard_stmt + | grant_stmt + | prepare_stmt + | revoke_stmt + | savepoint_stmt + | reassign_owned_by_stmt + | drop_owned_by_stmt + | release_stmt + | refresh_stmt + | nonpreparable_set_stmt + | transaction_stmt + | close_cursor_stmt + | declare_cursor_stmt + | fetch_cursor_stmt + | move_cursor_stmt diff --git a/pkg/gen/docs.bzl b/pkg/gen/docs.bzl index 6a9a2f35dd4f..857c96830732 100644 --- a/pkg/gen/docs.bzl +++ b/pkg/gen/docs.bzl @@ -88,6 +88,7 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:create_database_stmt.bnf", "//docs/generated/sql/bnf:create_ddl_stmt.bnf", "//docs/generated/sql/bnf:create_extension_stmt.bnf", + "//docs/generated/sql/bnf:create_func_stmt.bnf", "//docs/generated/sql/bnf:create_index_stmt.bnf", "//docs/generated/sql/bnf:create_index_with_storage_param.bnf", "//docs/generated/sql/bnf:create_inverted_index_stmt.bnf", @@ -142,6 +143,9 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:insert_stmt.bnf", "//docs/generated/sql/bnf:iso_level.bnf", "//docs/generated/sql/bnf:joined_table.bnf", + "//docs/generated/sql/bnf:legacy_begin_stmt.bnf", + "//docs/generated/sql/bnf:legacy_end_stmt.bnf", + "//docs/generated/sql/bnf:legacy_transaction_stmt.bnf", "//docs/generated/sql/bnf:like_table_option_list.bnf", "//docs/generated/sql/bnf:limit_clause.bnf", "//docs/generated/sql/bnf:move_cursor_stmt.bnf", @@ -179,6 +183,8 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:resume_stmt.bnf", "//docs/generated/sql/bnf:revoke_stmt.bnf", "//docs/generated/sql/bnf:rollback_transaction.bnf", + "//docs/generated/sql/bnf:routine_body_stmt.bnf", + "//docs/generated/sql/bnf:routine_return_stmt.bnf", "//docs/generated/sql/bnf:row_source_extension_stmt.bnf", "//docs/generated/sql/bnf:savepoint_stmt.bnf", "//docs/generated/sql/bnf:scrub_database_stmt.bnf", @@ -242,6 +248,7 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:split_table_at.bnf", "//docs/generated/sql/bnf:stmt.bnf", "//docs/generated/sql/bnf:stmt_block.bnf", + "//docs/generated/sql/bnf:stmt_without_legacy_transaction.bnf", "//docs/generated/sql/bnf:table_clause.bnf", "//docs/generated/sql/bnf:table_constraint.bnf", "//docs/generated/sql/bnf:table_ref.bnf", diff --git a/pkg/sql/delegate/show_completions_test.go b/pkg/sql/delegate/show_completions_test.go index 94260270a6fa..28d51a9771a3 100644 --- a/pkg/sql/delegate/show_completions_test.go +++ b/pkg/sql/delegate/show_completions_test.go @@ -44,9 +44,9 @@ func TestCompletions(t *testing.T) { { stmt: "se", expectedCompletions: []string{ - "SEARCH", "SECOND", "SELECT", "SEQUENCE", "SEQUENCES", + "SEARCH", "SECOND", "SECURITY", "SELECT", "SEQUENCE", "SEQUENCES", "SERIALIZABLE", "SERVER", "SESSION", "SESSIONS", "SESSION_USER", - "SET", "SETS", "SETTING", "SETTINGS", + "SET", "SETOF", "SETS", "SETTING", "SETTINGS", }, }, { diff --git a/pkg/sql/logictest/testdata/logic_test/show_completions b/pkg/sql/logictest/testdata/logic_test/show_completions index 71dbf1a5e678..bf4a0f25623f 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_completions +++ b/pkg/sql/logictest/testdata/logic_test/show_completions @@ -45,6 +45,7 @@ SHOW COMPLETIONS AT OFFSET 4 FOR e'\'se\''; ---- SEARCH SECOND +SECURITY SELECT SEQUENCE SEQUENCES @@ -54,6 +55,7 @@ SESSION SESSIONS SESSION_USER SET +SETOF SETS SETTING SETTINGS diff --git a/pkg/sql/logictest/testdata/logic_test/udf b/pkg/sql/logictest/testdata/logic_test/udf new file mode 100644 index 000000000000..03cb98c91d11 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/udf @@ -0,0 +1,2 @@ +statement error pq: unimplemented: CREATE FUNCTION unimplemented +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL diff --git a/pkg/sql/opt/optbuilder/builder.go b/pkg/sql/opt/optbuilder/builder.go index 1cc7c6883e9e..069485ab7798 100644 --- a/pkg/sql/opt/optbuilder/builder.go +++ b/pkg/sql/opt/optbuilder/builder.go @@ -305,6 +305,10 @@ func (b *Builder) buildStmt( case *tree.CreateView: return b.buildCreateView(stmt, inScope) + case *tree.CreateFunction: + // TODO (Chengxiong): implement this + panic(unimplemented.NewWithIssue(83228, "CREATE FUNCTION unimplemented")) + case *tree.Explain: return b.buildExplain(stmt, inScope) diff --git a/pkg/sql/parser/parse.go b/pkg/sql/parser/parse.go index 77b9f260b18b..07a389f2ac92 100644 --- a/pkg/sql/parser/parse.go +++ b/pkg/sql/parser/parse.go @@ -152,13 +152,28 @@ func (p *Parser) scanOneStmt() (sql string, tokens []sqlSymType, done bool) { // We make the resulting token positions match the returned string. lval.pos = 0 tokens = append(tokens, lval) + var preValID int32 + // This is used to track the degree of nested `BEGIN ATOMIC ... END` function + // body context. When greater than zero, it means that we're scanning through + // the function body of a `CREATE FUNCTION` statement. ';' character is only + // a separator of sql statements within the body instead of a finishing line + // of the `CREATE FUNCTION` statement. + curFuncBodyCnt := 0 for { if lval.id == ERROR { return p.scanner.In()[startPos:], tokens, true } + preValID = lval.id posBeforeScan := p.scanner.Pos() p.scanner.Scan(&lval) - if lval.id == 0 || lval.id == ';' { + + if preValID == BEGIN && lval.id == ATOMIC { + curFuncBodyCnt++ + } + if curFuncBodyCnt > 0 && lval.id == END { + curFuncBodyCnt-- + } + if lval.id == 0 || (curFuncBodyCnt == 0 && lval.id == ';') { return p.scanner.In()[startPos:posBeforeScan], tokens, (lval.id == 0) } lval.pos -= startPos diff --git a/pkg/sql/parser/parse_test.go b/pkg/sql/parser/parse_test.go index 05f741332e56..222b5c706b0b 100644 --- a/pkg/sql/parser/parse_test.go +++ b/pkg/sql/parser/parse_test.go @@ -421,8 +421,6 @@ func TestUnimplementedSyntax(t *testing.T) { {`CREATE EXTENSION IF NOT EXISTS a WITH schema = 'public'`, 74777, `create extension if not exists with`, ``}, {`CREATE FOREIGN DATA WRAPPER a`, 0, `create fdw`, ``}, {`CREATE FOREIGN TABLE a`, 0, `create foreign table`, ``}, - {`CREATE FUNCTION a`, 17511, `create`, ``}, - {`CREATE OR REPLACE FUNCTION a`, 17511, `create`, ``}, {`CREATE LANGUAGE a`, 17511, `create language a`, ``}, {`CREATE OPERATOR a`, 65017, ``, ``}, {`CREATE PUBLICATION a`, 0, `create publication`, ``}, diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y index 1a139bef888c..11a2d298f81d 100644 --- a/pkg/sql/parser/sql.y +++ b/pkg/sql/parser/sql.y @@ -781,6 +781,27 @@ func (u *sqlSymUnion) cursorStmt() tree.CursorStmt { func (u *sqlSymUnion) asTenantClause() tree.TenantID { return u.val.(tree.TenantID) } +func (u *sqlSymUnion) functionOptions() tree.FunctionOptions { + return u.val.(tree.FunctionOptions) +} +func (u *sqlSymUnion) functionOption() tree.FunctionOption { + return u.val.(tree.FunctionOption) +} +func (u *sqlSymUnion) functionArgs() tree.FuncArgs { + return u.val.(tree.FuncArgs) +} +func (u *sqlSymUnion) functionArg() tree.FuncArg { + return u.val.(tree.FuncArg) +} +func (u *sqlSymUnion) functionArgClass() tree.FuncArgClass { + return u.val.(tree.FuncArgClass) +} +func (u *sqlSymUnion) stmts() tree.Statements { + return u.val.(tree.Statements) +} +func (u *sqlSymUnion) routineBody() *tree.RoutineBody { + return u.val.(*tree.RoutineBody) +} %} // NB: the %token definitions must come before the %type definitions in this @@ -802,23 +823,23 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { // Ordinary key words in alphabetical order. %token ABORT ABSOLUTE ACCESS ACTION ADD ADMIN AFTER AGGREGATE %token ALL ALTER ALWAYS ANALYSE ANALYZE AND AND_AND ANY ANNOTATE_TYPE ARRAY AS ASC -%token ASENSITIVE ASYMMETRIC AT ATTRIBUTE AUTHORIZATION AUTOMATIC AVAILABILITY +%token ASENSITIVE ASYMMETRIC AT ATOMIC ATTRIBUTE AUTHORIZATION AUTOMATIC AVAILABILITY %token BACKUP BACKUPS BACKWARD BEFORE BEGIN BETWEEN BIGINT BIGSERIAL BINARY BIT %token BUCKET_COUNT %token BOOLEAN BOTH BOX2D BUNDLE BY -%token CACHE CANCEL CANCELQUERY CASCADE CASE CAST CBRT CHANGEFEED CHAR +%token CACHE CALLED CANCEL CANCELQUERY CASCADE CASE CAST CBRT CHANGEFEED CHAR %token CHARACTER CHARACTERISTICS CHECK CLOSE %token CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT %token COMMITTED COMPACT COMPLETE COMPLETIONS CONCAT CONCURRENTLY CONFIGURATION CONFIGURATIONS CONFIGURE %token CONFLICT CONNECTION CONSTRAINT CONSTRAINTS CONTAINS CONTROLCHANGEFEED CONTROLJOB -%token CONVERSION CONVERT COPY COVERING CREATE CREATEDB CREATELOGIN CREATEROLE +%token CONVERSION CONVERT COPY COST COVERING CREATE CREATEDB CREATELOGIN CREATEROLE %token CROSS CSV CUBE CURRENT CURRENT_CATALOG CURRENT_DATE CURRENT_SCHEMA %token CURRENT_ROLE CURRENT_TIME CURRENT_TIMESTAMP %token CURRENT_USER CURSOR CYCLE -%token DATA DATABASE DATABASES DATE DAY DEBUG_PAUSE_ON DEC DECIMAL DEFAULT DEFAULTS +%token DATA DATABASE DATABASES DATE DAY DEBUG_PAUSE_ON DEC DECIMAL DEFAULT DEFAULTS DEFINER %token DEALLOCATE DECLARE DEFERRABLE DEFERRED DELETE DELIMITER DESC DESTINATION DETACHED %token DISCARD DISTINCT DO DOMAIN DOUBLE DROP @@ -826,7 +847,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %token EXISTS EXECUTE EXECUTION EXPERIMENTAL %token EXPERIMENTAL_FINGERPRINTS EXPERIMENTAL_REPLICA %token EXPERIMENTAL_AUDIT EXPERIMENTAL_RELOCATE -%token EXPIRATION EXPLAIN EXPORT EXTENSION EXTRACT EXTRACT_DURATION +%token EXPIRATION EXPLAIN EXPORT EXTENSION EXTERNAL EXTRACT EXTRACT_DURATION %token FAILURE FALSE FAMILY FETCH FETCHVAL FETCHTEXT FETCHVAL_PATH FETCHTEXT_PATH %token FILES FILTER @@ -840,19 +861,19 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %token HAVING HASH HEADER HIGH HISTOGRAM HOLD HOUR %token IDENTITY -%token IF IFERROR IFNULL IGNORE_FOREIGN_KEYS ILIKE IMMEDIATE IMPORT IN INCLUDE +%token IF IFERROR IFNULL IGNORE_FOREIGN_KEYS ILIKE IMMEDIATE IMMUTABLE IMPORT IN INCLUDE %token INCLUDING INCREMENT INCREMENTAL INCREMENTAL_LOCATION %token INET INET_CONTAINED_BY_OR_EQUALS %token INET_CONTAINS_OR_EQUALS INDEX INDEXES INHERITS INJECT INITIALLY -%token INNER INSENSITIVE INSERT INT INTEGER -%token INTERSECT INTERVAL INTO INTO_DB INVERTED IS ISERROR ISNULL ISOLATION +%token INNER INOUT INPUT INSENSITIVE INSERT INT INTEGER +%token INTERSECT INTERVAL INTO INTO_DB INVERTED INVOKER IS ISERROR ISNULL ISOLATION %token JOB JOBS JOIN JSON JSONB JSON_SOME_EXISTS JSON_ALL_EXISTS %token KEY KEYS KMS KV %token LANGUAGE LAST LATERAL LATEST LC_CTYPE LC_COLLATE -%token LEADING LEASE LEAST LEFT LESS LEVEL LIKE LIMIT +%token LEADING LEASE LEAST LEAKPROOF LEFT LESS LEVEL LIKE LIMIT %token LINESTRING LINESTRINGM LINESTRINGZ LINESTRINGZM %token LIST LOCAL LOCALITY LOCALTIME LOCALTIMESTAMP LOCKED LOGIN LOOKUP LOW LSHIFT @@ -869,7 +890,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %token OF OFF OFFSET OID OIDS OIDVECTOR OLD_KMS ON ONLY OPT OPTION OPTIONS OR %token ORDER ORDINALITY OTHERS OUT OUTER OVER OVERLAPS OVERLAY OWNED OWNER OPERATOR -%token PARENT PARTIAL PARTITION PARTITIONS PASSWORD PAUSE PAUSED PHYSICAL PLACEMENT PLACING +%token PARALLEL PARENT PARTIAL PARTITION PARTITIONS PASSWORD PAUSE PAUSED PHYSICAL PLACEMENT PLACING %token PLAN PLANS POINT POINTM POINTZ POINTZM POLYGON POLYGONM POLYGONZ POLYGONZM %token POSITION PRECEDING PRECISION PREPARE PRESERVE PRIMARY PRIOR PRIORITY PRIVILEGES %token PROCEDURAL PUBLIC PUBLICATION @@ -879,21 +900,21 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %token RANGE RANGES READ REAL REASON REASSIGN RECURSIVE RECURRING REF REFERENCES REFRESH %token REGCLASS REGION REGIONAL REGIONS REGNAMESPACE REGPROC REGPROCEDURE REGROLE REGTYPE REINDEX %token RELATIVE RELOCATE REMOVE_PATH RENAME REPEATABLE REPLACE REPLICATION -%token RELEASE RESET RESTART RESTORE RESTRICT RESTRICTED RESUME RETURNING RETRY REVISION_HISTORY +%token RELEASE RESET RESTART RESTORE RESTRICT RESTRICTED RESUME RETURNING RETURN RETURNS RETRY REVISION_HISTORY %token REVOKE RIGHT ROLE ROLES ROLLBACK ROLLUP ROUTINES ROW ROWS RSHIFT RULE RUNNING -%token SAVEPOINT SCANS SCATTER SCHEDULE SCHEDULES SCROLL SCHEMA SCHEMAS SCRUB SEARCH SECOND SELECT SEQUENCE SEQUENCES -%token SERIALIZABLE SERVER SESSION SESSIONS SESSION_USER SET SETS SETTING SETTINGS +%token SAVEPOINT SCANS SCATTER SCHEDULE SCHEDULES SCROLL SCHEMA SCHEMAS SCRUB SEARCH SECOND SECURITY SELECT SEQUENCE SEQUENCES +%token SERIALIZABLE SERVER SESSION SESSIONS SESSION_USER SET SETOF SETS SETTING SETTINGS %token SHARE SHOW SIMILAR SIMPLE SKIP SKIP_LOCALITIES_CHECK SKIP_MISSING_FOREIGN_KEYS %token SKIP_MISSING_SEQUENCES SKIP_MISSING_SEQUENCE_OWNERS SKIP_MISSING_VIEWS SMALLINT SMALLSERIAL SNAPSHOT SOME SPLIT SQL %token SQLLOGIN -%token START STATE STATISTICS STATUS STDIN STREAM STRICT STRING STORAGE STORE STORED STORING SUBSTRING SUPER -%token SURVIVE SURVIVAL SYMMETRIC SYNTAX SYSTEM SQRT SUBSCRIPTION STATEMENTS +%token STABLE START STATE STATISTICS STATUS STDIN STREAM STRICT STRING STORAGE STORE STORED STORING SUBSTRING SUPER +%token SUPPORT SURVIVE SURVIVAL SYMMETRIC SYNTAX SYSTEM SQRT SUBSCRIPTION STATEMENTS %token TABLE TABLES TABLESPACE TEMP TEMPLATE TEMPORARY TENANT TENANTS TESTING_RELOCATE TEXT THEN %token TIES TIME TIMETZ TIMESTAMP TIMESTAMPTZ TO THROTTLING TRAILING TRACE -%token TRANSACTION TRANSACTIONS TRANSFER TREAT TRIGGER TRIM TRUE +%token TRANSACTION TRANSACTIONS TRANSFER TRANSFORM TREAT TRIGGER TRIM TRUE %token TRUNCATE TRUSTED TYPE TYPES %token TRACING @@ -901,7 +922,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %token UPDATE UPSERT UNSET UNTIL USE USER USERS USING UUID %token VALID VALIDATE VALUE VALUES VARBIT VARCHAR VARIADIC VIEW VARYING VIEWACTIVITY VIEWACTIVITYREDACTED -%token VIEWCLUSTERSETTING VIRTUAL VISIBLE VOTERS +%token VIEWCLUSTERSETTING VIRTUAL VISIBLE VOLATILE VOTERS %token WHEN WHERE WINDOW WITH WITHIN WITHOUT WORK WRITE @@ -939,7 +960,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { } %type stmt_block -%type stmt +%type stmt stmt_without_legacy_transaction %type alter_stmt @@ -1053,6 +1074,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %type create_table_as_stmt %type create_view_stmt %type create_sequence_stmt +%type create_func_stmt %type create_stats_stmt %type <*tree.CreateStatsOptions> opt_create_stats_options @@ -1163,7 +1185,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %type session_var %type <*string> comment_text -%type transaction_stmt +%type transaction_stmt legacy_transaction_stmt legacy_begin_stmt legacy_end_stmt %type truncate_stmt %type update_stmt %type upsert_stmt @@ -1424,6 +1446,7 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %type region_or_regions %type unreserved_keyword type_func_name_keyword type_func_name_no_crdb_extra_keyword type_func_name_crdb_extra_keyword +%type bare_label_keywords bare_col_label %type col_name_keyword reserved_keyword cockroachdb_extra_reserved_keyword extra_var_value %type complex_type_name @@ -1495,6 +1518,19 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID { %type target_object_type %type opt_as_tenant_clause +// User defined function relevant components. +%type opt_or_replace opt_return_set +%type param_name func_as +%type opt_func_arg_with_default_list func_arg_with_default_list +%type func_arg_with_default func_arg +%type func_return_type func_arg_type +%type opt_create_func_opt_list create_func_opt_list +%type create_func_opt_item common_func_opt_item +%type func_arg_class +%type <*tree.UnresolvedObjectName> func_create_name +%type routine_return_stmt routine_body_stmt +%type routine_body_stmt_list +%type <*tree.RoutineBody> opt_routine_body // Precedence: lowest to highest %nonassoc VALUES // see value_clause @@ -1574,7 +1610,15 @@ stmt_block: stmt: HELPTOKEN { return helpWith(sqllex, "") } -| preparable_stmt // help texts in sub-rule +| stmt_without_legacy_transaction +| legacy_transaction_stmt +| /* EMPTY */ + { + $$.val = tree.Statement(nil) + } + +stmt_without_legacy_transaction: + preparable_stmt // help texts in sub-rule | analyze_stmt // EXTEND WITH HELP: ANALYZE | copy_from_stmt | comment_stmt @@ -1596,10 +1640,6 @@ stmt: | fetch_cursor_stmt // EXTEND WITH HELP: FETCH | move_cursor_stmt // EXTEND WITH HELP: MOVE | reindex_stmt -| /* EMPTY */ - { - $$.val = tree.Statement(nil) - } // %Help: ALTER // %Category: Group @@ -3796,6 +3836,256 @@ create_extension_stmt: } | CREATE EXTENSION error // SHOW HELP: CREATE EXTENSION +create_func_stmt: + CREATE opt_or_replace FUNCTION func_create_name '(' opt_func_arg_with_default_list ')' RETURNS opt_return_set func_return_type + opt_create_func_opt_list opt_routine_body + { + name := $4.unresolvedObjectName().ToFunctionName() + $$.val = &tree.CreateFunction{ + IsProcedure: false, + Replace: $2.bool(), + FuncName: name, + Args: $6.functionArgs(), + ReturnType: tree.FuncReturnType{ + Type: $10.typeReference(), + IsSet: $9.bool(), + }, + Options: $11.functionOptions(), + RoutineBody: $12.routineBody(), + } + } + +opt_or_replace: + OR REPLACE { $$.val = true } +| /* EMPTY */ { $$.val = false } + +opt_return_set: + SETOF { $$.val = true} +| /* EMPTY */ { $$.val = false } + +func_create_name: + db_object_name + +opt_func_arg_with_default_list: + func_arg_with_default_list { $$.val = $1.functionArgs() } +| /* Empty */ { $$.val = tree.FuncArgs{} } + +func_arg_with_default_list: + func_arg_with_default { $$.val = tree.FuncArgs{$1.functionArg()} } +| func_arg_with_default_list ',' func_arg_with_default + { + $$.val = append($1.functionArgs(), $3.functionArg()) + } + +func_arg_with_default: + func_arg +| func_arg DEFAULT a_expr + { + arg := $1.functionArg() + arg.DefaultVal = $3.expr() + $$.val = arg + } +| func_arg '=' a_expr + { + arg := $1.functionArg() + arg.DefaultVal = $3.expr() + $$.val = arg + } + +func_arg: + func_arg_class param_name func_arg_type + { + $$.val = tree.FuncArg{ + Name: tree.Name($2), + Type: $3.typeReference(), + Class: $1.functionArgClass(), + } + } +| param_name func_arg_class func_arg_type + { + $$.val = tree.FuncArg{ + Name: tree.Name($1), + Type: $3.typeReference(), + Class: $2.functionArgClass(), + } + } +| param_name func_arg_type + { + $$.val = tree.FuncArg{ + Name: tree.Name($1), + Type: $2.typeReference(), + Class: tree.FunctionArgIn, + } + } +| func_arg_class func_arg_type + { + $$.val = tree.FuncArg{ + Type: $2.typeReference(), + Class: $1.functionArgClass(), + } + } +| func_arg_type + { + $$.val = tree.FuncArg{ + Type: $1.typeReference(), + Class: tree.FunctionArgIn, + } + } + +func_arg_class: + IN { $$.val = tree.FunctionArgIn } +| OUT { return unimplemented(sqllex, "create function with 'OUT' argument class") } +| INOUT { return unimplemented(sqllex, "create function with 'INOUT' argument class") } +| IN OUT { return unimplemented(sqllex, "create function with 'IN OUT' argument class") } +| VARIADIC { return unimplemented(sqllex, "create function with 'VARIADIC' argument class") } + +func_arg_type: + typename + +func_return_type: + func_arg_type + +opt_create_func_opt_list: + create_func_opt_list { $$.val = $1.functionOptions() } +| /* EMPTY */ { $$.val = tree.FunctionOptions{} } + +create_func_opt_list: + create_func_opt_item { $$.val = tree.FunctionOptions{$1.functionOption()} } +| create_func_opt_list create_func_opt_item + { + $$.val = append($1.functionOptions(), $2.functionOption()) + } + +create_func_opt_item: + AS func_as + { + $$.val = tree.FunctionBodyStr($2) + } +| LANGUAGE non_reserved_word_or_sconst + { + lang, err := tree.AsFunctionLanguage($2) + if err != nil { + return setErr(sqllex, err) + } + $$.val = lang + } +| TRANSFORM { return unimplemented(sqllex, "create transform function") } +| WINDOW { return unimplemented(sqllex, "create window function") } +| common_func_opt_item + { + $$.val = $1.functionOption() + } + +common_func_opt_item: + CALLED ON NULL INPUT + { + $$.val = tree.FunctionCalledOnNullInput + } +| RETURNS NULL ON NULL INPUT + { + $$.val = tree.FunctionReturnsNullOnNullInput + } +| STRICT + { + $$.val = tree.FunctionStrict + } +| IMMUTABLE + { + $$.val = tree.FunctionImmutable + } +| STABLE + { + $$.val = tree.FunctionStable + } +| VOLATILE + { + $$.val = tree.FunctionVolatile + } +| EXTERNAL SECURITY DEFINER + { + return unimplemented(sqllex, "create function...security") + } +| EXTERNAL SECURITY INVOKER + { + return unimplemented(sqllex, "create function...security") + } +| SECURITY DEFINER + { + return unimplemented(sqllex, "create function...security") + } +| SECURITY INVOKER + { + return unimplemented(sqllex, "create function...security") + } +| LEAKPROOF + { + $$.val = tree.FunctionLeakProof(true) + } +| NOT LEAKPROOF + { + $$.val = tree.FunctionLeakProof(false) + } +| COST numeric_only + { + return unimplemented(sqllex, "create function...cost") + } +| ROWS numeric_only + { + return unimplemented(sqllex, "create function...rows") + } +| SUPPORT name + { + return unimplemented(sqllex, "create function...support") + } +// In theory we should parse the a whole set/reset statement here. But it's fine +// to just return fast on SET/RESET keyword for now since it's not supported +// yet. +| SET { return unimplemented(sqllex, "create function...set") } +| PARALLEL { return unimplemented(sqllex, "create function...parallel") } + +func_as: + SCONST + +routine_return_stmt: + RETURN a_expr +{ + $$.val = &tree.RoutineReturn{ + ReturnVal: $2.expr(), + } +} + +routine_body_stmt: + stmt_without_legacy_transaction +| routine_return_stmt + +routine_body_stmt_list: + routine_body_stmt_list routine_body_stmt ';' + { + $$.val = append($1.stmts(), $2.stmt()) + } +| /* Empty */ + { + $$.val = tree.Statements{} + } + +opt_routine_body: + routine_return_stmt + { + $$.val = &tree.RoutineBody{ + Stmts: tree.Statements{$1.stmt()}, + } + } +| BEGIN ATOMIC routine_body_stmt_list END + { + $$.val = &tree.RoutineBody{ + Stmts: $3.stmts(), + } + } +| /* Empty */ + { + $$.val = (*tree.RoutineBody)(nil) + } + create_unsupported: CREATE ACCESS METHOD error { return unimplemented(sqllex, "create access method") } | CREATE AGGREGATE error { return unimplementedWithIssueDetail(sqllex, 74775, "create aggregate") } @@ -3805,8 +4095,6 @@ create_unsupported: | CREATE DEFAULT CONVERSION error { return unimplemented(sqllex, "create def conv") } | CREATE FOREIGN TABLE error { return unimplemented(sqllex, "create foreign table") } | CREATE FOREIGN DATA error { return unimplemented(sqllex, "create fdw") } -| CREATE FUNCTION error { return unimplementedWithIssueDetail(sqllex, 17511, "create function") } -| CREATE OR REPLACE FUNCTION error { return unimplementedWithIssueDetail(sqllex, 17511, "create function") } | CREATE opt_or_replace opt_trusted opt_procedural LANGUAGE name error { return unimplementedWithIssueDetail(sqllex, 17511, "create language " + $6) } | CREATE OPERATOR error { return unimplementedWithIssue(sqllex, 65017) } | CREATE PUBLICATION error { return unimplemented(sqllex, "create publication") } @@ -3817,10 +4105,6 @@ create_unsupported: | CREATE TEXT error { return unimplementedWithIssueDetail(sqllex, 7821, "create text") } | CREATE TRIGGER error { return unimplementedWithIssueDetail(sqllex, 28296, "create trigger") } -opt_or_replace: - OR REPLACE {} -| /* EMPTY */ {} - opt_trusted: TRUSTED {} | /* EMPTY */ {} @@ -3861,6 +4145,7 @@ create_ddl_stmt: | create_type_stmt // EXTEND WITH HELP: CREATE TYPE | create_view_stmt // EXTEND WITH HELP: CREATE VIEW | create_sequence_stmt // EXTEND WITH HELP: CREATE SEQUENCE +| create_func_stmt // %Help: CREATE STATISTICS - create a new table statistic // %Category: Misc @@ -9387,12 +9672,7 @@ transaction_stmt: // // %SeeAlso: COMMIT, ROLLBACK, WEBDOCS/begin-transaction.html begin_stmt: - BEGIN opt_transaction begin_transaction - { - $$.val = $3.stmt() - } -| BEGIN error // SHOW HELP: BEGIN -| START TRANSACTION begin_transaction + START TRANSACTION begin_transaction { $$.val = $3.stmt() } @@ -9410,11 +9690,6 @@ commit_stmt: $$.val = &tree.CommitTransaction{} } | COMMIT error // SHOW HELP: COMMIT -| END opt_transaction - { - $$.val = &tree.CommitTransaction{} - } -| END error // SHOW HELP: COMMIT abort_stmt: ABORT opt_abort_mod @@ -9444,6 +9719,28 @@ rollback_stmt: } | ROLLBACK error // SHOW HELP: ROLLBACK +// "legacy" here doesn't mean we're deprecating the syntax. We inherit this +// concept from postgres. The idea is to avoid conflicts in "CREATE FUNCTION"'s +// "BEGIN ATOMIC...END" function body context. +legacy_transaction_stmt: + legacy_begin_stmt // EXTEND WITH HELP: BEGIN +| legacy_end_stmt // EXTEND WITH HELP: COMMIT + +legacy_begin_stmt: + BEGIN opt_transaction begin_transaction + { + $$.val = $3.stmt() + } +| BEGIN error // SHOW HELP: BEGIN + +legacy_end_stmt: + END opt_transaction + { + $$.val = &tree.CommitTransaction{} + } +| END error // SHOW HELP: COMMIT + + opt_transaction: TRANSACTION {} | /* EMPTY */ {} @@ -13546,7 +13843,7 @@ target_elem: // infix expression, or a postfix expression and a column label? We prefer // to resolve this as an infix expression, which we accomplish by assigning // IDENT a precedence higher than POSTFIXOP. -| a_expr IDENT +| a_expr bare_col_label { $$.val = tree.SelectExpr{Expr: $1.expr(), As: tree.UnrestrictedName($2)} } @@ -13559,6 +13856,10 @@ target_elem: $$.val = tree.StarSelectExpr() } +bare_col_label: + IDENT +| bare_label_keywords + // Names and constants. table_index_name_list: @@ -14055,6 +14356,9 @@ type_function_name_no_crdb_extra: | unreserved_keyword | type_func_name_no_crdb_extra_keyword +param_name: + type_function_name + // Any not-fully-reserved word --- these names can be, eg, variable names. non_reserved_word: IDENT @@ -14079,6 +14383,8 @@ unrestricted_name: // shift or reduce conflicts. The earlier lists define "less reserved" // categories of keywords. // +// Note: also add the new keyword to `bare_label` list to not break +// user queries using column label without `AS`. // "Unreserved" keywords --- available for use as any kind of name. unreserved_keyword: ABORT @@ -14093,6 +14399,7 @@ unreserved_keyword: | ALWAYS | ASENSITIVE | AT +| ATOMIC | ATTRIBUTE | AUTOMATIC | AVAILABILITY @@ -14106,6 +14413,7 @@ unreserved_keyword: | BUNDLE | BY | CACHE +| CALLED | CANCEL | CANCELQUERY | CASCADE @@ -14131,6 +14439,7 @@ unreserved_keyword: | CONVERSION | CONVERT | COPY +| COST | COVERING | CREATEDB | CREATELOGIN @@ -14150,6 +14459,7 @@ unreserved_keyword: | DELETE | DEFAULTS | DEFERRED +| DEFINER | DELIMITER | DESTINATION | DETACHED @@ -14176,6 +14486,7 @@ unreserved_keyword: | EXPLAIN | EXPORT | EXTENSION +| EXTERNAL | FAILURE | FILES | FILTER @@ -14208,6 +14519,7 @@ unreserved_keyword: | HOUR | IDENTITY | IMMEDIATE +| IMMUTABLE | IMPORT | INCLUDE | INCLUDING @@ -14217,10 +14529,12 @@ unreserved_keyword: | INDEXES | INHERITS | INJECT +| INPUT | INSERT | INTO_DB | INVERTED | ISOLATION +| INVOKER | JOB | JOBS | JSON @@ -14233,6 +14547,7 @@ unreserved_keyword: | LATEST | LC_COLLATE | LC_CTYPE +| LEAKPROOF | LEASE | LESS | LEVEL @@ -14310,6 +14625,7 @@ unreserved_keyword: | OVER | OWNED | OWNER +| PARALLEL | PARENT | PARTIAL | PARTITION @@ -14365,6 +14681,8 @@ unreserved_keyword: | RESTRICTED | RESUME | RETRY +| RETURN +| RETURNS | REVISION_HISTORY | REVOKE | ROLE @@ -14389,6 +14707,7 @@ unreserved_keyword: | SCRUB | SEARCH | SECOND +| SECURITY | SERIALIZABLE | SEQUENCE | SEQUENCES @@ -14410,6 +14729,7 @@ unreserved_keyword: | SPLIT | SQL | SQLLOGIN +| STABLE | START | STATE | STATEMENTS @@ -14423,6 +14743,7 @@ unreserved_keyword: | STRICT | SUBSCRIPTION | SUPER +| SUPPORT | SURVIVE | SURVIVAL | SYNTAX @@ -14441,6 +14762,7 @@ unreserved_keyword: | TRANSACTION | TRANSACTIONS | TRANSFER +| TRANSFORM | TRIGGER | TRUNCATE | TRUSTED @@ -14467,6 +14789,7 @@ unreserved_keyword: | VIEWACTIVITYREDACTED | VIEWCLUSTERSETTING | VISIBLE +| VOLATILE | VOTERS | WITHIN | WITHOUT @@ -14474,6 +14797,29 @@ unreserved_keyword: | YEAR | ZONE +// Column label --- keywords that can be column label that doesn't use "AS" +// before it. This is to guarantee that any new keyword won't break user +// query like "SELECT col label FROM table" where "label" is a new keyword. +// Any new keyword should be added to this list. +bare_label_keywords: + ATOMIC +| CALLED +| DEFINER +| EXTERNAL +| IMMUTABLE +| INPUT +| INVOKER +| LEAKPROOF +| PARALLEL +| RETURN +| RETURNS +| SECURITY +| STABLE +| SUPPORT +| TRANSFORM +| VOLATILE +| SETOF + // Column identifier --- keywords that can be column, table, etc names. // // Many of these keywords will in fact be recognized as type or function names @@ -14483,6 +14829,9 @@ unreserved_keyword: // The type names appearing here are not usable as function names because they // can be followed by '(' in typename productions, which looks too much like a // function call for an LR(1) parser. +// +// Note: also add the new keyword to `bare_label` list to not break +// user queries using column label without `AS`. col_name_keyword: ANNOTATE_TYPE | BETWEEN @@ -14507,6 +14856,7 @@ col_name_keyword: | IF | IFERROR | IFNULL +| INOUT | INT | INTEGER | INTERVAL @@ -14522,6 +14872,7 @@ col_name_keyword: | PRECISION | REAL | ROW +| SETOF | SMALLINT | STRING | SUBSTRING @@ -14596,6 +14947,8 @@ type_func_name_crdb_extra_keyword: // // See cockroachdb_extra_reserved_keyword below. // +// Note: also add the new keyword to `bare_label` list to not break +// user queries using column label without `AS`. reserved_keyword: ALL | ANALYSE diff --git a/pkg/sql/parser/testdata/create_function b/pkg/sql/parser/testdata/create_function new file mode 100644 index 000000000000..f6b2b6cc619d --- /dev/null +++ b/pkg/sql/parser/testdata/create_function @@ -0,0 +1,388 @@ +parse +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL +---- +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT 7) RETURNS INT8 LANGUAGE SQL AS $$SELECT 1$$ -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT (7)) RETURNS INT8 LANGUAGE SQL AS $$SELECT 1$$ -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT _) RETURNS INT8 LANGUAGE SQL AS $$SELECT 1$$ -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8 DEFAULT 7) RETURNS INT8 LANGUAGE SQL AS $$SELECT 1$$ -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(IN a INT=7) RETURNS INT CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS 'SELECT 1' +---- +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT 7) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT (7)) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT _) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8 DEFAULT 7) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(IN a INT=7) RETURNS INT AS 'SELECT 1' CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL +---- +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT 7) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT (7)) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT _) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8 DEFAULT 7) RETURNS INT8 CALLED ON NULL INPUT IMMUTABLE LEAKPROOF LANGUAGE SQL AS $$SELECT 1$$ -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(a INT DEFAULT 10) RETURNS INT RETURNS NULL ON NULL INPUT LANGUAGE SQL AS 'SELECT 1' +---- +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT 10) RETURNS INT8 RETURNS NULL ON NULL INPUT LANGUAGE SQL AS $$SELECT 1$$ -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT (10)) RETURNS INT8 RETURNS NULL ON NULL INPUT LANGUAGE SQL AS $$SELECT 1$$ -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8 DEFAULT _) RETURNS INT8 RETURNS NULL ON NULL INPUT LANGUAGE SQL AS $$SELECT 1$$ -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8 DEFAULT 10) RETURNS INT8 RETURNS NULL ON NULL INPUT LANGUAGE SQL AS $$SELECT 1$$ -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT a; END +---- +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT a; END -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT (1); SELECT (a); END -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT _; SELECT a; END -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT _; END -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT $1; END +---- +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT $1; END -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT (1); SELECT ($1); END -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT _; SELECT $1; END -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; SELECT $1; END -- identifiers removed + +parse +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1; CREATE OR REPLACE FUNCTION g() RETURNS INT BEGIN ATOMIC SELECT 2; END; END +---- +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; CREATE OR REPLACE FUNCTION g() RETURNS INT8 BEGIN ATOMIC SELECT 2; END; END -- normalized! +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT (1); CREATE OR REPLACE FUNCTION g() RETURNS INT8 BEGIN ATOMIC SELECT (2); END; END -- fully parenthesized +CREATE OR REPLACE FUNCTION f(IN a INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT _; CREATE OR REPLACE FUNCTION g() RETURNS INT8 BEGIN ATOMIC SELECT _; END; END -- literals removed +CREATE OR REPLACE FUNCTION _(IN _ INT8) RETURNS INT8 LANGUAGE SQL BEGIN ATOMIC SELECT 1; CREATE OR REPLACE FUNCTION _() RETURNS INT8 BEGIN ATOMIC SELECT 2; END; END -- identifiers removed + +error +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1 END +---- +at or near "end": syntax error +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1 END + ^ +HINT: try \h CREATE + +error +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1; CREATE OR REPLACE FUNCTION g() RETURNS INT BEGIN ATOMIC SELECT 2; END; +---- +at or near "EOF": syntax error +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a INT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC SELECT 1; CREATE OR REPLACE FUNCTION g() RETURNS INT BEGIN ATOMIC SELECT 2; END; + ^ +HINT: try \h CREATE + +error +CREATE OR REPLACE FUNCTION f(OUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "out": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(OUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(INOUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "inout": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(INOUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(IN OUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "out": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(IN OUT a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(VARIADIC a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "variadic": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(VARIADIC a int = 7) RETURNS INT AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT TRANSFORM AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "transform": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT TRANSFORM AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT WINDOW AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "window": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT WINDOW AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT EXTERNAL SECURITY DEFINER AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "definer": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT EXTERNAL SECURITY DEFINER AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT EXTERNAL SECURITY INVOKER AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "invoker": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT EXTERNAL SECURITY INVOKER AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SECURITY DEFINER AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "definer": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SECURITY DEFINER AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SECURITY INVOKER AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "invoker": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SECURITY INVOKER AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT ROWS 123 AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "123": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT ROWS 123 AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SUPPORT abc AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "abc": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SUPPORT abc AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SET a = 123 AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "set": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT SET a = 123 AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT PARALLEL RESTRICTED AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "parallel": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT PARALLEL RESTRICTED AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- + +error +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT COST 123 AS 'SELECT 1' LANGUAGE SQL +---- +---- +at or near "123": syntax error: unimplemented: this syntax +DETAIL: source SQL: +CREATE OR REPLACE FUNCTION f(a int = 7) RETURNS INT COST 123 AS 'SELECT 1' LANGUAGE SQL + ^ +HINT: You have attempted to use a feature that is not yet implemented. + +Please check the public issue tracker to check whether this problem is +already tracked. If you cannot find it there, please report the error +with details by creating a new issue. + +If you would rather not post publicly, please contact us directly +using the support form. + +We appreciate your feedback. +---- +---- diff --git a/pkg/sql/scanner/scan.go b/pkg/sql/scanner/scan.go index ed303af6c22f..90189d97c286 100644 --- a/pkg/sql/scanner/scan.go +++ b/pkg/sql/scanner/scan.go @@ -1038,12 +1038,26 @@ func (s *Scanner) scanOne(lval *fakeSym) (done, hasToks bool, err error) { } } + var preValID int32 + // This is used to track the degree of nested `BEGIN ATOMIC ... END` function + // body context. When greater than zero, it means that we're scanning through + // the function body of a `CREATE FUNCTION` statement. ';' character is only + // a separator of sql statements within the body instead of a finishing line + // of the `CREATE FUNCTION` statement. + curFuncBodyCnt := 0 for { if lval.id == lexbase.ERROR { return true, true, fmt.Errorf("scan error: %s", lval.s) } + preValID = lval.id s.Scan(lval) - if lval.id == 0 || lval.id == ';' { + if preValID == lexbase.BEGIN && lval.id == lexbase.ATOMIC { + curFuncBodyCnt++ + } + if curFuncBodyCnt > 0 && lval.id == lexbase.END { + curFuncBodyCnt-- + } + if lval.id == 0 || (curFuncBodyCnt == 0 && lval.id == ';') { return (lval.id == 0), true, nil } } diff --git a/pkg/sql/sem/tree/BUILD.bazel b/pkg/sql/sem/tree/BUILD.bazel index 5716e63d179a..74edb0590bf7 100644 --- a/pkg/sql/sem/tree/BUILD.bazel +++ b/pkg/sql/sem/tree/BUILD.bazel @@ -100,6 +100,7 @@ go_library( "type_check.go", "type_name.go", "typing.go", + "udf.go", "union.go", "unsupported_error.go", "update.go", diff --git a/pkg/sql/sem/tree/object_name.go b/pkg/sql/sem/tree/object_name.go index 52f706bf3c10..1a7797bf270f 100644 --- a/pkg/sql/sem/tree/object_name.go +++ b/pkg/sql/sem/tree/object_name.go @@ -205,13 +205,11 @@ func (u *UnresolvedObjectName) Format(ctx *FmtCtx) { func (u *UnresolvedObjectName) String() string { return AsString(u) } -// ToTableName converts the unresolved name to a table name. -// // TODO(radu): the schema and catalog names might not be in the right places; we // would only figure that out during name resolution. This method is temporary, // while we change all the code paths to only use TableName after resolution. -func (u *UnresolvedObjectName) ToTableName() TableName { - return TableName{objName{ +func (u *UnresolvedObjectName) toObjName() objName { + return objName{ ObjectName: Name(u.Parts[0]), ObjectNamePrefix: ObjectNamePrefix{ SchemaName: Name(u.Parts[1]), @@ -219,7 +217,17 @@ func (u *UnresolvedObjectName) ToTableName() TableName { ExplicitSchema: u.NumParts >= 2, ExplicitCatalog: u.NumParts >= 3, }, - }} + } +} + +// ToTableName converts the unresolved name to a table name. +func (u *UnresolvedObjectName) ToTableName() TableName { + return TableName{u.toObjName()} +} + +// ToFunctionName converts the unresolved name to a function name. +func (u *UnresolvedObjectName) ToFunctionName() FunctionName { + return FunctionName{u.toObjName()} } // ToUnresolvedName converts the unresolved object name to the more general diff --git a/pkg/sql/sem/tree/stmt.go b/pkg/sql/sem/tree/stmt.go index 9c703d9bd973..155f49c6cd58 100644 --- a/pkg/sql/sem/tree/stmt.go +++ b/pkg/sql/sem/tree/stmt.go @@ -85,6 +85,9 @@ const ( TypeTCL ) +// Statements represent a list of statements. +type Statements []Statement + // Statement represents a statement. type Statement interface { fmt.Stringer @@ -1779,6 +1782,24 @@ func (*ValuesClause) StatementType() StatementType { return TypeDML } // StatementTag returns a short string identifying the type of statement. func (*ValuesClause) StatementTag() string { return "VALUES" } +// StatementReturnType implements the Statement interface. +func (*CreateFunction) StatementReturnType() StatementReturnType { return DDL } + +// StatementType implements the Statement interface. +func (*CreateFunction) StatementType() StatementType { return TypeDDL } + +// StatementTag returns a short string identifying the type of statement. +func (*CreateFunction) StatementTag() string { return "CREATE FUNCTION" } + +// StatementReturnType implements the Statement interface. +func (*RoutineReturn) StatementReturnType() StatementReturnType { return Rows } + +// StatementType implements the Statement interface. +func (*RoutineReturn) StatementType() StatementType { return TypeDML } + +// StatementTag returns a short string identifying the type of statement. +func (*RoutineReturn) StatementTag() string { return "RETURN" } + func (n *AlterChangefeed) String() string { return AsString(n) } func (n *AlterChangefeedCmds) String() string { return AsString(n) } func (n *AlterBackup) String() string { return AsString(n) } @@ -1836,6 +1857,7 @@ func (n *CopyFrom) String() string { return AsString(n) } func (n *CreateChangefeed) String() string { return AsString(n) } func (n *CreateDatabase) String() string { return AsString(n) } func (n *CreateExtension) String() string { return AsString(n) } +func (n *CreateFunction) String() string { return AsString(n) } func (n *CreateIndex) String() string { return AsString(n) } func (n *CreateRole) String() string { return AsString(n) } func (n *CreateTable) String() string { return AsString(n) } @@ -1878,6 +1900,7 @@ func (n *ReparentDatabase) String() string { return AsString(n) } func (n *RenameIndex) String() string { return AsString(n) } func (n *RenameTable) String() string { return AsString(n) } func (n *Restore) String() string { return AsString(n) } +func (n *RoutineReturn) String() string { return AsString(n) } func (n *Revoke) String() string { return AsString(n) } func (n *RevokeRole) String() string { return AsString(n) } func (n *RollbackToSavepoint) String() string { return AsString(n) } diff --git a/pkg/sql/sem/tree/udf.go b/pkg/sql/sem/tree/udf.go new file mode 100644 index 000000000000..9e9b57e93b4c --- /dev/null +++ b/pkg/sql/sem/tree/udf.go @@ -0,0 +1,291 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tree + +import ( + "strings" + + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" + "github.com/cockroachdb/errors" +) + +// FunctionName represent a function name in a UDF relevant statement, either +// DDL or DML statement. Similar to TableName, it is constructed for incoming +// SQL queries from an UnresolvedObjectName. +type FunctionName struct { + objName +} + +// Format implements the NodeFormatter interface. +func (f *FunctionName) Format(ctx *FmtCtx) { + f.ObjectNamePrefix.Format(ctx) + if f.ExplicitSchema || ctx.alwaysFormatTablePrefix() { + ctx.WriteByte('.') + } + ctx.FormatNode(&f.ObjectName) +} + +// CreateFunction represents a CREATE FUNCTION statement. +type CreateFunction struct { + IsProcedure bool + Replace bool + FuncName FunctionName + Args FuncArgs + ReturnType FuncReturnType + Options FunctionOptions + RoutineBody *RoutineBody +} + +// Format implements the NodeFormatter interface. +func (node *CreateFunction) Format(ctx *FmtCtx) { + ctx.WriteString("CREATE ") + if node.Replace { + ctx.WriteString("OR REPLACE ") + } + ctx.WriteString("FUNCTION ") + ctx.FormatNode(&node.FuncName) + ctx.WriteString("(") + ctx.FormatNode(node.Args) + ctx.WriteString(") ") + ctx.WriteString("RETURNS ") + if node.ReturnType.IsSet { + ctx.WriteString("SETOF ") + } + ctx.WriteString(node.ReturnType.Type.SQLString()) + ctx.WriteString(" ") + var funcBody FunctionBodyStr + for _, option := range node.Options { + switch t := option.(type) { + case FunctionBodyStr: + funcBody = t + continue + } + ctx.FormatNode(option) + ctx.WriteString(" ") + } + if len(funcBody) > 0 { + ctx.FormatNode(funcBody) + } + if node.RoutineBody != nil { + ctx.WriteString("BEGIN ATOMIC ") + for _, stmt := range node.RoutineBody.Stmts { + ctx.FormatNode(stmt) + ctx.WriteString("; ") + } + ctx.WriteString("END") + } +} + +// RoutineBody represent a list of statements in a UDF body. +type RoutineBody struct { + Stmts Statements +} + +// RoutineReturn represent a RETURN statement in a UDF body. +type RoutineReturn struct { + ReturnVal Expr +} + +// Format implements the NodeFormatter interface. +func (node *RoutineReturn) Format(ctx *FmtCtx) { + ctx.WriteString("RETURN ") + ctx.FormatNode(node.ReturnVal) +} + +// FunctionOptions represent a list of function options. +type FunctionOptions []FunctionOption + +// FunctionOption is an interface representing UDF properties. +type FunctionOption interface { + functionOption() + NodeFormatter +} + +func (FunctionNullInputBehavior) functionOption() {} +func (FunctionVolatility) functionOption() {} +func (FunctionLeakProof) functionOption() {} +func (FunctionBodyStr) functionOption() {} +func (FunctionLanguage) functionOption() {} + +// FunctionNullInputBehavior represent the UDF property on null parameters. +type FunctionNullInputBehavior int + +const ( + // FunctionCalledOnNullInput indicates that the function will be given the + // chance to execute when presented with NULL input. + FunctionCalledOnNullInput FunctionNullInputBehavior = iota + // FunctionReturnsNullOnNullInput indicates that the function will result in + // NULL given any NULL parameter. + FunctionReturnsNullOnNullInput + // FunctionStrict is the same as FunctionReturnsNullOnNullInput + FunctionStrict +) + +// Format implements the NodeFormatter interface. +func (node FunctionNullInputBehavior) Format(ctx *FmtCtx) { + switch node { + case FunctionCalledOnNullInput: + ctx.WriteString("CALLED ON NULL INPUT") + case FunctionReturnsNullOnNullInput: + ctx.WriteString("RETURNS NULL ON NULL INPUT") + case FunctionStrict: + ctx.WriteString("STRICT") + default: + panic(pgerror.New(pgcode.InvalidParameterValue, "Unknown function option")) + } +} + +// FunctionVolatility represent UDF volatility property. +type FunctionVolatility int + +const ( + // FunctionVolatile see volatility.Volatile + FunctionVolatile FunctionVolatility = iota + // FunctionImmutable see volatility.Immutable + FunctionImmutable + // FunctionStable see volatility.Stable + FunctionStable +) + +// Format implements the NodeFormatter interface. +func (node FunctionVolatility) Format(ctx *FmtCtx) { + switch node { + case FunctionVolatile: + ctx.WriteString("VOLATILE") + case FunctionImmutable: + ctx.WriteString("IMMUTABLE") + case FunctionStable: + ctx.WriteString("STABLE") + default: + panic(pgerror.New(pgcode.InvalidParameterValue, "Unknown function option")) + } +} + +// FunctionLeakProof indicates whether if a UDF is leakproof or not. +type FunctionLeakProof bool + +// Format implements the NodeFormatter interface. +func (node FunctionLeakProof) Format(ctx *FmtCtx) { + if !node { + ctx.WriteString("NOT ") + } + ctx.WriteString("LEAKPROOF") +} + +// FunctionLanguage indicates the language of the statements in the UDF function +// body. +type FunctionLanguage int + +const ( + _ FunctionLanguage = iota + // FunctionLangSQL represent SQL language. + FunctionLangSQL +) + +// Format implements the NodeFormatter interface. +func (node FunctionLanguage) Format(ctx *FmtCtx) { + ctx.WriteString("LANGUAGE ") + switch node { + case FunctionLangSQL: + ctx.WriteString("SQL") + default: + panic(pgerror.New(pgcode.InvalidParameterValue, "Unknown function option")) + } +} + +// AsFunctionLanguage converts a string to a FunctionLanguage if applicable. +// Error is returned if string does not represent a valid UDF language. +func AsFunctionLanguage(lang string) (FunctionLanguage, error) { + switch strings.ToLower(lang) { + case "sql": + return FunctionLangSQL, nil + } + return 0, errors.Newf("language %q does not exist", lang) +} + +// FunctionBodyStr is a string containing all statements in a UDF body. +type FunctionBodyStr string + +// Format implements the NodeFormatter interface. +func (node FunctionBodyStr) Format(ctx *FmtCtx) { + ctx.WriteString("AS ") + ctx.WriteString("$$") + ctx.WriteString(string(node)) + ctx.WriteString("$$") +} + +// FuncArgs represents a list of FuncArg. +type FuncArgs []FuncArg + +// Format implements the NodeFormatter interface. +func (node FuncArgs) Format(ctx *FmtCtx) { + for i, arg := range node { + if i > 0 { + ctx.WriteString(", ") + } + ctx.FormatNode(&arg) + } +} + +// FuncArg represents an argument from a UDF signature. +type FuncArg struct { + Name Name + Type ResolvableTypeReference + Class FuncArgClass + DefaultVal Expr +} + +// Format implements the NodeFormatter interface. +func (node *FuncArg) Format(ctx *FmtCtx) { + switch node.Class { + case FunctionArgIn: + ctx.WriteString("IN") + case FunctionArgOut: + ctx.WriteString("OUT") + case FunctionArgInOut: + ctx.WriteString("INOUT") + case FunctionArgVariadic: + ctx.WriteString("VARIADIC") + default: + panic(pgerror.New(pgcode.InvalidParameterValue, "Unknown function option")) + } + ctx.WriteString(" ") + if node.Name != "" { + ctx.FormatNode(&node.Name) + ctx.WriteString(" ") + } + ctx.WriteString(node.Type.SQLString()) + if node.DefaultVal != nil { + ctx.WriteString(" DEFAULT ") + ctx.FormatNode(node.DefaultVal) + } +} + +// FuncArgClass indicates what type of argument an arg is. +type FuncArgClass int + +const ( + // FunctionArgIn args can only be used as input. + FunctionArgIn FuncArgClass = iota + // FunctionArgOut args can only be used as output. + FunctionArgOut + // FunctionArgInOut args can be used as both input and output. + FunctionArgInOut + // FunctionArgVariadic args are variadic. + FunctionArgVariadic +) + +// FuncReturnType represent the return type of UDF. +type FuncReturnType struct { + Type ResolvableTypeReference + IsSet bool +} From 4ad11bd5cf554e6f22d79f37a8987a5df889333e Mon Sep 17 00:00:00 2001 From: Michael Erickson Date: Mon, 4 Jul 2022 23:31:54 -0700 Subject: [PATCH 6/7] sql/stats: convert between histograms and quantile functions To predict histograms in statistics forecasts, we will use linear regression over quantile functions. (Quantile functions are another representation of histogram data, in a form more amenable to statistical manipulation.) This commit defines quantile functions and adds methods to convert between histograms and quantile functions. This code was originally part of #77070 but has been pulled out to simplify that PR. A few changes have been made: - Common code has been factored into closures. - More checks have been added for positive values. - In `makeQuantile` we now trim leading empty buckets as well as trailing empty buckets. - The logic in `quantile.toHistogram` to steal from `NumRange` if `NumEq` is zero now checks that `NumRange` will still be >= 1. - More tests have been added. Assists: #79872 Release note: None --- pkg/sql/stats/histogram.go | 19 ++ pkg/sql/stats/quantile.go | 323 ++++++++++++++++++- pkg/sql/stats/quantile_test.go | 566 ++++++++++++++++++++++++++++++++- 3 files changed, 888 insertions(+), 20 deletions(-) diff --git a/pkg/sql/stats/histogram.go b/pkg/sql/stats/histogram.go index a920218eb25e..1dd3dc1d66bc 100644 --- a/pkg/sql/stats/histogram.go +++ b/pkg/sql/stats/histogram.go @@ -11,8 +11,10 @@ package stats import ( + "fmt" "math" "sort" + "strings" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" @@ -420,6 +422,23 @@ func (h histogram) toHistogramData(colType *types.T) (HistogramData, error) { return histogramData, nil } +// String prints a histogram to a string. +func (h histogram) String() string { + var b strings.Builder + b.WriteString("{[") + for i, bucket := range h.buckets { + if i > 0 { + b.WriteRune(' ') + } + fmt.Fprintf( + &b, "{%v %v %v %v}", + bucket.NumEq, bucket.NumRange, bucket.DistinctRange, bucket.UpperBound.String(), + ) + } + b.WriteString("]}") + return b.String() +} + // estimatedDistinctValuesInRange returns the estimated number of distinct // values in the range [lowerBound, upperBound), given that the total number // of values is numRange. diff --git a/pkg/sql/stats/quantile.go b/pkg/sql/stats/quantile.go index 93dd8a2b656c..33b39fee1f11 100644 --- a/pkg/sql/stats/quantile.go +++ b/pkg/sql/stats/quantile.go @@ -14,6 +14,8 @@ import ( "math" "time" + "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" + "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -21,10 +23,89 @@ import ( "github.com/cockroachdb/errors" ) -// CanMakeQuantile returns true if a quantile function can be created for a +// quantile is a piecewise quantile function with float64 values. +// +// A quantile function is a way of representing a probability distribution. It +// is a function from p to v, over (p=0, p=1], where p is the probability that +// an item in the distribution will have value <= v. The quantile function for a +// probability distribution is the inverse of the cumulative distribution +// function for the same probability distribution. See +// https://en.wikipedia.org/wiki/Quantile_function for more background. +// +// We use quantile functions within our modeling for a few reasons: +// * Unlike histograms, quantile functions are independent of the absolute +// counts. They are a "shape" not a "size". +// * Unlike cumulative distribution functions or probability density functions, +// we can always take the definite integral of a quantile function from p=0 to +// p=1. We use this when performing linear regression over quantiles. +// +// Type quantile represents a piecewise quantile function with float64 values as +// a series of quantilePoints from p=0 (exclusive) to p=1 (inclusive). A +// well-formed quantile is non-decreasing in both p and v. A quantile must have +// at least two points. The first point must have p=0, and the last point must +// have p=1. The pieces of the quantile function are line segments between +// subsequent points (exclusive and inclusive, respectively). +// +// Subsequent points may have the same p (a vertical line, or discontinuity), +// meaning the probability of finding a value > v₁ and <= v₂ is zero. Subsequent +// points may have the same v (a horizontal line), meaning the probability of +// finding exactly that v is p₂ - p₁. To put it in terms of our histograms: +// NumRange = 0 becomes a vertical line, NumRange > 0 becomes a slanted line +// with positive slope, NumEq = 0 goes away, and NumEq > 0 becomes a horizontal +// line. +// +// For example, given this population of 10 values: +// +// {200, 200, 210, 210, 210, 211, 212, 221, 222, 230} +// +// One possible histogram might be: +// +// {{UpperBound: 200, NumRange: 0, NumEq: 2}, +// {UpperBound: 210, NumRange: 0, NumEq: 3}, +// {UpperBound: 220, NumRange: 2, NumEq: 0}, +// {UpperBound: 230, NumRange: 2, NumEq: 1}} +// +// And the corresponding quantile function would be: +// +// {{0, 200}, {0.2, 200}, {0.2, 210}, {0.5, 210}, {0.7, 220}, {0.9, 230}, {1, 230}} +// +// 230 | *-* +// | / +// 220 | * +// | / +// 210 | o-----* +// | +// 200 o---* +// | +// 190 + - - - - - - - - - - +// 0 .2 .4 .6 .8 1 +// +type quantile []quantilePoint + +// quantilePoint is an endpoint of a piece (line segment) in a piecewise +// quantile function. +type quantilePoint struct { + p, v float64 +} + +// quantileIndex is the ordinal position of a quantilePoint within a +// quantile. +type quantileIndex = int + +// zeroQuantile is what we use for empty tables. Technically it says nothing +// about the number of rows in the table / items in the probability +// distribution, only that they all equal the zero value. +var zeroQuantile = quantile{{p: 0, v: 0}, {p: 1, v: 0}} + +// If you are introducing a new histogram version, please check whether +// makeQuantile and quantile.toHistogram need to change, and then increase the +// version number in this check. +const _ uint = 1 - uint(histVersion) + +// canMakeQuantile returns true if a quantile function can be created for a // histogram of the given type. // TODO(michae2): Add support for DECIMAL, TIME, TIMETZ, and INTERVAL. -func CanMakeQuantile(colType *types.T) bool { +func canMakeQuantile(colType *types.T) bool { if colType.UserDefined() { return false } @@ -34,20 +115,246 @@ func CanMakeQuantile(colType *types.T) bool { types.DateFamily, types.TimestampFamily, types.TimestampTZFamily: + // TODO(michae2): Even if the column is one of these types, explicit or + // implicit constraints could make it behave like an ENUM. For example, the + // hidden shard column of a hash-sharded index is INT8 and yet will only + // ever contain a few specific values which do not make sense to compare + // using < or > operators. Histogram prediction will likely not work well + // for these de facto ENUM columns, so we should skip them. + // + // One way to detect these columns would be to watch out for histograms with + // NumRange == 0 in every bucket. return true default: return false } } -// ToQuantileValue converts from a datum to a float suitable for use in a quantile +// makeQuantile converts a histogram to a quantile function, or returns an error +// if it cannot. The histogram must not contain a bucket for NULL values, and +// the row count must not include NULL values. The first bucket of the histogram +// must have NumRange == 0. +func makeQuantile(hist histogram, rowCount float64) (quantile, error) { + if !isValidCount(rowCount) { + return nil, errors.AssertionFailedf("invalid rowCount: %v", rowCount) + } + + // Empty table cases. + if len(hist.buckets) == 0 || rowCount < 1 { + return zeroQuantile, nil + } + + // To produce a quantile with first point at p=0 and at least two points, we + // need the first bucket to have NumRange == 0. + if hist.buckets[0].NumRange != 0 { + return nil, errors.AssertionFailedf( + "histogram with non-zero NumRange in first bucket: %v", hist.buckets[0].NumRange, + ) + } + + var ( + // qfTrimLo and qfTrimHi are indexes to slice the quantile to when trimming + // zero-row buckets from the beginning and end of the histogram. + qfTrimLo, qfTrimHi quantileIndex + qf = make(quantile, 0, len(hist.buckets)*2) + prevV = math.Inf(-1) + p float64 + ) + + // Add a point counting num rows with value <= v. + addPoint := func(num, v float64) error { + if !isValidCount(num) { + return errors.AssertionFailedf("invalid histogram num: %v", num) + } + // Advance p by the proportion of rows counted by num. + p += num / rowCount + // Fix any floating point errors or histogram errors (e.g. sum of bucket row + // counts > total row count) causing p to go above 1. + if p > 1 { + p = 1 + } + qf = append(qf, quantilePoint{p: p, v: v}) + if p == 0 { + qfTrimLo = len(qf) - 1 + } + if num > 0 { + qfTrimHi = len(qf) + } + return nil + } + + // For each histogram bucket, add two points to the quantile: (1) an endpoint + // for NumRange and (2) an endpoint for NumEq. If NumEq == 0 we can skip the + // second point, but we must always add the first point even if NumRange == 0. + for i := range hist.buckets { + if hist.buckets[i].NumRange < 0 || hist.buckets[i].NumEq < 0 { + return nil, errors.AssertionFailedf("histogram bucket with negative row count") + } + v, err := toQuantileValue(hist.buckets[i].UpperBound) + if err != nil { + return nil, err + } + if v <= prevV { + return nil, errors.AssertionFailedf("non-increasing quantile values") + } + prevV = v + + if err := addPoint(hist.buckets[i].NumRange, v); err != nil { + return nil, err + } + if hist.buckets[i].NumEq == 0 { + // Small optimization: skip adding a duplicate point to the quantile. + continue + } + if err := addPoint(hist.buckets[i].NumEq, v); err != nil { + return nil, err + } + } + + if qfTrimHi <= qfTrimLo { + // In the unlikely case that every bucket had zero rows we simply return the + // zeroQuantile. + qf = zeroQuantile + } else { + // Trim any zero-row buckets from the beginning and end. + qf = qf[qfTrimLo:qfTrimHi] + // Fix any floating point errors or histogram errors (e.g. sum of bucket row + // counts < total row count) causing p to be below 1 at the end. + qf[len(qf)-1].p = 1 + } + return qf, nil +} + +// toHistogram converts a quantile into a histogram, using the provided type and +// row count. It returns an error if the conversion fails. +func (qf quantile) toHistogram( + evalCtx *eval.Context, colType *types.T, rowCount float64, +) (histogram, error) { + if len(qf) < 2 || qf[0].p != 0 || qf[len(qf)-1].p != 1 { + return histogram{}, errors.AssertionFailedf("invalid quantile: %v", qf) + } + + // Empty table case. + if rowCount < 1 { + return histogram{}, nil + } + + hist := histogram{buckets: make([]cat.HistogramBucket, 0, len(qf)-1)} + + var i quantileIndex + // Skip any leading p=0 points instead of emitting zero-row buckets. + for qf[i].p == 0 { + i++ + } + + // Create the first bucket of the histogram. The first bucket must always have + // NumRange == 0. Sometimes we will emit a zero-row bucket to make this true. + var currentLowerBound tree.Datum + currentUpperBound, err := fromQuantileValue(colType, qf[i-1].v) + if err != nil { + return histogram{}, err + } + currentBucket := cat.HistogramBucket{ + NumEq: 0, + NumRange: 0, + DistinctRange: 0, + UpperBound: currentUpperBound, + } + + var pEq float64 + + // Set NumEq of the current bucket before creating a new current bucket. + closeCurrentBucket := func() error { + numEq := pEq * rowCount + if !isValidCount(numEq) { + return errors.AssertionFailedf("invalid histogram NumEq: %v", numEq) + } + if numEq < 1 && currentBucket.NumRange+numEq >= 2 { + // Steal from NumRange so that NumEq is at least 1, if it wouldn't make + // NumRange 0. This makes the histogram look more like something + // EquiDepthHistogram would produce. + currentBucket.NumRange -= 1 - numEq + numEq = 1 + } + currentBucket.NumEq = numEq + + // Calculate DistinctRange for this bucket now that NumRange is finalized. + distinctRange := estimatedDistinctValuesInRange( + evalCtx, currentBucket.NumRange, currentLowerBound, currentUpperBound, + ) + if !isValidCount(distinctRange) { + return errors.AssertionFailedf("invalid histogram DistinctRange: %v", distinctRange) + } + currentBucket.DistinctRange = distinctRange + + hist.buckets = append(hist.buckets, currentBucket) + pEq = 0 + return nil + } + + // For each point in the quantile, if its value is equal to the current + // upperBound then add to NumEq of the current bucket. Otherwise close the + // current bucket and add to NumRange of a new current bucket. + for ; i < len(qf); i++ { + upperBound, err := fromQuantileValue(colType, qf[i].v) + if err != nil { + return histogram{}, err + } + cmp, err := upperBound.CompareError(evalCtx, currentUpperBound) + if err != nil { + return histogram{}, err + } + if cmp < 0 { + return histogram{}, errors.AssertionFailedf("decreasing histogram values") + } + if cmp == 0 { + pEq += qf[i].p - qf[i-1].p + } else { + if err := closeCurrentBucket(); err != nil { + return histogram{}, err + } + + // Start a new current bucket. + pRange := qf[i].p - qf[i-1].p + numRange := pRange * rowCount + if !isValidCount(numRange) { + return histogram{}, errors.AssertionFailedf("invalid histogram NumRange: %v", numRange) + } + currentLowerBound = getNextLowerBound(evalCtx, currentUpperBound) + currentUpperBound = upperBound + currentBucket = cat.HistogramBucket{ + NumEq: 0, + NumRange: numRange, + DistinctRange: 0, + UpperBound: currentUpperBound, + } + } + // Skip any trailing p=1 points instead of emitting zero-row buckets. + if qf[i].p == 1 { + break + } + } + + // Close the last bucket. + if err := closeCurrentBucket(); err != nil { + return histogram{}, err + } + + return hist, nil +} + +func isValidCount(x float64) bool { + return x >= 0 && !math.IsInf(x, 0) && !math.IsNaN(x) +} + +// toQuantileValue converts from a datum to a float suitable for use in a quantile // function. It differs from eval.PerformCast in a few ways: // 1. It supports conversions that are not legal casts (e.g. DATE to FLOAT). // 2. It errors on NaN and infinite values because they will break our model. -// FromQuantileValue is the inverse of this function, and together they should +// fromQuantileValue is the inverse of this function, and together they should // support round-trip conversions. // TODO(michae2): Add support for DECIMAL, TIME, TIMETZ, and INTERVAL. -func ToQuantileValue(d tree.Datum) (float64, error) { +func toQuantileValue(d tree.Datum) (float64, error) { switch v := d.(type) { case *tree.DInt: return float64(*v), nil @@ -90,8 +397,8 @@ var ( quantileMaxTimestampSec = float64(quantileMaxTimestamp.Unix()) ) -// FromQuantileValue converts from a quantile value back to a datum suitable for -// use in a histogram. It is the inverse of ToQuantileValue. It differs from +// fromQuantileValue converts from a quantile value back to a datum suitable for +// use in a histogram. It is the inverse of toQuantileValue. It differs from // eval.PerformCast in a few ways: // 1. It supports conversions that are not legal casts (e.g. FLOAT to DATE). // 2. It errors on NaN and infinite values because they indicate a problem with @@ -99,7 +406,7 @@ var ( // 3. On overflow or underflow it clamps to maximum or minimum finite values // rather than failing the conversion (and thus the entire histogram). // TODO(michae2): Add support for DECIMAL, TIME, TIMETZ, and INTERVAL. -func FromQuantileValue(colType *types.T, val float64) (tree.Datum, error) { +func fromQuantileValue(colType *types.T, val float64) (tree.Datum, error) { if math.IsNaN(val) || math.IsInf(val, 0) { return nil, tree.ErrFloatOutOfRange } diff --git a/pkg/sql/stats/quantile_test.go b/pkg/sql/stats/quantile_test.go index ea4027fe7c70..194d0eb7d4de 100644 --- a/pkg/sql/stats/quantile_test.go +++ b/pkg/sql/stats/quantile_test.go @@ -11,19 +11,561 @@ package stats import ( + "fmt" "math" + "math/bits" + "math/rand" + "reflect" + "sort" "strconv" "testing" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/util/randutil" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil/pgdate" ) -// TODO(michae2): Test that a random histogram can round-trip to quantile -// and back. +// TestRandomQuantileRoundTrip creates a random histogram of each type, and +// tests that each can be converted to a quantile function and back without +// changing. +func TestRandomQuantileRoundTrip(t *testing.T) { + colTypes := []*types.T{ + // Types not in types.Scalar. + types.Int4, + types.Int2, + types.Float4, + } + colTypes = append(colTypes, types.Scalar...) + evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) + rng, seed := randutil.NewTestRand() + for _, colType := range colTypes { + if canMakeQuantile(colType) { + for i := 0; i < 5; i++ { + t.Run(fmt.Sprintf("%v/%v", colType.Name(), i), func(t *testing.T) { + hist, rowCount := randHist(evalCtx, colType, rng) + qfun, err := makeQuantile(hist, rowCount) + if err != nil { + t.Errorf("seed: %v unexpected makeQuantile error: %v", seed, err) + return + } + hist2, err := qfun.toHistogram(evalCtx, colType, rowCount) + if err != nil { + t.Errorf("seed: %v unexpected quantile.toHistogram error: %v", seed, err) + return + } + if !reflect.DeepEqual(hist, hist2) { + t.Errorf("seed: %v incorrect histogram:\n%v\nexpected:\n%v", seed, hist2, hist) + } + }) + } + } + } +} + +// randHist makes a random histogram of the specified type, with [1, 200] +// buckets. Not all types are supported. Every bucket will have NumEq > 0 but +// could have NumRange == 0. +func randHist(evalCtx *eval.Context, colType *types.T, rng *rand.Rand) (histogram, float64) { + numBuckets := rng.Intn(200) + 1 + buckets := make([]cat.HistogramBucket, numBuckets) + bounds := randBounds(evalCtx, colType, rng, numBuckets) + buckets[0].NumEq = float64(rng.Intn(100) + 1) + buckets[0].UpperBound = bounds[0] + rowCount := buckets[0].NumEq + for i := 1; i < len(buckets); i++ { + buckets[i].NumEq = float64(rng.Intn(100) + 1) + buckets[i].NumRange = float64(rng.Intn(1000)) + buckets[i].UpperBound = bounds[i] + rowCount += buckets[i].NumEq + buckets[i].NumRange + } + // Adjust counts so that we have a power-of-two rowCount to avoid floating point errors. + targetRowCount := 1 << int(math.Ceil(math.Log2(rowCount))) + for rowCount < float64(targetRowCount) { + rows := float64(rng.Intn(targetRowCount - int(rowCount) + 1)) + bucket := rng.Intn(numBuckets) + if bucket == 0 || rng.Float32() < 0.1 { + buckets[bucket].NumEq += rows + } else { + buckets[bucket].NumRange += rows + } + rowCount += rows + } + // Set DistinctRange in all buckets. + for i := 1; i < len(buckets); i++ { + lowerBound := getNextLowerBound(evalCtx, buckets[i-1].UpperBound) + buckets[i].DistinctRange = estimatedDistinctValuesInRange( + evalCtx, buckets[i].NumRange, lowerBound, buckets[i].UpperBound, + ) + } + return histogram{buckets: buckets}, rowCount +} + +// randBounds creates an ordered slice of num distinct Datums of the specified +// type. Not all types are supported. This differs from randgen.RandDatum in +// that it generates no "interesting" Datums, and differs from +// randgen.RandDatumSimple in that it generates distinct Datums without repeats. +func randBounds(evalCtx *eval.Context, colType *types.T, rng *rand.Rand, num int) tree.Datums { + datums := make(tree.Datums, num) + + // randInts creates an ordered slice of num distinct random ints in the closed + // interval [lo, hi]. + randInts := func(num, lo, hi int) []int { + vals := make([]int, 0, num) + set := make(map[int]struct{}, num) + span := hi - lo + 1 + for len(vals) < num { + val := rng.Intn(span) + lo + if _, ok := set[val]; !ok { + set[val] = struct{}{} + vals = append(vals, val) + } + } + sort.Ints(vals) + return vals + } + + // randFloat64s creates an ordered slice of num distinct random float64s in + // the half-open interval [lo, hi). If single is true they will also work as + // float32s. + randFloat64s := func(num int, lo, hi float64, single bool) []float64 { + vals := make([]float64, 0, num) + set := make(map[float64]struct{}, num) + span := hi - lo + for len(vals) < num { + val := rng.Float64()*span + lo + if single { + val = float64(float32(val)) + } + if _, ok := set[val]; !ok { + set[val] = struct{}{} + vals = append(vals, val) + } + } + sort.Float64s(vals) + return vals + } + + switch colType.Family() { + case types.IntFamily: + // First make sure we won't overflow in randInts (i.e. make sure that + // hi - lo + 1 <= math.MaxInt which requires -2 for hi). + w := int(bits.UintSize) - 2 + lo := -1 << w + hi := (1 << w) - 2 + // Then make sure hi and lo are representable by float64. + if w > 53 { + w = 53 + lo = -1 << w + hi = 1 << w + } + // Finally make sure hi and low are representable by this type. + if w > int(colType.Width())-1 { + w = int(colType.Width()) - 1 + lo = -1 << w + hi = (1 << w) - 1 + } + vals := randInts(num, lo, hi) + for i := range datums { + datums[i] = tree.NewDInt(tree.DInt(int64(vals[i]))) + } + case types.FloatFamily: + var lo, hi float64 + if colType.Width() == 32 { + lo = -math.MaxFloat32 + hi = math.MaxFloat32 + } else { + // Divide by 2 to make sure we won't overflow in randFloat64s. + lo = -math.MaxFloat64 / 2 + hi = math.MaxFloat64 / 2 + } + vals := randFloat64s(num, lo, hi, colType.Width() == 32) + for i := range datums { + datums[i] = tree.NewDFloat(tree.DFloat(vals[i])) + } + case types.DateFamily: + lo := int(pgdate.LowDate.PGEpochDays()) + hi := int(pgdate.HighDate.PGEpochDays()) + vals := randInts(num, lo, hi) + for i := range datums { + datums[i] = tree.NewDDate(pgdate.MakeDateFromPGEpochClampFinite(int32(vals[i]))) + } + case types.TimestampFamily, types.TimestampTZFamily: + roundTo := tree.TimeFamilyPrecisionToRoundDuration(colType.Precision()) + var lo, hi int + if quantileMaxTimestampSec < math.MaxInt/2 { + lo = int(quantileMinTimestampSec) + hi = int(quantileMaxTimestampSec) + } else { + // Make sure we won't overflow in randInts (i.e. make sure that + // hi - lo + 1 <= math.MaxInt which requires -2 for hi). + w := int(bits.UintSize) - 2 + lo = -1 << w + hi = (1 << w) - 2 + } + secs := randInts(num, lo, hi) + for i := range datums { + t := timeutil.Unix(int64(secs[i]), 0) + var err error + if colType.Family() == types.TimestampFamily { + datums[i], err = tree.MakeDTimestamp(t, roundTo) + } else { + datums[i], err = tree.MakeDTimestampTZ(t, roundTo) + } + if err != nil { + panic(err) + } + } + default: + panic(colType.Name()) + } + return datums +} + +// Test basic conversions from histogram to quantile. +func TestMakeQuantile(t *testing.T) { + // We use all floats here. TestToQuantileValue and TestFromQuantileValue test + // conversions to other datatypes. + testCases := []struct { + hist testHistogram + rows float64 + qfun quantile + err bool + }{ + { + hist: nil, + rows: 0, + qfun: zeroQuantile, + }, + { + hist: testHistogram{}, + rows: 0, + qfun: zeroQuantile, + }, + { + hist: testHistogram{{0, 0, 0, 0}}, + rows: 0, + qfun: zeroQuantile, + }, + { + hist: testHistogram{{0, 0, 0, 100}, {0, 0, 0, 200}}, + rows: 10, + qfun: zeroQuantile, + }, + { + hist: testHistogram{{1, 0, 0, 0}}, + rows: 1, + qfun: zeroQuantile, + }, + { + hist: testHistogram{{2, 0, 0, 0}}, + rows: 2, + qfun: zeroQuantile, + }, + { + hist: testHistogram{{1, 0, 0, 100}}, + rows: 1, + qfun: quantile{{0, 100}, {1, 100}}, + }, + { + hist: testHistogram{{1, 0, 0, 100}, {0, 1, 1, 200}}, + rows: 2, + qfun: quantile{{0, 100}, {0.5, 100}, {1, 200}}, + }, + { + hist: testHistogram{{1, 0, 0, 100}, {2, 1, 1, 200}}, + rows: 4, + qfun: quantile{{0, 100}, {0.25, 100}, {0.5, 200}, {1, 200}}, + }, + { + hist: testHistogram{{0, 0, 0, 100}, {6, 2, 2, 200}}, + rows: 8, + qfun: quantile{{0, 100}, {0.25, 200}, {1, 200}}, + }, + { + hist: testHistogram{{0, 0, 0, 100}, {6, 2, 2, 200}}, + rows: 8, + qfun: quantile{{0, 100}, {0.25, 200}, {1, 200}}, + }, + { + hist: testHistogram{{2, 0, 0, 100}, {6, 2, 2, 200}, {2, 0, 0, 300}, {0, 4, 4, 400}}, + rows: 16, + qfun: quantile{{0, 100}, {0.125, 100}, {0.25, 200}, {0.625, 200}, {0.625, 300}, {0.75, 300}, {1, 400}}, + }, + // Cases where we trim leading and trailing zero buckets. + { + hist: testHistogram{{0, 0, 0, 0}, {0, 0, 0, 100}, {2, 2, 2, 200}}, + rows: 4, + qfun: quantile{{0, 100}, {0.5, 200}, {1, 200}}, + }, + { + hist: testHistogram{{0, 0, 0, 100}, {2, 6, 6, 200}, {0, 0, 0, 300}}, + rows: 8, + qfun: quantile{{0, 100}, {0.75, 200}, {1, 200}}, + }, + { + hist: testHistogram{{0, 0, 0, 0}, {4, 0, 0, 100}, {1, 3, 3, 200}, {0, 0, 0, 300}}, + rows: 8, + qfun: quantile{{0, 100}, {0.5, 100}, {0.875, 200}, {1, 200}}, + }, + // Cases where we clamp p to 1 to fix histogram errors. + { + hist: testHistogram{{2, 0, 0, 100}}, + rows: 1, + qfun: quantile{{0, 100}, {1, 100}}, + }, + { + hist: testHistogram{{1, 0, 0, 100}, {0, 1, 1, 200}}, + rows: 1, + qfun: quantile{{0, 100}, {1, 100}, {1, 200}}, + }, + // Error cases. + { + hist: testHistogram{}, + rows: math.Inf(1), + err: true, + }, + { + hist: testHistogram{}, + rows: math.NaN(), + err: true, + }, + { + hist: testHistogram{}, + rows: -1, + err: true, + }, + { + hist: testHistogram{{0, 1, 1, 100}}, + rows: 1, + err: true, + }, + { + hist: testHistogram{{-1, 0, 0, 100}}, + rows: 1, + err: true, + }, + { + hist: testHistogram{{math.Inf(1), 0, 0, 100}}, + rows: 1, + err: true, + }, + { + hist: testHistogram{{1, 0, 0, 100}, {1, 0, 0, 99}}, + rows: 2, + err: true, + }, + { + hist: testHistogram{{1, 0, 0, 100}, {0, 1, 1, 100}}, + rows: 2, + err: true, + }, + } + for i, tc := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + qfun, err := makeQuantile(tc.hist.toHistogram(), tc.rows) + if err != nil { + if !tc.err { + t.Errorf("test case %d unexpected makeQuantile err: %v", i, err) + } + return + } + if tc.err { + t.Errorf("test case %d expected makeQuantile err", i) + return + } + if !reflect.DeepEqual(qfun, tc.qfun) { + t.Errorf("test case %d incorrect quantile %v expected %v", i, qfun, tc.qfun) + } + }) + } +} + +// Test basic conversions from quantile to histogram. +func TestQuantileToHistogram(t *testing.T) { + // We use all floats here. TestToQuantileValue and TestFromQuantileValue test + // conversions to other datatypes. + testCases := []struct { + qfun quantile + rows float64 + hist testHistogram + err bool + }{ + { + qfun: zeroQuantile, + rows: 0, + hist: nil, + }, + { + qfun: zeroQuantile, + rows: 1, + hist: testHistogram{{1, 0, 0, 0}}, + }, + { + qfun: zeroQuantile, + rows: 2, + hist: testHistogram{{2, 0, 0, 0}}, + }, + { + qfun: quantile{{0, 100}, {1, 100}}, + rows: 1, + hist: testHistogram{{1, 0, 0, 100}}, + }, + { + qfun: quantile{{0, 0}, {0, 100}, {1, 100}}, + rows: 1, + hist: testHistogram{{1, 0, 0, 100}}, + }, + { + qfun: quantile{{0, 100}, {1, 100}, {1, 100}}, + rows: 1, + hist: testHistogram{{1, 0, 0, 100}}, + }, + { + qfun: quantile{{0, 100}, {1, 100}, {1, 200}}, + rows: 1, + hist: testHistogram{{1, 0, 0, 100}}, + }, + { + qfun: quantile{{0, 0}, {1, 100}}, + rows: 1, + hist: testHistogram{{0, 0, 0, 0}, {0, 1, 1, 100}}, + }, + { + qfun: quantile{{0, 0}, {0.5, 100}, {1, 100}}, + rows: 2, + hist: testHistogram{{0, 0, 0, 0}, {1, 1, 1, 100}}, + }, + { + qfun: quantile{{0, 0}, {0.9, 100}, {1, 100}}, + rows: 10, + hist: testHistogram{{0, 0, 0, 0}, {1, 9, 9, 100}}, + }, + { + qfun: quantile{{0, 100}, {0.25, 100}, {0.75, 200}, {1, 200}}, + rows: 16, + hist: testHistogram{{4, 0, 0, 100}, {4, 8, 8, 200}}, + }, + { + qfun: quantile{{0, 100}, {0.25, 100}, {0.5, 200}, {0.75, 200}, {0.75, 300}, {1, 300}}, + rows: 16, + hist: testHistogram{{4, 0, 0, 100}, {4, 4, 4, 200}, {4, 0, 0, 300}}, + }, + { + qfun: quantile{{0, 500}, {0.125, 500}, {0.25, 600}, {0.5, 600}, {0.75, 800}, {1, 800}}, + rows: 16, + hist: testHistogram{{2, 0, 0, 500}, {4, 2, 2, 600}, {4, 4, 4, 800}}, + }, + { + qfun: quantile{{0, 300}, {0, 310}, {0.125, 310}, {0.125, 320}, {0.25, 320}, {0.25, 330}, {0.5, 330}, {0.5, 340}, {0.625, 340}, {0.625, 350}, {0.75, 350}, {0.75, 360}, {0.875, 360}, {0.875, 370}, {1, 370}}, + rows: 32, + hist: testHistogram{{4, 0, 0, 310}, {4, 0, 0, 320}, {8, 0, 0, 330}, {4, 0, 0, 340}, {4, 0, 0, 350}, {4, 0, 0, 360}, {4, 0, 0, 370}}, + }, + // Cases where we steal a row from NumRange to give to NumEq. + { + qfun: quantile{{0, 0}, {1, 100}}, + rows: 2, + hist: testHistogram{{0, 0, 0, 0}, {1, 1, 1, 100}}, + }, + { + qfun: quantile{{0, 100}, {0.5, 100}, {1, 200}, {1, 300}}, + rows: 4, + hist: testHistogram{{2, 0, 0, 100}, {1, 1, 1, 200}}, + }, + { + qfun: quantile{{0, 0}, {0.875, 87.5}, {1, 100}}, + rows: 8, + hist: testHistogram{{0, 0, 0, 0}, {1, 6, 6, 87.5}, {0, 1, 1, 100}}, + }, + { + qfun: quantile{{0, 400}, {0.5, 600}, {0.75, 700}, {1, 800}}, + rows: 16, + hist: testHistogram{{0, 0, 0, 400}, {1, 7, 7, 600}, {1, 3, 3, 700}, {1, 3, 3, 800}}, + }, + // Error cases. + { + qfun: quantile{}, + rows: 1, + err: true, + }, + { + qfun: quantile{{0, 0}}, + rows: 1, + err: true, + }, + { + qfun: quantile{{1, 0}, {0, 0}}, + rows: 1, + err: true, + }, + { + qfun: quantile{{0, 100}, {0, 200}}, + rows: 1, + err: true, + }, + { + qfun: quantile{{0, 100}, {math.NaN(), 100}, {1, 100}}, + rows: 1, + err: true, + }, + { + qfun: quantile{{0, 0}, {0.75, 25}, {0.25, 75}, {1, 100}}, + rows: 1, + err: true, + }, + { + qfun: quantile{{0, 100}, {1, 99}}, + rows: 1, + err: true, + }, + } + evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) + for i, tc := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + hist, err := tc.qfun.toHistogram(evalCtx, types.Float, tc.rows) + if err != nil { + if !tc.err { + t.Errorf("test case %d unexpected quantile.toHistogram err: %v", i, err) + } + return + } + if tc.err { + t.Errorf("test case %d expected quantile.toHistogram err", i) + return + } + hist2 := tc.hist.toHistogram() + if !reflect.DeepEqual(hist, hist2) { + t.Errorf("test case %d incorrect histogram %v expected %v", i, hist, hist2) + } + }) + } +} + +// testHistogram is a float64-only histogram, which is a little more convenient +// to construct than a normal histogram with tree.Datum UpperBounds. +type testHistogram []testBucket + +type testBucket struct { + NumEq, NumRange, DistinctRange, UpperBound float64 +} + +func (th testHistogram) toHistogram() histogram { + if th == nil { + return histogram{} + } + h := histogram{buckets: make([]cat.HistogramBucket, len(th))} + for i := range th { + h.buckets[i].NumEq = th[i].NumEq + h.buckets[i].NumRange = th[i].NumRange + h.buckets[i].DistinctRange = th[i].DistinctRange + h.buckets[i].UpperBound = tree.NewDFloat(tree.DFloat(th[i].UpperBound)) + } + return h +} // Test conversions from datum to quantile value and back. func TestQuantileValueRoundTrip(t *testing.T) { @@ -261,15 +803,15 @@ func TestQuantileValueRoundTrip(t *testing.T) { evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { - val, err := ToQuantileValue(tc.dat) + val, err := toQuantileValue(tc.dat) if err != nil { if !tc.err { - t.Errorf("test case %d (%v) unexpected ToQuantileValue err: %v", i, tc.typ.Name(), err) + t.Errorf("test case %d (%v) unexpected toQuantileValue err: %v", i, tc.typ.Name(), err) } return } if tc.err { - t.Errorf("test case %d (%v) expected ToQuantileValue err", i, tc.typ.Name()) + t.Errorf("test case %d (%v) expected toQuantileValue err", i, tc.typ.Name()) return } if val != tc.val { @@ -277,9 +819,9 @@ func TestQuantileValueRoundTrip(t *testing.T) { return } // Check that we can make the round trip. - res, err := FromQuantileValue(tc.typ, val) + res, err := fromQuantileValue(tc.typ, val) if err != nil { - t.Errorf("test case %d (%v) unexpected FromQuantileValue err: %v", i, tc.typ.Name(), err) + t.Errorf("test case %d (%v) unexpected fromQuantileValue err: %v", i, tc.typ.Name(), err) return } cmp, err := res.CompareError(evalCtx, tc.dat) @@ -538,15 +1080,15 @@ func TestQuantileValueRoundTripOverflow(t *testing.T) { evalCtx := eval.NewTestingEvalContext(cluster.MakeTestingClusterSettings()) for i, tc := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { - d, err := FromQuantileValue(tc.typ, tc.val) + d, err := fromQuantileValue(tc.typ, tc.val) if err != nil { if !tc.err { - t.Errorf("test case %d (%v) unexpected FromQuantileValue err: %v", i, tc.typ.Name(), err) + t.Errorf("test case %d (%v) unexpected fromQuantileValue err: %v", i, tc.typ.Name(), err) } return } if tc.err { - t.Errorf("test case %d (%v) expected FromQuantileValue err", i, tc.typ.Name()) + t.Errorf("test case %d (%v) expected fromQuantileValue err", i, tc.typ.Name()) return } cmp, err := d.CompareError(evalCtx, tc.dat) @@ -559,9 +1101,9 @@ func TestQuantileValueRoundTripOverflow(t *testing.T) { return } // Check that we can make the round trip with the clamped value. - res, err := ToQuantileValue(d) + res, err := toQuantileValue(d) if err != nil { - t.Errorf("test case %d (%v) unexpected ToQuantileValue err: %v", i, tc.typ.Name(), err) + t.Errorf("test case %d (%v) unexpected toQuantileValue err: %v", i, tc.typ.Name(), err) return } if res != tc.res { From 4caddd44b4820324d5d1c08870a3b235dc2a7ca0 Mon Sep 17 00:00:00 2001 From: Ben Bardin Date: Wed, 6 Jul 2022 13:07:36 -0400 Subject: [PATCH 7/7] pkg/ui: Don't force tracez tags to uppercase. Also, deprecate uses of db-console/.../Badge in favor of the identical version in cluster-ui. Release note: None --- .../cluster-ui/src/badge/badge.module.scss | 5 +- .../workspaces/cluster-ui/src/badge/badge.tsx | 13 +++- .../src/components/badge/badge.module.styl | 75 ------------------- .../src/components/badge/badge.stories.tsx | 23 ------ .../db-console/src/components/badge/badge.tsx | 46 ------------ .../db-console/src/components/badge/index.ts | 11 --- .../db-console/src/components/index.ts | 1 - .../src/views/app/containers/layout/index.tsx | 2 +- .../containers/nodesOverview/index.tsx | 4 +- .../db-console/src/views/tracez/tracez.tsx | 1 + 10 files changed, 20 insertions(+), 161 deletions(-) delete mode 100644 pkg/ui/workspaces/db-console/src/components/badge/badge.module.styl delete mode 100644 pkg/ui/workspaces/db-console/src/components/badge/badge.stories.tsx delete mode 100644 pkg/ui/workspaces/db-console/src/components/badge/badge.tsx delete mode 100644 pkg/ui/workspaces/db-console/src/components/badge/index.ts diff --git a/pkg/ui/workspaces/cluster-ui/src/badge/badge.module.scss b/pkg/ui/workspaces/cluster-ui/src/badge/badge.module.scss index c0e86a9ffbb4..ad1ff51d23a0 100644 --- a/pkg/ui/workspaces/cluster-ui/src/badge/badge.module.scss +++ b/pkg/ui/workspaces/cluster-ui/src/badge/badge.module.scss @@ -5,12 +5,15 @@ display: flex; flex-direction: row; border-radius: 3px; - text-transform: uppercase; width: max-content; padding: $spacing-xx-small $spacing-x-small; cursor: default; } +.badge--uppercase { + text-transform: uppercase; +} + .badge--size-small, .badge--size-large, .badge--size-medium { diff --git a/pkg/ui/workspaces/cluster-ui/src/badge/badge.tsx b/pkg/ui/workspaces/cluster-ui/src/badge/badge.tsx index 33ccbf45185e..1f7d465bc51f 100644 --- a/pkg/ui/workspaces/cluster-ui/src/badge/badge.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/badge/badge.tsx @@ -21,13 +21,21 @@ export interface BadgeProps { status?: BadgeStatus; icon?: React.ReactNode; iconPosition?: "left" | "right"; + forceUpperCase: boolean; } const cx = classNames.bind(styles); export function Badge(props: BadgeProps) { - const { size, status, icon, iconPosition, text } = props; - const classes = cx("badge", `badge--size-${size}`, `badge--status-${status}`); + const { size, status, icon, iconPosition, text, forceUpperCase } = props; + const classes = cx( + "badge", + `badge--size-${size}`, + `badge--status-${status}`, + { + "badge--uppercase": forceUpperCase, + }, + ); const iconClasses = cx( "badge__icon", `badge__icon--position-${iconPosition || "left"}`, @@ -43,4 +51,5 @@ export function Badge(props: BadgeProps) { Badge.defaultProps = { size: "medium", status: "default", + forceUpperCase: true, }; diff --git a/pkg/ui/workspaces/db-console/src/components/badge/badge.module.styl b/pkg/ui/workspaces/db-console/src/components/badge/badge.module.styl deleted file mode 100644 index c3149f1cbf04..000000000000 --- a/pkg/ui/workspaces/db-console/src/components/badge/badge.module.styl +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -@require '~src/components/core/index.styl' -@require "~@cockroachlabs/design-tokens/dist/web/tokens"; - -.badge - display flex - flex-direction row - border-radius 3px - text-transform uppercase - width max-content - padding $spacing-xx-small $spacing-x-small - cursor default - -.badge--size-small - -.badge--size-large - -.badge--size-medium - height $line-height--medium - font-size $font-size--small - font-weight $font-weight--medium - line-height $line-height--xx-small - font-family $font-family--semi-bold - letter-spacing 1.5px - -.badge--status-success - color $color-intent-success-4 - background-color $color-intent-success-1 - border-radius 3px - -.badge--status-danger - color $colors--functional-red-4 - background-color $colors--functional-red-1 - background $colors--functional-red-1 - -.badge--status-default - background-color $colors--neutral-2 - color $colors--neutral-6 - -.badge--status-info - color $colors--primary-blue-4 - background-color $colors--primary-blue-1 - -.badge--status-warning - color $colors--functional-orange-4 - background-color $colors--functional-orange-1 - -.badge__icon - margin 0 $spacing-base - -.badge__icon--position-left - order 0 - margin-right $spacing-x-small - -.badge__icon--position-right - order 2 - margin-left $spacing-x-small - -.badge__text - order 1 - margin auto - -.badge__text--no-wrap - text-overflow ellipsis - white-space nowrap - overflow hidden diff --git a/pkg/ui/workspaces/db-console/src/components/badge/badge.stories.tsx b/pkg/ui/workspaces/db-console/src/components/badge/badge.stories.tsx deleted file mode 100644 index 84c6a5d7490e..000000000000 --- a/pkg/ui/workspaces/db-console/src/components/badge/badge.stories.tsx +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2020 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -import React from "react"; -import { storiesOf } from "@storybook/react"; - -import { Badge } from "./index"; -import { styledWrapper } from "src/util/decorators"; - -storiesOf("Badge", module) - .addDecorator(styledWrapper({ padding: "24px" })) - .add("with small size", () => ) - .add("with medium (default) size", () => ( - - )) - .add("with large size", () => ); diff --git a/pkg/ui/workspaces/db-console/src/components/badge/badge.tsx b/pkg/ui/workspaces/db-console/src/components/badge/badge.tsx deleted file mode 100644 index 87e0037af47c..000000000000 --- a/pkg/ui/workspaces/db-console/src/components/badge/badge.tsx +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -import * as React from "react"; -import classNames from "classnames/bind"; - -import styles from "./badge.module.styl"; - -export type BadgeStatus = "success" | "danger" | "default" | "info" | "warning"; - -export interface BadgeProps { - text: React.ReactNode; - size?: "small" | "medium" | "large"; - status?: BadgeStatus; - icon?: React.ReactNode; - iconPosition?: "left" | "right"; -} - -Badge.defaultProps = { - size: "medium", - status: "default", -}; - -const cx = classNames.bind(styles); - -export function Badge(props: BadgeProps) { - const { size, status, icon, iconPosition, text } = props; - const classes = cx("badge", `badge--size-${size}`, `badge--status-${status}`); - const iconClasses = cx( - "badge__icon", - `badge__icon--position-${iconPosition || "left"}`, - ); - return ( -
- {icon &&
{icon}
} -
{text}
-
- ); -} diff --git a/pkg/ui/workspaces/db-console/src/components/badge/index.ts b/pkg/ui/workspaces/db-console/src/components/badge/index.ts deleted file mode 100644 index daa0a6776ce1..000000000000 --- a/pkg/ui/workspaces/db-console/src/components/badge/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -export * from "./badge"; diff --git a/pkg/ui/workspaces/db-console/src/components/index.ts b/pkg/ui/workspaces/db-console/src/components/index.ts index 204ca72fe30d..d4e5b7c65c68 100644 --- a/pkg/ui/workspaces/db-console/src/components/index.ts +++ b/pkg/ui/workspaces/db-console/src/components/index.ts @@ -8,7 +8,6 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -export * from "./badge"; export * from "./button"; export * from "./icon"; export * from "./inlineAlert/inlineAlert"; diff --git a/pkg/ui/workspaces/db-console/src/views/app/containers/layout/index.tsx b/pkg/ui/workspaces/db-console/src/views/app/containers/layout/index.tsx index 388798a85386..d3345eff1e74 100644 --- a/pkg/ui/workspaces/db-console/src/views/app/containers/layout/index.tsx +++ b/pkg/ui/workspaces/db-console/src/views/app/containers/layout/index.tsx @@ -34,8 +34,8 @@ import { PageHeader, Text, TextTypes, - Badge, } from "src/components"; +import { Badge } from "@cockroachlabs/cluster-ui"; import "./layout.styl"; import "./layoutPanel.styl"; diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodesOverview/index.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodesOverview/index.tsx index d40f40b43d64..c2d620e0e3f1 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodesOverview/index.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodesOverview/index.tsx @@ -26,8 +26,10 @@ import { AdminUIState } from "src/redux/state"; import { refreshNodes, refreshLiveness } from "src/redux/apiReducers"; import { LocalSetting } from "src/redux/localsettings"; import { INodeStatus, MetricConstants } from "src/util/proto"; -import { Text, TextTypes, Tooltip, Badge, BadgeProps } from "src/components"; +import { Text, TextTypes, Tooltip } from "src/components"; import { + Badge, + BadgeProps, ColumnsConfig, Table, SortSetting, diff --git a/pkg/ui/workspaces/db-console/src/views/tracez/tracez.tsx b/pkg/ui/workspaces/db-console/src/views/tracez/tracez.tsx index d47bdf3937cd..495deaa5f595 100644 --- a/pkg/ui/workspaces/db-console/src/views/tracez/tracez.tsx +++ b/pkg/ui/workspaces/db-console/src/views/tracez/tracez.tsx @@ -188,6 +188,7 @@ const TagBadge = ({ size="small" status={badgeStatus} icon={icon} + forceUpperCase={false} /> );