Merge #120490 #120505 #120547 #120562

120490: ui: show license expiration alert in Db Console r=koorosh a=koorosh

With this change, new alert message is shown in Db Console when license is expired or less than 15 days left before it will expire.
This change doesn't affect clusters that doesn't have any license set.

Release note (ui change): show alert message in Db Console when license is expired or close to expire.

Depends on: #120475

Resolves: #98589

Epic: None

Screens:
1. Less than 15 days before license expires
<img width="1215" alt="Screenshot 2024-03-14 at 13 26 18" src="https://github.com/cockroachdb/cockroach/assets/3106437/54f18792-d16f-43d1-a439-bd04e7a91abd">
2. License expired
<img width="1215" alt="Screenshot 2024-03-14 at 13 25 26" src="https://github.com/cockroachdb/cockroach/assets/3106437/ec9b924a-7800-4cf9-a164-9f4f5b49e91f">
3. License expired today
<img width="1215" alt="Screenshot 2024-03-14 at 13 25 59" src="https://github.com/cockroachdb/cockroach/assets/3106437/38a29b0d-47c3-447a-beb5-d557b58bcfc9">



120505: sql: deflake TestTrackOnlyUserOpenTransactionsAndActiveStatements r=rafiss a=rafiss

This changes the test to block in AfterExecute rather than OnTxnFinish, which should make the active statements assertion less flaky.

It also fixes a testing bug where the SELECT FOR UPDATE was not in a txn.

fixes #120042
fixes #120235
fixes #119829

Release note: None

120547: ccl/cliccl: avoid opening Engine in debug encryption-decrypt r=sumeerbhola a=jbowens

Adapt the `debug encryption-decrypt` command to avoid actually opening the Engine and instead only open the filesystem environment. This allows the command to be used even when missing or corrupt files prevent the Engine from being opened.

Epic: none
Fix #96699.
Release note: none

120562: build: update `rules_go` r=jlinder a=rickystewart

... to pull in `0e7e4e31aa49f1afbb402fbb4895f38bc702c88c`.

See bazel-contrib/rules_go#3890

This reverts a change in bazel-contrib/rules_go#3824 which makes it much more difficult to see build errors.

Epic: none
Release note: None

Co-authored-by: Andrii Vorobiov <[email protected]>
Co-authored-by: Rafi Shamim <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>
Co-authored-by: Ricky Stewart <[email protected]>

WORKSPACE

@@ -8,12 +8,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 # Load go bazel tools. This gives us access to the go bazel SDK/toolchains.
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "d667ccdf672d5190d33f91ad0460cb7b69a21dc39124a4a9ded8621265302df7",
-    strip_prefix = "cockroachdb-rules_go-d2ae515",
+    sha256 = "c47bbf3fedda339cf41e9a17964bc76a404e4cb2f1213067cc42b453e8a2eb38",
+    strip_prefix = "cockroachdb-rules_go-0e7e4e3",
     urls = [
-        # cockroachdb/rules_go as of d2ae51519b29e078c7affe4918ba3b297e38ef6e
+        # cockroachdb/rules_go as of 0e7e4e31aa49f1afbb402fbb4895f38bc702c88c
         # (upstream release-0.46 plus a few patches).
-        "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-457-gd2ae515.tar.gz",
+        "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-458-g0e7e4e3.tar.gz",
     ],
 )
 

build/bazelutil/distdir_files.bzl

@@ -1179,7 +1179,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/bmatcuk-doublestar-v4.0.1-0-gf7a8118.tar.gz": "d11c3b3a45574f89d6a6b2f50e53feea50df60407b35f36193bf5815d32c79d1",
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-protobuf-3f5d91f.tar.gz": "6d4e7fe1cbd958dee69ce9becbf8892d567f082b6782d3973a118d0aa00807a8",
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_foreign_cc-8d34d77.tar.gz": "03afebfc3f173666a3820a29512265c710c3a08d0082ba77469779d3e3af5a11",
-    "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-457-gd2ae515.tar.gz": "d667ccdf672d5190d33f91ad0460cb7b69a21dc39124a4a9ded8621265302df7",
+    "https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-458-g0e7e4e3.tar.gz": "c47bbf3fedda339cf41e9a17964bc76a404e4cb2f1213067cc42b453e8a2eb38",
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/google-starlark-go-e043a3d.tar.gz": "a35c6468e0e0921833a63290161ff903295eaaf5915200bbce272cbc8dfd1c1c",
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/googleapis-83c3605afb5a39952bf0a0809875d41cf2a558ca.zip": "ba694861340e792fd31cb77274eacaf6e4ca8bda97707898f41d8bebfd8a4984",
     "https://storage.googleapis.com/public-bazel-artifacts/bazel/platforms-0.0.4.tar.gz": "079945598e4b6cc075846f7fd6a9d0857c33a7afc0de868c2ccb96405225135d",

pkg/ccl/cliccl/BUILD.bazel

@@ -43,7 +43,6 @@ go_library(
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_errors//oserror",
-        "@com_github_cockroachdb_pebble//vfs",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_lestrrat_go_jwx//jwk",
         "@com_github_olekukonko_tablewriter//:tablewriter",

pkg/ccl/cliccl/ear.go

@@ -10,59 +10,53 @@ package cliccl
 
 import (
 	"bytes"
-	"context"
+	"cmp"
 	"fmt"
 	"io"
-	"os"
-	"sort"
+	"slices"
 
 	"github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl"
 	"github.com/cockroachdb/cockroach/pkg/cli"
-	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
-	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/pebble/vfs"
 	"github.com/spf13/cobra"
 )
 
-func runDecrypt(_ *cobra.Command, args []string) (returnErr error) {
+func runDecrypt(cmd *cobra.Command, args []string) (returnErr error) {
 	dir, inPath := args[0], args[1]
 	var outPath string
 	if len(args) > 2 {
 		outPath = args[2]
 	}
 
-	stopper := stop.NewStopper()
-	defer stopper.Stop(context.Background())
-
-	db, err := cli.OpenEngine(dir, stopper, fs.ReadOnly, storage.MustExist)
+	env, err := cli.OpenFilesystemEnv(dir, fs.ReadOnly)
 	if err != nil {
 		return errors.Wrap(err, "could not open store")
 	}
+	defer func() { returnErr = errors.CombineErrors(returnErr, env.Close()) }()
 
 	// Open the specified file through the FS, decrypting it.
-	f, err := db.Open(inPath)
+	f, err := env.DefaultFS.Open(inPath)
 	if err != nil {
 		return errors.Wrapf(err, "could not open input file %s", inPath)
 	}
 	defer f.Close()
 
 	// Copy the raw bytes into the destination file.
-	outFile := os.Stdout
+	out := cmd.OutOrStdout()
 	if outPath != "" {
-		outFile, err = os.OpenFile(outPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+		outFile, err := env.UnencryptedFS.Create(outPath)
 		if err != nil {
 			return errors.Wrapf(err, "could not open output file %s", outPath)
 		}
 		defer outFile.Close()
+		out = outFile
 	}
-	if _, err = io.Copy(outFile, f); err != nil {
+	if _, err = io.Copy(out, f); err != nil {
 		return errors.Wrapf(err, "could not write to output file")
 	}
-
 	return nil
 }
 
@@ -87,23 +81,22 @@ func (f fileEntry) String() string {
 	return b.String()
 }
 
-func runList(cmd *cobra.Command, args []string) error {
+func runList(cmd *cobra.Command, args []string) (returnErr error) {
 	dir := args[0]
 
-	fr := &fs.FileRegistry{
-		FS:                  vfs.Default,
-		DBDir:               dir,
-		ReadOnly:            true,
-		NumOldRegistryFiles: fs.DefaultNumOldFileRegistryFiles,
+	env, err := cli.OpenFilesystemEnv(dir, fs.ReadOnly)
+	if err != nil {
+		return errors.Wrap(err, "could not open store")
 	}
-	if err := fr.Load(cmd.Context()); err != nil {
-		return errors.Wrapf(err, "could not load file registry")
+	defer func() { returnErr = errors.CombineErrors(returnErr, env.Close()) }()
+
+	if env.Registry == nil {
+		return errors.Newf("encryption-at-rest not enabled")
 	}
-	defer func() { _ = fr.Close() }()
 
 	// List files and print to stdout.
 	var entries []fileEntry
-	for name, entry := range fr.List() {
+	for name, entry := range env.Registry.List() {
 		var encSettings enginepbccl.EncryptionSettings
 		settings := entry.EncryptionSettings
 		if err := protoutil.Unmarshal(settings, &encSettings); err != nil {
@@ -115,12 +108,9 @@ func runList(cmd *cobra.Command, args []string) error {
 			settings: encSettings,
 		})
 	}
-	sort.Slice(entries, func(i, j int) bool {
-		return entries[i].name < entries[j].name
-	})
+	slices.SortFunc(entries, func(a, b fileEntry) int { return cmp.Compare(a.name, b.name) })
 	for _, e := range entries {
 		_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", e)
 	}
-
 	return nil
 }

pkg/ccl/cliccl/ear_test.go

@@ -160,8 +160,8 @@ func TestList(t *testing.T) {
 		var b bytes.Buffer
 		cmd.SetOut(&b)
 		cmd.SetErr(&b)
-		err = runList(cmd, []string{dir})
-		require.NoError(t, err)
+		require.NoError(t, cmd.Flags().Set("enterprise-encryption", encSpecStr))
+		require.NoError(t, runList(cmd, []string{dir}))
 		return b.String()
 	})
 }

pkg/cli/debug.go

@@ -172,26 +172,34 @@ func (f *keyFormat) Type() string {
 	return "hex|base64"
 }
 
+// OpenFilesystemEnv opens the filesystem environment at 'dir'. Note that
+// opening the fs.Env will acquire the directory lock and prevent opening of the
+// engine through [OpenEngine]. If the caller wishes to then open the storage
+// engine, they should manually open it using storage.Open. The returned Env has
+// 1 reference and the caller must ensure it's closed.
+func OpenFilesystemEnv(dir string, rw fs.RWMode) (*fs.Env, error) {
+	envConfig := fs.EnvConfig{RW: rw}
+	if PopulateEnvConfigHook != nil {
+		if err := PopulateEnvConfigHook(dir, &envConfig); err != nil {
+			return nil, err
+		}
+	}
+	return fs.InitEnv(context.Background(), vfs.Default, dir, envConfig)
+}
+
 // OpenEngine opens the engine at 'dir'. Depending on the supplied options,
 // an empty engine might be initialized.
 func OpenEngine(
 	dir string, stopper *stop.Stopper, rw fs.RWMode, opts ...storage.ConfigOption,
 ) (storage.Engine, error) {
-	maxOpenFiles, err := server.SetOpenFileLimitForOneStore()
+	env, err := OpenFilesystemEnv(dir, rw)
 	if err != nil {
 		return nil, err
 	}
-	envConfig := fs.EnvConfig{RW: rw}
-	if PopulateEnvConfigHook != nil {
-		if err := PopulateEnvConfigHook(dir, &envConfig); err != nil {
-			return nil, err
-		}
-	}
-	env, err := fs.InitEnv(context.Background(), vfs.Default, dir, envConfig)
+	maxOpenFiles, err := server.SetOpenFileLimitForOneStore()
 	if err != nil {
 		return nil, err
 	}
-
 	db, err := storage.Open(
 		context.Background(),
 		env,

pkg/jobs/registry_external_test.go

@@ -753,7 +753,7 @@ func TestWaitWithRetryableError(t *testing.T) {
 		Knobs: base.TestingKnobs{
 			SQLExecutor: &sql.ExecutorTestingKnobs{
 				DisableAutoCommitDuringExec: true,
-				AfterExecute: func(ctx context.Context, stmt string, err error) {
+				AfterExecute: func(ctx context.Context, stmt string, isInternal bool, err error) {
 					if targetJobID.Load() > 0 &&
 						strings.Contains(stmt, "SELECT count(*) FROM system.jobs") &&
 						strings.Contains(stmt, fmt.Sprintf("%d", targetJobID.Load())) {

pkg/sql/catalog/lease/lease_test.go

@@ -3447,7 +3447,7 @@ func TestLeaseBulkInsertWithImplicitTxn(t *testing.T) {
 				beforeExecute.Unlock()
 			}
 		},
-		AfterExecute: func(ctx context.Context, stmt string, err error) {
+		AfterExecute: func(ctx context.Context, stmt string, isInternal bool, err error) {
 			beforeExecute.Lock()
 			if stmt == beforeExecuteResumeStmt {
 				beforeExecute.Unlock()

pkg/sql/conn_executor_exec.go

@@ -2048,7 +2048,7 @@ func (ex *connExecutor) dispatchToExecutionEngine(
 	}
 
 	if ex.server.cfg.TestingKnobs.AfterExecute != nil {
-		ex.server.cfg.TestingKnobs.AfterExecute(ctx, stmt.String(), res.Err())
+		ex.server.cfg.TestingKnobs.AfterExecute(ctx, stmt.String(), ex.executorType == executorTypeInternal, res.Err())
 	}
 
 	if limitsErr := ex.handleTxnRowsWrittenReadLimits(ctx); limitsErr != nil && res.Err() == nil {

pkg/sql/conn_executor_test.go

@@ -42,7 +42,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/rowexec"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
-	"github.com/cockroachdb/cockroach/pkg/sql/sessionphase"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlliveness"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/sqllivenesstestutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
@@ -1660,15 +1659,14 @@ func TestInjectRetryOnCommitErrors(t *testing.T) {
 func TestTrackOnlyUserOpenTransactionsAndActiveStatements(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderStress(t, "slow test")
 
 	ctx := context.Background()
 	var shouldBlock syncutil.AtomicBool
 	blockingInternalTxns := make(chan struct{})
 	g := ctxgroup.WithContext(ctx)
 	params := base.TestServerArgs{}
 	params.Knobs.SQLExecutor = &sql.ExecutorTestingKnobs{
-		OnRecordTxnFinish: func(isInternal bool, _ *sessionphase.Times, _ string) {
+		AfterExecute: func(ctx context.Context, stmt string, isInternal bool, err error) {
 			if isInternal && shouldBlock.Get() {
 				<-blockingInternalTxns
 			}
@@ -1701,7 +1699,7 @@ func TestTrackOnlyUserOpenTransactionsAndActiveStatements(t *testing.T) {
 		prevInternalActiveStatements := sqlServer.InternalMetrics.EngineMetrics.SQLActiveStatements.Value()
 
 		// Begin a user-initiated transaction.
-		testDB.Exec(t, "BEGIN")
+		testTx := testDB.Begin(t)
 
 		// Check that the number of open transactions has incremented, but not the
 		// internal metric.
@@ -1711,7 +1709,7 @@ func TestTrackOnlyUserOpenTransactionsAndActiveStatements(t *testing.T) {
 		// Create a state of contention. Use a cancellable context so that the
 		// other queries that get blocked on this one don't deadlock if the test
 		// aborts.
-		_, err := sqlDB.ExecContext(ctx, "SELECT * FROM t.foo WHERE i = 1 FOR UPDATE")
+		_, err := testTx.ExecContext(ctx, "SELECT * FROM t.foo WHERE i = 1 FOR UPDATE")
 		require.NoError(t, err)
 
 		// Execute internal statement (this case is identical to opening an internal
@@ -1752,16 +1750,16 @@ func TestTrackOnlyUserOpenTransactionsAndActiveStatements(t *testing.T) {
 			if v := sqlServer.InternalMetrics.EngineMetrics.SQLTxnsOpen.Value(); v <= prevInternalTxnsOpen {
 				return errors.Newf("Wrong InternalSQLTxnsOpen value. Expected: greater than %d. Actual: %d", prevInternalTxnsOpen, v)
 			}
-			if v := sqlServer.InternalMetrics.EngineMetrics.SQLActiveStatements.Value(); v != prevInternalActiveStatements+1 {
-				return errors.Newf("Wrong InternalSQLActiveStatements value. Expected: %d. Actual: %d", prevInternalActiveStatements+1, v)
+			if v := sqlServer.InternalMetrics.EngineMetrics.SQLActiveStatements.Value(); v <= prevInternalActiveStatements {
+				return errors.Newf("Wrong InternalSQLActiveStatements value. Expected: greater than %d. Actual: %d", prevInternalActiveStatements, v)
 			}
 			return nil
 		})
 
 		require.Equal(t, int64(1), sqlServer.Metrics.EngineMetrics.SQLTxnsOpen.Value())
 		require.Equal(t, int64(0), sqlServer.Metrics.EngineMetrics.SQLActiveStatements.Value())
 		require.Less(t, prevInternalTxnsOpen, sqlServer.InternalMetrics.EngineMetrics.SQLTxnsOpen.Value())
-		require.Equal(t, prevInternalActiveStatements+1, sqlServer.InternalMetrics.EngineMetrics.SQLActiveStatements.Value())
+		require.Less(t, prevInternalActiveStatements, sqlServer.InternalMetrics.EngineMetrics.SQLActiveStatements.Value())
 
 		// Create active user-initiated statement.
 		g.GoCtx(func(ctx context.Context) error {
@@ -1799,7 +1797,7 @@ func TestTrackOnlyUserOpenTransactionsAndActiveStatements(t *testing.T) {
 
 		// Commit the initial user-initiated transaction. The internal and user
 		// select queries are no longer in contention.
-		testDB.Exec(t, "COMMIT")
+		require.NoError(t, testTx.Commit())
 	}()
 
 	// Check that both the internal & user statements are no longer active.

pkg/sql/exec_util.go

@@ -1521,7 +1521,7 @@ type ExecutorTestingKnobs struct {
 
 	// AfterExecute is like StatementFilter, but it runs in the same goroutine of the
 	// statement.
-	AfterExecute func(ctx context.Context, stmt string, err error)
+	AfterExecute func(ctx context.Context, stmt string, isInternal bool, err error)
 
 	// AfterExecCmd is called after successful execution of any command.
 	AfterExecCmd func(ctx context.Context, cmd Command, buf *StmtBuf)

pkg/sql/sqlstats/sslocal/sql_stats_test.go

@@ -736,7 +736,7 @@ func TestTransactionServiceLatencyOnExtendedProtocol(t *testing.T) {
 
 	var params base.TestServerArgs
 	params.Knobs.SQLExecutor = &sql.ExecutorTestingKnobs{
-		AfterExecute: func(ctx context.Context, stmt string, err error) {
+		AfterExecute: func(ctx context.Context, stmt string, isInternal bool, err error) {
 			if currentTestCaseIdx < len(testData) && testData[currentTestCaseIdx].query == stmt {
 				finishedExecute.Set(true)
 			}

pkg/ui/workspaces/cluster-ui/src/util/dataFromServer.ts

@@ -22,6 +22,8 @@ export interface DataFromServer {
   OIDCButtonText: string;
   OIDCGenerateJWTAuthTokenEnabled: boolean;
   FeatureFlags: FeatureFlags;
+  LicenseType: string;
+  SecondsUntilLicenseExpiry: number;
 }
 
 // Tell TypeScript about `window.dataFromServer`, which is set in a script