Merge #76611 #76897 #77247 #77483 #77493

76611: roachpb: add lock table metadata structures r=AlexTalks a=AlexTalks

This change adds the protobuf structures, as well as the method, needed
to capture the state of a replica's lock table.  This is part of the
work coming out of #75541, and is needed to be able to implement the
`QueryLocks` RPC.

Release justification: (Category 2) The structures created here are
entirely new, for the purposes of observability, and do not represent a
risk to any existing code or functionality.

Release note: None


76897: release: use bazel to build release binaries r=rail,jlinder a=rickystewart

This is a pretty large swath of related changes that update the release
pipeline to use Bazel for builds.

1. Update `.bazelrc` to add a new `*base` config for each cross-config.
   The existing cross configs will continue to work as before. Each
   `*base` config has the same contents as the corresponding cross
   config EXCEPT the `--workspace_status_command` argument is missing.
2. Stamping. `build/bazelutil/stamp.sh` now takes up to 3 more
   arguments in addition to the target triple: `build-channel`,
   `build-tag`, and `build-type`. These are documented in the script and
   have appropriate defaults for development builds if not specified.
3. Refactor `pkg/release`; the new API is more high-level rather than
   allowing injecting arbitrary `gotags`, etc. In the future it won't
   be impossible to add this functionality back in if we want it.
4. Prepare to fork the `Make and Publish Build` build configuration in
   TC by moving its script into `build/teamcity` and update the script
   to use Bazel.
5. Update `bazci` to make sure it reads the Windows `geos` libraries
   from the appropriate location in `bazel-bin`.
6. Update `publish-artifacts` and `publish-provisional-artifacts` to use
   the new `pkg/release` API.

Closes #67147.
Closes #67161.
Closes #67279.
Closes #67322.

Release note (general change): Use Bazel to build release binaries
Release justification: update release process

77247: server: Allow bypass of tsdump's strict checks r=abarganier a=rimadeodhar

tsdump performs strict checks that verify that it
has data for exactly the stores it expects to see
according to the node->store mapping. It can happen
that there legitimately isn't data there, for example
when nodes are down during the tsdump window. We still
want the strict checks, but now tsdump will defer
returning an error and allow restarting with a "-"
in lieu of the tsimport file name  to display the data
anyway.

Additionally, cluster settings are set via SET CLUSTER SETTING
as opposed to overriding settings directly.

Release note (cli change): debug tsdump command allows
viewing timeseries data even in case of node failures
by rerunning the command with the import filename set
to "-".

Addresses #75993

Release justification: low risk, high benefit changes to existing functionality


77483: sql/catalog: add return error to RunPostDeserializationChanges r=ajwerner a=rafiss

Release justification: low risk enhancement to existing functionality.

Release note: None

77493: sql: make session_revival_token.enabled tenant-ro r=otan,jaylim-crl a=rafiss

I was hoping to wait for the new cluster setting syntax to be completed,
but since it's getting close to the branch cut time I'd rather merge
this now so we don't forget at the last minute.

Release justification: low risk change to new functionality.

Release note: None

Co-authored-by: Alex Sarkesian <[email protected]>
Co-authored-by: Ricky Stewart <[email protected]>
Co-authored-by: rimadeodhar <[email protected]>
Co-authored-by: Rafi Shamim <[email protected]>

.bazelrc

@@ -26,18 +26,27 @@ build:cross --stamp
 build:cross --define cockroach_cross=y
 
 # Cross-compilation configurations. Add e.g. --config=crosslinux to turn these on.
-build:crosslinux --platforms=//build/toolchains:cross_linux
+# Generally these should be used for development builds. Each cross config has
+# a corresponding `base` config that is the same thing but without the
+# `--workspace_status_command`; if using these `base` configs, you need to
+# specify an appropriate `--workspace_status_command`. These `base` configs are
+# used by the release process which needs to have more control over stamping.
 build:crosslinux '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu'
-build:crosslinux --config=cross
-build:crosswindows --platforms=//build/toolchains:cross_windows
+build:crosslinux --config=crosslinuxbase
+build:crosslinuxbase --platforms=//build/toolchains:cross_linux
+build:crosslinuxbase --config=cross
 build:crosswindows '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32'
-build:crosswindows --config=cross
-build:crossmacos --platforms=//build/toolchains:cross_macos
+build:crosswindows --config=crosswindowsbase
+build:crosswindowsbase --platforms=//build/toolchains:cross_windows
+build:crosswindowsbase --config=cross
 build:crossmacos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19'
-build:crossmacos --config=cross
-build:crosslinuxarm --platforms=//build/toolchains:cross_linux_arm
+build:crossmacos --config=crossmacosbase
+build:crossmacosbase --platforms=//build/toolchains:cross_macos
+build:crossmacosbase --config=cross
 build:crosslinuxarm '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu'
-build:crosslinuxarm --config=cross
+build:crosslinuxarm --config=crosslinuxarmbase
+build:crosslinuxarmbase --platforms=//build/toolchains:cross_linux_arm
+build:crosslinuxarmbase --config=cross
 
 # Developer configurations. Add e.g. --config=devdarwinx86_64 to turn these on.
 # NB: This is consumed in `BUILD` files (see build/toolchains/BUILD.bazel).

Makefile

@@ -1721,7 +1721,6 @@ bins = \
   bin/cockroach-oss \
   bin/cockroach-short \
   bin/cockroach-sql \
-  bin/compile-builds \
   bin/docgen \
   bin/execgen \
   bin/fuzz \

build/bazelutil/stamp.sh

@@ -3,6 +3,15 @@
 # This command is used by bazel as the workspace_status_command
 # to implement build stamping with git information.
 
+# Usage: stamp.sh [target-triple] [build-channel] [build-tag] [build-type]
+# All arguments are optional and have appropriate defaults. In this way,
+# stamp.sh with no arguments is appropriate as the `workplace_status_command`
+# for a development build.
+#  target-triple: defaults to the value of `cc -dumpmachine`
+#  build-channel: defaults to `unknown`, but can be `official-binary`
+#  build-tag: defaults to a value that is gleaned from `git rev-parse`
+#  build-type: defaults to `development`, but can be `release`
+
 set -euo pipefail
 
 # Do not use plumbing commands, like git diff-index, in this target. Our build
@@ -14,21 +23,52 @@ set -euo pipefail
 # For details, see the "Possible timestamp problems with diff-files?" thread on
 # the Git mailing list (http://marc.info/?l=git&m=131687596307197).
 
-GIT_BUILD_TYPE="development"
-GIT_COMMIT=$(git rev-parse HEAD)
-GIT_TAG=$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev-parse --short HEAD;)
-GIT_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
-
+# Handle target-triple.
 if [ -z "${1+x}" ]
 then
     TARGET_TRIPLE=$(cc -dumpmachine)
 else
     TARGET_TRIPLE="$1"
+    shift 1
+fi
+
+# Handle build-channel.
+if [ -z "${1+x}" ]
+then
+    BUILD_CHANNEL="unknown"
+else
+    BUILD_CHANNEL="$1"
+    shift 1
+fi
+
+# Handle build-tag.
+if [ -z "${1+x}" ]
+then
+    BUILD_TAG=$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev-parse --short HEAD;)
+else
+    BUILD_TAG="$1"
+    shift 1
+fi
+
+# Handle build-type.
+if [ -z "${1+x}" ]
+then
+    BUILD_TYPE="development"
+else
+    BUILD_TYPE="$1"
+    shift 1
 fi
 
-# TODO(ricky): Also provide a way to stamp the following variables:
-# - github.com/cockroachdb/cockroach/pkg/build.channel
-# - github.com/cockroachdb/cockroach/pkg/util/log/logcrash.crashReportEnv
+if [ "$BUILD_TYPE" = "release" ]
+then
+    CRASH_REPORT_ENV="$BUILD_TAG"
+else
+    CRASH_REPORT_ENV="development"
+fi
+
+BUILD_REV=$(git rev-parse HEAD)
+BUILD_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
+
 
 # Variables beginning with "STABLE" will be written to stable-status.txt, and
 # others will be written to volatile-status.txt.
@@ -38,9 +78,11 @@ fi
 # * https://docs.bazel.build/versions/main/user-manual.html#workspace_status
 # * https://github.com/bazelbuild/rules_go/blob/master/go/core.rst#defines-and-stamping
 cat <<EOF
-STABLE_BUILD_GIT_BUILD_TYPE ${GIT_BUILD_TYPE-}
+STABLE_BUILD_CHANNEL ${BUILD_CHANNEL-}
 STABLE_BUILD_TARGET_TRIPLE ${TARGET_TRIPLE-}
-BUILD_GIT_COMMIT ${GIT_COMMIT-}
-BUILD_GIT_TAG ${GIT_TAG-}
-BUILD_GIT_UTCTIME ${GIT_UTCTIME-}
+STABLE_BUILD_TYPE ${BUILD_TYPE-}
+STABLE_CRASH_REPORT_ENV ${CRASH_REPORT_ENV-}
+BUILD_REV ${BUILD_REV-}
+BUILD_TAG ${BUILD_TAG-}
+BUILD_UTCTIME ${BUILD_UTCTIME-}
 EOF

build/release/teamcity-support.sh

@@ -1,7 +1,7 @@
 # Common helpers for teamcity-*.sh scripts.
 
 # root is the absolute path to the root directory of the repository.
-root=$(cd "$(dirname "$0")/../.." && pwd)
+root="$(dirname $(dirname $(cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )))"
 source "$root/build/teamcity-common-support.sh"
 
 remove_files_on_exit() {

build/teamcity-compile-builds.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 export BUILDER_HIDE_GOPATH_SRC=1
 
-build/builder.sh go install ./pkg/cmd/compile-build
-build/builder.sh env \
-  compile-build --all --buildtype=development
+build/builder.sh mkrelease linux-gnu SUFFIX=.linux-2.6.32-gnu-amd64
+build/builder.sh mkrelease darwin    SUFFIX=.darwin-10.9-amd64
+build/builder.sh mkrelease windows   SUFFIX=.windows-6.2-amd64.exe
 cp cockroach.* artifacts

build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# This script is called by the build configuration
+# "Cockroach > Post Merge > Publish Bleeding Edge" in TeamCity.
+
+set -euxo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-support.sh"
+source "$dir/teamcity-bazel-support.sh"
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH" run_bazel << 'EOF'
+bazel build --config ci //pkg/cmd/publish-artifacts
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts
+EOF

build/release/teamcity-make-and-publish-build.sh → build/teamcity/internal/release/process/make-and-publish-build.sh

@@ -2,13 +2,13 @@
 
 set -euo pipefail
 
-source "$(dirname "${0}")/teamcity-support.sh"
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+source "$dir/release/teamcity-support.sh"
+source "$dir/teamcity-bazel-support.sh"  # for run_bazel
 
 tc_start_block "Variable Setup"
-export BUILDER_HIDE_GOPATH_SRC=1
 
-build/builder.sh make .buildinfo/tag
-build_name="${TAG_NAME:-$(cat .buildinfo/tag)}"
+build_name=$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev-parse --short HEAD;)
 
 # On no match, `grep -Eo` returns 1. `|| echo""` makes the script not error.
 release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")"
@@ -44,21 +44,14 @@ tc_start_block "Tag the release"
 git tag "${build_name}"
 tc_end_block "Tag the release"
 
-
-tc_start_block "Compile publish-provisional-artifacts"
-build/builder.sh go install ./pkg/cmd/publish-provisional-artifacts
-tc_end_block "Compile publish-provisional-artifacts"
-
-
 tc_start_block "Compile and publish S3 artifacts"
-build/builder.sh env \
-  AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-  AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
-  TC_BUILD_BRANCH="$build_name" \
-  publish-provisional-artifacts -provisional -release -bucket "$bucket"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+bazel build --config ci //pkg/cmd/publish-provisional-artifacts
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
+EOF
 tc_end_block "Compile and publish S3 artifacts"
 
-
 tc_start_block "Make and push docker image"
 configure_docker_creds
 docker_login_with_google
@@ -74,7 +67,6 @@ docker build --no-cache --tag="${gcr_repository}:${build_name}" build/deploy
 docker push "${gcr_repository}:${build_name}"
 tc_end_block "Make and push docker image"
 
-
 tc_start_block "Push release tag to github.com/cockroachdb/cockroach"
 github_ssh_key="${GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY}"
 configure_git_ssh_key

build/release/teamcity-publish-release.sh → build/teamcity/internal/release/process/publish-cockroach-release.sh

@@ -2,10 +2,11 @@
 
 set -euxo pipefail
 
-source "$(dirname "${0}")/teamcity-support.sh"
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+source "$dir/release/teamcity-support.sh"
+source "$dir/teamcity-bazel-support.sh"  # for run_bazel
 
 tc_start_block "Variable Setup"
-export BUILDER_HIDE_GOPATH_SRC=1
 
 # Matching the version name regex from within the cockroach code except
 # for the `metadata` part at the end because Docker tags don't support
@@ -73,19 +74,14 @@ git tag "${build_name}"
 tc_end_block "Tag the release"
 
 
-tc_start_block "Compile publish-provisional-artifacts"
-build/builder.sh go install ./pkg/cmd/publish-provisional-artifacts
-tc_end_block "Compile publish-provisional-artifacts"
-
-
 tc_start_block "Make and publish release S3 artifacts"
 # Using publish-provisional-artifacts here is funky. We're directly publishing
 # the official binaries, not provisional ones. Legacy naming. To clean up...
-build/builder.sh env \
-  AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-  AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
-  TC_BUILD_BRANCH="$build_name" \
-  publish-provisional-artifacts -provisional -release -bucket "$bucket"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+bazel build --config ci //pkg/cmd/publish-provisional-artifacts
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket"
+EOF
 tc_end_block "Make and publish release S3 artifacts"
 
 
@@ -131,11 +127,12 @@ tc_start_block "Publish S3 binaries and archive as latest"
 # Only push the "latest" for our most recent release branch.
 # https://github.com/cockroachdb/cockroach/issues/41067
 if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then
-  build/builder.sh env \
-    AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-    AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
-    TC_BUILD_BRANCH="$build_name" \
-    publish-provisional-artifacts -bless -release -bucket "${bucket}"
+    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket" run_bazel << 'EOF'
+bazel build --config ci //pkg/cmd/publish-provisional-artifacts
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket"
+EOF
+
 else
   echo "The latest S3 binaries and archive were _not_ updated."
 fi

docs/generated/redact_safe.md

@@ -12,6 +12,8 @@ pkg/kv/bulk/sst_batcher.go | `sz`
 pkg/kv/bulk/sst_batcher.go | `timing`
 pkg/kv/kvserver/closedts/ctpb/service.go | `LAI`
 pkg/kv/kvserver/closedts/ctpb/service.go | `SeqNum`
+pkg/kv/kvserver/concurrency/lock/locking.go | `Durability`
+pkg/kv/kvserver/concurrency/lock/locking.go | `Strength`
 pkg/kv/kvserver/concurrency/lock/locking.go | `WaitPolicy`
 pkg/kv/kvserver/kvserverpb/raft.go | `SnapshotRequest_Type`
 pkg/roachpb/data.go | `LeaseSequence`

pkg/BUILD.bazel

@@ -139,6 +139,7 @@ ALL_TESTS = [
     "//pkg/kv/kvserver/closedts/sidetransport:sidetransport_test",
     "//pkg/kv/kvserver/closedts/tracker:tracker_test",
     "//pkg/kv/kvserver/closedts:closedts_test",
+    "//pkg/kv/kvserver/concurrency/lock:lock_test",
     "//pkg/kv/kvserver/concurrency/poison:poison_test",
     "//pkg/kv/kvserver/concurrency:concurrency_test",
     "//pkg/kv/kvserver/gc:gc_test",

pkg/build/BUILD.bazel

@@ -15,10 +15,11 @@ go_library(
     visibility = ["//visibility:public"],
     x_defs = {
         "github.com/cockroachdb/cockroach/pkg/build.cgoTargetTriple": "{STABLE_BUILD_TARGET_TRIPLE}",
-        "github.com/cockroachdb/cockroach/pkg/build.typ": "{STABLE_BUILD_GIT_BUILD_TYPE}",
-        "github.com/cockroachdb/cockroach/pkg/build.rev": "{BUILD_GIT_COMMIT}",
-        "github.com/cockroachdb/cockroach/pkg/build.tag": "{BUILD_GIT_TAG}",
-        "github.com/cockroachdb/cockroach/pkg/build.utcTime": "{BUILD_GIT_UTCTIME}",
+        "github.com/cockroachdb/cockroach/pkg/build.channel": "{STABLE_BUILD_CHANNEL}",
+        "github.com/cockroachdb/cockroach/pkg/build.rev": "{BUILD_REV}",
+        "github.com/cockroachdb/cockroach/pkg/build.tag": "{BUILD_TAG}",
+        "github.com/cockroachdb/cockroach/pkg/build.typ": "{STABLE_BUILD_TYPE}",
+        "github.com/cockroachdb/cockroach/pkg/build.utcTime": "{BUILD_UTCTIME}",
     },
     deps = [
         "//pkg/util/envutil",

pkg/ccl/backupccl/restore_planning.go

@@ -868,7 +868,9 @@ func maybeUpgradeDescriptors(descs []catalog.Descriptor, skipFKsWithNoMatchingTa
 		} else {
 			b = desc.NewBuilder()
 		}
-		b.RunPostDeserializationChanges()
+		if err := b.RunPostDeserializationChanges(); err != nil {
+			return errors.NewAssertionErrorWithWrappedErrf(err, "error during RunPostDeserializationChanges")
+		}
 		err := b.RunRestoreChanges(func(id descpb.ID) catalog.Descriptor {
 			for _, d := range descs {
 				if d.GetID() == id {

pkg/ccl/testccl/sqlccl/session_revival_test.go

@@ -43,7 +43,9 @@ func TestAuthenticateWithSessionRevivalToken(t *testing.T) {
 
 	_, err := tenantDB.Exec("CREATE USER testuser WITH PASSWORD 'hunter2'")
 	require.NoError(t, err)
-	_, err = tenantDB.Exec("SET CLUSTER SETTING server.user_login.session_revival_token.enabled = true")
+	// TODO(rafi): use ALTER TENANT ALL when available.
+	_, err = mainDB.Exec(`INSERT INTO system.tenant_settings (tenant_id, name, value, value_type) VALUES
+		(0, 'server.user_login.session_revival_token.enabled', 'true', 'b')`)
 	require.NoError(t, err)
 
 	var token string