Merge cockroachdb#67035 cockroachdb#67039 cockroachdb#67046 cockroach…

…db#67073 67035: AUTHORS: Added bradjacobs to authors r=rail a=bradwastaken Release note: None 67039: bazel: add toolchain for cross-compiling to linux on aarch64 r=rail a=rickystewart This is just another `crosstool-ng` toolchain, so we use all the pre-existing stuff added in b782105. Release note: None 67046: build: install 2 go versions on TeamCity agents r=rail a=rail Previously, to run the acceptance test we compiled the test binary using the go version from the builder docker image, which depends on the branch. TeamCity agents have only one version installed and with the go 1.16 upgrade it caused some issues. See cockroachdb#66977 for the details. This patch installs 2 go versions in parallel. The older version is installed in a location which is not added to `PATH` and has to be called using its absolute path. Release note: None 67073: jobs/jobspb,sql/stats: move constant from stats to jobspb r=rytaft a=ajwerner It seemed crazy for jobspb to depend on stats over this one string constant. Release note: None Co-authored-by: Brad Jacobs <[email protected]> Co-authored-by: Ricky Stewart <[email protected]> Co-authored-by: Rail Aliiev <[email protected]> Co-authored-by: Andrew Werner <[email protected]>
stevendanna · Jun 30, 2021 · 2f41ff5 · 2f41ff5
5 parents 5bee963 + 2b8059b + a7924aa + 79a683d + 59756c4
commit 2f41ff5
Show file tree

Hide file tree

Showing 17 changed files with 85 additions and 32 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -22,3 +22,5 @@ build:crosswindows --platforms=//build/toolchains:cross_windows
 build:crosswindows --crosstool_top=@toolchain_cross_x86_64-w64-mingw32//:suite
 build:crossmacos --platforms=//build/toolchains:cross_macos
 build:crossmacos --crosstool_top=@toolchain_cross_x86_64-apple-darwin19//:suite
+build:crosslinuxarm --platforms=//build/toolchains:cross_linux_arm
+build:crosslinuxarm --crosstool_top=@toolchain_cross_aarch64-unknown-linux-gnu//:suite
diff --git a/AUTHORS b/AUTHORS
@@ -83,6 +83,7 @@ Bob Ferris <[email protected]>
 Bob Potter <[email protected]>
 Bob Vawter <[email protected]>
 Bogdan Batog <[email protected]> bogdanbatog <[email protected]>
+Brad Jacobs <[email protected]>
 Brad Seiler <[email protected]>
 Bram Gruneir <[email protected]> <[email protected]> <[email protected]> <[email protected]>
 Brandon Gonzalez <[email protected]>

diff --git a/WORKSPACE b/WORKSPACE
@@ -181,6 +181,7 @@ toolchain_dependencies()
 
 register_toolchains(
     "//build/toolchains:cross_linux_toolchain",
+    "//build/toolchains:cross_linux_arm_toolchain",
     "//build/toolchains:cross_macos_toolchain",
     "//build/toolchains:cross_windows_toolchain",
 )

diff --git a/build/packer/teamcity-agent.sh b/build/packer/teamcity-agent.sh
@@ -49,6 +49,15 @@ b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061 /tmp/go.tgz
 EOF
 tar -C /usr/local -zxf /tmp/go.tgz && rm /tmp/go.tgz
 
+# Install the older version in parallel in order to run the acceptance test on older branches
+# TODO: Remove this when 21.1 is EOL
+curl -fsSL https://dl.google.com/go/go1.15.11.linux-amd64.tar.gz > /tmp/go_old.tgz
+sha256sum -c - <<EOF
+8825b72d74b14e82b54ba3697813772eb94add3abf70f021b6bdebe193ed01ec /tmp/go_old.tgz
+EOF
+mkdir -p /usr/local/go1.15
+tar -C /usr/local/go1.15 --strip-components=1 -zxf /tmp/go_old.tgz && rm /tmp/go_old.tgz
+
 # Explicitly symlink the pinned version to /usr/bin.
 for f in `ls /usr/local/go/bin`; do
     ln -s /usr/local/go/bin/$f /usr/bin

diff --git a/build/teamcity-bazel-arm-build.sh b/build/teamcity-bazel-arm-build.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/teamcity-support.sh"  # For $root
+source "$(dirname "${0}")/teamcity-bazel-support.sh"  # For run_bazel
+
+tc_prepare
+
+tc_start_block "Run Bazel build"
+run_bazel build/bazelutil/bazelbuild.sh crosslinuxarm
+tc_end_block "Run Bazel build"
diff --git a/build/toolchains/BUILD.bazel b/build/toolchains/BUILD.bazel
@@ -63,3 +63,25 @@ platform(
         "@platforms//cpu:x86_64",
     ],
 )
+
+toolchain(
+    name = "cross_linux_arm_toolchain",
+    exec_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+    target_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:arm64",
+    ],
+    toolchain = "@toolchain_cross_aarch64-unknown-linux-gnu//:toolchain",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+platform(
+    name = "cross_linux_arm",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:arm64",
+    ],
+)
diff --git a/c-deps/BUILD.bazel b/c-deps/BUILD.bazel
@@ -12,6 +12,7 @@ configure_make(
     ] + select({
         "@io_bazel_rules_go//go/platform:windows": ["--host=x86_64-w64-mingw32"],
         "@io_bazel_rules_go//go/platform:darwin": ["--host=x86_64-apple-darwin19"],
+        "@io_bazel_rules_go//go/platform:linux_arm64": ["--host=aarch64-unknown-linux-gnu"],
         "//conditions:default": [],
     }),
     lib_source = "@jemalloc//:all",

diff --git a/pkg/jobs/jobspb/BUILD.bazel b/pkg/jobs/jobspb/BUILD.bazel
@@ -10,7 +10,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/sql/catalog/descpb",
-        "//pkg/sql/stats",
         "//pkg/storage/cloud",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_gogo_protobuf//jsonpb",

diff --git a/pkg/jobs/jobspb/wrap.go b/pkg/jobs/jobspb/wrap.go
@@ -16,7 +16,6 @@ import (
 	"strings"
 
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
-	"github.com/cockroachdb/cockroach/pkg/sql/stats"
 	"github.com/cockroachdb/cockroach/pkg/storage/cloud"
 	"github.com/cockroachdb/errors"
 	"github.com/gogo/protobuf/jsonpb"
@@ -59,6 +58,11 @@ func (p *Payload) Type() Type {
 	return DetailsType(p.Details)
 }
 
+// AutoStatsName is the name to use for statistics created automatically.
+// The name is chosen to be something that users are unlikely to choose when
+// running CREATE STATISTICS manually.
+const AutoStatsName = "__auto__"
+
 // DetailsType returns the type for a payload detail.
 func DetailsType(d isPayload_Details) Type {
 	switch d := d.(type) {
@@ -74,7 +78,7 @@ func DetailsType(d isPayload_Details) Type {
 		return TypeChangefeed
 	case *Payload_CreateStats:
 		createStatsName := d.CreateStats.Name
-		if createStatsName == stats.AutoStatsName {
+		if createStatsName == AutoStatsName {
 			return TypeAutoCreateStats
 		}
 		return TypeCreateStats

diff --git a/pkg/sql/create_stats.go b/pkg/sql/create_stats.go
@@ -124,7 +124,7 @@ func (n *createStatsNode) startJob(ctx context.Context, resultsCh chan<- tree.Da
 		return err
 	}
 
-	if n.Name == stats.AutoStatsName {
+	if n.Name == jobspb.AutoStatsName {
 		// Don't start the job if there is already a CREATE STATISTICS job running.
 		// (To handle race conditions we check this again after the job starts,
 		// but this check is used to prevent creating a large number of jobs that
@@ -266,7 +266,7 @@ func (n *createStatsNode) makeJobRecord(ctx context.Context) (*jobs.Record, erro
 	statement := tree.AsStringWithFQNames(n, n.p.EvalContext().Annotations)
 	eventLogStatement := statement
 	var description string
-	if n.Name == stats.AutoStatsName {
+	if n.Name == jobspb.AutoStatsName {
 		// Use a user-friendly description for automatic statistics.
 		description = fmt.Sprintf("Table statistics refresh for %s", fqTableName)
 	} else {
@@ -495,7 +495,7 @@ var _ jobs.Resumer = &createStatsResumer{}
 func (r *createStatsResumer) Resume(ctx context.Context, execCtx interface{}) error {
 	p := execCtx.(JobExecContext)
 	details := r.job.Details().(jobspb.CreateStatsDetails)
-	if details.Name == stats.AutoStatsName {
+	if details.Name == jobspb.AutoStatsName {
 		// We want to make sure that an automatic CREATE STATISTICS job only runs if
 		// there are no other CREATE STATISTICS jobs running, automatic or manual.
 		if err := checkRunningJobs(ctx, r.job, p); err != nil {

diff --git a/pkg/sql/opt/testutils/opttester/BUILD.bazel b/pkg/sql/opt/testutils/opttester/BUILD.bazel
@@ -15,6 +15,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/build/bazel",
+        "//pkg/jobs/jobspb",
         "//pkg/roachpb",
         "//pkg/security",
         "//pkg/settings/cluster",

diff --git a/pkg/sql/opt/testutils/opttester/opt_tester.go b/pkg/sql/opt/testutils/opttester/opt_tester.go
@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/build/bazel"
+	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
@@ -1893,7 +1894,7 @@ func (ot *OptTester) makeStat(
 	columns []string, rowCount, distinctCount, nullCount uint64,
 ) stats.JSONStatistic {
 	return stats.JSONStatistic{
-		Name: stats.AutoStatsName,
+		Name: jobspb.AutoStatsName,
 		CreatedAt: tree.AsStringWithFlags(
 			&tree.DTimestamp{Time: timeutil.Now()}, tree.FmtBareStrings,
 		),

diff --git a/pkg/sql/stats/BUILD.bazel b/pkg/sql/stats/BUILD.bazel
@@ -18,6 +18,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/gossip",
+        "//pkg/jobs/jobspb",
         "//pkg/keys",
         "//pkg/kv",
         "//pkg/kv/kvclient/rangefeed",

diff --git a/pkg/sql/stats/automatic_stats.go b/pkg/sql/stats/automatic_stats.go
@@ -17,6 +17,7 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
@@ -116,11 +117,6 @@ var bufferedChanFullLogLimiter = log.Every(time.Second)
 // Constants for automatic statistics collection.
 // TODO(rytaft): Should these constants be configurable?
 const (
-	// AutoStatsName is the name to use for statistics created automatically.
-	// The name is chosen to be something that users are unlikely to choose when
-	// running CREATE STATISTICS manually.
-	AutoStatsName = "__auto__"
-
 	// defaultAverageTimeBetweenRefreshes is the default time to use as the
 	// "average" time between refreshes when there is no information for a given
 	// table.
@@ -487,7 +483,7 @@ func (r *Refresher) refreshStats(ctx context.Context, tableID descpb.ID, asOf ti
 		nil, /* txn */
 		fmt.Sprintf(
 			"CREATE STATISTICS %s FROM [%d] WITH OPTIONS THROTTLING %g AS OF SYSTEM TIME '-%s'",
-			AutoStatsName,
+			jobspb.AutoStatsName,
 			tableID,
 			AutomaticStatisticsMaxIdleTime.Get(&r.st.SV),
 			asOf.String(),
@@ -501,7 +497,7 @@ func (r *Refresher) refreshStats(ctx context.Context, tableID descpb.ID, asOf ti
 func mostRecentAutomaticStat(tableStats []*TableStatistic) *TableStatistic {
 	// Stats are sorted with the most recent first.
 	for _, stat := range tableStats {
-		if stat.Name == AutoStatsName {
+		if stat.Name == jobspb.AutoStatsName {
 			return stat
 		}
 	}
@@ -522,7 +518,7 @@ func avgRefreshTime(tableStats []*TableStatistic) time.Duration {
 	var sum time.Duration
 	var count int
 	for _, stat := range tableStats {
-		if stat.Name != AutoStatsName {
+		if stat.Name != jobspb.AutoStatsName {
 			continue
 		}
 		if reference == nil {

diff --git a/pkg/sql/stats/automatic_stats_test.go b/pkg/sql/stats/automatic_stats_test.go
@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
@@ -275,7 +276,7 @@ func TestAverageRefreshTime(t *testing.T) {
 			if err != nil {
 				return err
 			}
-			if err := insertStat(txn, AutoStatsName, columnIDsVal, createdAt); err != nil {
+			if err := insertStat(txn, jobspb.AutoStatsName, columnIDsVal, createdAt); err != nil {
 				return err
 			}
 		}
@@ -326,7 +327,7 @@ func TestAverageRefreshTime(t *testing.T) {
 			if err != nil {
 				return err
 			}
-			if err := insertStat(txn, AutoStatsName, columnIDsVal, createdAt); err != nil {
+			if err := insertStat(txn, jobspb.AutoStatsName, columnIDsVal, createdAt); err != nil {
 				return err
 			}
 		}

diff --git a/pkg/sql/stats/delete_stats.go b/pkg/sql/stats/delete_stats.go
@@ -13,6 +13,7 @@ package stats
 import (
 	"context"
 
+	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -64,7 +65,7 @@ func DeleteOldStatsForColumns(
                    LIMIT $4
                )`,
 		tableID,
-		AutoStatsName,
+		jobspb.AutoStatsName,
 		columnIDsVal,
 		keepCount,
 	)