Merge cockroachdb#120752

120752: roachprod-microbench,build: ignore packages r=renatolabs,srosenberg a=herkolategan

Recently, `pkg/testutils/lint/lint_test.go` (cockroachdb#119589) added logic to `init`, that requires
a `GO_SDK` env var that causes a failure when listing the benchmarks for that
package, since on the target microbenchmark env there is no such env var
specified. It's possible to add the env var to avoid failure. But in the future
more logic that could probe the env var further could still cause a failure as
go will not be installed on these machines. These packages should be excluded
from any interaction, to avoid complete execution failure.

This is slightly different from the other `exclude` flag that is meant to not
execute specific benchmarks within a given package.

See: cockroachdb#119589

Epic: None
Release Note: None

Co-authored-by: Herko Lategan <[email protected]>

build/teamcity/cockroach/nightlies/microbenchmark_weekly.sh

@@ -13,6 +13,7 @@
 #   SHEET_DESCRIPTION: Adds a description to the name of the published spreadsheets (e.g., "22.2 -> 22.1")
 #   BENCH_TIMEOUT: timeout for each microbenchmark on a function level (default: 20m)
 #   BENCH_EXCLUDE: comma-separated list of benchmarks to exclude (default: none)
+#   BENCH_IGNORE_PACKAGES: comma-separated list of packages to exclude completely from listing and execution (default: none)
 #   TEST_ARGS: additional arguments to pass to the test binary (default: none)
 #   ROACHPROD_CREATE_ARGS: additional arguments to pass to `roachprod create` (default: none)
 #   MICROBENCH_SLACK_TOKEN: token to use to post to slack (default: none)
@@ -82,6 +83,7 @@ fi
   ${bench_compare_binaries:+--compare-binaries="$bench_compare_binaries"} \
   ${BENCH_TIMEOUT:+--timeout="$BENCH_TIMEOUT"} \
   ${BENCH_EXCLUDE:+--exclude="$BENCH_EXCLUDE"} \
+  ${BENCH_IGNORE_PACKAGES:+--ignore-package="$BENCH_IGNORE_PACKAGES"} \
   --quiet \
   -- "$TEST_ARGS" \
   || exit_status=$?

pkg/cmd/roachprod-microbench/executor.go

@@ -30,21 +30,22 @@ import (
 )
 
 type executorConfig struct {
-	cluster         string
-	binaries        string
-	compareBinaries string
-	excludeList     []string
-	outputDir       string
-	libDir          string
-	remoteDir       string
-	timeout         string
-	shellCommand    string
-	testArgs        []string
-	iterations      int
-	copyBinaries    bool
-	lenient         bool
-	affinity        bool
-	quiet           bool
+	cluster           string
+	binaries          string
+	compareBinaries   string
+	excludeList       []string
+	ignorePackageList []string
+	outputDir         string
+	libDir            string
+	remoteDir         string
+	timeout           string
+	shellCommand      string
+	testArgs          []string
+	iterations        int
+	copyBinaries      bool
+	lenient           bool
+	affinity          bool
+	quiet             bool
 }
 
 type executor struct {
@@ -73,11 +74,25 @@ type benchmarkExtractionResult struct {
 
 func newExecutor(config executorConfig) (*executor, error) {
 	// Gather package info from the primary binary.
-	packages, err := readArchivePackages(config.binaries)
+	archivePackages, err := readArchivePackages(config.binaries)
 	if err != nil {
 		return nil, err
 	}
 
+	// Exclude packages that should not to be probed. This is useful for excluding
+	// packages that have known issues and unable to list its benchmarks, or are
+	// not relevant to the current benchmarking effort.
+	ignorePackages := make(map[string]struct{})
+	for _, pkg := range config.ignorePackageList {
+		ignorePackages[pkg] = struct{}{}
+	}
+	packages := make([]string, 0, len(archivePackages))
+	for _, pkg := range archivePackages {
+		if _, ok := ignorePackages[pkg]; !ok {
+			packages = append(packages, pkg)
+		}
+	}
+
 	config.outputDir = strings.TrimRight(config.outputDir, "/")
 	err = os.MkdirAll(config.outputDir, os.ModePerm)
 	if err != nil {

pkg/cmd/roachprod-microbench/main.go

@@ -89,7 +89,8 @@ func makeRunCommand() *cobra.Command {
 	cmd.Flags().StringVar(&config.remoteDir, "remote-dir", config.remoteDir, "working directory on the target cluster")
 	cmd.Flags().StringVar(&config.timeout, "timeout", config.timeout, "timeout for each benchmark e.g. 10m")
 	cmd.Flags().StringVar(&config.shellCommand, "shell", config.shellCommand, "additional shell command to run on node before benchmark execution")
-	cmd.Flags().StringSliceVar(&config.excludeList, "exclude", []string{}, "comma-separated regex of packages and benchmarks to exclude e.g. 'pkg/util/.*:BenchmarkIntPool,pkg/sql:.*'")
+	cmd.Flags().StringSliceVar(&config.excludeList, "exclude", []string{}, "benchmarks to exclude, in the form <pkg regex:benchmark regex> e.g. 'pkg/util/.*:BenchmarkIntPool,pkg/sql:.*'")
+	cmd.Flags().StringSliceVar(&config.ignorePackageList, "ignore-package", []string{}, "packages to completely exclude from listing or execution'")
 	cmd.Flags().IntVar(&config.iterations, "iterations", config.iterations, "number of iterations to run each benchmark")
 	cmd.Flags().BoolVar(&config.copyBinaries, "copy", config.copyBinaries, "copy and extract test binaries and libraries to the target cluster")
 	cmd.Flags().BoolVar(&config.lenient, "lenient", config.lenient, "tolerate errors while running benchmarks")