Merge #26187

26187: build: speed it up r=petermattis,jordanlewis a=benesch

Employ some tricks to speed Make up. Especially noticeable on noop or nearly-noop builds.

Co-authored-by: Nikhil Benesch <[email protected]>

build/.gitignore

@@ -7,3 +7,6 @@
 # NPM
 node_modules
 yarn.installed
+
+# Build system.
+defs.mk*

build/go-version-check.sh

@@ -3,31 +3,23 @@
 # Detect whether the installed version of Go can build this version of
 # CockroachDB.
 #
-# To bump the required version of Go, edit the last conditional appropriately.
-#
-# Note: this script is invoked by Make, so it has odd error reporting semantics.
-# Errors are printed to stdout; the Go version is compatible iff nothing is
-# printed to stdout. The exit code is meaningless.
-
-# Ensure stray error messages are printed to stdout, or they'll go unnoticed by
-# Make.
-exec 2>&1
+# To bump the required version of Go, edit the last conditional appropriately.s
 
 go=${1-go}
 
 if ! raw_version=$("$go" version 2>&1); then
-  echo "unable to detect go version: $raw_version"
-  exit
+  echo "unable to detect go version: $raw_version" >&2
+  exit 1
 fi
 
 if ! version=$(grep -oE "[0-9]+\.[0-9]+" <<< "$raw_version" | head -n1); then
-  echo "unable to parse go version '$raw_version'"
-  exit
+  echo "unable to parse go version '$raw_version'" >&2
+  exit 1
 fi
 
 version_major=$(cut -f1 -d. <<< "$version")
 version_minor=$(cut -f2 -d. <<< "$version")
 if (( version_major != 1 )) || (( version_minor < 10 )); then
-  echo "go1.10+ required (detected go$version)"
-  exit
+  echo "go1.10+ required (detected go$version)" >&2
+  exit 1
 fi

build/variables.mk

@@ -52,6 +52,7 @@ define VALID_VARS
   GITHOOKSDIR
   GIT_DIR
   GO
+  GOEXE
   GOFLAGS
   GOGOPROTO_PROTO
   GOGO_PROTOBUF_PATH
@@ -75,8 +76,15 @@ define VALID_VARS
   JS_PROTOS_CCL
   KARMA
   LC_COLLATE
+  LIBCRYPTOPP
+  LIBJEMALLOC
+  LIBPROTOBUF
+  LIBROACH
+  LIBROACHCCL
   LIBROACH_DIR
   LIBROACH_SRC_DIR
+  LIBROCKSDB
+  LIBSNAPPY
   LINKFLAGS
   MACOS
   MACOSX_DEPLOYMENT_TARGET
@@ -88,6 +96,7 @@ define VALID_VARS
   NATIVE_SPECIFIER
   NATIVE_SPECIFIER_TAG
   NATIVE_SUFFIX
+  NCPUS
   NODE_RUN
   OPTGEN_TARGETS
   PATH
@@ -150,7 +159,7 @@ define VALID_VARS
   cyan
   go-targets
   go-targets-ccl
-  go-version-check
+  have-defs
   langgen-package
   logictest-package
   logictestccl-package
@@ -159,6 +168,7 @@ define VALID_VARS
   optgen-package
   optgen-xform-rules
   prefix
+  sig
   space
   sse
   term-reset

pkg/cmd/uptodate/uptodate.go

@@ -0,0 +1,78 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// uptodate efficiently computes whether an output file is up-to-date with
+// regard to its input files.
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/MichaelTJones/walk"
+	"github.com/spf13/pflag"
+)
+
+var debug = pflag.BoolP("debug", "d", false, "debug mode")
+
+func die(err error) {
+	fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
+	os.Exit(2)
+}
+
+func main() {
+	pflag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "usage: %s [-d] OUTPUT INPUT\n", os.Args[0])
+	}
+	pflag.Parse()
+	if pflag.NArg() != 2 {
+		pflag.Usage()
+		os.Exit(1)
+	}
+	if !*debug {
+		log.SetOutput(ioutil.Discard)
+	}
+	output, input := pflag.Arg(0), pflag.Arg(1)
+
+	fi, err := os.Stat(output)
+	if os.IsNotExist(err) {
+		log.Printf("output %q is missing", output)
+		os.Exit(1)
+	} else if err != nil {
+		die(err)
+	}
+	outputModTime := fi.ModTime()
+
+	err = walk.Walk(input, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.IsDir() {
+			return nil
+		}
+		if !fi.ModTime().Before(outputModTime) {
+			log.Printf("input %q (mtime %s) not older than output %q (mtime %s)",
+				path, fi.ModTime(), output, outputModTime)
+			os.Exit(1)
+		}
+		return nil
+	})
+	if err != nil {
+		die(err)
+	}
+	log.Printf("all inputs older than output %q (mtime %s)\n", output, outputModTime)
+	os.Exit(0)
+}