tools/bin/ut: accelerate building test binary

Makefile

@@ -125,7 +125,7 @@ devgotest: failpoint-enable
 	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
-ut: failpoint-enable tools/bin/ut
+ut: failpoint-enable tools/bin/ut tools/bin/xprog
 	tools/bin/ut $(X);
 	@$(FAILPOINT_DISABLE)
 
@@ -216,6 +216,10 @@ tools/bin/ut: tools/check/ut.go
 	cd tools/check; \
 	$(GO) build -o ../bin/ut ut.go
 
+tools/bin/xprog: tools/check/xprog.go
+	cd tools/check; \
+	$(GO) build -o ../bin/xprog xprog.go
+
 tools/bin/megacheck: tools/check/go.mod
 	cd tools/check; \
 	$(GO) build -o ../bin/megacheck honnef.co/go/tools/cmd/megacheck

tools/check/ut.go

@@ -133,12 +133,10 @@ func cmdBuild(args ...string) bool {
 
 	// build all packages
 	if len(args) == 0 {
-		for _, pkg := range pkgs {
-			err := buildTestBinary(pkg)
-			if err != nil {
-				fmt.Println("build package error", pkg, err)
-				return false
-			}
+		err := buildTestBinaryMulti(pkgs)
+		if err != nil {
+			fmt.Println("build package error", pkgs, err)
+			return false
 		}
 		return true
 	}
@@ -163,16 +161,16 @@ func cmdRun(args ...string) bool {
 		return false
 	}
 	tasks := make([]task, 0, 5000)
+	start := time.Now()
 	// run all tests
 	if len(args) == 0 {
-		for _, pkg := range pkgs {
-			fmt.Println("handling package", pkg)
-			err := buildTestBinary(pkg)
-			if err != nil {
-				fmt.Println("build package error", pkg, err)
-				return false
-			}
+		err := buildTestBinaryMulti(pkgs)
+		if err != nil {
+			fmt.Println("build package error", pkgs, err)
+			return false
+		}
 
+		for _, pkg := range pkgs {
 			exist, err := testBinaryExist(pkg)
 			if err != nil {
 				fmt.Println("check test binary existance error", err)
@@ -248,7 +246,7 @@ func cmdRun(args ...string) bool {
 		}
 		tasks = tmp
 	}
-	fmt.Println("building task finish...", len(tasks))
+	fmt.Printf("building task finish, count=%d, takes=%v\n", len(tasks), time.Since(start))
 
 	numactl := numactlExist()
 	taskCh := make(chan task, 100)
@@ -446,7 +444,27 @@ func buildTestBinary(pkg string) error {
 		return withTrace(err)
 	}
 	return nil
+}
 
+// buildTestBinaryMulti is much faster than build the test packages one by one.
+func buildTestBinaryMulti(pkgs []string) error {
+       // go test --exec=xprog -cover -vet=off --count=0 $(pkgs)
+       xprogPath := path.Join(workDir, "tools/bin/xprog")
+       packages := make([]string, 0, len(pkgs))
+       for _, pkg := range pkgs {
+               packages = append(packages, path.Join(modulePath, pkg))
+       }
+
+       var cmd *exec.Cmd
+	cmd = exec.Command("go", "test", "--exec", xprogPath, "-vet", "off", "-count", "0")
+       cmd.Args = append(cmd.Args, packages...)
+       cmd.Dir = workDir
+       cmd.Stdout = os.Stdout
+       cmd.Stderr = os.Stderr
+       if err := cmd.Run(); err != nil {
+               return withTrace(err)
+       }
+       return nil
 }
 
 func testBinaryExist(pkg string) (bool, error) {

tools/check/xprog.go

@@ -0,0 +1,118 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	// See https://github.com/golang/go/issues/15513#issuecomment-773994959
+	// go test --exec=xprog ./...
+	// Command line args looks like:
+	// '$CWD/xprog /tmp/go-build2662369829/b1382/aggfuncs.test -test.paniconexit0 -test.timeout=10m0s'
+	// This program moves the test binary /tmp/go-build2662369829/b1382/aggfuncs.test to someplace else for later use.
+
+	// Extract the current work directory
+	cwd := os.Args[0]
+	cwd = cwd[:len(cwd)-len("tools/bin/xprog")]
+
+	testBinaryPath := os.Args[1]
+	dir, _ := filepath.Split(testBinaryPath)
+
+	// Extract the package info from /tmp/go-build2662369829/b1382/importcfg.link
+	pkg := getPackageInfo(dir)
+
+	const prefix = "github.com/pingcap/tidb/"
+	if !strings.HasPrefix(pkg, prefix) {
+		os.Exit(-3)
+	}
+
+	// github.com/pingcap/tidb/util/topsql.test => util/topsql
+	pkg = pkg[len(prefix) : len(pkg)-len(".test")]
+
+	_, file := filepath.Split(pkg)
+
+	// The path of the destination file looks like $CWD/util/topsql/topsql.test.bin
+	newName := filepath.Join(cwd, pkg, file+".test.bin")
+
+	if err1 := os.Rename(testBinaryPath, newName); err1 != nil {
+		// Rename fail, handle error like "invalid cross-device linkcd tools/check"
+		err1 = MoveFile(testBinaryPath, newName)
+		if err1 != nil {
+			os.Exit(-4)
+		}
+	}
+}
+
+func getPackageInfo(dir string) string {
+	// Read the /tmp/go-build2662369829/b1382/importcfg.link file to get the package information
+	f, err := os.Open(filepath.Join(dir, "importcfg.link"))
+	if err != nil {
+		os.Exit(-1)
+	}
+	defer f.Close()
+
+	r := bufio.NewReader(f)
+	// packagefile github.com/pingcap/tidb/session.test=/home/genius/.cache/go-build/fb/fb1587cce5727fa9461131eab8260a52878da04f5c8da49dd3c7b2d941430c63-d
+	line, _, err := r.ReadLine()
+	if err != nil {
+		os.Exit(-2)
+	}
+	start := strings.IndexByte(string(line), ' ')
+	end := strings.IndexByte(string(line), '=')
+	pkg := string(line[start+1 : end])
+	return pkg
+}
+
+func MoveFile(sourcePath, destPath string) error {
+	inputFile, err := os.Open(sourcePath)
+	if err != nil {
+		return fmt.Errorf("Couldn't open source file: %s", err)
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return fmt.Errorf("Couldn't open dest file: %s", err)
+	}
+	defer outputFile.Close()
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return fmt.Errorf("Writing to output file failed: %s", err)
+	}
+
+	// Handle the permissions
+	si, err := os.Stat(sourcePath)
+	if err != nil {
+		return fmt.Errorf("Stat error: %s", err)
+	}
+	err = os.Chmod(destPath, si.Mode())
+	if err != nil {
+		return fmt.Errorf("Chmod error: %s", err)
+	}
+
+	// The copy was successful, so now delete the original file
+	err = os.Remove(sourcePath)
+	if err != nil {
+		return fmt.Errorf("Failed removing original file: %s", err)
+	}
+	return nil
+}