From a973f517eba4257f2de9cb77ed7b5c34925670f5 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 27 Feb 2024 19:47:13 +0800 Subject: [PATCH 1/4] Makefile: adpot staged build to make ut a little bit faster --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 35bc50fce8f28..14ab46cb488c8 100644 --- a/Makefile +++ b/Makefile @@ -129,7 +129,11 @@ integrationtest: server_check ddltest: @cd cmd/ddltest && $(GO) test --tags=deadllock,intest -o ../../bin/ddltest -c -ut: tools/bin/ut tools/bin/xprog failpoint-enable +generate-test-build-cache: + @echo "generate test build cache" + @cd cmd/tidb-server && go test -tags intest -exec true -vet off + +ut: tools/bin/ut tools/bin/xprog failpoint-enable generate-test-build-cache tools/bin/ut $(X) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) @$(CLEAN_UT_BINARY) From aaa14c2f5ffa6caf2eb632b3c1b3e1cdb78c30d9 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 27 Feb 2024 20:38:56 +0800 Subject: [PATCH 2/4] change to make it in ut.go --- Makefile | 6 +----- tools/check/ut.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 14ab46cb488c8..35bc50fce8f28 100644 --- a/Makefile +++ b/Makefile @@ -129,11 +129,7 @@ integrationtest: server_check ddltest: @cd cmd/ddltest && $(GO) test --tags=deadllock,intest -o ../../bin/ddltest -c -generate-test-build-cache: - @echo "generate test build cache" - @cd cmd/tidb-server && go test -tags intest -exec true -vet off - -ut: tools/bin/ut tools/bin/xprog failpoint-enable generate-test-build-cache +ut: tools/bin/ut tools/bin/xprog failpoint-enable tools/bin/ut $(X) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) @$(CLEAN_UT_BINARY) diff --git a/tools/check/ut.go b/tools/check/ut.go index c28c535377c40..fee7a19eb25d6 100644 --- a/tools/check/ut.go +++ b/tools/check/ut.go @@ -869,8 +869,24 @@ func buildTestBinary(pkg string) error { return nil } +func generateBuildCache() error { + // cd cmd/tidb-server && go test -tags intest -exec true -vet off + cmd := exec.Command("go", "test", "-tags=intest", "-exec", "true", "-vet", "off") + cmd.Dir = path.Join(workDir, "cmd/tidb-server") + if err := cmd.Run(); err != nil { + return withTrace(err) + } + return nil +} + // buildTestBinaryMulti is much faster than build the test packages one by one. func buildTestBinaryMulti(pkgs []string) error { + // staged build, generate the build cache for all the tests first, then generate the test binary. + // This way is faster than generating test binaries directly, because the cache can be used. + if err := generateBuildCache(); err != nil { + return withTrace(err) + } + // go test --exec=xprog -cover -vet=off --count=0 $(pkgs) xprogPath := path.Join(workDir, "tools/bin/xprog") packages := make([]string, 0, len(pkgs)) From f3ca7b3e433e5feb36332e06137dc13a0f160f70 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 27 Feb 2024 22:28:32 +0800 Subject: [PATCH 3/4] address comment --- tools/check/go-compile-without-link.sh | 33 ++++++++++++++++++++++++++ tools/check/ut.go | 4 +++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tools/check/go-compile-without-link.sh diff --git a/tools/check/go-compile-without-link.sh b/tools/check/go-compile-without-link.sh new file mode 100644 index 0000000000000..9ad1059747ec6 --- /dev/null +++ b/tools/check/go-compile-without-link.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2024 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://gist.github.com/howardjohn/c0f5d0bc293ef7d7fada533a2c9ffaf4 +# Usage: go test -exec=true -toolexec=go-compile-without-link -vet=off ./... +# Preferably as an alias like `alias go-test-compile='go test -exec=true -toolexec=go-compile-without-link -vet=off'` +# This will compile all tests, but not link them (which is the least cacheable part) + +if [[ "${2}" == "-V=full" ]]; then + "$@" + exit 0 +fi +case "$(basename ${1})" in + link) + # Output a dummy file + touch "${3}" + ;; + # We could skip vet as well, but it can be done with -vet=off if desired + *) + "$@" +esac diff --git a/tools/check/ut.go b/tools/check/ut.go index fee7a19eb25d6..da9d48e9dfef3 100644 --- a/tools/check/ut.go +++ b/tools/check/ut.go @@ -871,7 +871,9 @@ func buildTestBinary(pkg string) error { func generateBuildCache() error { // cd cmd/tidb-server && go test -tags intest -exec true -vet off - cmd := exec.Command("go", "test", "-tags=intest", "-exec", "true", "-vet", "off") + cmd := exec.Command("go", "test", "-tags=intest", "-exec=true", "-vet=off") + goCompileWithoutLink := fmt.Sprintf("-toolexec=%s/tools/check/go-compile-without-link.sh", workDir) + cmd.Args = append(cmd.Args, goCompileWithoutLink) cmd.Dir = path.Join(workDir, "cmd/tidb-server") if err := cmd.Run(); err != nil { return withTrace(err) From 77c33bc01d4fcfdf616f4b4ab173efb843ada29c Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 27 Feb 2024 22:30:13 +0800 Subject: [PATCH 4/4] update a comment --- tools/check/ut.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check/ut.go b/tools/check/ut.go index da9d48e9dfef3..34f826f957f6f 100644 --- a/tools/check/ut.go +++ b/tools/check/ut.go @@ -870,7 +870,7 @@ func buildTestBinary(pkg string) error { } func generateBuildCache() error { - // cd cmd/tidb-server && go test -tags intest -exec true -vet off + // cd cmd/tidb-server && go test -tags intest -exec true -vet off -toolexec=go-compile-without-link cmd := exec.Command("go", "test", "-tags=intest", "-exec=true", "-vet=off") goCompileWithoutLink := fmt.Sprintf("-toolexec=%s/tools/check/go-compile-without-link.sh", workDir) cmd.Args = append(cmd.Args, goCompileWithoutLink)