From 03c084689322892a92bb1226437014a93f6db236 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 27 Mar 2020 11:03:00 -0600 Subject: [PATCH 1/5] Remove deadcode in tests Signed-off-by: Morgan Tocker --- Makefile | 14 ------- test.go | 117 ------------------------------------------------------- 2 files changed, 131 deletions(-) diff --git a/Makefile b/Makefile index b47e3e7bd19..b64aae8f116 100644 --- a/Makefile +++ b/Makefile @@ -74,14 +74,6 @@ install: build # binaries mkdir -p "$${PREFIX}/bin" cp "$${VTROOT}/bin/"{mysqlctld,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/" - # config files - mkdir -p "$${PREFIX}/src/vitess.io/vitess" - cp -R config "$${PREFIX}/src/vitess.io/vitess/" - # also symlink config files in the old location - ln -sf src/vitess.io/vitess/config "$${PREFIX}/config" - # vtctld web UI files - mkdir -p "$${PREFIX}/src/vitess.io/vitess/web/vtctld2" - cp -R web/vtctld2/app "$${PREFIX}/src/vitess.io/vitess/web/vtctld2/" parser: make -C go/vt/sqlparser @@ -289,12 +281,6 @@ docker_test: docker_unit_test: go run test.go -flavor $(flavor) unit -# This can be used to rebalance the total average runtime of each group of -# tests in Travis. The results are saved in test/config.json, which you can -# then commit and push. -rebalance_tests: - go run test.go -rebalance 5 - # Release a version. # This will generate a tar.gz file into the releases folder with the current source release: docker_base diff --git a/test.go b/test.go index a31c1ce153e..284e747078a 100755 --- a/test.go +++ b/test.go @@ -39,7 +39,6 @@ package main import ( "bytes" "encoding/json" - "errors" "flag" "fmt" "io" @@ -86,7 +85,6 @@ var ( shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N") tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list") exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)") - reshard = flag.Int("rebalance", 0, "if N>0, check the stats and group tests into N similarly-sized bins by average run time") keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") @@ -299,22 +297,6 @@ func main() { log.Fatalf("Can't parse config file: %v", err) } - // Resharding. - if *reshard > 0 { - if err := reshardTests(&config, *reshard); err != nil { - log.Fatalf("resharding error: %v", err) - } - log.Printf("Saving updated config...") - data, err := json.MarshalIndent(config, "", "\t") - if err != nil { - log.Fatalf("can't save new config: %v", err) - } - if err := ioutil.WriteFile(configFileName, data, 0644); err != nil { - log.Fatalf("can't write new config: %v", err) - } - return - } - flavors := []string{"local"} if *docker { @@ -697,105 +679,6 @@ func updateTestStats(name string, update func(*TestStats)) { } } -func reshardTests(config *Config, numShards int) error { - var stats Stats - - var data []byte - if *remoteStats != "" { - log.Printf("Using remote stats for resharding: %v", *remoteStats) - resp, err := http.Get(*remoteStats) - if err != nil { - return err - } - defer resp.Body.Close() - if data, err = ioutil.ReadAll(resp.Body); err != nil { - return err - } - } else { - var err error - data, err = ioutil.ReadFile(statsFileName) - if err != nil { - return errors.New("can't read stats file") - } - } - - if err := json.Unmarshal(data, &stats); err != nil { - return fmt.Errorf("can't parse stats file: %v", err) - } - - // Sort tests by PassTime. - var tests []TestStats - var totalTime int64 - for name, test := range stats.TestStats { - test.name = name - tests = append(tests, test) - totalTime += int64(test.PassTime) - } - sort.Sort(ByPassTime(tests)) - - // Group into shards. - max := totalTime / int64(numShards) - shards := make([][]TestStats, numShards) - sums := make([]int64, numShards) - // First pass: greedy approximation. -firstPass: - for len(tests) > 0 { - v := int64(tests[0].PassTime) - - for n := range shards { - if sums[n]+v < max { - shards[n] = append(shards[n], tests[0]) - sums[n] += v - tests = tests[1:] - continue firstPass - } - } - // None of the bins has room. Go to second pass. - break - } - // Second pass: distribute the remainder. - for len(tests) > 0 { - nmin := 0 - min := sums[0] - - for n := range sums { - if sums[n] < min { - nmin = n - min = sums[n] - } - } - - shards[nmin] = append(shards[nmin], tests[0]) - sums[nmin] += int64(tests[0].PassTime) - tests = tests[1:] - } - - // Update config and print results. - for i, tests := range shards { - for _, t := range tests { - ct, ok := config.Tests[t.name] - if !ok { - log.Printf("WARNING: skipping unknown test: %v", t.name) - continue - } - ct.Shard = i - if ct.Args == nil { - ct.Args = []string{} - } - if ct.Command == nil { - ct.Command = []string{} - } - if ct.Tags == nil { - ct.Tags = []string{} - } - log.Printf("%v:\t%v\n", t.name, t.PassTime) - } - log.Printf("Shard %v total: %v\n", i, time.Duration(sums[i])) - } - - return nil -} - type ByPassTime []TestStats func (a ByPassTime) Len() int { return len(a) } From 863c10e6cc74d98c6d76627bf3731ab0c9a960da Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 27 Mar 2020 12:18:27 -0600 Subject: [PATCH 2/5] Fix small bug in build.env file Signed-off-by: Morgan Tocker --- build.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.env b/build.env index 0cc095fe651..a09807bdf5a 100755 --- a/build.env +++ b/build.env @@ -17,7 +17,7 @@ source ./tools/shell_functions.inc go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions." -goversion_min 1.13 || fail "Go version reported: ${go version}. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions." +goversion_min 1.13 || fail "Go version reported: `go version`. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions." mkdir -p dist mkdir -p bin From b3cd9ca10b971acbf13b985da47dc98d60546ebb Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 27 Mar 2020 12:37:27 -0600 Subject: [PATCH 3/5] Add k3s to dependency check Signed-off-by: Morgan Tocker --- tools/dependency_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dependency_check.sh b/tools/dependency_check.sh index 7d5179c1616..058e702ae67 100755 --- a/tools/dependency_check.sh +++ b/tools/dependency_check.sh @@ -24,6 +24,6 @@ function fail() { # These binaries are required to 'make test' # mysqld might be in /usr/sbin which will not be in the default PATH PATH="/usr/sbin:$PATH" -for binary in mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do +for binary in k3s mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/contributing/build-from-source for install instructions." done; From d7f03d268e709c49956c867970a37fc9f1a4281f Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 27 Mar 2020 12:46:33 -0600 Subject: [PATCH 4/5] Move sonar test from master to a special branch Signed-off-by: Morgan Tocker --- .github/workflows/sonar_analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sonar_analysis.yml b/.github/workflows/sonar_analysis.yml index f901499a853..3381d93b223 100644 --- a/.github/workflows/sonar_analysis.yml +++ b/.github/workflows/sonar_analysis.yml @@ -2,7 +2,7 @@ name: sonar_analysis on: push: branches: - - 'master' + - 'sonartest' jobs: build: From 45d92a41597dacacd80edfdc61047e234648cb3a Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 31 Mar 2020 20:12:23 -0600 Subject: [PATCH 5/5] Kick CI (job is queued eternally) Signed-off-by: Morgan Tocker