jobs: add verbose logging of progress to RunAndWaitForTerminalState

Hook it up to the various acceptanceccl tests. Also only special case Restore2TB onto gce, which is a bug introduced in #17956. The others should remain on Azure. Also also change 10 to 15 nodes on the right test. Whoops!
cockroachdb · Aug 28, 2017 · 66c6d7a · 66c6d7a
1 parent 9857a53
commit 66c6d7a
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 11 deletions.
diff --git a/build/teamcity-nightly-acceptance.sh b/build/teamcity-nightly-acceptance.sh
@@ -63,10 +63,15 @@ case $TESTNAME in
     TESTTIMEOUT=6h
     COCKROACH_EXTRA_FLAGS+=' -nodes 4'
     ;;
-  BenchmarkRestoreBig|BenchmarkRestoreTPCH10/numNodes=1|BenchmarkRestoreTPCH10/numNodes=3|BenchmarkRestoreTPCH10/numNodes=10|BenchmarkRestore2TB|BenchmarkBackup2TB)
+  BenchmarkRestoreBig|BenchmarkRestoreTPCH10/numNodes=1|BenchmarkRestoreTPCH10/numNodes=3|BenchmarkRestoreTPCH10/numNodes=10|BenchmarkBackup2TB)
     PKG=./pkg/ccl/acceptanceccl
     TESTTIMEOUT=2h
-    COCKROACH_EXTRA_FLAGS+=" -tf.cockroach-env=COCKROACH_PREEMPTIVE_SNAPSHOT_RATE=8388608 -tf.storage-location=westus -cwd=$PWD/pkg/acceptance/terraform/gce"
+    COCKROACH_EXTRA_FLAGS+=" -tf.cockroach-env=COCKROACH_PREEMPTIVE_SNAPSHOT_RATE=8388608 -tf.storage-location=westus --vmodule=jobs=1"
+    ;;
+  BenchmarkRestore2TB)
+    PKG=./pkg/ccl/acceptanceccl
+    TESTTIMEOUT=2h
+    COCKROACH_EXTRA_FLAGS+=" -tf.cockroach-env=COCKROACH_PREEMPTIVE_SNAPSHOT_RATE=8388608 -tf.storage-location=westus -cwd=$PWD/pkg/acceptance/terraform/gce --vmodule=jobs=1"
     ;;
   *)
     echo "unknown test name $TESTNAME"

diff --git a/pkg/ccl/acceptanceccl/backup_test.go b/pkg/ccl/acceptanceccl/backup_test.go
@@ -314,8 +314,11 @@ func BenchmarkRestore2TB(b *testing.B) {
 	}
 
 	bt := benchmarkTest{
-		b:      b,
-		nodes:  10,
+		b: b,
+		// TODO(dan): Switch this back to 10 machines when this test goes back
+		// to Azure. Each GCE machine disk is 375GB, so with our current need
+		// for 2x the restore size in disk space, 2tb unless we up this a bit.
+		nodes:  15,
 		prefix: "restore2tb",
 	}
 
@@ -361,11 +364,8 @@ func BenchmarkBackup2TB(b *testing.B) {
 	}
 
 	bt := benchmarkTest{
-		b: b,
-		// TODO(dan): Switch this back to 10 machines when this test goes back
-		// to Azure. Each GCE machine disk is 375GB, so with our current need
-		// for 2x the restore size in disk space, 2tb unless we up this a bit.
-		nodes:           15,
+		b:               b,
+		nodes:           10,
 		storeFixture:    acceptance.FixtureURL(bulkArchiveStoreFixture),
 		prefix:          "backup2tb",
 		skipClusterInit: true,

diff --git a/pkg/sql/jobs/jobs.go b/pkg/sql/jobs/jobs.go
@@ -717,13 +717,18 @@ func RunAndWaitForTerminalState(
 
 		var status Status
 		var jobErr gosql.NullString
-		err := sqlDB.QueryRow(`SELECT status, error FROM [SHOW JOBS] WHERE id = $1`, jobID).Scan(
-			&status, &jobErr,
+		var fractionCompleted float64
+		err := sqlDB.QueryRow(`SELECT status, error, fraction_completed FROM [SHOW JOBS] WHERE id = $1`, jobID).Scan(
+			&status, &jobErr, &fractionCompleted,
 		)
 		if err != nil {
 			return jobID, "", errors.Wrapf(err, "getting status of job %d", jobID)
 		}
 		if !status.Terminal() {
+			if log.V(1) {
+				log.Infof(ctx, "job %d: status=%s, progress=%0.3f, created %s ago",
+					jobID, status, fractionCompleted, timeutil.Since(begin))
+			}
 			continue
 		}
 		if jobErr.Valid && len(jobErr.String) > 0 {