workload/schemachange: fix watchdog stack dumping behaviour

Previously, the watch dog threads inside the schema
changer workload could dump the stacks way incorrectly,
since it didn't give connections time to terminate. This
change adapts the watch dog logic to have a delay.

Release note: None

pkg/ccl/testccl/workload/schemachange/BUILD.bazel

@@ -3,11 +3,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_test(
     name = "schemachange_test",
+    size = "large",
     srcs = [
         "main_test.go",
         "schema_change_external_test.go",
     ],
-    args = ["-test.timeout=295s"],
+    args = ["-test.timeout=895s"],
     data = [
         "//c-deps:libgeos",
     ],

pkg/workload/schemachange/watch_dog.go

@@ -93,7 +93,15 @@ func (w *schemaChangeWatchDog) watchLoop(ctx context.Context) {
 			close(responseChannel)
 			return
 		case <-ctx.Done():
-			panic("dumping stacks, we failed to terminate threads on time.")
+			// Give the connections a small amount of time to clean up, if they fail
+			// to do so, we will dump stacks.
+			select {
+			case <-w.cmdChannel:
+				return
+			case <-time.After(time.Second * 4):
+				panic("dumping stacks, we failed to terminate threads on time.")
+
+			}
 		case <-time.After(time.Second):
 			// If the connection is making progress, the watch dog timer can be reset
 			// again.