cli: unskip test tenant zip test

This patch unskips and re-records the datadriven `TestTenantZip` as it
was fixed in cockroachdb#96553, but was not unskipped or recorded. The test was run
locally using `--stress` and did not flake:
```
101 runs so far, 0 failures, over 5m0s
```

Fixes cockroachdb#87141

Release note: None

pkg/cli/testdata/zip/testzip

@@ -78,7 +78,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] requesting tenant ranges: creating error output: debug/tenant_ranges.err.txt... done
 [cluster] requesting CPU profiles
 [cluster] profiles generated
-[cluster] profile for node 1... writing binary output: debug//nodes/1/cpu.pprof... done
+[cluster] profile for node 1... writing binary output: debug/nodes/1/cpu.pprof... done
 [node 1] node status... converting to JSON... writing binary output: debug/nodes/1/status.json... done
 [node 1] using SQL connection URL: postgresql://...
 [node 1] retrieving SQL data for crdb_internal.active_range_feeds... writing output: debug/nodes/1/crdb_internal.active_range_feeds.txt... done

pkg/cli/testdata/zip/testzip_tenant

@@ -1,19 +1,13 @@
 zip
 ----
 debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
+[cluster] discovering tenants on cluster... done
+[cluster] creating output file /dev/null... done
 [cluster] establishing RPC connection to ...
-[cluster] retrieving the node status to get the SQL address... done
 [cluster] using SQL address: ...
-[cluster] creating output file /dev/null... done
-[cluster] requesting data for debug/events... received response...
-[cluster] requesting data for debug/events: last request failed: rpc error: ...
-[cluster] requesting data for debug/events: creating error output: debug/events.json.err.txt... done
-[cluster] requesting data for debug/rangelog... received response...
-[cluster] requesting data for debug/rangelog: last request failed: rpc error: ...
-[cluster] requesting data for debug/rangelog: creating error output: debug/rangelog.json.err.txt... done
-[cluster] requesting data for debug/settings... received response...
-[cluster] requesting data for debug/settings: last request failed: rpc error: ...
-[cluster] requesting data for debug/settings: creating error output: debug/settings.json.err.txt... done
+[cluster] requesting data for debug/events... received response... converting to JSON... writing binary output: debug/events.json... done
+[cluster] requesting data for debug/rangelog... received response... converting to JSON... writing binary output: debug/rangelog.json... done
+[cluster] requesting data for debug/settings... received response... converting to JSON... writing binary output: debug/settings.json... done
 [cluster] requesting data for debug/reports/problemranges... received response...
 [cluster] requesting data for debug/reports/problemranges: last request failed: rpc error: ...
 [cluster] requesting data for debug/reports/problemranges: creating error output: debug/reports/problemranges.json.err.txt... done
@@ -30,6 +24,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for crdb_internal.cluster_sessions... writing output: debug/crdb_internal.cluster_sessions.txt... done
 [cluster] retrieving SQL data for crdb_internal.cluster_settings... writing output: debug/crdb_internal.cluster_settings.txt... done
 [cluster] retrieving SQL data for crdb_internal.cluster_transactions... writing output: debug/crdb_internal.cluster_transactions.txt... done
+[cluster] retrieving SQL data for crdb_internal.cluster_txn_execution_insights... writing output: debug/crdb_internal.cluster_txn_execution_insights.txt... done
 [cluster] retrieving SQL data for crdb_internal.default_privileges... writing output: debug/crdb_internal.default_privileges.txt... done
 [cluster] retrieving SQL data for crdb_internal.index_usage_statistics... writing output: debug/crdb_internal.index_usage_statistics.txt... done
 [cluster] retrieving SQL data for crdb_internal.invalid_objects... writing output: debug/crdb_internal.invalid_objects.txt... done
@@ -55,7 +50,6 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for system.descriptor... writing output: debug/system.descriptor.txt... done
 [cluster] retrieving SQL data for system.eventlog... writing output: debug/system.eventlog.txt... done
 [cluster] retrieving SQL data for system.external_connections... writing output: debug/system.external_connections.txt... done
-[cluster] retrieving SQL data for system.job_info... writing output: debug/system.job_info.txt... done
 [cluster] retrieving SQL data for system.jobs... writing output: debug/system.jobs.txt... done
 [cluster] retrieving SQL data for system.lease... writing output: debug/system.lease.txt... done
 [cluster] retrieving SQL data for system.locations... writing output: debug/system.locations.txt... done
@@ -82,19 +76,25 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for system.statement_diagnostics... writing output: debug/system.statement_diagnostics.txt... done
 [cluster] retrieving SQL data for system.statement_diagnostics_requests... writing output: debug/system.statement_diagnostics_requests.txt... done
 [cluster] retrieving SQL data for system.table_statistics... writing output: debug/system.table_statistics.txt... done
+[cluster] retrieving SQL data for system.task_payloads... writing output: debug/system.task_payloads.txt...
+[cluster] retrieving SQL data for system.task_payloads: last request failed: ERROR: relation "system.task_payloads" does not exist (SQLSTATE 42P01)
+[cluster] retrieving SQL data for system.task_payloads: creating error output: debug/system.task_payloads.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenant_settings... writing output: debug/system.tenant_settings.txt...
 [cluster] retrieving SQL data for system.tenant_settings: last request failed: ERROR: relation "system.tenant_settings" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenant_settings: creating error output: debug/system.tenant_settings.txt.err.txt... done
+[cluster] retrieving SQL data for system.tenant_tasks... writing output: debug/system.tenant_tasks.txt...
+[cluster] retrieving SQL data for system.tenant_tasks: last request failed: ERROR: relation "system.tenant_tasks" does not exist (SQLSTATE 42P01)
+[cluster] retrieving SQL data for system.tenant_tasks: creating error output: debug/system.tenant_tasks.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenant_usage... writing output: debug/system.tenant_usage.txt...
 [cluster] retrieving SQL data for system.tenant_usage: last request failed: ERROR: relation "system.tenant_usage" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenant_usage: creating error output: debug/system.tenant_usage.txt.err.txt... done
 [cluster] retrieving SQL data for system.tenants... writing output: debug/system.tenants.txt...
 [cluster] retrieving SQL data for system.tenants: last request failed: ERROR: relation "system.tenants" does not exist (SQLSTATE 42P01)
 [cluster] retrieving SQL data for system.tenants: creating error output: debug/system.tenants.txt.err.txt... done
-[cluster] requesting nodes... received response... converting to JSON... writing binary output: debug/nodes.json... done
-[cluster] requesting liveness... received response...
-[cluster] requesting liveness: last request failed: rpc error: ...
-[cluster] requesting liveness: creating error output: debug/liveness.json.err.txt... done
+[cluster] requesting nodes... received response...
+[cluster] requesting nodes: last request failed: rpc error: ...
+[cluster] requesting nodes: creating error output: debug/nodes.json.err.txt... done
+[cluster] requesting liveness... received response... converting to JSON... writing binary output: debug/liveness.json... done
 [cluster] requesting tenant ranges... received response...
 [cluster] requesting tenant ranges: last request failed: rpc error: ...
 [cluster] requesting tenant ranges: creating error output: debug/tenant_ranges.err.txt... done
@@ -120,13 +120,18 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] retrieving SQL data for crdb_internal.node_distsql_flows... writing output: debug/nodes/1/crdb_internal.node_distsql_flows.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_execution_insights... writing output: debug/nodes/1/crdb_internal.node_execution_insights.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_inflight_trace_spans... writing output: debug/nodes/1/crdb_internal.node_inflight_trace_spans.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_memory_monitors... writing output: debug/nodes/1/crdb_internal.node_memory_monitors.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_metrics... writing output: debug/nodes/1/crdb_internal.node_metrics.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_queries... writing output: debug/nodes/1/crdb_internal.node_queries.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_runtime_info... writing output: debug/nodes/1/crdb_internal.node_runtime_info.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_sessions... writing output: debug/nodes/1/crdb_internal.node_sessions.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_statement_statistics... writing output: debug/nodes/1/crdb_internal.node_statement_statistics.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache... writing output: debug/nodes/1/crdb_internal.node_tenant_capabilities_cache.txt...
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache: last request failed: ERROR: operation node_tenant_capabilities_cache supported only by system tenant (SQLSTATE XXUUU)
+[node 1] retrieving SQL data for crdb_internal.node_tenant_capabilities_cache: creating error output: debug/nodes/1/crdb_internal.node_tenant_capabilities_cache.txt.err.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_transaction_statistics... writing output: debug/nodes/1/crdb_internal.node_transaction_statistics.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_transactions... writing output: debug/nodes/1/crdb_internal.node_transactions.txt... done
+[node 1] retrieving SQL data for crdb_internal.node_txn_execution_insights... writing output: debug/nodes/1/crdb_internal.node_txn_execution_insights.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_txn_stats... writing output: debug/nodes/1/crdb_internal.node_txn_stats.txt... done
 [node 1] requesting data for debug/nodes/1/details... received response... converting to JSON... writing binary output: debug/nodes/1/details.json... done
 [node 1] requesting data for debug/nodes/1/gossip... received response...
@@ -141,10 +146,9 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] requesting heap file list... received response... done
 [node ?] ? heap profiles found
 [node 1] requesting goroutine dump list... received response... done
-[node 1] 0 goroutine dumps found
-[node 1] requesting log file ...
-[node 1] requesting log file ...
+[node ?] ? goroutine dumps found
 [node 1] requesting log file ...
+[node 1] 0 log file ...
 [node 1] requesting ranges... received response...
 [node 1] requesting ranges: last request failed: rpc error: ...
 [node 1] requesting ranges: creating error output: debug/nodes/1/ranges.err.txt... done

pkg/cli/zip_per_node.go

@@ -133,7 +133,7 @@ func (zc *debugZipContext) collectCPUProfiles(
 			continue // skipped node
 		}
 		nodeID := nodeList[i].NodeID
-		prefix := fmt.Sprintf("%s/%s/%s", zc.prefix, nodesPrefix, fmt.Sprintf("%d", nodeID))
+		prefix := fmt.Sprintf("%s%s/%s", zc.prefix, nodesPrefix, fmt.Sprintf("%d", nodeID))
 		s := zc.clusterPrinter.start("profile for node %d", nodeID)
 		if err := zc.z.createRawOrError(s, prefix+"/cpu.pprof", pd.data, pd.err); err != nil {
 			return err

pkg/cli/zip_tenant_test.go

@@ -31,7 +31,6 @@ var _ = kvtenantccl.Connector{}
 // TestTenantZip tests the operation of zip for a tenant server.
 func TestTenantZip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
-	skip.WithIssue(t, 89192, "flaky test")
 
 	skip.UnderRace(t, "test too slow under race")
 	tenantDir, tenantDirCleanupFn := testutils.TempDir(t)

pkg/cli/zip_test.go

@@ -365,7 +365,6 @@ func eraseNonDeterministicZipOutput(out string) string {
 	re = regexp.MustCompile(`(?m)^\[node \d+\] writing dump.*$` + "\n")
 	out = re.ReplaceAllString(out, ``)
 
-	//out = strings.ReplaceAll(out, "\n\n", "\n")
 	return out
 }