diff --git a/pkg/cmd/roachtest/clusterstats/helpers.go b/pkg/cmd/roachtest/clusterstats/helpers.go index 2cf45b69df3f..721f35e41e73 100644 --- a/pkg/cmd/roachtest/clusterstats/helpers.go +++ b/pkg/cmd/roachtest/clusterstats/helpers.go @@ -17,7 +17,6 @@ import ( "sort" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/prometheus" "github.com/cockroachdb/cockroach/pkg/util/search" @@ -37,12 +36,7 @@ func SetupCollectorPromClient( if err != nil { return nil, err } - httpclient, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, l, c.Node(1), fmt.Sprintf("http://%s:9090/api/v1/query", prometheusNodeIP[0])) - if err != nil { - return nil, err - } client, err := promapi.NewClient(promapi.Config{ - Client: &httpclient, Address: fmt.Sprintf("http://%s:9090", prometheusNodeIP[0]), }) if err != nil { diff --git a/pkg/cmd/roachtest/roachtestutil/utils.go b/pkg/cmd/roachtest/roachtestutil/utils.go index 711c5c6628a8..0b44d6940f98 100644 --- a/pkg/cmd/roachtest/roachtestutil/utils.go +++ b/pkg/cmd/roachtest/roachtestutil/utils.go @@ -39,9 +39,13 @@ func SystemInterfaceSystemdUnitName() string { // DefaultPGUrl is a wrapper over roachprod.PgUrl that calls it with the arguments // that *almost* all roachtests want: single tenant and only a single node. func DefaultPGUrl( - ctx context.Context, c cluster.Cluster, l *logger.Logger, node option.NodeListOption, + ctx context.Context, + c cluster.Cluster, + l *logger.Logger, + node option.NodeListOption, + auth install.PGAuthMode, ) (string, error) { - opts := roachprod.PGURLOptions{Secure: c.IsSecure()} + opts := roachprod.PGURLOptions{Auth: auth, Secure: c.IsSecure()} pgurl, err := roachprod.PgURL(ctx, l, c.MakeNodes(node), "certs", opts) if err != nil { return "", err diff --git a/pkg/cmd/roachtest/tests/activerecord.go b/pkg/cmd/roachtest/tests/activerecord.go index dbf23553dd8e..95ab0e1f161d 100644 --- a/pkg/cmd/roachtest/tests/activerecord.go +++ b/pkg/cmd/roachtest/tests/activerecord.go @@ -51,7 +51,8 @@ func registerActiveRecord(r registry.Registry) { t.Status("setting up cockroach") startOpts := option.DefaultStartOptsInMemory() startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + // Activerecord uses root user with ssl disabled. + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { diff --git a/pkg/cmd/roachtest/tests/admission_control_elastic_io.go b/pkg/cmd/roachtest/tests/admission_control_elastic_io.go index 6ae40aa1fc78..cdf0a98fb9ae 100644 --- a/pkg/cmd/roachtest/tests/admission_control_elastic_io.go +++ b/pkg/cmd/roachtest/tests/admission_control_elastic_io.go @@ -65,10 +65,6 @@ func registerElasticIO(r registry.Registry) { WithGrafanaDashboardJSON(grafana.ChangefeedAdmissionControlGrafana) err := c.StartGrafana(ctx, t.L(), promCfg) require.NoError(t, err) - promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg) - require.NoError(t, err) - statCollector := clusterstats.NewStatsCollector(ctx, promClient) - c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(workAndPromNode)) startOpts := option.DefaultStartOptsNoBackups() roachtestutil.SetDefaultAdminUIPort(c, &startOpts.RoachprodOpts) @@ -76,6 +72,9 @@ func registerElasticIO(r registry.Registry) { "--vmodule=io_load_listener=2") settings := install.MakeClusterSettings() c.Start(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes)) + promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg) + require.NoError(t, err) + statCollector := clusterstats.NewStatsCollector(ctx, promClient) setAdmissionControl(ctx, t, c, true) duration := 30 * time.Minute t.Status("running workload") @@ -85,7 +84,7 @@ func registerElasticIO(r registry.Registry) { url := fmt.Sprintf(" {pgurl:1-%d}", crdbNodes) cmd := "./workload run kv --init --histograms=perf/stats.json --concurrency=512 " + "--splits=1000 --read-percent=0 --min-block-bytes=65536 --max-block-bytes=65536 " + - "--txn-qos=background --tolerate-errors" + dur + url + "--txn-qos=background --tolerate-errors --secure" + dur + url c.Run(ctx, option.WithNodes(c.Node(workAndPromNode)), cmd) return nil }) diff --git a/pkg/cmd/roachtest/tests/admission_control_intent_resolution.go b/pkg/cmd/roachtest/tests/admission_control_intent_resolution.go index de66ebba16f9..a74aaa3f0a96 100644 --- a/pkg/cmd/roachtest/tests/admission_control_intent_resolution.go +++ b/pkg/cmd/roachtest/tests/admission_control_intent_resolution.go @@ -64,17 +64,18 @@ func registerIntentResolutionOverload(r registry.Registry) { WithGrafanaDashboardJSON(grafana.ChangefeedAdmissionControlGrafana) err := c.StartGrafana(ctx, t.L(), promCfg) require.NoError(t, err) - promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg) - require.NoError(t, err) - statCollector := clusterstats.NewStatsCollector(ctx, promClient) startOpts := option.DefaultStartOptsNoBackups() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=io_load_listener=2") - roachtestutil.SetDefaultSQLPort(c, &startOpts.RoachprodOpts) roachtestutil.SetDefaultAdminUIPort(c, &startOpts.RoachprodOpts) settings := install.MakeClusterSettings() c.Start(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes)) + + promClient, err := clusterstats.SetupCollectorPromClient(ctx, c, t.L(), promCfg) + require.NoError(t, err) + statCollector := clusterstats.NewStatsCollector(ctx, promClient) + setAdmissionControl(ctx, t, c, true) t.Status("running txn") m := c.NewMonitor(ctx, c.Range(1, crdbNodes)) diff --git a/pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go b/pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go index a87ddec96b80..55727c52392b 100644 --- a/pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go +++ b/pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go @@ -124,7 +124,7 @@ func verifyNodeLiveness( if err := retry.WithMaxAttempts(ctx, retry.Options{ MaxBackoff: 500 * time.Millisecond, }, 60, func() (err error) { - response, err = getMetrics(ctx, adminURLs[0], now.Add(-runDuration), now, []tsQuery{ + response, err = getMetrics(ctx, c, t, adminURLs[0], now.Add(-runDuration), now, []tsQuery{ { name: "cr.node.liveness.heartbeatfailures", queryType: total, diff --git a/pkg/cmd/roachtest/tests/allocator.go b/pkg/cmd/roachtest/tests/allocator.go index 67e7dea59562..702e0416fa89 100644 --- a/pkg/cmd/roachtest/tests/allocator.go +++ b/pkg/cmd/roachtest/tests/allocator.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/clusterstats" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -48,17 +47,12 @@ func registerAllocator(r registry.Registry) { db := c.Conn(ctx, t.L(), 1) defer db.Close() - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - m := c.NewMonitor(ctx, c.Range(1, start)) m.Go(func(ctx context.Context) error { t.Status("loading fixture") if err := c.RunE( ctx, option.WithNodes(c.Node(1)), - "./cockroach", "workload", "fixtures", "import", "tpch", "--scale-factor", "10", pgurl, + "./cockroach", "workload", "fixtures", "import", "tpch", "--scale-factor", "10", "{pgurl:1}", ); err != nil { t.Fatal(err) } @@ -73,7 +67,7 @@ func registerAllocator(r registry.Registry) { WithCluster(clusNodes.InstallNodes()). WithPrometheusNode(promNode.InstallNodes()[0]) - err = c.StartGrafana(ctx, t.L(), cfg) + err := c.StartGrafana(ctx, t.L(), cfg) require.NoError(t, err) cleanupFunc := func() { @@ -91,7 +85,7 @@ func registerAllocator(r registry.Registry) { // Start the remaining nodes to kick off upreplication/rebalancing. c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(start+1, nodes)) - c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf("./cockroach workload init kv --drop '%s'", pgurl)) + c.Run(ctx, option.WithNodes(c.Node(1)), "./cockroach workload init kv --drop {pgurl:1}") for node := 1; node <= nodes; node++ { node := node // TODO(dan): Ideally, the test would fail if this queryload failed, @@ -457,13 +451,9 @@ FROM crdb_internal.kv_store_status t.Fatalf("expected 0 mis-replicated ranges, but found %d", n) } - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } decom := func(id int) { c.Run(ctx, option.WithNodes(c.Node(1)), - fmt.Sprintf("./cockroach node decommission --insecure --url=%s --wait=none %d", pgurl, id)) + fmt.Sprintf("./cockroach node decommission --certs-dir=certs --port={pgport%s} --wait=none %d", c.Node(id), id)) } // Decommission a node. The ranges should down-replicate to 7 replicas. diff --git a/pkg/cmd/roachtest/tests/alterpk.go b/pkg/cmd/roachtest/tests/alterpk.go index 942b067556c5..f0ab215feda1 100644 --- a/pkg/cmd/roachtest/tests/alterpk.go +++ b/pkg/cmd/roachtest/tests/alterpk.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -105,14 +104,9 @@ func registerAlterPK(r registry.Registry) { const duration = 10 * time.Minute roachNodes, loadNode := setupTest(ctx, t, c) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1)) - if err != nil { - t.Fatal(err) - } cmd := fmt.Sprintf( - "./cockroach workload fixtures import tpcc --warehouses=%d --db=tpcc '%s'", + "./cockroach workload fixtures import tpcc --warehouses=%d --db=tpcc {pgurl:1}", warehouses, - pgurl, ) if err := c.RunE(ctx, option.WithNodes(c.Node(roachNodes[0])), cmd); err != nil { t.Fatal(err) diff --git a/pkg/cmd/roachtest/tests/asyncpg.go b/pkg/cmd/roachtest/tests/asyncpg.go index f0edb349509f..a0f9016ce546 100644 --- a/pkg/cmd/roachtest/tests/asyncpg.go +++ b/pkg/cmd/roachtest/tests/asyncpg.go @@ -12,6 +12,7 @@ package tests import ( "context" + "fmt" "regexp" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" @@ -22,11 +23,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/install" ) -const asyncpgRunTestCmd = ` -source venv/bin/activate && +var asyncpgRunTestCmd = fmt.Sprintf(` +source venv/bin/activate && cd /mnt/data1/asyncpg && -PGPORT={pgport:1} PGHOST=localhost PGUSER=test_admin PGDATABASE=defaultdb python3 setup.py test > asyncpg.stdout -` +PGPORT={pgport:1} PGHOST=localhost PGUSER=%s PGPASSWORD=%s PGSSLROOTCERT=$HOME/certs/ca.crt PGSSLMODE=require PGDATABASE=defaultdb python3 setup.py test > asyncpg.stdout +`, install.DefaultUser, install.DefaultPassword) var asyncpgReleaseTagRegex = regexp.MustCompile(`^(?Pv\d+)\.(?P\d+)\.(?P\d+)$`) diff --git a/pkg/cmd/roachtest/tests/asyncpg_blocklist.go b/pkg/cmd/roachtest/tests/asyncpg_blocklist.go index ea9c63423330..c22cc398faa5 100644 --- a/pkg/cmd/roachtest/tests/asyncpg_blocklist.go +++ b/pkg/cmd/roachtest/tests/asyncpg_blocklist.go @@ -38,6 +38,9 @@ var asyncpgBlocklist = blocklist{ `test_codecs.TestCodecs.test_unknown_type_text_fallback`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/54516", `test_codecs.TestCodecs.test_void`: "unknown", `test_connect.TestSettings.test_get_settings_01`: "unknown", + `test_copy.TestCopyFrom.test_copy_from_query_cancellation_explicit`: "known asyncpg ssl issue - https://github.com/MagicStack/asyncpg/issues/240", + `test_copy.TestCopyFrom.test_copy_from_query_timeout_1`: "known asyncpg ssl issue - https://github.com/MagicStack/asyncpg/issues/240", + `test_copy.TestCopyFrom.test_copy_from_query_to_sink`: "known asyncpg ssl issue - https://github.com/MagicStack/asyncpg/issues/240", `test_copy.TestCopyFrom.test_copy_from_table_basics`: "no support for COPY TO - https://github.com/cockroachdb/cockroach/issues/85571", `test_copy.TestCopyTo.test_copy_to_table_basics`: "unknown", `test_cursor.TestCursor.test_cursor_02`: "unknown", @@ -60,7 +63,6 @@ var asyncpgBlocklist = blocklist{ `test_introspection.TestIntrospection.test_introspection_on_large_db`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/22456", `test_introspection.TestIntrospection.test_introspection_retries_after_cache_bust`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/27796", `test_introspection.TestIntrospection.test_introspection_sticks_for_ps`: "unknown type: pg_catalog.json", - `test_listeners.TestListeners.test_dangling_listener_warns`: "LISTEN - https://github.com/cockroachdb/cockroach/issues/41522", `test_listeners.TestListeners.test_listen_01`: "LISTEN - https://github.com/cockroachdb/cockroach/issues/41522", `test_listeners.TestListeners.test_listen_02`: "LISTEN - https://github.com/cockroachdb/cockroach/issues/41522", `test_listeners.TestListeners.test_listen_notletters`: "LISTEN - https://github.com/cockroachdb/cockroach/issues/41522", @@ -68,7 +70,6 @@ var asyncpgBlocklist = blocklist{ `test_listeners.TestLogListeners.test_log_listener_02`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/17511", `test_listeners.TestLogListeners.test_log_listener_03`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/17511", `test_pool.TestPool.test_pool_remote_close`: "unsupported pg_terminate_backend() function", - `test_prepare.TestPrepare.test_prepare_08_big_result`: "unknown", `test_prepare.TestPrepare.test_prepare_09_raise_error`: "unsupported feature - https://github.com/cockroachdb/cockroach/issues/17511", `test_prepare.TestPrepare.test_prepare_14_explain`: "unknown", `test_prepare.TestPrepare.test_prepare_16_command_result`: "unknown", diff --git a/pkg/cmd/roachtest/tests/autoupgrade.go b/pkg/cmd/roachtest/tests/autoupgrade.go index bf4bb792d23e..35ff010efb53 100644 --- a/pkg/cmd/roachtest/tests/autoupgrade.go +++ b/pkg/cmd/roachtest/tests/autoupgrade.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/testutils" @@ -74,12 +73,8 @@ func registerAutoUpgrade(r registry.Registry) { decommissionAndStop := func(node int) error { t.WorkerStatus("decommission") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(node)) - if err != nil { - return err - } if err := c.RunE(ctx, option.WithNodes(c.Node(node)), - fmt.Sprintf("./cockroach node decommission %d --insecure --url=%s", node, pgurl)); err != nil { + fmt.Sprintf("./cockroach node decommission %d --certs-dir=certs --port={pgport%s}", node, c.Node(node))); err != nil { return err } t.WorkerStatus("stop") diff --git a/pkg/cmd/roachtest/tests/awsdms.go b/pkg/cmd/roachtest/tests/awsdms.go index 1cda78eb1b5f..a81dcd5f0e5e 100644 --- a/pkg/cmd/roachtest/tests/awsdms.go +++ b/pkg/cmd/roachtest/tests/awsdms.go @@ -766,11 +766,9 @@ func setupDMSEndpointsAndTask( PostgreSQLSettings: &dmstypes.PostgreSQLSettings{ DatabaseName: proto.String(awsdmsCRDBDatabase), Username: proto.String(awsdmsCRDBUser), - // Password is a required field, but CockroachDB doesn't take passwords in - // --insecure mode. As such, put in some garbage. - Password: proto.String("garbage"), - Port: proto.Int32(26257), - ServerName: proto.String(externalCRDBAddr[0]), + Password: proto.String(awsdmsPassword), + Port: proto.Int32(26257), + ServerName: proto.String(externalCRDBAddr[0]), }, }, endpoint: dmsEndpoints.defaultTarget, diff --git a/pkg/cmd/roachtest/tests/backup.go b/pkg/cmd/roachtest/tests/backup.go index 55e249cdac34..12cc64d6337b 100644 --- a/pkg/cmd/roachtest/tests/backup.go +++ b/pkg/cmd/roachtest/tests/backup.go @@ -335,15 +335,11 @@ func registerBackup(r registry.Registry) { m := c.NewMonitor(ctx) m.Go(func(ctx context.Context) error { t.Status(`running backup`) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1)) - if err != nil { - return err - } // Tick once before starting the backup, and once after to capture the // total elapsed time. This is used by roachperf to compute and display // the average MB/sec per node. tick() - c.Run(ctx, option.WithNodes(c.Node(1)), `./cockroach sql --insecure --url=`+pgurl+` -e " + c.Run(ctx, option.WithNodes(c.Node(1)), `./cockroach sql --url={pgurl:1} -e " BACKUP bank.bank TO 'gs://`+backupTestingBucket+`/`+dest+`?AUTH=implicit'"`) tick() diff --git a/pkg/cmd/roachtest/tests/build_info.go b/pkg/cmd/roachtest/tests/build_info.go index 232d3c7ae9a0..6c9afee53667 100644 --- a/pkg/cmd/roachtest/tests/build_info.go +++ b/pkg/cmd/roachtest/tests/build_info.go @@ -12,11 +12,11 @@ package tests import ( "context" - "net/http" "strings" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/server/serverpb" @@ -33,7 +33,11 @@ func RunBuildInfo(ctx context.Context, t test.Test, c cluster.Cluster) { t.Fatal(err) } url := `http://` + adminUIAddrs[0] + `/_status/details/local` - err = httputil.GetJSON(http.Client{}, url, &details) + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.Node(1), url) + if err != nil { + t.Fatal(err) + } + err = httputil.GetJSON(client, url, &details) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/clearrange.go b/pkg/cmd/roachtest/tests/clearrange.go index 31625bf6421d..76a2b8e4f9ad 100644 --- a/pkg/cmd/roachtest/tests/clearrange.go +++ b/pkg/cmd/roachtest/tests/clearrange.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -66,16 +65,12 @@ func registerClearRange(r registry.Registry) { func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressiveChecks bool) { t.Status("restoring fixture") c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } m := c.NewMonitor(ctx) m.Go(func(ctx context.Context) error { // NB: on a 10 node cluster, this should take well below 3h. tBegin := timeutil.Now() c.Run(ctx, option.WithNodes(c.Node(1)), "./cockroach", "workload", "fixtures", "import", "bank", - "--payload-bytes=10240", "--ranges=10", "--rows=65104166", "--seed=4", "--db=bigbank", pgurl) + "--payload-bytes=10240", "--ranges=10", "--rows=65104166", "--seed=4", "--db=bigbank", "{pgurl:1}") t.L().Printf("import took %.2fs", timeutil.Since(tBegin).Seconds()) return nil }) @@ -104,10 +99,9 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi // Use a 120s connect timeout to work around the fact that the server will // declare itself ready before it's actually 100% ready. See: // https://github.com/cockroachdb/cockroach/issues/34897#issuecomment-465089057 - c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf( - `COCKROACH_CONNECT_TIMEOUT=120 ./cockroach sql --url=%s --insecure -e "DROP DATABASE IF EXISTS tinybank"`, pgurl)) + c.Run(ctx, option.WithNodes(c.Node(1)), `COCKROACH_CONNECT_TIMEOUT=120 ./cockroach sql --url={pgurl:1} -e "DROP DATABASE IF EXISTS tinybank"`) c.Run(ctx, option.WithNodes(c.Node(1)), "./cockroach", "workload", "fixtures", "import", "bank", "--db=tinybank", - "--payload-bytes=100", "--ranges=10", "--rows=800", "--seed=1", pgurl) + "--payload-bytes=100", "--ranges=10", "--rows=800", "--seed=1", "{pgurl:1}") t.Status() @@ -139,7 +133,7 @@ ORDER BY raw_start_key ASC LIMIT 1`, }() m.Go(func(ctx context.Context) error { - c.Run(ctx, option.WithNodes(c.Node(1)), `./cockroach workload init kv`, pgurl) + c.Run(ctx, option.WithNodes(c.Node(1)), `./cockroach workload init kv {pgurl:1}`) c.Run(ctx, option.WithNodes(c.All()), fmt.Sprintf(`./cockroach workload run kv --concurrency=32 --duration=1h --tolerate-errors {pgurl%s}`, c.All())) return nil }) diff --git a/pkg/cmd/roachtest/tests/cli.go b/pkg/cmd/roachtest/tests/cli.go index 9f6165ed5539..31d54070a14b 100644 --- a/pkg/cmd/roachtest/tests/cli.go +++ b/pkg/cmd/roachtest/tests/cli.go @@ -47,7 +47,7 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { } nodeStatus := func() (_ string, _ []string, err error) { - result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(1)), "./cockroach node status --insecure -p {pgport:1}") + result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(1)), "./cockroach node status --certs-dir=certs -p {pgport:1}") if err != nil { return "", nil, err } diff --git a/pkg/cmd/roachtest/tests/cluster_init.go b/pkg/cmd/roachtest/tests/cluster_init.go index dd3d653e17fb..6c48e1b91a95 100644 --- a/pkg/cmd/roachtest/tests/cluster_init.go +++ b/pkg/cmd/roachtest/tests/cluster_init.go @@ -13,7 +13,6 @@ package tests import ( "context" gosql "database/sql" - "fmt" "io" "net/http" "strings" @@ -48,10 +47,14 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { // via the join targets. startOpts.RoachprodOpts.JoinTargets = c.All() + // Start the cluster in insecure mode to allow it to test both + // authenticated and unauthenticated code paths. + settings := install.MakeClusterSettings(install.SecureOption(false)) + for _, initNode := range []int{2, 1} { c.Wipe(ctx, false /* preserveCerts */) t.L().Printf("starting test with init node %d", initNode) - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings()) + c.Start(ctx, t.L(), startOpts, settings) urlMap := make(map[int]string) adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.All()) @@ -150,8 +153,7 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { } t.L().Printf("sending init command to node %d", initNode) - c.Run(ctx, option.WithNodes(c.Node(initNode)), - fmt.Sprintf(`./cockroach init --insecure --port={pgport:%d}`, initNode)) + c.Run(ctx, option.WithNodes(c.Node(initNode)), `./cockroach init --url={pgurl:1}`) // This will only succeed if 3 nodes joined the cluster. err = WaitFor3XReplication(ctx, t, dbs[0]) @@ -160,8 +162,7 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { execCLI := func(runNode int, extraArgs ...string) (string, error) { args := []string{"./cockroach"} args = append(args, extraArgs...) - args = append(args, "--insecure") - args = append(args, fmt.Sprintf("--port={pgport:%d}", runNode)) + args = append(args, "--url={pgurl:1}") result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(runNode)), args...) combinedOutput := result.Stdout + result.Stderr t.L().Printf("%s\n", combinedOutput) diff --git a/pkg/cmd/roachtest/tests/copyfrom.go b/pkg/cmd/roachtest/tests/copyfrom.go index c07fdb42e196..7ceb8967c969 100644 --- a/pkg/cmd/roachtest/tests/copyfrom.go +++ b/pkg/cmd/roachtest/tests/copyfrom.go @@ -146,7 +146,7 @@ func runCopyFromCRDB(ctx context.Context, t test.Test, c cluster.Cluster, sf int db, err := c.ConnE(ctx, t.L(), 1) require.NoError(t, err) stmts := []string{ - "CREATE USER importer", + "CREATE USER importer WITH PASSWORD '123'", fmt.Sprintf("ALTER ROLE importer SET copy_from_atomic_enabled = %t", atomic), } for _, stmt := range stmts { @@ -155,7 +155,7 @@ func runCopyFromCRDB(ctx context.Context, t test.Test, c cluster.Cluster, sf int t.Fatal(err) } } - urls, err := c.InternalPGUrl(ctx, t.L(), c.Node(1), roachprod.PGURLOptions{}) + urls, err := c.InternalPGUrl(ctx, t.L(), c.Node(1), roachprod.PGURLOptions{Auth: install.AuthUserPassword}) require.NoError(t, err) m := c.NewMonitor(ctx, c.All()) m.Go(func(ctx context.Context) error { @@ -163,10 +163,10 @@ func runCopyFromCRDB(ctx context.Context, t test.Test, c cluster.Cluster, sf int urlstr := strings.Replace(urls[0], "?", "/defaultdb?", 1) u, err := url.Parse(urlstr) require.NoError(t, err) - u.User = url.User("importer") + u.User = url.UserPassword("importer", "123") urlstr = u.String() - c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf("psql %s -c 'SELECT 1'", urlstr)) - c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf("psql %s -c '%s'", urlstr, lineitemSchema)) + c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf("psql '%s' -c 'SELECT 1'", urlstr)) + c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf("psql '%s' -c '%s'", urlstr, lineitemSchema)) runTest(ctx, t, c, fmt.Sprintf("psql '%s'", urlstr)) return nil }) diff --git a/pkg/cmd/roachtest/tests/decommission.go b/pkg/cmd/roachtest/tests/decommission.go index 6cb7f23bc780..ee22c01bdb57 100644 --- a/pkg/cmd/roachtest/tests/decommission.go +++ b/pkg/cmd/roachtest/tests/decommission.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/testutils" @@ -151,12 +150,7 @@ func runDrainAndDecommission( for i := 1; i <= nodes; i++ { c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(i)) } - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(pinnedNode)), - fmt.Sprintf(`./cockroach workload init kv --drop --splits 1000 '%s'`, pgurl)) + c.Run(ctx, option.WithNodes(c.Node(pinnedNode)), `./cockroach workload init kv --drop --splits 1000 {pgurl:1}`) run := func(stmt string) { db := c.Conn(ctx, t.L(), pinnedNode) @@ -206,11 +200,7 @@ func runDrainAndDecommission( m.Go(func() error { drain := func(id int) error { t.Status(fmt.Sprintf("draining node %d", id)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(id)) - if err != nil { - t.Fatal(err) - } - return c.RunE(ctx, option.WithNodes(c.Node(id)), fmt.Sprintf("./cockroach node drain --insecure --url=%s", pgurl)) + return c.RunE(ctx, option.WithNodes(c.Node(id)), fmt.Sprintf("./cockroach node drain --certs-dir=certs --port={pgport:%d}", id)) } return drain(id) }) @@ -224,11 +214,7 @@ func runDrainAndDecommission( id := nodes - 3 decom := func(id int) error { t.Status(fmt.Sprintf("decommissioning node %d", id)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(id)) - if err != nil { - t.Fatal(err) - } - return c.RunE(ctx, option.WithNodes(c.Node(id)), fmt.Sprintf("./cockroach node decommission --self --insecure --url=%s", pgurl)) + return c.RunE(ctx, option.WithNodes(c.Node(id)), fmt.Sprintf("./cockroach node decommission --self --certs-dir=certs --port={pgport:%d}", id)) } return decom(id) }) @@ -891,7 +877,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust t.L().Printf("wiping n%d and adding it back to the cluster as a new node\n", targetNode) c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(targetNode)) - c.Wipe(ctx, false /*preserveCerts */, c.Node(targetNode)) + c.Wipe(ctx, true /*preserveCerts */, c.Node(targetNode)) joinNode := h.getRandNode() internalAddrs, err := c.InternalAddr(ctx, t.L(), c.Node(joinNode)) @@ -1136,13 +1122,9 @@ func runDecommissionSlow(ctx context.Context, t test.Test, c cluster.Cluster) { m.Go(func(ctx context.Context) error { decom := func(id int) error { t.Status(fmt.Sprintf("decommissioning node %d", id)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(id)) - if err != nil { - t.Fatal(err) - } return c.RunE(ctx, option.WithNodes(c.Node(id)), - fmt.Sprintf("./cockroach node decommission %d --insecure --checks=skip --url=%s", id, pgurl), + fmt.Sprintf("./cockroach node decommission %d --checks=skip --certs-dir=certs --port={pgport:%d}", id, id), ) } return decom(id) @@ -1479,12 +1461,8 @@ func execCLI( ) (string, error) { args := []string{"./cockroach"} args = append(args, extraArgs...) - args = append(args, "--insecure") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(runNode)) - if err != nil { - t.Fatal(err) - } - args = append(args, fmt.Sprintf("--url=%s", pgurl)) + args = append(args, fmt.Sprintf("--port={pgport:%d}", runNode)) + args = append(args, "--certs-dir=certs") result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(runNode)), args...) t.L().Printf("%s\n", result.Stdout) return result.Stdout, err diff --git a/pkg/cmd/roachtest/tests/decommissionbench.go b/pkg/cmd/roachtest/tests/decommissionbench.go index fa4125885ecb..e3ed80e8c293 100644 --- a/pkg/cmd/roachtest/tests/decommissionbench.go +++ b/pkg/cmd/roachtest/tests/decommissionbench.go @@ -16,7 +16,6 @@ import ( gosql "database/sql" "encoding/json" "fmt" - t "log" "math" "os" "path/filepath" @@ -79,7 +78,7 @@ type decommissionBenchSpec struct { drainFirst bool // When true, the test will add a node to the cluster prior to decommission, - // so that the upreplication will overlap with the the decommission. + // so that the upreplication will overlap with the decommission. whileUpreplicating bool // When true, attempts to simulate decommissioning a node with high read @@ -401,11 +400,8 @@ func setupDecommissionBench( t.Status(fmt.Sprintf("initializing cluster with %d warehouses", benchSpec.warehouses)) // Add the connection string here as the port is not decided until c.Start() is called. - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - importCmd = fmt.Sprintf("%s '%s'", importCmd, pgurl) + + importCmd = fmt.Sprintf("%s {pgurl:1}", importCmd) c.Run(ctx, option.WithNodes(c.Node(pinnedNode)), importCmd) if benchSpec.snapshotRate != 0 { @@ -437,7 +433,7 @@ func setupDecommissionBench( } // Wait for initial up-replication. - err = WaitFor3XReplication(ctx, t, db) + err := WaitFor3XReplication(ctx, t, db) require.NoError(t, err) } } @@ -929,11 +925,7 @@ func runSingleDecommission( if drainFirst { h.t.Status(fmt.Sprintf("draining node%d", target)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, h.t.L(), c.Node(target)) - if err != nil { - t.Fatal(err) - } - cmd := fmt.Sprintf("./cockroach node drain --url=%s --self --insecure", pgurl) + cmd := fmt.Sprintf("./cockroach node drain --certs-dir=certs --port={pgport%s} --self", c.Node(target)) if err := h.c.RunE(ctx, option.WithNodes(h.c.Node(target)), cmd); err != nil { return err } diff --git a/pkg/cmd/roachtest/tests/disk_stall.go b/pkg/cmd/roachtest/tests/disk_stall.go index 5788ddfb444e..32f7ae9cce99 100644 --- a/pkg/cmd/roachtest/tests/disk_stall.go +++ b/pkg/cmd/roachtest/tests/disk_stall.go @@ -153,7 +153,7 @@ func runDiskStalledDetection( } stallAt := timeutil.Now() - response := mustGetMetrics(ctx, t, adminURL, workloadStartAt, stallAt, []tsQuery{ + response := mustGetMetrics(ctx, c, t, adminURL, workloadStartAt, stallAt, []tsQuery{ {name: "cr.node.sql.query.count", queryType: total}, }) cum := response.Results[0].Datapoints @@ -205,7 +205,7 @@ func runDiskStalledDetection( { now := timeutil.Now() - response := mustGetMetrics(ctx, t, adminURL, workloadStartAt, now, []tsQuery{ + response := mustGetMetrics(ctx, c, t, adminURL, workloadStartAt, now, []tsQuery{ {name: "cr.node.sql.query.count", queryType: total}, }) cum := response.Results[0].Datapoints diff --git a/pkg/cmd/roachtest/tests/django.go b/pkg/cmd/roachtest/tests/django.go index 0a7b0e0febe1..5f2139945f31 100644 --- a/pkg/cmd/roachtest/tests/django.go +++ b/pkg/cmd/roachtest/tests/django.go @@ -230,7 +230,7 @@ source venv/bin/activate && cd /mnt/data1/django/tests && python3 runtests.py %[1]s --settings cockroach_settings -v 2 > %[1]s.stdout ` -const cockroachDjangoSettings = ` +var cockroachDjangoSettings = fmt.Sprintf(` from django.test.runner import DiscoverRunner @@ -238,16 +238,16 @@ DATABASES = { 'default': { 'ENGINE': 'django_cockroachdb', 'NAME': 'django_tests', - 'USER': 'test_admin', - 'PASSWORD': '', + 'USER': '%[1]s', + 'PASSWORD': '%[2]s', 'HOST': 'localhost', 'PORT': {pgport:1}, }, 'other': { 'ENGINE': 'django_cockroachdb', 'NAME': 'django_tests2', - 'USER': 'root', - 'PASSWORD': '', + 'USER': '%[1]s', + 'PASSWORD': '%[2]s', 'HOST': 'localhost', 'PORT': {pgport:1}, }, @@ -269,4 +269,4 @@ class NonDescribingDiscoverRunner(DiscoverRunner): } USE_TZ = False -` +`, install.DefaultUser, install.DefaultPassword) diff --git a/pkg/cmd/roachtest/tests/drain.go b/pkg/cmd/roachtest/tests/drain.go index 91215289bf3d..b42f3427c632 100644 --- a/pkg/cmd/roachtest/tests/drain.go +++ b/pkg/cmd/roachtest/tests/drain.go @@ -24,7 +24,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -128,10 +127,6 @@ func runEarlyExitInConnectionWait(ctx context.Context, t test.Test, c cluster.Cl m.Go(func(ctx context.Context) error { t.Status(fmt.Sprintf("start draining node %d", nodeToDrain)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(nodeToDrain)) - if err != nil { - t.Fatal(err) - } results, err := c.RunWithDetailsSingleNode( ctx, t.L(), @@ -141,7 +136,7 @@ func runEarlyExitInConnectionWait(ctx context.Context, t test.Test, c cluster.Cl // of server.shutdown.initial_wait, server.shutdown.connections.timeout, // server.shutdown.transactions.timeout times two, and // server.shutdown.lease_transfer_iteration.timeout. - fmt.Sprintf("./cockroach node drain --self --insecure --drain-wait=10s --url=%s", pgurl), + fmt.Sprintf("./cockroach node drain --self --drain-wait=10s --certs-dir=certs --port={pgport:%d}", nodeToDrain), ) if err != nil { return err @@ -261,13 +256,9 @@ func runWarningForConnWait(ctx context.Context, t test.Test, c cluster.Cluster) m := c.NewMonitor(ctx, c.Node(nodeToDrain)) m.Go(func(ctx context.Context) error { t.Status(fmt.Sprintf("draining node %d", nodeToDrain)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(nodeToDrain)) - if err != nil { - t.Fatal(err) - } return c.RunE(ctx, option.WithNodes(c.Node(nodeToDrain)), - fmt.Sprintf("./cockroach node drain --self --insecure --drain-wait=600s --url=%s", pgurl), + fmt.Sprintf("./cockroach node drain --self --drain-wait=600s --certs-dir=certs --port={pgport:%d}", nodeToDrain), ) }) @@ -342,15 +333,11 @@ func runClusterNotAtQuorum(ctx context.Context, t test.Test, c cluster.Cluster) c.Stop(ctx, t.L(), stopOpts, c.Node(2)) t.Status("start draining node 3") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(3)) - if err != nil { - t.Fatal(err) - } // Ignore the error, since the command is expected to time out. results, _ := c.RunWithDetailsSingleNode( ctx, t.L(), - option.WithNodes(c.Node(3)), fmt.Sprintf("./cockroach node drain --self --insecure --drain-wait=10s --url=%s", pgurl)) + option.WithNodes(c.Node(3)), "./cockroach node drain --self --drain-wait=10s --certs-dir=certs --port={pgport:3}") t.L().Printf("drain output:\n%s\n%s\n", results.Stdout, results.Stderr) require.Regexp(t, "(cluster settings require a value of at least|could not check drain related cluster settings)", results.Stderr) } diff --git a/pkg/cmd/roachtest/tests/drop.go b/pkg/cmd/roachtest/tests/drop.go index 74c53da2a90b..a4c5a2e48a8f 100644 --- a/pkg/cmd/roachtest/tests/drop.go +++ b/pkg/cmd/roachtest/tests/drop.go @@ -43,11 +43,8 @@ func registerDrop(r registry.Registry) { m := c.NewMonitor(ctx, c.Range(1, nodes)) m.Go(func(ctx context.Context) error { t.WorkerStatus("importing TPCC fixture") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(1)), tpccImportCmd(warehouses, pgurl)) + + c.Run(ctx, option.WithNodes(c.Node(1)), tpccImportCmd(warehouses, "{pgurl:1}")) // Don't open the DB connection until after the data has been imported. // Otherwise the ALTER TABLE query below might fail to find the diff --git a/pkg/cmd/roachtest/tests/follower_reads.go b/pkg/cmd/roachtest/tests/follower_reads.go index 83bd2fa50960..6bed709c45f3 100644 --- a/pkg/cmd/roachtest/tests/follower_reads.go +++ b/pkg/cmd/roachtest/tests/follower_reads.go @@ -16,7 +16,6 @@ import ( gosql "database/sql" "fmt" "math/rand" - "net/http" "reflect" "regexp" "strconv" @@ -26,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/mixedversion" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" @@ -681,8 +681,12 @@ func verifySQLLatency( SourceAggregator: tspb.TimeSeriesQueryAggregator_MAX.Enum(), }}, } + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.All(), url) + if err != nil { + t.Fatal(err) + } var response tspb.TimeSeriesQueryResponse - if err := httputil.PostProtobuf(ctx, http.Client{}, url, &request, &response); err != nil { + if err := httputil.PostProtobuf(ctx, client, url, &request, &response); err != nil { t.Fatal(err) } perTenSeconds := response.Results[0].Datapoints @@ -751,9 +755,12 @@ func verifyHighFollowerReadRatios( Derivative: tspb.TimeSeriesQueryDerivative_NON_NEGATIVE_DERIVATIVE.Enum(), }) } - + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.All(), url) + if err != nil { + t.Fatal(err) + } var response tspb.TimeSeriesQueryResponse - if err := httputil.PostProtobuf(ctx, http.Client{}, url, &request, &response); err != nil { + if err := httputil.PostProtobuf(ctx, client, url, &request, &response); err != nil { t.Fatal(err) } @@ -840,7 +847,11 @@ func getFollowerReadCounts(ctx context.Context, t test.Test, c cluster.Cluster) return err } url := "http://" + adminUIAddrs[0] + "/_status/vars" - resp, err := httputil.Get(ctx, url) + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.All(), url) + if err != nil { + return err + } + resp, err := client.Get(url) if err != nil { return err } @@ -907,6 +918,7 @@ func parsePrometheusMetric(s string) (*prometheusMetric, bool) { func runFollowerReadsMixedVersionSingleRegionTest( ctx context.Context, t test.Test, c cluster.Cluster, ) { + topology := topologySpec{multiRegion: false} runFollowerReadsMixedVersionTest(ctx, t, c, topology, exactStaleness) } @@ -941,14 +953,6 @@ func runFollowerReadsMixedVersionTest( rc readConsistency, opts ...mixedversion.CustomOption, ) { - // The http requests to the admin UI performed by the test don't play - // well with secure clusters. As of the time of writing, they return - // either of the following errors: - // tls: failed to verify certificate: x509: “node” certificate is not standards compliant - // tls: failed to verify certificate: x509: certificate signed by unknown authority - // - // Disable secure mode for simplicity. - opts = append(opts, mixedversion.ClusterSettingOption(install.SecureOption(false))) mvt := mixedversion.NewTest(ctx, t, t.L(), c, c.All(), opts...) var data map[int]int64 diff --git a/pkg/cmd/roachtest/tests/gopg.go b/pkg/cmd/roachtest/tests/gopg.go index e43c472ff23b..20ba713a42cf 100644 --- a/pkg/cmd/roachtest/tests/gopg.go +++ b/pkg/cmd/roachtest/tests/gopg.go @@ -53,7 +53,12 @@ func registerGopg(r registry.Registry) { } node := c.Node(1) t.Status("setting up cockroach") - c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(), c.All()) + // go-pg does not support reading in the password from the environment + // in v10.9.0. + // See: https://github.com/go-pg/pg/pull/1996 + // TODO(darrylwong): once the above change is part of a release, + // upgrade support to that version and enable secure mode. + c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(install.SecureOption(false)), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) diff --git a/pkg/cmd/roachtest/tests/gorm.go b/pkg/cmd/roachtest/tests/gorm.go index 6d90307f7bee..17fbfddee892 100644 --- a/pkg/cmd/roachtest/tests/gorm.go +++ b/pkg/cmd/roachtest/tests/gorm.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/stretchr/testify/require" @@ -105,11 +104,7 @@ func registerGORM(r registry.Registry) { ignorelistName, ignoredFailures := "gormIgnorelist", gormIgnorelist t.L().Printf("Running cockroach version %s, using blocklist %s, using ignorelist %s", version, blocklistName, ignorelistName) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1)) - if err != nil { - t.Fatal(err) - } - err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`./cockroach sql -e "CREATE DATABASE gorm" --insecure --url=%s`, pgurl)) + err = c.RunE(ctx, option.WithNodes(node), `./cockroach sql -e "CREATE DATABASE gorm" --url={pgurl:1}`) require.NoError(t, err) t.Status("downloading go dependencies for tests") @@ -130,7 +125,7 @@ func registerGORM(r registry.Registry) { ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && rm migrate_test.go && - GORM_DIALECT="postgres" GORM_DSN="user=test_admin password= dbname=gorm host=localhost port={pgport:1} sslmode=disable" + GORM_DIALECT="postgres" GORM_DSN="user=roach password=sytem dbname=gorm host=localhost port={pgport:1} sslmode=require" go test -v ./... 2>&1 | %s/bin/go-junit-report > %s`, gormTestPath, goPath, resultsPath), ) diff --git a/pkg/cmd/roachtest/tests/gossip.go b/pkg/cmd/roachtest/tests/gossip.go index 92b919f54bfc..c013d7380baf 100644 --- a/pkg/cmd/roachtest/tests/gossip.go +++ b/pkg/cmd/roachtest/tests/gossip.go @@ -15,7 +15,6 @@ import ( gosql "database/sql" "fmt" "net" - "net/http" "net/url" "sort" "strconv" @@ -25,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachprod" @@ -180,12 +180,20 @@ type checkGossipFunc func(map[string]gossip.Info) error // checkGossip fetches the gossip infoStore from each node and invokes the // given function. The test passes if the function returns 0 for every node, // retrying for up to the given duration. -func (g *gossipUtil) check(ctx context.Context, c cluster.Cluster, f checkGossipFunc) error { +func (g *gossipUtil) check( + ctx context.Context, c cluster.Cluster, f checkGossipFunc, l *logger.Logger, +) error { return retry.ForDuration(g.waitTime, func() error { var infoStatus gossip.InfoStatus for i := 1; i <= c.Spec().NodeCount; i++ { url := g.urlMap[i] + `/_status/gossip/local` - if err := httputil.GetJSON(http.Client{}, url, &infoStatus); err != nil { + // runGossipRestartNodeOne restarts node 1. Just use node 2 for all tests + // to simplify things since using node 1 is arbitrary anyway. + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, l, c.Node(2), url) + if err != nil { + return err + } + if err := httputil.GetJSON(client, url, &infoStatus); err != nil { return errors.Wrapf(err, "failed to get gossip status from node %d", i) } if err := f(infoStatus.Infos); err != nil { @@ -234,13 +242,13 @@ func (g *gossipUtil) checkConnectedAndFunctional( ctx context.Context, t test.Test, c cluster.Cluster, ) { t.L().Printf("waiting for gossip to be connected\n") - if err := g.check(ctx, c, g.hasPeers(c.Spec().NodeCount)); err != nil { + if err := g.check(ctx, c, g.hasPeers(c.Spec().NodeCount), t.L()); err != nil { t.Fatal(err) } - if err := g.check(ctx, c, g.hasClusterID); err != nil { + if err := g.check(ctx, c, g.hasClusterID, t.L()); err != nil { t.Fatal(err) } - if err := g.check(ctx, c, g.hasSentinel); err != nil { + if err := g.check(ctx, c, g.hasSentinel, t.L()); err != nil { t.Fatal(err) } @@ -288,13 +296,13 @@ func runGossipPeerings(ctx context.Context, t test.Test, c cluster.Cluster) { for i := 1; timeutil.Now().Before(deadline); i++ { WaitForReady(ctx, t, c, c.All()) - if err := g.check(ctx, c, g.hasPeers(c.Spec().NodeCount)); err != nil { + if err := g.check(ctx, c, g.hasPeers(c.Spec().NodeCount), t.L()); err != nil { t.Fatal(err) } - if err := g.check(ctx, c, g.hasClusterID); err != nil { + if err := g.check(ctx, c, g.hasClusterID, t.L()); err != nil { t.Fatal(err) } - if err := g.check(ctx, c, g.hasSentinel); err != nil { + if err := g.check(ctx, c, g.hasSentinel, t.L()); err != nil { t.Fatal(err) } t.L().Printf("%d: OK\n", i) @@ -435,10 +443,11 @@ SELECT count(replicas) // connections. This will require node 1 to reach out to the other nodes in // the cluster for gossip info. err = c.RunE(ctx, option.WithNodes(c.Node(1)), - ` ./cockroach start --insecure --background --store={store-dir} `+ + ` ./cockroach start --background --store={store-dir} `+ `--log-dir={log-dir} --cache=10% --max-sql-memory=10% `+ fmt.Sprintf(`--listen-addr=:$[{pgport:1}+1000] --http-port=%d `, adminPorts[0])+ `--join={pghost:1}:{pgport:1} `+ + `--certs-dir=certs `+ `--advertise-addr={pghost:1}:$[{pgport:1}+1000] `+ `> {log-dir}/cockroach.stdout 2> {log-dir}/cockroach.stderr`) if err != nil { diff --git a/pkg/cmd/roachtest/tests/hibernate.go b/pkg/cmd/roachtest/tests/hibernate.go index b3e22436fd27..6fb6b44dab21 100644 --- a/pkg/cmd/roachtest/tests/hibernate.go +++ b/pkg/cmd/roachtest/tests/hibernate.go @@ -96,7 +96,8 @@ func registerHibernate(r registry.Registry, opt hibernateOptions) { t.Status("setting up cockroach") startOpts := option.DefaultStartOptsInMemory() startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + // Hibernate uses a hardcoded connection string with ssl disabled. + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All()) if opt.dbSetupFunc != nil { opt.dbSetupFunc(ctx, t, c) diff --git a/pkg/cmd/roachtest/tests/import.go b/pkg/cmd/roachtest/tests/import.go index bc0c414b6911..3c42ef1185db 100644 --- a/pkg/cmd/roachtest/tests/import.go +++ b/pkg/cmd/roachtest/tests/import.go @@ -141,15 +141,11 @@ func registerImportTPCC(r registry.Registry) { m.Go(hc.Runner) tick, perfBuf := initBulkJobPerfArtifacts(testName, timeout) - workloadStr := `./cockroach workload fixtures import tpcc --warehouses=%d --csv-server='http://localhost:8081' '%s'` + workloadStr := `./cockroach workload fixtures import tpcc --warehouses=%d --csv-server='http://localhost:8081' {pgurl:1}` m.Go(func(ctx context.Context) error { defer dul.Done() defer hc.Done() - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - cmd := fmt.Sprintf(workloadStr, warehouses, pgurl) + cmd := fmt.Sprintf(workloadStr, warehouses) // Tick once before starting the import, and once after to capture the // total elapsed time. This is used by roachperf to compute and display // the average MB/sec per node. @@ -359,22 +355,13 @@ func registerImportDecommissioned(r registry.Registry) { // Decommission a node. nodeToDecommission := 2 t.Status(fmt.Sprintf("decommissioning node %d", nodeToDecommission)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(nodeToDecommission)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(nodeToDecommission)), - fmt.Sprintf(`./cockroach node decommission --insecure --self --wait=all --url=%s`, pgurl)) + c.Run(ctx, option.WithNodes(c.Node(nodeToDecommission)), `./cockroach node decommission --self --wait=all --url={pgurl:2}`) // Wait for a bit for node liveness leases to expire. time.Sleep(10 * time.Second) t.Status("running workload") - pgurl, err = roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(1)), tpccImportCmd(warehouses, pgurl)) + c.Run(ctx, option.WithNodes(c.Node(1)), tpccImportCmd(warehouses, "{pgurl:1}")) } r.Add(registry.TestSpec{ diff --git a/pkg/cmd/roachtest/tests/jasyncsql.go b/pkg/cmd/roachtest/tests/jasyncsql.go index d04ee8823059..8f0a81233e96 100644 --- a/pkg/cmd/roachtest/tests/jasyncsql.go +++ b/pkg/cmd/roachtest/tests/jasyncsql.go @@ -30,7 +30,10 @@ func registerJasyncSQL(r registry.Registry) { } node := c.Node(1) t.Status("setting up cockroach") - c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(), c.All()) + // jasync does not support changing the default sslmode for postgresql, defaulting + // sslmode=disable. See: https://github.com/jasync-sql/jasync-sql/issues/422 + // TODO(darrylwong): If the above issue is addressed we can enable secure mode + c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(install.SecureOption(false)), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { @@ -93,7 +96,7 @@ func registerJasyncSQL(r registry.Registry) { _ = c.RunE( ctx, option.WithNodes(node), - `cd /mnt/data1/jasync-sql && PGUSER=test_admin PGHOST=localhost PGPORT={pgport:1} PGDATABASE=defaultdb ./gradlew :postgresql-async:test`, + `cd /mnt/data1/jasync-sql && PGHOST=localhost PGUSER=test_admin PGPORT={pgport:1} PGDATABASE=defaultdb ./gradlew :postgresql-async:test`, ) _ = c.RunE(ctx, option.WithNodes(node), `mkdir -p ~/logs/report/jasyncsql-results`) diff --git a/pkg/cmd/roachtest/tests/knex.go b/pkg/cmd/roachtest/tests/knex.go index 3510643e0c2f..790293809ed4 100644 --- a/pkg/cmd/roachtest/tests/knex.go +++ b/pkg/cmd/roachtest/tests/knex.go @@ -12,13 +12,13 @@ package tests import ( "context" + "fmt" "strings" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" - "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/stretchr/testify/require" ) @@ -36,14 +36,12 @@ func registerKnex(r registry.Registry) { t test.Test, c cluster.Cluster, ) { - if c.IsLocal() { - t.Fatal("cannot be run in local mode") - } + //if c.IsLocal() { + // t.Fatal("cannot be run in local mode") + //} node := c.Node(1) t.Status("setting up cockroach") - startOpts := option.DefaultStartOptsInMemory() - startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) require.NoError(t, err) @@ -57,7 +55,7 @@ func registerKnex(r registry.Registry) { c, node, "create sql database", - `./cockroach sql --insecure -e "CREATE DATABASE test"`, + `./cockroach sql --url={pgurl:1} -e "CREATE DATABASE test"`, ) require.NoError(t, err) @@ -121,12 +119,19 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co ) require.NoError(t, err) + // Write the knexfile test config into the test suite to use. + // The default test config does not support ssl connections. + testConfigFile := "./pkg/cmd/roachtest/tests/knexfile.js" + err = c.PutE(ctx, t.L(), testConfigFile, "/mnt/data1/knex/knexfile.js") + require.NoError(t, err) + t.Status("running knex tests") result, err := c.RunWithDetailsSingleNode( ctx, t.L(), option.WithNodes(node), - `cd /mnt/data1/knex/ && DB='cockroachdb' npm test`, + fmt.Sprintf(`cd /mnt/data1/knex/ && PGUSER=%s PGPASSWORD=%s PGPORT={pgport:1} PGSSLROOTCERT=$HOME/certs/ca.crt \ + KNEX_TEST='/mnt/data1/knex/knexfile.js' DB='cockroachdb' npm test`, install.DefaultUser, install.DefaultPassword), ) rawResultsStr := result.Stdout + result.Stderr t.L().Printf("Test Results: %s", rawResultsStr) diff --git a/pkg/cmd/roachtest/tests/knexfile.js b/pkg/cmd/roachtest/tests/knexfile.js new file mode 100644 index 000000000000..27aee4618f28 --- /dev/null +++ b/pkg/cmd/roachtest/tests/knexfile.js @@ -0,0 +1,35 @@ +'use strict'; +/* eslint no-var: 0 */ + +const _ = require('lodash'); + +console.log(`Using custom cockroachdb test config`); + +const testIntegrationDialects = ( + process.env.DB || + 'cockroachdb' +).match(/[\w-]+/g); + +const testConfigs = { + cockroachdb: { + adapter: 'cockroachdb', + port: process.env.PGPORT, + host: 'localhost', + database: 'test', + user: process.env.PGUSER, + password: process.env.PGPASSWORD, + ssl: { + rejectUnauthorized: false, + ca: process.env.PGSSLROOTCERT + } + }, +}; + +module.exports = _.reduce( + testIntegrationDialects, + function (res, dialectName) { + res[dialectName] = testConfigs[dialectName]; + return res; + }, + {} +); diff --git a/pkg/cmd/roachtest/tests/kv.go b/pkg/cmd/roachtest/tests/kv.go index f77b4b42473c..28a89099d6a5 100644 --- a/pkg/cmd/roachtest/tests/kv.go +++ b/pkg/cmd/roachtest/tests/kv.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/kv" @@ -119,16 +118,7 @@ func registerKV(r registry.Registry) { } } if opts.sharedProcessMT { - createInMemoryTenant(ctx, t, c, appTenantName, c.Range(1, nodes), false /* secure */) - } - - // Create a user and grant them admin privileges so they can freely - // interact with the cluster. - if _, err := db.ExecContext(ctx, `CREATE USER testuser WITH PASSWORD 'password'`); err != nil { - t.Fatal(err) - } - if _, err := db.ExecContext(ctx, `GRANT admin TO testuser`); err != nil { - t.Fatal(err) + createInMemoryTenant(ctx, t, c, appTenantName, c.Range(1, nodes), true /* secure */) } t.Status("running workload") @@ -181,7 +171,9 @@ func registerKV(r registry.Registry) { if opts.sharedProcessMT { url = fmt.Sprintf(" {pgurl:1-%d:%s}", nodes, appTenantName) } - cmd := "./workload run kv --tolerate-errors --init --user=testuser --password=password" + + cmd := fmt.Sprintf( + "./workload run kv --tolerate-errors --init --user=%s --password=%s", install.DefaultUser, install.DefaultPassword, + ) + histograms + concurrency + splits + duration + readPercent + batchSize + blockSize + sequential + envFlags + url c.Run(ctx, option.WithNodes(c.Node(nodes+1)), cmd) @@ -541,12 +533,7 @@ func registerKVGracefulDraining(r registry.Registry) { // Initialize the database with a lot of ranges so that there are // definitely a large number of leases on the node that we shut down // before it starts draining. - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(1)), - fmt.Sprintf("./cockroach workload init kv --splits 100 '%s'", pgurl)) + c.Run(ctx, option.WithNodes(c.Node(1)), "./cockroach workload init kv --splits 100 {pgurl:1}") m := c.NewMonitor(ctx, c.Nodes(1, nodes)) m.ExpectDeath() diff --git a/pkg/cmd/roachtest/tests/libpq.go b/pkg/cmd/roachtest/tests/libpq.go index 76022bf25165..98d2bafebc3d 100644 --- a/pkg/cmd/roachtest/tests/libpq.go +++ b/pkg/cmd/roachtest/tests/libpq.go @@ -104,7 +104,9 @@ func registerLibPQ(r registry.Registry) { result, err := c.RunWithDetailsSingleNode( ctx, t.L(), option.WithNodes(node), - fmt.Sprintf(`cd %s && PGPORT={pgport:1} PGUSER=root PGSSLMODE=disable PGDATABASE=postgres go test -list "%s"`, libPQPath, testListRegex), + fmt.Sprintf( + `cd %s && PGPORT={pgport:1} PGUSER=%s PGPASSWORD=%s PGSSLMODE=require PGDATABASE=postgres go test -list "%s"`, + libPQPath, install.DefaultUser, install.DefaultPassword, testListRegex), ) require.NoError(t, err) @@ -131,8 +133,8 @@ func registerLibPQ(r registry.Registry) { _ = c.RunE( ctx, option.WithNodes(node), - fmt.Sprintf("cd %s && PGPORT={pgport:1} PGUSER=test_admin PGSSLMODE=disable PGDATABASE=postgres go test -run %s -v 2>&1 | %s/bin/go-junit-report > %s", - libPQPath, allowedTestsRegExp, goPath, resultsPath), + fmt.Sprintf("cd %s && PGPORT={pgport:1} PGUSER=%s PGPASSWORD=%s PGSSLMODE=require PGDATABASE=postgres go test -run %s -v 2>&1 | %s/bin/go-junit-report > %s", + libPQPath, install.DefaultUser, install.DefaultPassword, allowedTestsRegExp, goPath, resultsPath), ) parseAndSummarizeJavaORMTestsResults( diff --git a/pkg/cmd/roachtest/tests/liquibase.go b/pkg/cmd/roachtest/tests/liquibase.go index a515857ad5dc..2800064f8e33 100644 --- a/pkg/cmd/roachtest/tests/liquibase.go +++ b/pkg/cmd/roachtest/tests/liquibase.go @@ -38,7 +38,8 @@ func registerLiquibase(r registry.Registry) { t.Status("setting up cockroach") startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + // TODO(darrylwong): if https://github.com/liquibase/liquibase-test-harness/pull/724 is merged, enable secure mode + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { @@ -98,6 +99,7 @@ func registerLiquibase(r registry.Registry) { if err = c.RunE(ctx, option.WithNodes(node), `sudo mkdir /cockroach && sudo ln -sf /home/ubuntu/cockroach /cockroach/cockroach.sh`); err != nil { t.Fatal(err) } + // TODO(darrylwong): once secure mode is enabled, add --certs-dir=certs if err = c.RunE(ctx, option.WithNodes(node), `/mnt/data1/liquibase-test-harness/src/test/resources/docker/setup_db.sh localhost`); err != nil { t.Fatal(err) } @@ -112,6 +114,7 @@ func registerLiquibase(r registry.Registry) { resultsPath = repoDir + "/target/surefire-reports/TEST-liquibase.harness.LiquibaseHarnessSuiteTest.xml" ) + // TODO(darrylwong): once secure mode is enabled, add -DdbUsername=roach -DdbPassword=system cmd := fmt.Sprintf("cd /mnt/data1/liquibase-test-harness/ && "+ "mvn surefire-report:report-only test -Dtest=LiquibaseHarnessSuiteTest "+ "-DdbName=cockroachdb -DdbVersion=20.2 -DoutputDirectory=%s", repoDir) diff --git a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go index 4edcba146083..8422cbb89d84 100644 --- a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go +++ b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go @@ -305,7 +305,7 @@ func runRecoverLossOfQuorum(ctx context.Context, t test.Test, c cluster.Cluster, if err := timeutil.RunWithTimeout(ctx, "mark-nodes-decommissioned", 5*time.Minute, func(ctx context.Context) error { decommissionCmd := fmt.Sprintf( - "./cockroach node decommission --wait none --insecure --url={pgurl:%d} 2 3", 1) + "./cockroach node decommission --wait none --url={pgurl:%d} 2 3", 1) return c.RunE(ctx, option.WithNodes(c.Node(controller)), decommissionCmd) }); err != nil { // Timeout means we failed to recover ranges especially system ones @@ -331,8 +331,8 @@ func runRecoverLossOfQuorum(ctx context.Context, t test.Test, c cluster.Cluster, if err := timeutil.RunWithTimeout(ctx, "decommission-removed-nodes", 5*time.Minute, func(ctx context.Context) error { decommissionCmd := fmt.Sprintf( - "./cockroach node decommission --wait all --insecure --url={pgurl:%d} 2 3", 1) - return c.RunE(ctx, option.WithNodes(c.Node(controller)), decommissionCmd) + "./cockroach node decommission --wait all --url={pgurl:%d} 2 3", 1) + return c.RunE(ctx, option.WithNodes(c.Nodes(controller)), decommissionCmd) }); err != nil { // Timeout means we failed to drain all ranges from failed nodes, possibly // because some ranges were not recovered. @@ -447,7 +447,7 @@ func runHalfOnlineRecoverLossOfQuorum( require.NoError(t, err, "infra failure, can't get IP addr of cluster node") require.NotEmpty(t, addrs, "infra failure, can't get IP addr of cluster node") addr := addrs[0] - planCmd := "./cockroach debug recover make-plan --confirm y --insecure --host " + addr + " -o " + planName + planCmd := "./cockroach debug recover make-plan --confirm y --host " + addr + " -o " + planName if err = c.RunE(ctx, option.WithNodes(c.Node(controller)), planCmd); err != nil { t.L().Printf("failed to create plan, test can't proceed assuming unrecoverable cluster: %s", @@ -461,7 +461,7 @@ func runHalfOnlineRecoverLossOfQuorum( } t.L().Printf("staging recovery plan") - applyCommand := "./cockroach debug recover apply-plan --confirm y --insecure --host " + addr + " " + planName + applyCommand := "./cockroach debug recover apply-plan --confirm y --host " + addr + " " + planName c.Run(ctx, option.WithNodes(c.Nodes(controller)), applyCommand) // Ignore node failures because they could fail if recovered ranges @@ -477,7 +477,7 @@ func runHalfOnlineRecoverLossOfQuorum( } t.L().Printf("waiting for nodes to process recovery") - verifyCommand := "./cockroach debug recover verify --insecure --host " + addr + " " + planName + verifyCommand := "./cockroach debug recover verify --host " + addr + " " + planName if err = timeutil.RunWithTimeout(ctx, "wait-for-restart", 2*time.Minute, func(ctx context.Context) error { for { diff --git a/pkg/cmd/roachtest/tests/mixed_version_decommission.go b/pkg/cmd/roachtest/tests/mixed_version_decommission.go index 9f31c08d0eac..7135017c6709 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_decommission.go +++ b/pkg/cmd/roachtest/tests/mixed_version_decommission.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/clusterupgrade" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -135,12 +134,8 @@ func preloadDataStep(target int) versionStep { // Load data into cluster to ensure we have a large enough number of replicas // to move on decommissioning. c := u.c - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } c.Run(ctx, option.WithNodes(c.Node(target)), - `./cockroach workload fixtures import tpcc --warehouses=100`, pgurl) + `./cockroach workload fixtures import tpcc --warehouses=100 {pgurl:1}`) db := c.Conn(ctx, t.L(), target) defer db.Close() if err := WaitFor3XReplication(ctx, t, db); err != nil { @@ -156,7 +151,7 @@ func partialDecommissionStep(target, from int, binaryVersion *clusterupgrade.Ver return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { c := u.c c.Run(ctx, option.WithNodes(c.Node(from)), clusterupgrade.CockroachPathForVersion(t, binaryVersion), "node", "decommission", - "--wait=none", "--insecure", strconv.Itoa(target), "--port", fmt.Sprintf("{pgport:%d}", from)) + "--wait=none", strconv.Itoa(target), "--port", fmt.Sprintf("{pgport:%d}", from), "--certs-dir=certs") } } @@ -167,7 +162,7 @@ func recommissionAllStep(from int, binaryVersion *clusterupgrade.Version) versio return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { c := u.c c.Run(ctx, option.WithNodes(c.Node(from)), clusterupgrade.CockroachPathForVersion(t, binaryVersion), "node", "recommission", - "--insecure", c.All().NodeIDsString(), "--port", fmt.Sprintf("{pgport:%d}", from)) + c.All().NodeIDsString(), "--port", fmt.Sprintf("{pgport:%d}", from), "--certs-dir=certs") } } @@ -176,12 +171,8 @@ func recommissionAllStep(from int, binaryVersion *clusterupgrade.Version) versio func fullyDecommissionStep(target, from int, binaryVersion *clusterupgrade.Version) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { c := u.c - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(from)) - if err != nil { - t.Fatal(err) - } c.Run(ctx, option.WithNodes(c.Node(from)), clusterupgrade.CockroachPathForVersion(t, binaryVersion), "node", "decommission", - "--wait=all", "--insecure", strconv.Itoa(target), fmt.Sprintf("--url=%s", pgurl)) + "--wait=all", strconv.Itoa(target), "--port={pgport:1}", "--certs-dir=certs") // If we are decommissioning a target node from the same node, the drain // step will be skipped. In this case, we should not consider the step done diff --git a/pkg/cmd/roachtest/tests/mvcc_gc.go b/pkg/cmd/roachtest/tests/mvcc_gc.go index 7667bc5ff2ab..88aa6a279bc2 100644 --- a/pkg/cmd/roachtest/tests/mvcc_gc.go +++ b/pkg/cmd/roachtest/tests/mvcc_gc.go @@ -123,15 +123,11 @@ func runMVCCGC(ctx context.Context, t test.Test, c cluster.Cluster) { t.Fatalf("failed to up-replicate cluster: %s", err) } - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } m := c.NewMonitor(ctx) m.Go(func(ctx context.Context) error { cmd := roachtestutil.NewCommand("./cockroach workload init kv"). Flag("cycle-length", 20000). - Arg("%s", pgurl). + Arg("{pgurl:1}"). String() c.Run(ctx, option.WithNodes(c.Node(1)), cmd) @@ -642,7 +638,7 @@ func sendBatchRequest( } cmd := roachtestutil.NewCommand("./cockroach debug send-kv-batch"). Arg(requestFileName). - Option("insecure"). + Flag("certs-dir", "certs"). Flag("host", fmt.Sprintf("localhost:{pgport:%d}", node)). String() res, err := c.RunWithDetailsSingleNode( diff --git a/pkg/cmd/roachtest/tests/nodejs_postgres.go b/pkg/cmd/roachtest/tests/nodejs_postgres.go index 68b03e754c8c..72fd3ced61d2 100644 --- a/pkg/cmd/roachtest/tests/nodejs_postgres.go +++ b/pkg/cmd/roachtest/tests/nodejs_postgres.go @@ -148,9 +148,9 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(node), fmt.Sprintf( `cd /mnt/data1/node-postgres/ && sudo \ -PGPORT={pgport:1} PGUSER=%s PGSSLMODE=require PGDATABASE=postgres_node_test \ -PGSSLCERT=$HOME/certs/client.%s.crt PGSSLKEY=$HOME/certs/client.%s.key PGSSLROOTCERT=$HOME/certs/ca.crt yarn test`, - user, user, user, +PGPORT={pgport:1} PGUSER=%[1]s PGSSLMODE=require PGDATABASE=postgres_node_test \ +PGSSLCERT=$HOME/certs/client.%[1]s.crt PGSSLKEY=$HOME/certs/client.%[1]s.key PGSSLROOTCERT=$HOME/certs/ca.crt yarn test`, + user, ), ) diff --git a/pkg/cmd/roachtest/tests/pgx.go b/pkg/cmd/roachtest/tests/pgx.go index 920fb0c0a14c..f365d8c2a198 100644 --- a/pkg/cmd/roachtest/tests/pgx.go +++ b/pkg/cmd/roachtest/tests/pgx.go @@ -116,9 +116,9 @@ func registerPgx(r registry.Registry) { result, err := repeatRunWithDetailsSingleNode( ctx, c, t, node, "run pgx test suite", - "cd /mnt/data1/pgx && "+ - "PGX_TEST_DATABASE='postgresql://test_admin:@localhost:{pgport:1}/pgx_test' go test -v 2>&1 | "+ - "`go env GOPATH`/bin/go-junit-report", + fmt.Sprintf("cd /mnt/data1/pgx && "+ + "PGX_TEST_DATABASE='postgresql://%s:%s@localhost:{pgport:1}/pgx_test?sslmode=require' go test -v 2>&1 | "+ + "`go env GOPATH`/bin/go-junit-report", install.DefaultUser, install.DefaultPassword), ) // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. diff --git a/pkg/cmd/roachtest/tests/pop.go b/pkg/cmd/roachtest/tests/pop.go index 33abf3121d1d..bf1d468c1be9 100644 --- a/pkg/cmd/roachtest/tests/pop.go +++ b/pkg/cmd/roachtest/tests/pop.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" - "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/stretchr/testify/require" ) @@ -35,7 +34,7 @@ func registerPop(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") startOpts := option.DefaultStartOptsInMemory() - startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort + startOpts.RoachprodOpts.SQLPort = 26259 c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { @@ -81,22 +80,28 @@ func registerPop(r registry.Registry) { t.Status("building and setting up tests") + // pop expects to find certificates in a specific path. + err = c.RunE(ctx, option.WithNodes(node), "mkdir -p /mnt/data1/pop/crdb/certs") + require.NoError(t, err) + err = c.RunE(ctx, option.WithNodes(node), "cp -r certs /mnt/data1/pop/crdb/") + require.NoError(t, err) + err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && go build -v -tags sqlite -o tsoda ./soda`, popPath)) require.NoError(t, err) - err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda drop -e cockroach -c ./database.yml -p ./testdata/migrations`, popPath)) + err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda drop -e cockroach_ssl -c ./database.yml -p ./testdata/migrations`, popPath)) require.NoError(t, err) - err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda create -e cockroach -c ./database.yml -p ./testdata/migrations`, popPath)) + err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda create -e cockroach_ssl -c ./database.yml -p ./testdata/migrations`, popPath)) require.NoError(t, err) - err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda migrate -e cockroach -c ./database.yml -p ./testdata/migrations`, popPath)) + err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && ./tsoda migrate -e cockroach_ssl -c ./database.yml -p ./testdata/migrations`, popPath)) require.NoError(t, err) t.Status("running pop test suite") // No tests are expected to fail. - err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && SODA_DIALECT=cockroach go test -race -tags sqlite -v ./... -count=1`, popPath)) + err = c.RunE(ctx, option.WithNodes(node), fmt.Sprintf(`cd %s && SODA_DIALECT=cockroach_ssl go test -race -tags sqlite -v ./... -count=1`, popPath)) require.NoError(t, err, "error while running pop tests") } diff --git a/pkg/cmd/roachtest/tests/psycopg.go b/pkg/cmd/roachtest/tests/psycopg.go index 9a4afbc7568f..5369f789d2f7 100644 --- a/pkg/cmd/roachtest/tests/psycopg.go +++ b/pkg/cmd/roachtest/tests/psycopg.go @@ -108,14 +108,15 @@ func registerPsycopg(r registry.Registry) { t.Status("running psycopg test suite") - result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(node), + result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(node), fmt.Sprintf( `cd /mnt/data1/psycopg/ && export PSYCOPG2_TESTDB=defaultdb && - export PSYCOPG2_TESTDB_USER=test_admin && + export PSYCOPG2_TESTDB_USER=%s && + export PSYCOPG2_TESTDB_PASSWORD=%s && export PSYCOPG2_TESTDB_PORT={pgport:1} && export PSYCOPG2_TESTDB_HOST=localhost && make check PYTHON_VERSION=3`, - ) + install.DefaultUser, install.DefaultPassword)) // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. // Proceed for any other (command) errors diff --git a/pkg/cmd/roachtest/tests/query_comparison_util.go b/pkg/cmd/roachtest/tests/query_comparison_util.go index f73721c9e64c..851484b7eada 100644 --- a/pkg/cmd/roachtest/tests/query_comparison_util.go +++ b/pkg/cmd/roachtest/tests/query_comparison_util.go @@ -108,7 +108,7 @@ func runQueryComparison( return } c.Stop(clusterCtx, t.L(), option.DefaultStopOpts()) - c.Wipe(clusterCtx, false /* preserveCerts */) + c.Wipe(clusterCtx, true /* preserveCerts */) } } diff --git a/pkg/cmd/roachtest/tests/quit.go b/pkg/cmd/roachtest/tests/quit.go index b3913d84b1f2..36ffd61e4022 100644 --- a/pkg/cmd/roachtest/tests/quit.go +++ b/pkg/cmd/roachtest/tests/quit.go @@ -14,6 +14,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -257,15 +258,27 @@ func (q *quitTest) checkNoLeases(ctx context.Context, nodeID int) { if err != nil { q.Fatal(err) } - result, err := q.c.RunWithDetailsSingleNode(ctx, q.t.L(), option.WithNodes(q.c.Node(i)), - "curl", "-s", fmt.Sprintf("http://%s/_status/ranges/local", - adminAddrs[0])) + url := fmt.Sprintf("https://%s/_status/ranges/local", adminAddrs[0]) + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, q.c, q.t.L(), q.c.Node(i), url) if err != nil { q.Fatal(err) } + var data []byte + func() { + response, err := client.Get(url) + if err != nil { + q.Fatal(err) + } + defer response.Body.Close() + data, err = io.ReadAll(response.Body) + if err != nil { + q.Fatal(err) + } + }() + // Persist the response to artifacts to aid debugging. See #75438. _ = os.WriteFile(filepath.Join(q.t.ArtifactsDir(), fmt.Sprintf("status_ranges_n%d.json", i)), - []byte(result.Stdout), 0644, + data, 0644, ) // We need just a subset of the response. Make an ad-hoc // struct with just the bits of interest. @@ -286,7 +299,7 @@ func (q *quitTest) checkNoLeases(ctx context.Context, nodeID int) { } `json:"ranges"` } var details jsonOutput - if err := json.Unmarshal([]byte(result.Stdout), &details); err != nil { + if err := json.Unmarshal(data, &details); err != nil { q.Fatal(err) } // Some sanity check. @@ -385,13 +398,10 @@ func registerQuitTransfersLeases(r registry.Registry) { // kill. If the drain is successful, the leases are transferred // successfully even if if the process terminates non-gracefully. registerTest("drain", "v20.1.0", func(ctx context.Context, t test.Test, c cluster.Cluster, nodeID int) { - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(nodeID)) - if err != nil { - t.Fatal(err) - } result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(nodeID)), - "./cockroach", "node", "drain", "--insecure", "--logtostderr=INFO", - fmt.Sprintf("--url=%s", pgurl), + "./cockroach", "node", "drain", "--logtostderr=INFO", + fmt.Sprintf("--port={pgport:%d}", nodeID), + "--certs-dir certs", ) t.L().Printf("cockroach node drain:\n%s\n", result.Stdout+result.Stdout) if err != nil { @@ -432,13 +442,10 @@ func registerQuitTransfersLeases(r registry.Registry) { // - we add one to bring the value back between 1 and NodeCount // inclusive. otherNodeID := (nodeID % c.Spec().NodeCount) + 1 - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(otherNodeID)) - if err != nil { - t.Fatal(err) - } result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(otherNodeID)), - "./cockroach", "node", "drain", "--insecure", "--logtostderr=INFO", - fmt.Sprintf("--url=%s", pgurl), + "./cockroach", "node", "drain", "--logtostderr=INFO", + fmt.Sprintf("--port={pgport:%d}", otherNodeID), + "--certs-dir certs", fmt.Sprintf("%d", nodeID), ) t.L().Printf("cockroach node drain:\n%s\n", result.Stdout+result.Stderr) diff --git a/pkg/cmd/roachtest/tests/rebalance_load.go b/pkg/cmd/roachtest/tests/rebalance_load.go index 11021a407b8d..cf89c9cd9dfd 100644 --- a/pkg/cmd/roachtest/tests/rebalance_load.go +++ b/pkg/cmd/roachtest/tests/rebalance_load.go @@ -95,14 +95,6 @@ func registerRebalanceLoad(r registry.Registry) { if mixedVersion { mvt := mixedversion.NewTest(ctx, t, t.L(), c, roachNodes, mixedversion.NeverUseFixtures, - // The http requests to the admin UI performed by the test don't play - // well with secure clusters. As of the time of writing, they return - // either of the following errors: - // tls: failed to verify certificate: x509: “node” certificate is not standards compliant - // tls: failed to verify certificate: x509: certificate signed by unknown authority - // - // Disable secure mode for simplicity. - mixedversion.ClusterSettingOption(install.SecureOption(false)), mixedversion.ClusterSettingOption(install.ClusterSettingsOption(settings.ClusterSettings)), ) mvt.InMixedVersion("rebalance load run", @@ -337,7 +329,7 @@ func makeStoreCPUFn( return func(ctx context.Context) ([]float64, error) { now := timeutil.Now() resp, err := getMetricsWithSamplePeriod( - ctx, url, startTime, now, statSamplePeriod, tsQueries) + ctx, c, t, url, startTime, now, statSamplePeriod, tsQueries) if err != nil { return nil, err } diff --git a/pkg/cmd/roachtest/tests/restart.go b/pkg/cmd/roachtest/tests/restart.go index a73edcf4bead..85a3c67ae097 100644 --- a/pkg/cmd/roachtest/tests/restart.go +++ b/pkg/cmd/roachtest/tests/restart.go @@ -18,7 +18,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -37,13 +36,8 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio // We don't really need tpcc, we just need a good amount of traffic and a good // amount of data. t.Status("importing tpcc fixture") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } c.Run(ctx, option.WithNodes(workloadNode), - "./cockroach workload fixtures import tpcc --warehouses=100 --fks=false --checks=false", - pgurl, + "./cockroach workload fixtures import tpcc --warehouses=100 --fks=false --checks=false {pgurl:1}", ) // Wait a full scanner cycle (10m) for the raft log queue to truncate the @@ -59,7 +53,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio // lastIndex will be helpful because that drives the log size down eagerly // when things are healthy. t.Status("waiting for addsstable truncations") - time.Sleep(11 * time.Minute) + //time.Sleep(11 * time.Minute) // Stop a node. c.Stop(ctx, t.L(), option.DefaultStopOpts(), restartNode) @@ -71,8 +65,8 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio c.Run(ctx, option.WithNodes(workloadNode), "./cockroach workload run tpcc --warehouses=100 "+ fmt.Sprintf("--tolerate-errors --wait=false --duration=%s {pgurl:1-2}", downDuration)) - // Bring it back up and make sure it can serve a query within a reasonable - // time limit. For now, less time than it was down for. + //Bring it back up and make sure it can serve a query within a reasonable + //time limit. For now, less time than it was down for. c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), restartNode) // Dialing the formerly down node may still be prevented by the circuit breaker @@ -90,11 +84,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio SELECT count(*) FROM tpcc.order_line; SET TRACING = OFF; SHOW TRACE FOR SESSION;` - pgurl, err = roachtestutil.DefaultPGUrl(ctx, c, t.L(), restartNode) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(restartNode), fmt.Sprintf(`./cockroach sql --insecure --url=%s -e "%s"`, pgurl, tracedQ)) + c.Run(ctx, option.WithNodes(restartNode), fmt.Sprintf(`./cockroach sql --url={pgurl:1} -e "%s"`, tracedQ)) if took := timeutil.Since(start); took > downDuration { t.Fatalf(`expected to recover within %s took %s`, downDuration, took) } else { diff --git a/pkg/cmd/roachtest/tests/roachmart.go b/pkg/cmd/roachtest/tests/roachmart.go index c9d4605358df..ef005d8540cc 100644 --- a/pkg/cmd/roachtest/tests/roachmart.go +++ b/pkg/cmd/roachtest/tests/roachmart.go @@ -17,7 +17,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -51,12 +50,9 @@ func registerRoachmart(r registry.Registry) { } } t.Status("initializing workload") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } + // See https://github.com/cockroachdb/cockroach/issues/94062 for the --data-loader. - roachmartRun(ctx, 0, "./workload", "init", "roachmart", "--data-loader=INSERT", pgurl) + roachmartRun(ctx, 0, "./workload", "init", "roachmart", "--data-loader=INSERT", "{pgurl:1}") duration := " --duration=" + ifLocal(c, "10s", "10m") diff --git a/pkg/cmd/roachtest/tests/ruby_pg.go b/pkg/cmd/roachtest/tests/ruby_pg.go index 413a893336f8..9231dfcbf4cf 100644 --- a/pkg/cmd/roachtest/tests/ruby_pg.go +++ b/pkg/cmd/roachtest/tests/ruby_pg.go @@ -52,7 +52,9 @@ func registerRubyPG(r registry.Registry) { t.Status("setting up cockroach") startOpts := option.DefaultStartOptsInMemory() startOpts.RoachprodOpts.SQLPort = config.DefaultSQLPort - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + // TODO(darrylwong): ruby-pg is currently being updated to run on Ubuntu 22.04. + // Once complete, fix up ruby_pg_helpers to accept a tls connection. + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All()) version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { diff --git a/pkg/cmd/roachtest/tests/rust_postgres.go b/pkg/cmd/roachtest/tests/rust_postgres.go index 024f4c1bd3fa..6a778a50631e 100644 --- a/pkg/cmd/roachtest/tests/rust_postgres.go +++ b/pkg/cmd/roachtest/tests/rust_postgres.go @@ -35,7 +35,11 @@ func registerRustPostgres(r registry.Registry) { // We hardcode port 5433 since that's the port rust-postgres expects. startOpts := option.DefaultStartOptsInMemory() startOpts.RoachprodOpts.SQLPort = 5433 - c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + // rust-postgres currently doesn't support changing the config through + // the environment, which means we can't pass it ssl connection details + // and must run the cluster in insecure mode. + // See: https://github.com/sfackler/rust-postgres/issues/654 + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.All()) db := c.Conn(ctx, t.L(), 1) _, err := db.Exec("create user postgres with createdb createlogin createrole cancelquery") if err != nil { diff --git a/pkg/cmd/roachtest/tests/schemachange.go b/pkg/cmd/roachtest/tests/schemachange.go index d9e6cf4ffe77..6bf24d297845 100644 --- a/pkg/cmd/roachtest/tests/schemachange.go +++ b/pkg/cmd/roachtest/tests/schemachange.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -59,11 +58,7 @@ func registerSchemaChangeDuringKV(r registry.Registry) { }) m.Wait() - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(c.Node(1)), `./workload init kv --drop --db=test`, pgurl) + c.Run(ctx, option.WithNodes(c.Node(1)), `./workload init kv --drop --db=test {pgurl:1}`) for node := 1; node <= c.Spec().NodeCount; node++ { node := node // TODO(dan): Ideally, the test would fail if this queryload failed, diff --git a/pkg/cmd/roachtest/tests/schemachange_random_load.go b/pkg/cmd/roachtest/tests/schemachange_random_load.go index 4924984175c1..1303770204d3 100644 --- a/pkg/cmd/roachtest/tests/schemachange_random_load.go +++ b/pkg/cmd/roachtest/tests/schemachange_random_load.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -98,11 +97,8 @@ func runSchemaChangeRandomLoad( t.Status("starting cockroach nodes") c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(loadNode), fmt.Sprintf("./workload init schemachange '%s'", pgurl)) + + c.Run(ctx, option.WithNodes(loadNode), "./workload init schemachange {pgurl:1}") result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(1)), "echo", "-n", "{store-dir}") if err != nil { diff --git a/pkg/cmd/roachtest/tests/slow_drain.go b/pkg/cmd/roachtest/tests/slow_drain.go index 55786168147a..966816dce637 100644 --- a/pkg/cmd/roachtest/tests/slow_drain.go +++ b/pkg/cmd/roachtest/tests/slow_drain.go @@ -20,7 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/testutils" @@ -119,13 +118,9 @@ func runSlowDrain(ctx context.Context, t test.Test, c cluster.Cluster, duration m.Go(func(ctx context.Context) error { drain := func(id int) error { t.Status(fmt.Sprintf("draining node %d", id)) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(id)) - if err != nil { - t.Fatal(err) - } return c.RunE(ctx, option.WithNodes(c.Node(id)), - fmt.Sprintf("./cockroach node drain %d --insecure --drain-wait=%s --url=%s", id, duration.String(), pgurl), + fmt.Sprintf("./cockroach node drain %d --drain-wait=%s --certs-dir=certs --port={pgport:%d}", id, duration.String(), id), ) } return drain(id) diff --git a/pkg/cmd/roachtest/tests/sqlalchemy.go b/pkg/cmd/roachtest/tests/sqlalchemy.go index b92ba5b63b20..56c9538d6b1d 100644 --- a/pkg/cmd/roachtest/tests/sqlalchemy.go +++ b/pkg/cmd/roachtest/tests/sqlalchemy.go @@ -143,10 +143,10 @@ func runSQLAlchemy(ctx context.Context, t test.Test, c cluster.Cluster) { // Note that this is expected to return an error, since the test suite // will fail. And it is safe to swallow it here. result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(node), - `source venv/bin/activate && cd /mnt/data1/sqlalchemy-cockroachdb/ && pytest --maxfail=0 \ - --dburi='cockroachdb://test_admin@localhost:{pgport:1}/defaultdb?sslmode=disable&disable_cockroachdb_telemetry=true' \ + fmt.Sprintf(`source venv/bin/activate && cd /mnt/data1/sqlalchemy-cockroachdb/ && pytest --maxfail=0 \ + --dburi='cockroachdb://%s:%s@localhost:{pgport:1}/defaultdb?sslmode=require&disable_cockroachdb_telemetry=true' \ test/test_suite_sqlalchemy.py - `) + `, install.DefaultUser, install.DefaultPassword)) // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. // Proceed for any other (command) errors diff --git a/pkg/cmd/roachtest/tests/status_server.go b/pkg/cmd/roachtest/tests/status_server.go index c513848e371d..cdd2f00bebae 100644 --- a/pkg/cmd/roachtest/tests/status_server.go +++ b/pkg/cmd/roachtest/tests/status_server.go @@ -15,10 +15,12 @@ import ( "fmt" "io" "net/http" + "net/url" "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -43,8 +45,12 @@ func runStatusServer(ctx context.Context, t test.Test, c cluster.Cluster) { // Use a retry-loop when populating the maps because we might be trying to // talk to the servers before they are responding to status requests // (resulting in 404's). + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.Node(1), url) + if err != nil { + t.Fatal(err) + } if err := retry.ForDuration(10*time.Second, func() error { - return httputil.GetJSON(http.Client{}, url, &details) + return httputil.GetJSON(client, url, &details) }); err != nil { t.Fatal(err) } @@ -52,14 +58,13 @@ func runStatusServer(ctx context.Context, t test.Test, c cluster.Cluster) { urlMap[i+1] = `http://` + addr } - // The status endpoints below may take a while to produce their answer, maybe more - // than the 3 second timeout of the default http client. - httpClient := httputil.NewClientWithTimeout(15 * time.Second) - // get performs an HTTP GET to the specified path for a specific node. - get := func(base, rel string) []byte { - url := base + rel - resp, err := httpClient.Get(context.TODO(), url) + get := func(path string, httpClient *http.Client) []byte { + url, err := url.Parse(path) + if err != nil { + t.Fatal(err) + } + resp, err := httpClient.Get(url.String()) if err != nil { t.Fatalf("could not GET %s - %s", url, err) } @@ -85,7 +90,22 @@ func runStatusServer(ctx context.Context, t test.Test, c cluster.Cluster) { } var details serverpb.DetailsResponse for _, urlID := range urlIDs { - if err := httputil.GetJSON(http.Client{}, url+`/_status/details/`+urlID, &details); err != nil { + endpoints := []string{ + fmt.Sprintf("%s/_status/details/%s", url, urlID), + fmt.Sprintf("%s/_status/gossip/%s", url, urlID), + fmt.Sprintf("%s/_status/nodes/%s", url, urlID), + fmt.Sprintf("%s/_status/logfiles/%s", url, urlID), + fmt.Sprintf("%s/_status/logs/%s", url, urlID), + fmt.Sprintf("%s/_status/stacks/%s", url, urlID), + } + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.Node(1), endpoints...) + if err != nil { + t.Fatal(err) + } + // The status endpoints below may take a while to produce their answer, maybe more + // than the 3 second timeout of the default http client. + client.Timeout = 15 * time.Second + if err := httputil.GetJSON(client, url+`/_status/details/`+urlID, &details); err != nil { t.Fatalf("unable to parse details - %s", err) } if details.NodeID != expectedNodeID { @@ -93,21 +113,30 @@ func runStatusServer(ctx context.Context, t test.Test, c cluster.Cluster) { nodeID, urlID, expectedNodeID, details.NodeID) } - get(url, fmt.Sprintf("/_status/gossip/%s", urlID)) - get(url, fmt.Sprintf("/_status/nodes/%s", urlID)) - get(url, fmt.Sprintf("/_status/logfiles/%s", urlID)) - get(url, fmt.Sprintf("/_status/logs/%s", urlID)) - get(url, fmt.Sprintf("/_status/stacks/%s", urlID)) + // Skip the first endpoint as that was used above + for _, endpoint := range endpoints[1:] { + get(endpoint, &client) + } } - get(url, "/_status/vars") + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.Node(1), url+"/_status/vars") + if err != nil { + t.Fatal(err) + } + client.Timeout = 15 * time.Second + get(url+"/_status/vars", &client) } // Check local response for the every node. for i := 1; i <= c.Spec().NodeCount; i++ { id := idMap[i] checkNode(urlMap[i], id, id, id) - get(urlMap[i], "/_status/nodes") + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.Node(1), urlMap[i]+"/_status/nodes") + if err != nil { + t.Fatal(err) + } + client.Timeout = 15 * time.Second + get(urlMap[i]+"/_status/nodes", &client) } // Proxy from the first node to the last node. diff --git a/pkg/cmd/roachtest/tests/sysbench.go b/pkg/cmd/roachtest/tests/sysbench.go index 9235252a7d20..e721761c2ba5 100644 --- a/pkg/cmd/roachtest/tests/sysbench.go +++ b/pkg/cmd/roachtest/tests/sysbench.go @@ -75,8 +75,8 @@ func (o *sysbenchOptions) cmd(haproxy bool) string { --db-driver=pgsql \ --pgsql-host=%s \ --pgsql-port=%s \ - --pgsql-user=root \ - --pgsql-password= \ + --pgsql-user=%s \ + --pgsql-password=%s \ --pgsql-db=sysbench \ --report-interval=1 \ --time=%d \ @@ -87,6 +87,8 @@ func (o *sysbenchOptions) cmd(haproxy bool) string { %s`, pghost, pgport, + install.DefaultUser, + install.DefaultPassword, int(o.duration.Seconds()), o.concurrency, o.tables, @@ -109,7 +111,12 @@ func runSysbench(ctx context.Context, t test.Test, c cluster.Cluster, opts sysbe if err = c.Install(ctx, t.L(), loadNode, "haproxy"); err != nil { t.Fatal(err) } - c.Run(ctx, option.WithNodes(loadNode), "./cockroach gen haproxy --insecure --url {pgurl:1}") + // cockroach gen haproxy does not support specifying a non root user + pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1), install.AuthRootCert) + if err != nil { + t.Fatal(err) + } + c.Run(ctx, option.WithNodes(loadNode), fmt.Sprintf("./cockroach gen haproxy --url %s", pgurl)) c.Run(ctx, option.WithNodes(loadNode), "haproxy -f haproxy.cfg -D") t.Status("installing sysbench") @@ -120,13 +127,12 @@ func runSysbench(ctx context.Context, t test.Test, c cluster.Cluster, opts sysbe m := c.NewMonitor(ctx, roachNodes) m.Go(func(ctx context.Context) error { t.Status("preparing workload") - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1)) + pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1), install.AuthRootCert) if err != nil { t.Fatal(err) } - c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf(`./cockroach sql --insecure --url=%s -e "CREATE DATABASE sysbench"`, pgurl)) + c.Run(ctx, option.WithNodes(c.Node(1)), fmt.Sprintf(`./cockroach sql --url=%s -e "CREATE DATABASE sysbench"`, pgurl)) c.Run(ctx, option.WithNodes(loadNode), opts.cmd(false /* haproxy */)+" prepare") - t.Status("running workload") cmd := opts.cmd(true /* haproxy */) + " run" result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(loadNode), cmd) diff --git a/pkg/cmd/roachtest/tests/tlp.go b/pkg/cmd/roachtest/tests/tlp.go index b02987d99fcc..75e999b70eb1 100644 --- a/pkg/cmd/roachtest/tests/tlp.go +++ b/pkg/cmd/roachtest/tests/tlp.go @@ -80,7 +80,7 @@ func runTLP(ctx context.Context, t test.Test, c cluster.Cluster) { return } c.Stop(ctx, t.L(), option.DefaultStopOpts()) - c.Wipe(ctx, false /* preserveCerts */) + c.Wipe(ctx, true /* preserveCerts */) } } diff --git a/pkg/cmd/roachtest/tests/tpcc.go b/pkg/cmd/roachtest/tests/tpcc.go index cc02a4e7c0ec..c508387373be 100644 --- a/pkg/cmd/roachtest/tests/tpcc.go +++ b/pkg/cmd/roachtest/tests/tpcc.go @@ -179,11 +179,7 @@ func setupTPCC( // Do nothing. case usingImport: t.Status("loading fixture" + estimatedSetupTimeStr) - pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } - c.Run(ctx, option.WithNodes(crdbNodes[:1]), tpccImportCmd(opts.Warehouses, opts.ExtraSetupArgs, pgurl)) + c.Run(ctx, option.WithNodes(crdbNodes[:1]), tpccImportCmd(opts.Warehouses, opts.ExtraSetupArgs, "{pgurl:1}")) case usingInit: t.Status("initializing tables" + estimatedSetupTimeStr) extraArgs := opts.ExtraSetupArgs @@ -1312,10 +1308,7 @@ func loadTPCCBench( if b.SharedProcessMT { pgurl = fmt.Sprintf("{pgurl%s:%s}", roachNodes[:1], appTenantName) } else { - pgurl, err = roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Nodes(1)) - if err != nil { - t.Fatal(err) - } + pgurl = "{pgurl:1}" } cmd := tpccImportCmd(loadWarehouses, loadArgs, pgurl) if err = c.RunE(ctx, option.WithNodes(roachNodes[:1]), cmd); err != nil { @@ -1404,7 +1397,12 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen if err := c.Install(ctx, t.L(), loadNodes, "haproxy"); err != nil { t.Fatal(err) } - c.Run(ctx, option.WithNodes(loadNodes), "./cockroach gen haproxy --insecure --url {pgurl:1}") + // cockroach gen haproxy does not support specifying a non root user + pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), c.Node(1), install.AuthRootCert) + if err != nil { + t.Fatal(err) + } + c.Run(ctx, option.WithNodes(loadNodes), fmt.Sprintf("./cockroach gen haproxy --url %s", pgurl)) // Increase the maximum connection limit to ensure that no TPC-C // load gen workers get stuck during connection initialization. // 10k warehouses requires at least 20,000 connections, so add a diff --git a/pkg/cmd/roachtest/tests/tpce.go b/pkg/cmd/roachtest/tests/tpce.go index 85e70bec2bcb..89ac316c3d35 100644 --- a/pkg/cmd/roachtest/tests/tpce.go +++ b/pkg/cmd/roachtest/tests/tpce.go @@ -54,8 +54,8 @@ type tpceConnectionOpts struct { const ( defaultFixtureBucket = "gs://cockroach-fixtures-us-east1/tpce-csv" - defaultUser = "root" - defaultPassword = "" + defaultUser = "roach" + defaultPassword = "system" ) func defaultTPCEConnectionOpts() tpceConnectionOpts { diff --git a/pkg/cmd/roachtest/tests/tpchvec.go b/pkg/cmd/roachtest/tests/tpchvec.go index f821095db0c3..f799494be6be 100644 --- a/pkg/cmd/roachtest/tests/tpchvec.go +++ b/pkg/cmd/roachtest/tests/tpchvec.go @@ -26,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/util/binfetcher" @@ -567,17 +568,25 @@ func smithcmpTestRun( ) { runConfig := tc.getRunConfig() smithcmpPreTestRunHook(ctx, t, c, conn, runConfig.clusterSetups[0]) - const ( - configFile = `tpchvec_smithcmp.toml` - configURL = `https://raw.githubusercontent.com/cockroachdb/cockroach/master/pkg/cmd/roachtest/tests/` + configFile - ) + const configFile = `tpchvec_smithcmp.toml` + firstNode := c.Node(1) - if err := c.RunE(ctx, option.WithNodes(firstNode), fmt.Sprintf("curl %s > %s", configURL, configFile)); err != nil { + if err := c.PutE(ctx, t.L(), "./pkg/cmd/roachtest/tests/"+configFile, configFile); err != nil { + t.Fatal(err) + } + // We don't know the pgurl ahead of time, so we must edit it into the config file like this. + pgurl, err := roachtestutil.DefaultPGUrl(ctx, c, t.L(), firstNode, install.AuthRootCert) + if err != nil { t.Fatal(err) } - // smithcmp cannot access the pgport env variable, so we must edit the config file here - // to tell it the port to use. - if err := c.RunE(ctx, option.WithNodes(firstNode), fmt.Sprintf(`port=$(echo -n {pgport:1}) && sed -i "s|26257|$port|g" %s`, configFile)); err != nil { + // smithcmp expects postgresql instead of postgres + pgurl = strings.Replace(pgurl, "postgres", "postgresql", 1) + // sed treats & as a special character to mean "replace with the matched + // string", so we need to escape it. + pgurl = strings.Replace(pgurl, "&", `\&`, -1) + // The config file wants double quotes, not the single quote returned by the helper. + pgurl = strings.Trim(pgurl, "'") + if err := c.RunE(ctx, option.WithNodes(firstNode), fmt.Sprintf(`sed -i "s|PG_Connection_String|%s|g" %s`, pgurl, configFile)); err != nil { t.Fatal(err) } cmd := fmt.Sprintf("./%s %s", tpchVecSmithcmp, configFile) @@ -605,7 +614,7 @@ func runTPCHVec( if _, err := singleTenantConn.Exec("SET CLUSTER SETTING kv.range_merge.queue_enabled = false;"); err != nil { t.Fatal(err) } - conn = createInMemoryTenantWithConn(ctx, t, c, appTenantName, c.All(), false /* secure */) + conn = createInMemoryTenantWithConn(ctx, t, c, appTenantName, c.All(), c.IsSecure() /* secure */) } else { conn = c.Conn(ctx, t.L(), 1) disableMergeQueue = true @@ -613,7 +622,7 @@ func runTPCHVec( t.Status("restoring TPCH dataset for Scale Factor 1") if err := loadTPCHDataset( - ctx, t, c, conn, 1 /* sf */, c.NewMonitor(ctx), c.All(), disableMergeQueue, false, /* secure */ + ctx, t, c, conn, 1 /* sf */, c.NewMonitor(ctx), c.All(), disableMergeQueue, c.IsSecure(), /* secure */ ); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/tpchvec_smithcmp.toml b/pkg/cmd/roachtest/tests/tpchvec_smithcmp.toml index 951103a22d53..00940ef3d23a 100644 --- a/pkg/cmd/roachtest/tests/tpchvec_smithcmp.toml +++ b/pkg/cmd/roachtest/tests/tpchvec_smithcmp.toml @@ -504,15 +504,17 @@ ORDER BY # thus be executing the same query each time. Queries that don't change # should be tested in other places; smithcmp is for random testing. +# We use a placeholder for addr so we can easily +# find and replace it with a valid connection string. [databases.vec-off] -addr = "postgresql://root@localhost:26257/tpch?sslmode=disable" +addr = "PG_Connection_String" allowmutations = true initsql = """ set vectorize=off; """ [databases.vec-on] -addr = "postgresql://root@localhost:26257/tpch?sslmode=disable" +addr = "PG_Connection_String" allowmutations = true initsql = """ set vectorize=on; diff --git a/pkg/cmd/roachtest/tests/ts_util.go b/pkg/cmd/roachtest/tests/ts_util.go index 3330318b6643..a14b516f37ce 100644 --- a/pkg/cmd/roachtest/tests/ts_util.go +++ b/pkg/cmd/roachtest/tests/ts_util.go @@ -12,11 +12,11 @@ package tests import ( "context" - "net/http" "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/ts/tspb" "github.com/cockroachdb/cockroach/pkg/util/httputil" @@ -48,9 +48,14 @@ type tsQuery struct { } func mustGetMetrics( - ctx context.Context, t test.Test, adminURL string, start, end time.Time, tsQueries []tsQuery, + ctx context.Context, + c cluster.Cluster, + t test.Test, + adminURL string, + start, end time.Time, + tsQueries []tsQuery, ) tspb.TimeSeriesQueryResponse { - response, err := getMetrics(ctx, adminURL, start, end, tsQueries) + response, err := getMetrics(ctx, c, t, adminURL, start, end, tsQueries) if err != nil { t.Fatal(err) } @@ -58,13 +63,20 @@ func mustGetMetrics( } func getMetrics( - ctx context.Context, adminURL string, start, end time.Time, tsQueries []tsQuery, + ctx context.Context, + c cluster.Cluster, + t test.Test, + adminURL string, + start, end time.Time, + tsQueries []tsQuery, ) (tspb.TimeSeriesQueryResponse, error) { - return getMetricsWithSamplePeriod(ctx, adminURL, start, end, defaultSamplePeriod, tsQueries) + return getMetricsWithSamplePeriod(ctx, c, t, adminURL, start, end, defaultSamplePeriod, tsQueries) } func getMetricsWithSamplePeriod( ctx context.Context, + c cluster.Cluster, + t test.Test, adminURL string, start, end time.Time, samplePeriod time.Duration, @@ -103,7 +115,12 @@ func getMetricsWithSamplePeriod( Queries: queries, } var response tspb.TimeSeriesQueryResponse - err := httputil.PostProtobuf(ctx, http.Client{Timeout: 500 * time.Millisecond}, url, &request, &response) + client, err := roachtestutil.DefaultHttpClientWithSessionCookie(ctx, c, t.L(), c.All(), url) + if err != nil { + return tspb.TimeSeriesQueryResponse{}, err + } + client.Timeout = 500 * time.Millisecond + err = httputil.PostProtobuf(ctx, client, url, &request, &response) return response, err } @@ -122,7 +139,7 @@ func verifyTxnPerSecond( t.Fatal(err) } adminURL := adminUIAddrs[0] - response := mustGetMetrics(ctx, t, adminURL, start, end, []tsQuery{ + response := mustGetMetrics(ctx, c, t, adminURL, start, end, []tsQuery{ {name: "cr.node.txn.commits", queryType: rate}, {name: "cr.node.txn.commits", queryType: total}, }) @@ -173,7 +190,7 @@ func verifyLookupsPerSec( t.Fatal(err) } adminURL := adminUIAddrs[0] - response := mustGetMetrics(ctx, t, adminURL, start, end, []tsQuery{ + response := mustGetMetrics(ctx, c, t, adminURL, start, end, []tsQuery{ {name: "cr.node.distsender.rangelookups", queryType: rate}, }) diff --git a/pkg/cmd/roachtest/tests/typeorm.go b/pkg/cmd/roachtest/tests/typeorm.go index 6f37e255e3db..40e32976ea97 100644 --- a/pkg/cmd/roachtest/tests/typeorm.go +++ b/pkg/cmd/roachtest/tests/typeorm.go @@ -176,9 +176,11 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co t.Fatal(err) } - t.Status("running TypeORM test suite - approx 12 mins") + // We have to pass in the root cert with NODE_EXTRA_CA_CERTS because the JSON + // config only accepts the actual certificate contents and not a path. + t.Status("running TypeORM test suite - approx 2 mins") result, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(node), - `cd /mnt/data1/typeorm/ && npm test`, + `cd /mnt/data1/typeorm/ && NODE_EXTRA_CA_CERTS=$HOME/certs/ca.crt npm test`, ) rawResults := result.Stdout + result.Stderr t.L().Printf("Test Results: %s", rawResults) @@ -226,7 +228,7 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co // This full config is required, but notice that all the non-cockroach databases // are set to skip. Some of the unit tests look for a specific config, like // sqlite and will fail if it is not present. -const typeORMConfigJSON = ` +var typeORMConfigJSON = fmt.Sprintf(` [ { "skip": true, @@ -295,9 +297,15 @@ const typeORMConfigJSON = ` "type": "cockroachdb", "host": "localhost", "port": {pgport:1}, - "username": "test_admin", - "password": "", - "database": "defaultdb" + "username": "%s", + "password": "%s", + "database": "defaultdb", + "ssl": true, + "extra": { + "ssl": { + "rejectUnauthorized": true + } + } }, { "skip": true, @@ -309,4 +317,4 @@ const typeORMConfigJSON = ` "useNewUrlParser": true } ] -` +`, install.DefaultUser, install.DefaultPassword) diff --git a/pkg/cmd/roachtest/tests/validate_system_schema_after_version_upgrade.go b/pkg/cmd/roachtest/tests/validate_system_schema_after_version_upgrade.go index 87d8bce438e0..7d8867d5ebd0 100644 --- a/pkg/cmd/roachtest/tests/validate_system_schema_after_version_upgrade.go +++ b/pkg/cmd/roachtest/tests/validate_system_schema_after_version_upgrade.go @@ -68,7 +68,7 @@ func runValidateSystemSchemaAfterVersionUpgrade( t.Fatal(err) } expected = obtainSystemSchema(ctx, t.L(), c, 1) - c.Wipe(ctx, false /* preserveCerts */, c.All()) + c.Wipe(ctx, true /* preserveCerts */, c.All()) mvt := mixedversion.NewTest(ctx, t, t.L(), c, c.All(), // Fixtures are generated on a version that's too old for this test. diff --git a/pkg/roachprod/install/cluster_settings.go b/pkg/roachprod/install/cluster_settings.go index 6984f1b32306..3583e97c6d4f 100644 --- a/pkg/roachprod/install/cluster_settings.go +++ b/pkg/roachprod/install/cluster_settings.go @@ -110,7 +110,7 @@ func MakeClusterSettings(opts ...ClusterSettingOption) ClusterSettings { Binary: config.Binary, Tag: "", PGUrlCertsDir: "./certs", - Secure: false, + Secure: true, UseTreeDist: true, Env: config.DefaultEnvVars(), NumRacks: 0, diff --git a/pkg/roachprod/install/cluster_synced.go b/pkg/roachprod/install/cluster_synced.go index 391573230eb4..7c818151cee2 100644 --- a/pkg/roachprod/install/cluster_synced.go +++ b/pkg/roachprod/install/cluster_synced.go @@ -1641,6 +1641,7 @@ fi %[1]s cert create-ca --certs-dir=certs --ca-key=certs/ca.key %[1]s cert create-client root --certs-dir=certs --ca-key=certs/ca.key $TENANT_SCOPE_OPT %[1]s cert create-client testuser --certs-dir=certs --ca-key=certs/ca.key $TENANT_SCOPE_OPT +%[1]s cert create-client roach --certs-dir=certs --ca-key=certs/ca.key $TENANT_SCOPE_OPT %[1]s cert create-node %[2]s --certs-dir=certs --ca-key=certs/ca.key tar cvf %[3]s certs `, cockroachNodeBinary(c, 1), strings.Join(nodeNames, " "), certsTarName) diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go index b3820467498d..de7165fa6a56 100644 --- a/pkg/roachprod/install/cockroach.go +++ b/pkg/roachprod/install/cockroach.go @@ -529,7 +529,7 @@ func (c *SyncedCluster) NodeURL( user := DefaultUser // TODO(DarrylWong): Support authentication for multitenant, // since they do not use roach:system. - password := SystemInterfaceName + password := DefaultPassword switch auth { case AuthRootCert: @@ -1113,7 +1113,7 @@ func (c *SyncedCluster) createAdminUserForSecureCluster( stmts := strings.Join([]string{ fmt.Sprintf("CREATE USER IF NOT EXISTS %s WITH LOGIN PASSWORD '%s'", DefaultUser, password), - fmt.Sprintf("GRANT ADMIN TO %s", DefaultUser), + fmt.Sprintf("GRANT ADMIN TO %s WITH ADMIN OPTION", DefaultUser), }, "; ") // We retry a few times here because cockroach process might not be