diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go index 6113f1ae925..c04990ab267 100644 --- a/nomad/rpc_test.go +++ b/nomad/rpc_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/go-sockaddr" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" @@ -30,6 +31,7 @@ import ( "github.com/hashicorp/nomad/testutil" "github.com/hashicorp/raft" "github.com/hashicorp/yamux" + "github.com/shoenig/test/must" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1314,9 +1316,9 @@ type tlsTestHelper struct { nodeID int mtlsServer1 *Server - mtlsServer1Cleanup func() + mtlsServerCleanup1 func() mtlsServer2 *Server - mtlsServer2Cleanup func() + mtlsServerCleanup2 func() nonVerifyServer *Server nonVerifyServerCleanup func() @@ -1335,57 +1337,41 @@ func newTLSTestHelper(t *testing.T) tlsTestHelper { // Generate CA certificate and write it to disk. h.caPEM, h.pk, err = tlsutil.GenerateCA(tlsutil.CAOpts{Days: 5, Domain: "nomad"}) - require.NoError(t, err) + must.NoError(t, err) err = os.WriteFile(filepath.Join(h.dir, "ca.pem"), []byte(h.caPEM), 0600) - require.NoError(t, err) + must.NoError(t, err) // Generate servers and their certificate. h.serverCert = h.newCert(t, "server.global.nomad") - h.mtlsServer1, h.mtlsServer1Cleanup = TestServer(t, func(c *Config) { - c.BootstrapExpect = 2 - c.TLSConfig = &config.TLSConfig{ - EnableRPC: true, - VerifyServerHostname: true, - CAFile: filepath.Join(h.dir, "ca.pem"), - CertFile: h.serverCert + ".pem", - KeyFile: h.serverCert + ".key", - } - }) - h.mtlsServer2, h.mtlsServer2Cleanup = TestServer(t, func(c *Config) { - c.BootstrapExpect = 2 - c.TLSConfig = &config.TLSConfig{ - EnableRPC: true, - VerifyServerHostname: true, - CAFile: filepath.Join(h.dir, "ca.pem"), - CertFile: h.serverCert + ".pem", - KeyFile: h.serverCert + ".key", - } - }) - TestJoin(t, h.mtlsServer1, h.mtlsServer2) - testutil.WaitForLeader(t, h.mtlsServer1.RPC) - testutil.WaitForLeader(t, h.mtlsServer2.RPC) + makeServer := func(bootstrapExpect int, verifyServerHostname bool) (*Server, func()) { + return TestServer(t, func(c *Config) { + c.Logger.SetLevel(hclog.Off) + c.BootstrapExpect = bootstrapExpect + c.TLSConfig = &config.TLSConfig{ + EnableRPC: true, + VerifyServerHostname: verifyServerHostname, + CAFile: filepath.Join(h.dir, "ca.pem"), + CertFile: h.serverCert + ".pem", + KeyFile: h.serverCert + ".key", + } + }) + } - h.nonVerifyServer, h.nonVerifyServerCleanup = TestServer(t, func(c *Config) { - c.TLSConfig = &config.TLSConfig{ - EnableRPC: true, - VerifyServerHostname: false, - CAFile: filepath.Join(h.dir, "ca.pem"), - CertFile: h.serverCert + ".pem", - KeyFile: h.serverCert + ".key", - } - }) - testutil.WaitForLeader(t, h.nonVerifyServer.RPC) + h.mtlsServer1, h.mtlsServerCleanup1 = makeServer(3, true) + h.mtlsServer2, h.mtlsServerCleanup2 = makeServer(3, true) + h.nonVerifyServer, h.nonVerifyServerCleanup = makeServer(3, false) + TestJoin(t, h.mtlsServer1, h.mtlsServer2, h.nonVerifyServer) + testutil.WaitForLeaders(t, h.mtlsServer1.RPC, h.mtlsServer2.RPC, h.nonVerifyServer.RPC) return h } func (h tlsTestHelper) cleanup() { - h.mtlsServer1Cleanup() - h.mtlsServer2Cleanup() + h.mtlsServerCleanup1() + h.mtlsServerCleanup2() h.nonVerifyServerCleanup() - os.RemoveAll(h.dir) } func (h tlsTestHelper) newCert(t *testing.T, name string) string { diff --git a/nomad/testing.go b/nomad/testing.go index d25e6fe5e6b..b1da2e20dcf 100644 --- a/nomad/testing.go +++ b/nomad/testing.go @@ -1,6 +1,7 @@ package nomad import ( + "errors" "fmt" "math/rand" "net" @@ -16,7 +17,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/version" - "github.com/stretchr/testify/require" + "github.com/shoenig/test/must" ) var ( @@ -40,7 +41,7 @@ func TestACLServer(t *testing.T, cb func(*Config)) (*Server, *structs.ACLToken, func TestServer(t *testing.T, cb func(*Config)) (*Server, func()) { s, c, err := TestServerErr(t, cb) - require.NoError(t, err, "failed to start test server") + must.NoError(t, err, must.Sprint("failed to start test server")) return s, c } @@ -127,7 +128,7 @@ func TestServerErr(t *testing.T, cb func(*Config)) (*Server, func(), error) { defer close(ch) // Shutdown server - err := server.Shutdown() + err = server.Shutdown() if err != nil { ch <- fmt.Errorf("failed to shutdown server: %w", err) } @@ -142,9 +143,7 @@ func TestServerErr(t *testing.T, cb func(*Config)) (*Server, func(), error) { t.Fatal("timed out while shutting down server") } }, nil - } else if i == 0 { - return nil, nil, err - } else { + } else if i > 0 { if server != nil { _ = server.Shutdown() } @@ -153,7 +152,7 @@ func TestServerErr(t *testing.T, cb func(*Config)) (*Server, func(), error) { } } - return nil, nil, nil + return nil, nil, errors.New("unable to acquire ports for test server") } func TestJoin(t *testing.T, servers ...*Server) { diff --git a/testutil/wait.go b/testutil/wait.go index e96b3966012..e29947a4e2d 100644 --- a/testutil/wait.go +++ b/testutil/wait.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/kr/pretty" "github.com/shoenig/test/must" + "github.com/shoenig/test/wait" "github.com/stretchr/testify/require" ) @@ -113,6 +114,25 @@ func WaitForLeader(t testing.TB, rpc rpcFn) { }) } +// WaitForLeaders blocks until each serverRPC knows the leader. +func WaitForLeaders(t testing.TB, serverRPCs ...rpcFn) { + t.Helper() + + for i := 0; i < len(serverRPCs); i++ { + ok := func() (bool, error) { + args := &structs.GenericRequest{} + var leader string + err := serverRPCs[i]("Status.Leader", args, &leader) + return leader != "", err + } + must.Wait(t, wait.InitialSuccess( + wait.TestFunc(ok), + wait.Timeout(10*time.Second), + wait.Gap(1*time.Second), + )) + } +} + // WaitForClient blocks until the client can be found func WaitForClient(t testing.TB, rpc rpcFn, nodeID string, region string) { t.Helper()