From 037f3bf34e86aea98191f0bae5ede5a0278aaa35 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 29 Aug 2023 08:33:33 +0200 Subject: [PATCH] Handle errors in integration test setups Thanks @kev-the-dev Closes #1460 Signed-off-by: Kristoffer Dalby --- cmd/headscale/headscale.go | 2 +- integration/auth_oidc_test.go | 43 +++++++------- integration/auth_web_flow_test.go | 38 +++++++----- integration/control.go | 2 +- integration/embedded_derp_test.go | 27 ++++----- integration/general_test.go | 20 ++++--- integration/hsic/hsic.go | 4 +- integration/scenario.go | 98 +++++++++++++++---------------- integration/scenario_test.go | 4 +- integration/tailscale.go | 12 ++-- integration/tsic/tsic.go | 59 +++++++++++++------ 11 files changed, 172 insertions(+), 137 deletions(-) diff --git a/cmd/headscale/headscale.go b/cmd/headscale/headscale.go index 26055f05e0d..dfaf512f129 100644 --- a/cmd/headscale/headscale.go +++ b/cmd/headscale/headscale.go @@ -48,7 +48,7 @@ func main() { zerolog.TimeFieldFormat = zerolog.TimeFormatUnix log.Logger = log.Output(zerolog.ConsoleWriter{ - Out: os.Stdout, + Out: os.Stderr, TimeFormat: time.RFC3339, NoColor: !colors, }) diff --git a/integration/auth_oidc_test.go b/integration/auth_oidc_test.go index 9b05af9324f..f0be3e8a8d4 100644 --- a/integration/auth_oidc_test.go +++ b/integration/auth_oidc_test.go @@ -171,7 +171,10 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) { t.Logf("%d successful pings out of %d (before expiry)", success, len(allClients)*len(allIps)) // await all nodes being logged out after OIDC token expiry - scenario.WaitForTailscaleLogout() + err = scenario.WaitForTailscaleLogout() + if err != nil { + t.Errorf("failed to log out tailscale nodes: %s", err) + } err = scenario.Shutdown() if err != nil { @@ -188,7 +191,7 @@ func (s *AuthOIDCScenario) CreateHeadscaleEnv( return err } - err = headscale.WaitForReady() + err = headscale.WaitForRunning() if err != nil { return err } @@ -311,13 +314,9 @@ func (s *AuthOIDCScenario) runTailscaleUp( log.Printf("running tailscale up for user %s", userStr) if user, ok := s.users[userStr]; ok { for _, client := range user.Clients { - user.joinWaitGroup.Add(1) - - go func(c TailscaleClient) { - defer user.joinWaitGroup.Done() - - // TODO(juanfont): error handle this - loginURL, err := c.UpWithLoginURL(loginServer) + c := client + user.joinWaitGroup.Go(func() error { + loginURL, err := c.LoginWithURL(loginServer) if err != nil { log.Printf("failed to run tailscale up: %s", err) } @@ -336,9 +335,14 @@ func (s *AuthOIDCScenario) runTailscaleUp( req, _ := http.NewRequestWithContext(ctx, http.MethodGet, loginURL.String(), nil) resp, err := httpClient.Do(req) if err != nil { - log.Printf("%s failed to get login url %s: %s", c.Hostname(), loginURL, err) - - return + log.Printf( + "%s failed to get login url %s: %s", + c.Hostname(), + loginURL, + err, + ) + + return err } defer resp.Body.Close() @@ -347,24 +351,23 @@ func (s *AuthOIDCScenario) runTailscaleUp( if err != nil { log.Printf("%s failed to read response body: %s", c.Hostname(), err) - return + return err } log.Printf("Finished request for %s to join tailnet", c.Hostname()) - }(client) - err = client.WaitForReady() - if err != nil { - log.Printf("error waiting for client %s to be ready: %s", client.Hostname(), err) - } + return nil + }) log.Printf("client %s is ready", client.Hostname()) } - user.joinWaitGroup.Wait() + if err := user.joinWaitGroup.Wait(); err != nil { + return err + } for _, client := range user.Clients { - err := client.WaitForReady() + err := client.WaitForRunning() if err != nil { log.Printf("client %s was not ready: %s", client.Hostname(), err) diff --git a/integration/auth_web_flow_test.go b/integration/auth_web_flow_test.go index 7e497f68075..2129b35827e 100644 --- a/integration/auth_web_flow_test.go +++ b/integration/auth_web_flow_test.go @@ -134,7 +134,10 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) { } } - scenario.WaitForTailscaleLogout() + err = scenario.WaitForTailscaleLogout() + if err != nil { + t.Errorf("failed to log out tailscale nodes: %s", err) + } t.Logf("all clients logged out") @@ -218,7 +221,7 @@ func (s *AuthWebFlowScenario) CreateHeadscaleEnv( return err } - err = headscale.WaitForReady() + err = headscale.WaitForRunning() if err != nil { return err } @@ -250,32 +253,37 @@ func (s *AuthWebFlowScenario) runTailscaleUp( log.Printf("running tailscale up for user %s", userStr) if user, ok := s.users[userStr]; ok { for _, client := range user.Clients { - user.joinWaitGroup.Add(1) - - go func(c TailscaleClient) { - defer user.joinWaitGroup.Done() - - // TODO(juanfont): error handle this - loginURL, err := c.UpWithLoginURL(loginServer) + c := client + user.joinWaitGroup.Go(func() error { + loginURL, err := c.LoginWithURL(loginServer) if err != nil { - log.Printf("failed to run tailscale up: %s", err) + log.Printf("failed to run tailscale up (%s): %s", c.Hostname(), err) + + return err } err = s.runHeadscaleRegister(userStr, loginURL) if err != nil { - log.Printf("failed to register client: %s", err) + log.Printf("failed to register client (%s): %s", c.Hostname(), err) + + return err } - }(client) - err := client.WaitForReady() + return nil + }) + + err := client.WaitForRunning() if err != nil { log.Printf("error waiting for client %s to be ready: %s", client.Hostname(), err) } } - user.joinWaitGroup.Wait() + + if err := user.joinWaitGroup.Wait(); err != nil { + return err + } for _, client := range user.Clients { - err := client.WaitForReady() + err := client.WaitForRunning() if err != nil { log.Printf("client %s was not ready: %s", client.Hostname(), err) diff --git a/integration/control.go b/integration/control.go index 65432b3ff29..b41100705a6 100644 --- a/integration/control.go +++ b/integration/control.go @@ -13,7 +13,7 @@ type ControlServer interface { ConnectToNetwork(network *dockertest.Network) error GetHealthEndpoint() string GetEndpoint() string - WaitForReady() error + WaitForRunning() error CreateUser(user string) error CreateAuthKey(user string, reusable bool, ephemeral bool) (*v1.PreAuthKey, error) ListMachinesInUser(user string) ([]*v1.Machine, error) diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index e9183cdcbd0..93b227190a7 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -2,7 +2,6 @@ package integration import ( "fmt" - "log" "net/url" "testing" @@ -105,7 +104,7 @@ func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv( headscaleURL.Host = fmt.Sprintf("%s:%s", hsServer.GetHostname(), headscaleURL.Port()) - err = hsServer.WaitForReady() + err = hsServer.WaitForRunning() if err != nil { return err } @@ -186,16 +185,11 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser( cert := hsServer.GetCert() - user.createWaitGroup.Add(1) - opts = append(opts, tsic.WithHeadscaleTLS(cert), ) - go func() { - defer user.createWaitGroup.Done() - - // TODO(kradalby): error handle this + user.createWaitGroup.Go(func() error { tsClient, err := tsic.New( s.pool, version, @@ -203,20 +197,23 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser( opts..., ) if err != nil { - // return fmt.Errorf("failed to add tailscale node: %w", err) - log.Printf("failed to create tailscale node: %s", err) + return fmt.Errorf("failed to create tailscale node: %w", err) } - err = tsClient.WaitForReady() + err = tsClient.WaitForRunning() if err != nil { - // return fmt.Errorf("failed to add tailscale node: %w", err) - log.Printf("failed to wait for tailscaled: %s", err) + return fmt.Errorf("failed to wait for tailscaled to need login: %w", err) } user.Clients[tsClient.Hostname()] = tsClient - }() + + return nil + }) + } + + if err := user.createWaitGroup.Wait(); err != nil { + return err } - user.createWaitGroup.Wait() return nil } diff --git a/integration/general_test.go b/integration/general_test.go index f3187e3a937..56c897d6c39 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -108,7 +108,10 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) { } } - scenario.WaitForTailscaleLogout() + err = scenario.WaitForTailscaleLogout() + if err != nil { + t.Errorf("failed to log out tailscale nodes: %s", err) + } t.Logf("all clients logged out") @@ -261,7 +264,10 @@ func TestEphemeral(t *testing.T) { } } - scenario.WaitForTailscaleLogout() + err = scenario.WaitForTailscaleLogout() + if err != nil { + t.Errorf("failed to log out tailscale nodes: %s", err) + } t.Logf("all clients logged out") @@ -488,7 +494,7 @@ func TestResolveMagicDNS(t *testing.T) { spec := map[string]int{ // Omit 1.16.2 (-1) because it does not have the FQDN field "magicdns1": len(TailscaleVersions) - 1, - "magicdns2": len(TailscaleVersions) - 1, + // "magicdns2": len(TailscaleVersions) - 1, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns")) @@ -553,10 +559,10 @@ func TestResolveMagicDNS(t *testing.T) { } } - err = scenario.Shutdown() - if err != nil { - t.Errorf("failed to tear down scenario: %s", err) - } + // err = scenario.Shutdown() + // if err != nil { + // t.Errorf("failed to tear down scenario: %s", err) + // } } func TestExpireNode(t *testing.T) { diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index d27eb06fcde..e13b7273548 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -428,9 +428,9 @@ func (t *HeadscaleInContainer) GetHostname() string { return t.hostname } -// WaitForReady blocks until the Headscale instance is ready to +// WaitForRunning blocks until the Headscale instance is ready to // serve clients. -func (t *HeadscaleInContainer) WaitForReady() error { +func (t *HeadscaleInContainer) WaitForRunning() error { url := t.GetHealthEndpoint() log.Printf("waiting for headscale to be ready at %s", url) diff --git a/integration/scenario.go b/integration/scenario.go index 927d6c80cbc..2d3f28368a9 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -16,6 +16,7 @@ import ( "github.com/juanfont/headscale/integration/tsic" "github.com/ory/dockertest/v3" "github.com/puzpuzpuz/xsync/v2" + "golang.org/x/sync/errgroup" ) const ( @@ -33,6 +34,9 @@ var ( tailscaleVersions2021 = []string{ "head", "unstable", + "1.46.1", + "1.44.2", + "1.42.0", "1.40.0", "1.38.4", "1.36.2", @@ -79,9 +83,9 @@ var ( type User struct { Clients map[string]TailscaleClient - createWaitGroup sync.WaitGroup - joinWaitGroup sync.WaitGroup - syncWaitGroup sync.WaitGroup + createWaitGroup errgroup.Group + joinWaitGroup errgroup.Group + syncWaitGroup errgroup.Group } // Scenario is a representation of an environment with one ControlServer and @@ -213,7 +217,7 @@ func (s *Scenario) Headscale(opts ...hsic.Option) (ControlServer, error) { return nil, fmt.Errorf("failed to create headscale container: %w", err) } - err = headscale.WaitForReady() + err = headscale.WaitForRunning() if err != nil { return nil, fmt.Errorf("failed reach headscale container: %w", err) } @@ -286,17 +290,12 @@ func (s *Scenario) CreateTailscaleNodesInUser( cert := headscale.GetCert() hostname := headscale.GetHostname() - user.createWaitGroup.Add(1) - opts = append(opts, tsic.WithHeadscaleTLS(cert), tsic.WithHeadscaleName(hostname), ) - go func() { - defer user.createWaitGroup.Done() - - // TODO(kradalby): error handle this + user.createWaitGroup.Go(func() error { tsClient, err := tsic.New( s.pool, version, @@ -304,20 +303,26 @@ func (s *Scenario) CreateTailscaleNodesInUser( opts..., ) if err != nil { - // return fmt.Errorf("failed to add tailscale node: %w", err) - log.Printf("failed to create tailscale node: %s", err) + return fmt.Errorf("failed to create tailscale node: %w", err) + // log.Printf("failed to create tailscale node: %s", err) } - err = tsClient.WaitForReady() + err = tsClient.WaitForNeedsLogin() if err != nil { - // return fmt.Errorf("failed to add tailscale node: %w", err) - log.Printf("failed to wait for tailscaled: %s", err) + return fmt.Errorf( + "failed to wait for tailscaled to be ready for login: %w", + err, + ) } user.Clients[tsClient.Hostname()] = tsClient - }() + + return nil + }) + } + if err := user.createWaitGroup.Wait(); err != nil { + return err } - user.createWaitGroup.Wait() return nil } @@ -332,25 +337,18 @@ func (s *Scenario) RunTailscaleUp( ) error { if user, ok := s.users[userStr]; ok { for _, client := range user.Clients { - user.joinWaitGroup.Add(1) - - go func(c TailscaleClient) { - defer user.joinWaitGroup.Done() - - // TODO(kradalby): error handle this - _ = c.Up(loginServer, authKey) - }(client) - - err := client.WaitForReady() - if err != nil { - log.Printf("error waiting for client %s to be ready: %s", client.Hostname(), err) - } + c := client + user.joinWaitGroup.Go(func() error { + return c.Login(loginServer, authKey) + }) } - user.joinWaitGroup.Wait() + if err := user.joinWaitGroup.Wait(); err != nil { + return err + } for _, client := range user.Clients { - err := client.WaitForReady() + err := client.WaitForRunning() if err != nil { log.Printf("client %s was not ready: %s", client.Hostname(), err) @@ -383,16 +381,14 @@ func (s *Scenario) WaitForTailscaleSync() error { for _, user := range s.users { for _, client := range user.Clients { - user.syncWaitGroup.Add(1) - - go func(c TailscaleClient) { - defer user.syncWaitGroup.Done() - - // TODO(kradalby): error handle this - _ = c.WaitForPeers(tsCount) - }(client) + c := client + user.syncWaitGroup.Go(func() error { + return c.WaitForPeers(tsCount) + }) + } + if err := user.syncWaitGroup.Wait(); err != nil { + return err } - user.syncWaitGroup.Wait() } return nil @@ -555,18 +551,18 @@ func (s *Scenario) ListTailscaleClientsFQDNs(users ...string) ([]string, error) // WaitForTailscaleLogout blocks execution until all TailscaleClients have // logged out of the ControlServer. -func (s *Scenario) WaitForTailscaleLogout() { +func (s *Scenario) WaitForTailscaleLogout() error { for _, user := range s.users { for _, client := range user.Clients { - user.syncWaitGroup.Add(1) - - go func(c TailscaleClient) { - defer user.syncWaitGroup.Done() - - // TODO(kradalby): error handle this - _ = c.WaitForLogout() - }(client) + c := client + user.syncWaitGroup.Go(func() error { + return c.WaitForLogout() + }) + } + if err := user.syncWaitGroup.Wait(); err != nil { + return err } - user.syncWaitGroup.Wait() } + + return nil } diff --git a/integration/scenario_test.go b/integration/scenario_test.go index 31f69530485..c2e563eaae9 100644 --- a/integration/scenario_test.go +++ b/integration/scenario_test.go @@ -44,7 +44,7 @@ func TestHeadscale(t *testing.T) { t.Errorf("failed to create start headcale: %s", err) } - err = headscale.WaitForReady() + err = headscale.WaitForRunning() if err != nil { t.Errorf("headscale failed to become ready: %s", err) } @@ -137,7 +137,7 @@ func TestTailscaleNodesJoiningHeadcale(t *testing.T) { t.Errorf("failed to create start headcale: %s", err) } - err = headscale.WaitForReady() + err = headscale.WaitForRunning() if err != nil { t.Errorf("headscale failed to become ready: %s", err) } diff --git a/integration/tailscale.go b/integration/tailscale.go index 166b851fcea..46c87c47ca9 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -14,14 +14,18 @@ type TailscaleClient interface { Hostname() string Shutdown() error Version() string - Execute(command []string, options ...dockertestutil.ExecuteCommandOption) (string, string, error) - Up(loginServer, authKey string) error - UpWithLoginURL(loginServer string) (*url.URL, error) + Execute( + command []string, + options ...dockertestutil.ExecuteCommandOption, + ) (string, string, error) + Login(loginServer, authKey string) error + LoginWithURL(loginServer string) (*url.URL, error) Logout() error IPs() ([]netip.Addr, error) FQDN() (string, error) Status() (*ipnstate.Status, error) - WaitForReady() error + WaitForNeedsLogin() error + WaitForRunning() error WaitForLogout() error WaitForPeers(expected int) error Ping(hostnameOrIP string, opts ...tsic.PingOption) error diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index ffc7e0a90ed..ecf293bd6fd 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -270,7 +270,7 @@ func (t *TailscaleInContainer) Execute( options..., ) if err != nil { - log.Printf("command stderr: %s\n", stderr) + // log.Printf("command stderr: %s\n", stderr) if stdout != "" { log.Printf("command stdout: %s\n", stdout) @@ -288,18 +288,15 @@ func (t *TailscaleInContainer) Execute( // Up runs the login routine on the given Tailscale instance. // This login mechanism uses the authorised key for authentication. -func (t *TailscaleInContainer) Up( +func (t *TailscaleInContainer) Login( loginServer, authKey string, ) error { command := []string{ "tailscale", - "up", - "-login-server", - loginServer, - "--authkey", - authKey, - "--hostname", - t.hostname, + "login", + "--login-server=" + loginServer, + "--authkey=" + authKey, + "--hostname=" + t.hostname, } if t.withSSH { @@ -313,7 +310,11 @@ func (t *TailscaleInContainer) Up( } if _, _, err := t.Execute(command); err != nil { - return fmt.Errorf("failed to join tailscale client: %w", err) + return fmt.Errorf( + "failed to join tailscale client (%s): %w", + strings.Join(command, " "), + err, + ) } return nil @@ -321,16 +322,14 @@ func (t *TailscaleInContainer) Up( // Up runs the login routine on the given Tailscale instance. // This login mechanism uses web + command line flow for authentication. -func (t *TailscaleInContainer) UpWithLoginURL( +func (t *TailscaleInContainer) LoginWithURL( loginServer string, ) (*url.URL, error) { command := []string{ "tailscale", - "up", - "-login-server", - loginServer, - "--hostname", - t.hostname, + "login", + "--login-server=" + loginServer, + "--hostname=" + t.hostname, } _, stderr, err := t.Execute(command) @@ -432,9 +431,31 @@ func (t *TailscaleInContainer) FQDN() (string, error) { return status.Self.DNSName, nil } -// WaitForReady blocks until the Tailscale (tailscaled) instance is ready -// to login or be used. -func (t *TailscaleInContainer) WaitForReady() error { +// WaitForNeedsLogin blocks until the Tailscale (tailscaled) instance has +// started and needs to be logged into. +func (t *TailscaleInContainer) WaitForNeedsLogin() error { + return t.pool.Retry(func() error { + status, err := t.Status() + if err != nil { + return fmt.Errorf("failed to fetch tailscale status: %w", err) + } + + // ipnstate.Status.CurrentTailnet was added in Tailscale 1.22.0 + // https://github.com/tailscale/tailscale/pull/3865 + // + // Before that, we can check the BackendState to see if the + // tailscaled daemon is connected to the control system. + if status.BackendState == "NeedsLogin" { + return nil + } + + return errTailscaleNotConnected + }) +} + +// WaitForRunning blocks until the Tailscale (tailscaled) instance is logged in +// and ready to be used. +func (t *TailscaleInContainer) WaitForRunning() error { return t.pool.Retry(func() error { status, err := t.Status() if err != nil {