Skip to content

Commit

Permalink
Remove TestTwoClustersTunnel integration test.
Browse files Browse the repository at this point in the history 
TestTwoClustersTunnel has legitimately been failing for many months.
However, we've been re-running integration tests until it passes.

On master and branch/v8 we've fixed the issue by upgrading etcd and
grpc dependency.

#9656
#9607

Ideally we'd fix TestTwoClustersTunnel on branch/6.2 and branch/v7 by
updating etcd and grpc, but as these branches are meant to be fairly
stable at this point, and the risk of unintended consequences by
updating etcd and grpc feels too high.
  • Loading branch information
@russjones
russjones committed Jan 15, 2022
1 parent 9edeaf7 commit d7280e2
Showing 1 changed file with 0 additions and 211 deletions.
211 changes: 0 additions & 211 deletions integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"bufio"
"bytes"
"context"
"crypto/x509"
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -49,7 +48,6 @@ import (
"github.com/gravitational/teleport/api/types"
apievents "github.com/gravitational/teleport/api/types/events"
apiutils "github.com/gravitational/teleport/api/utils"
"github.com/gravitational/teleport/api/utils/keypaths"
"github.com/gravitational/teleport/lib"
"github.com/gravitational/teleport/lib/auth"
"github.com/gravitational/teleport/lib/auth/testauthority"
Expand Down Expand Up @@ -210,7 +208,6 @@ func TestIntegrations(t *testing.T) {
t.Run("TrustedClustersWithLabels", suite.bind(testTrustedClustersWithLabels))
t.Run("TrustedTunnelNode", suite.bind(testTrustedTunnelNode))
t.Run("TwoClustersProxy", suite.bind(testTwoClustersProxy))
t.Run("TwoClustersTunnel", suite.bind(testTwoClustersTunnel))
t.Run("UUIDBasedProxy", suite.bind(testUUIDBasedProxy))
t.Run("WindowChange", suite.bind(testWindowChange))
}
Expand Down Expand Up @@ -1358,214 +1355,6 @@ func testInvalidLogins(t *testing.T, suite *integrationTestSuite) {
require.Regexp(t, "cluster wrong-site not found", err.Error())
}

// TestTwoClustersTunnel creates two teleport clusters: "a" and "b" and creates a
// tunnel from A to B.
//
// Two tests are run, first is when both A and B record sessions at nodes. It
// executes an SSH command on A by connecting directly to A and by connecting
// to B via B<->A tunnel. All sessions should end up in A.
//
// In the second test, sessions are recorded at B. All sessions still show up on
// A (they are Teleport nodes) but in addition, two show up on B when connecting
// over the B<->A tunnel because sessions are recorded at the proxy.
func testTwoClustersTunnel(t *testing.T, suite *integrationTestSuite) {
tr := utils.NewTracer(utils.ThisFunction()).Start()
defer tr.Stop()

now := time.Now().In(time.UTC).Round(time.Second)

var tests = []struct {
inRecordLocation string
outExecCountSiteA int
outExecCountSiteB int
}{
// normal teleport. since all events are recorded at the node, all events
// end up on site-a and none on site-b.
{
types.RecordAtNode,
3,
0,
},
// recording proxy. since events are recorded at the proxy, 3 events end up
// on site-a (because it's a teleport node so it still records at the node)
// and 2 events end up on site-b because it's recording.
{
types.RecordAtProxy,
3,
2,
},
}

for _, tt := range tests {
t.Run(tt.inRecordLocation, func(t *testing.T) {
twoClustersTunnel(t, suite, now, tt.inRecordLocation, tt.outExecCountSiteA, tt.outExecCountSiteB)
})
}

log.Info("Tests done. Cleaning up.")
}

func twoClustersTunnel(t *testing.T, suite *integrationTestSuite, now time.Time, proxyRecordMode string, execCountSiteA, execCountSiteB int) {
// start the http proxy, we need to make sure this was not used
ps := &proxyServer{}
ts := httptest.NewServer(ps)
defer ts.Close()

// clear out any proxy environment variables
for _, v := range []string{"http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"} {
os.Setenv(v, "")
}

username := suite.me.Username

a := suite.newNamedTeleportInstance(t, "site-A")
b := suite.newNamedTeleportInstance(t, "site-B")

a.AddUser(username, []string{username})
b.AddUser(username, []string{username})

recConfig, err := types.NewSessionRecordingConfigFromConfigFile(types.SessionRecordingConfigSpecV2{
Mode: proxyRecordMode,
})
require.NoError(t, err)

acfg := suite.defaultServiceConfig()
acfg.Auth.Enabled = true
acfg.Proxy.Enabled = true
acfg.Proxy.DisableWebService = true
acfg.Proxy.DisableWebInterface = true
acfg.SSH.Enabled = true

bcfg := suite.defaultServiceConfig()
bcfg.Auth.Enabled = true
bcfg.Auth.SessionRecordingConfig = recConfig
bcfg.Proxy.Enabled = true
bcfg.Proxy.DisableWebService = true
bcfg.Proxy.DisableWebInterface = true
bcfg.SSH.Enabled = false

require.NoError(t, b.CreateEx(t, a.Secrets.AsSlice(), bcfg))
defer b.StopAll()
require.NoError(t, a.CreateEx(t, b.Secrets.AsSlice(), acfg))
defer a.StopAll()

require.NoError(t, b.Start())
require.NoError(t, a.Start())

// wait for both sites to see each other via their reverse tunnels (for up to 10 seconds)
clustersAreCrossConnected := func() bool {
return len(checkGetClusters(t, a.Tunnel)) >= 2 && len(checkGetClusters(t, b.Tunnel)) >= 2
}
require.Eventually(t,
clustersAreCrossConnected,
10*time.Second /* waitFor */, 200*time.Millisecond, /* tick */
"Two clusters do not see each other: tunnels are not working.")

var (
outputA bytes.Buffer
outputB bytes.Buffer
)

// make sure the direct dialer was used and not the proxy dialer
require.Zero(t, ps.Count())

// if we got here, it means two sites are cross-connected. lets execute SSH commands
sshPort := a.GetPortSSHInt()
cmd := []string{"echo", "hello world"}

// directly:
tc, err := a.NewClient(t, ClientConfig{
Login: username,
Cluster: a.Secrets.SiteName,
Host: Host,
Port: sshPort,
ForwardAgent: true,
})
tc.Stdout = &outputA
require.NoError(t, err)
err = tc.SSH(context.TODO(), cmd, false)
require.NoError(t, err)
require.Equal(t, "hello world\n", outputA.String())

// Update trusted CAs.
err = tc.UpdateTrustedCA(context.TODO(), a.Secrets.SiteName)
require.NoError(t, err)

// The known_hosts file should have two certificates, the way bytes.Split
// works that means the output will be 3 (2 certs + 1 empty).
buffer, err := ioutil.ReadFile(keypaths.KnownHostsPath(tc.KeysDir))
require.NoError(t, err)
parts := bytes.Split(buffer, []byte("\n"))
require.Len(t, parts, 3)

// The certs.pem file should have 2 certificates.
buffer, err = ioutil.ReadFile(keypaths.TLSCAsPath(tc.KeysDir, Host))
require.NoError(t, err)
roots := x509.NewCertPool()
ok := roots.AppendCertsFromPEM(buffer)
require.True(t, ok)
require.Len(t, roots.Subjects(), 2)

// wait for active tunnel connections to be established
waitForActiveTunnelConnections(t, b.Tunnel, a.Secrets.SiteName, 1)

// via tunnel b->a:
tc, err = b.NewClient(t, ClientConfig{
Login: username,
Cluster: a.Secrets.SiteName,
Host: Host,
Port: sshPort,
ForwardAgent: true,
})
tc.Stdout = &outputB
require.NoError(t, err)
err = tc.SSH(context.TODO(), cmd, false)
require.NoError(t, err)
require.Equal(t, outputA.String(), outputB.String())

// Stop "site-A" and try to connect to it again via "site-A" (expect a connection error)
require.NoError(t, a.StopAuth(false))
err = tc.SSH(context.TODO(), cmd, false)
require.IsType(t, err, trace.ConnectionProblem(nil, ""))

// Reset and start "Site-A" again
require.NoError(t, a.Reset())
require.NoError(t, a.Start())

// try to execute an SSH command using the same old client to Site-B
// "site-A" and "site-B" reverse tunnels are supposed to reconnect,
// and 'tc' (client) is also supposed to reconnect
tcHasReconnected := func() bool {
return tc.SSH(context.TODO(), cmd, false) == nil
}
require.Eventually(t, tcHasReconnected, 2500*time.Millisecond, 250*time.Millisecond,
"Timed out waiting for Site A to restart")

clientHasEvents := func(site auth.ClientI, count int) func() bool {
// only look for exec events
eventTypes := []string{events.ExecEvent}

return func() bool {
eventsInSite, _, err := site.SearchEvents(now, now.Add(1*time.Hour), apidefaults.Namespace, eventTypes, 0, types.EventOrderAscending, "")
require.NoError(t, err)
return len(eventsInSite) == count
}
}

siteA := a.GetSiteAPI(a.Secrets.SiteName)
require.Eventually(t, clientHasEvents(siteA, execCountSiteA), 5*time.Second, 500*time.Millisecond,
"Failed to find %d events on Site A after 5s", execCountSiteA)

siteB := b.GetSiteAPI(b.Secrets.SiteName)
require.Eventually(t, clientHasEvents(siteB, execCountSiteB), 5*time.Second, 500*time.Millisecond,
"Failed to find %d events on Site B after 5s", execCountSiteB)

// stop both sites for real
require.NoError(t, b.StopAll())

require.NoError(t, a.StopAll())
}

// TestTwoClustersProxy checks if the reverse tunnel uses a HTTP PROXY to
// establish a connection.
func testTwoClustersProxy(t *testing.T, suite *integrationTestSuite) {
Expand Down

0 comments on commit d7280e2

Please sign in to comment.