Real e2e (#573)

* Move e2e -> integration to make way for actual E2E tests

* Code-complete (but non-functional) E2E test

* Add credentials to execution

* Actually create an E2E test that stands up a cluster

intended to be used in conjection with hack/check{in,out}_account.py
This test:
- Stands up KIND
- Applies the CAPI and CAPA controllers and CRDs
- Creates a cluster and machine in CAPI
- Waits for the control plane to become healthy

* Reduce timeout

* Use interface for session instead of struct

* Use example YAML instead of struct literals for clusters and machines

config/BUILD.bazel

@@ -41,10 +41,12 @@ genrule(
         "//cmd/clusterctl/examples/aws:out/aws_manager_image_patch.yaml",
         "//cmd/clusterctl/examples/aws:test-credentials",
         "config",
+        ":credential_file",
         "//:WORKSPACE",
     ],
     outs = ["aws_provider.yaml"],
-    cmd = """CONFIG_SRCDIR={root_dir}/config && \\
+    cmd = """install -D $(location :credential_file) $(@D)/../cmd/clusterctl/examples/aws/out/credentials && \\
+             CONFIG_SRCDIR={root_dir}/config && \\
              cp -R $$CONFIG_SRCDIR/default $(@D)/default && \\
              cp -R $$CONFIG_SRCDIR/manager $(@D)/manager && \\
              {kustomize} build $(@D)/default > $@
@@ -55,3 +57,18 @@ genrule(
     tools = ["@io_k8s_sigs_kustomize//:kustomize"],
     visibility = ["//visibility:public"],
 )
+
+genrule(
+    name = "credential_file",
+    outs = ["credentials"],
+    visibility = ["//visibility:public"],
+    cmd = " && ".join([
+        "touch $@",
+        "export AWS_ACCESS_KEY_ID=$$(grep ^AWS_ACCESS_KEY_ID bazel-out/volatile-status.txt | cut -f2 -d\" \")",
+        "export AWS_SECRET_ACCESS_KEY=$$(grep ^AWS_SECRET_ACCESS_KEY bazel-out/volatile-status.txt | cut -f2 -d\" \")",
+        "echo '[default]' >> $@",
+        "echo aws_access_key_id = $$AWS_ACCESS_KEY_ID >> $@",
+        "echo aws_secret_access_key = $$AWS_SECRET_ACCESS_KEY >> $@",
+    ]),
+    stamp = 1,
+)

hack/checkin_account.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import httplib
 import urllib
 import os

hack/print-workspace-status.sh

@@ -80,4 +80,6 @@ GIT_VERSION ${GIT_VERSION-}
 GIT_BRANCH ${GIT_BRANCH-}
 GIT_RELEASE_TAG ${GIT_RELEASE_TAG-}
 GIT_RELEASE_COMMIT ${GIT_RELEASE_COMMIT-}
+AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID-}
+AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY-}
 EOF

pkg/cloud/aws/actuators/machine/actuator.go

@@ -66,7 +66,8 @@ func NewActuator(params ActuatorParams) *Actuator {
 	}
 }
 
-func (a *Actuator) getControlPlaneMachines(machineList *clusterv1.MachineList) []*clusterv1.Machine {
+// GetControlPlaneMachines retrieves all control plane nodes from a MachineList
+func GetControlPlaneMachines(machineList *clusterv1.MachineList) []*clusterv1.Machine {
 	var cpm []*clusterv1.Machine
 	for _, m := range machineList.Items {
 		if m.Spec.Versions.ControlPlane != "" {
@@ -134,7 +135,7 @@ func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machi
 	if err != nil {
 		return errors.Wrapf(err, "failed to retrieve machines in cluster %q", cluster.Name)
 	}
-	controlPlaneMachines := a.getControlPlaneMachines(clusterMachines)
+	controlPlaneMachines := GetControlPlaneMachines(clusterMachines)
 	isNodeJoin, err := a.isNodeJoin(controlPlaneMachines, machine, cluster)
 	if err != nil {
 		return errors.Wrapf(err, "failed to determine whether machine %q should join cluster %q", machine.Name, cluster.Name)

pkg/cloud/aws/actuators/machine/actuator_test.go

@@ -220,10 +220,9 @@ func TestGetControlPlaneMachines(t *testing.T) {
 			expectedOut: []clusterv1.Machine{},
 		},
 	}
-	testActuator := NewActuator(ActuatorParams{})
 
 	for _, tc := range testCases {
-		actual := testActuator.getControlPlaneMachines(tc.input)
+		actual := GetControlPlaneMachines(tc.input)
 		if len(actual) != len(tc.expectedOut) {
 			t.Fatalf("[%s] Unexpected number of controlplane machines returned. Got: %d, Want: %d", tc.name, len(actual), len(tc.expectedOut))
 		}

pkg/cloud/aws/actuators/machine_scope.go

@@ -43,7 +43,7 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
 		return nil, err
 	}
 
-	machineConfig, err := machineConfigFromProviderSpec(params.Client, params.Machine.Spec.ProviderSpec)
+	machineConfig, err := MachineConfigFromProviderSpec(params.Client, params.Machine.Spec.ProviderSpec)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get machine config")
 	}
@@ -135,7 +135,8 @@ func (m *MachineScope) Close() {
 	}
 }
 
-func machineConfigFromProviderSpec(clusterClient client.MachineClassesGetter, providerConfig clusterv1.ProviderSpec) (*v1alpha1.AWSMachineProviderSpec, error) {
+// MachineConfigFromProviderSpec tries to decode the JSON-encoded spec, falling back on getting a MachineClass if the value is absent.
+func MachineConfigFromProviderSpec(clusterClient client.MachineClassesGetter, providerConfig clusterv1.ProviderSpec) (*v1alpha1.AWSMachineProviderSpec, error) {
 	var config v1alpha1.AWSMachineProviderSpec
 	if providerConfig.Value != nil {
 		klog.V(4).Info("Decoding ProviderConfig from Value")

pkg/cloud/aws/services/ec2/instances_test.go

@@ -389,7 +389,7 @@ vuO9LYxDXLVY9F7W4ccyCqe27Cj1xyAvdZxwhITrib8Wg5CMqoRpqTw5V3+TpA==
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
 			mockCtrl := gomock.NewController(t)
-			defer mockCtrl.Finish()
+			// defer mockCtrl.Finish()
 			ec2Mock := mock_ec2iface.NewMockEC2API(mockCtrl)
 			elbMock := mock_elbiface.NewMockELBAPI(mockCtrl)
 
@@ -408,13 +408,13 @@ vuO9LYxDXLVY9F7W4ccyCqe27Cj1xyAvdZxwhITrib8Wg5CMqoRpqTw5V3+TpA==
 				},
 			})
 
-			scope.Scope.ClusterConfig = tc.clusterConfig
-			scope.Scope.ClusterStatus = tc.clusterStatus
-
 			if err != nil {
 				t.Fatalf("Failed to create test context: %v", err)
 			}
 
+			scope.Scope.ClusterConfig = tc.clusterConfig
+			scope.Scope.ClusterStatus = tc.clusterStatus
+
 			tc.expect(ec2Mock.EXPECT())
 
 			s := NewService(scope.Scope)

scripts/ci-bazel-e2e.sh

@@ -20,7 +20,7 @@ set -o pipefail
 REPO_ROOT=$(dirname "${BASH_SOURCE}")/..
 
 cd $REPO_ROOT
-bazel test --define='gotags=integration' --test_output all //test/e2e/...
+bazel test --define='gotags=integration' --test_output all //test/integration/...
 bazel_status=$?
 python hack/coalesce.py
 exit $bazel_status

test/e2e/BUILD

@@ -1,43 +1,59 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_test(
     name = "go_default_test",
     size = "large",
     srcs = [
+        "aws_test.go",
         "e2e_suite_test.go",
-        "metacluster_test.go",
     ],
     args = [
         "-kindBinary=$(location @io_k8s_sigs_kind//:kind)",
         "-kubectlBinary=$(location @io_k8s_kubernetes//cmd/kubectl:kubectl)",
         "-awsProviderYAML=$(location //config:aws-provider-yaml)",
         "-clusterAPIYAML=$(location //vendor/sigs.k8s.io/cluster-api/config:cluster-api-yaml)",
         "-managerImageTar=$(location //cmd/manager:manager-amd64.tar)",
+        "-credFile=$(location //config:credential_file)",
+        "-clusterYAML=$(location //cmd/clusterctl/examples/aws:out/cluster.yaml)",
+        "-machineYAML=$(location //cmd/clusterctl/examples/aws:out/machines.yaml)",
     ],
     data = [
+        "//cmd/clusterctl/examples/aws:out/cluster.yaml",
+        "//cmd/clusterctl/examples/aws:out/machines.yaml",
         "//config:aws-provider-yaml",
+        "//config:credential_file",
         "//vendor/sigs.k8s.io/cluster-api/config:cluster-api-yaml",
         "//cmd/manager:manager-amd64.tar",
         "@io_k8s_kubernetes//cmd/kubectl:kubectl",
         "@io_k8s_sigs_kind//:kind",
     ],
-    embed = [":go_default_library"],
     rundir = ".",
     deps = [
+        "//pkg/apis/awsprovider/v1alpha1:go_default_library",
+        "//pkg/cloud/aws/actuators:go_default_library",
+        "//pkg/cloud/aws/actuators/machine:go_default_library",
+        "//pkg/cloud/aws/services/awserrors:go_default_library",
+        "//pkg/cloud/aws/services/cloudformation:go_default_library",
+        "//pkg/cloud/aws/services/sts:go_default_library",
         "//test/e2e/util/kind:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/credentials:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/cloudformation:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/onsi/gomega/gstruct:go_default_library",
         "//vendor/github.com/onsi/gomega/types:go_default_library",
-        "//vendor/k8s.io/api/apps/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
+        "//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset:go_default_library",
+        "//vendor/sigs.k8s.io/cluster-api/pkg/controller/machine:go_default_library",
     ],
 )
-
-go_library(
-    name = "go_default_library",
-    srcs = ["e2e.go"],
-    importpath = "sigs.k8s.io/cluster-api-provider-aws/test/e2e",
-    visibility = ["//visibility:public"],
-)