diff --git a/Makefile b/Makefile index 916460c99f..564feba12d 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,19 @@ help: ## Display this help test: generate verify ## Run tests bazel test --nosandbox_debug //pkg/... //cmd/... $(BAZEL_ARGS) +.PHONY: integration +integration: generate verify ## Run integraion tests + bazel test --define='gotags=integration' --test_output all //test/integration/... + +JANITOR_ENABLED ?= 0 +.PHONY: e2e +e2e: generate verify ## Run e2e tests + JANITOR_ENABLED=$(JANITOR_ENABLED) ./hack/e2e.sh + +.PHONY: e2e-janitor +e2e-janitor: + ./hack/e2e-aws-janitor.sh + ## -------------------------------------- ## Docker ## -------------------------------------- diff --git a/docs/development.md b/docs/development.md index 9d216c946c..b8383b22d1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -21,6 +21,10 @@ - [Building and pushing dev images to GCR](#building-and-pushing-dev-images-to-gcr) - [Building and pushing dev images to custom (non GCR) container registry](#building-and-pushing-dev-images-to-custom-non-gcr-container-registry) - [Running clusterctl](#running-clusterctl) + - [Executing unit tests](#executing-unit-tests) + - [Executing integration tests](#executing-integration-tests) + - [Executing e2e tests](#executing-e2e-tests) + - [Executing e2e tests with Boskos](#executing-e2e-tests-with-boskos) - [Automated Testing](#automated-testing) - [Mocks](#mocks) - [Troubleshooting](#troubleshooting) @@ -130,6 +134,61 @@ manifests creating a target cluster in AWS. After this is finished you will have a kubeconfig copied locally. You can debug most issues by SSHing into the instances that have been created and reading `/var/log/cloud-init-output.log`. +#### Executing unit tests + +`make test` executes the project's unit tests. These tests do not stand up a +Kubernetes cluster, nor do they have external dependencies. + +#### Executing integration tests +`make integration` executes the project's integration tests. These tests stand +up a local Kubernetes cluster using Kind in order to deploy the project's CRDs. +The tested controller is **not** used to deploy Kubernetes to AWS. + +These tests depend on the following binaries in the system path: +* `kind` + +#### Executing e2e tests +`make e2e` executes the project's end-to-end tests with AWS account +information parsed from the environment. + +These tests stand up a local Kubernetes cluster using Kind. The project's CRDs +and controllers are deployed to the Kind cluster and are used to deploy +Kubernetes to AWS. + +The AWS janitor is disabled by default. `JANITOR_ENABLED=1 make e2e` executes +janitor immediately after running the e2e tests. + +Please keep in mind that the janitor is highly destructive and should not +be executed against shared AWS accounts or preferrably AWS accounts not +dedicated to testing this project. + +These tests depend on the following binaries in the system path: +* `kind` +* `aws-janitor` + +#### Executing e2e tests with Boskos +`BOSKOS_HOST=http://boskos make e2e` executes the project's end-to-end tests +with AWS account information acquired from a Boskos host. + +These tests stand up a local Kubernetes cluster using Kind. The project's CRDs +and controllers are deployed to the Kind cluster and are used to deploy +Kubernetes to AWS. + +The AWS janitor is disabled by default. +`BOSKOS_HOST=http://boskos JANITOR_ENABLED=1 make e2e` executes +the janitor immediately after running the e2e tests. + +Please keep in mind that the janitor is highly destructive and should not +be executed against shared AWS accounts or preferrably AWS accounts not +dedicated to testing this project. + +`BOSKOS_HOST=http://boskos make e2e` executes the project's +end-to-end tests with the janitor disabled. + +These tests depend on the following binaries in the system path: +* `kind` +* `aws-janitor` + ### Automated Testing #### Mocks diff --git a/hack/checkout_account.py b/hack/checkout_account.py index b750bd3022..44e92a8b12 100755 --- a/hack/checkout_account.py +++ b/hack/checkout_account.py @@ -44,7 +44,7 @@ body = json.load(resp) conn.close() - print 'export BOSKOS_RESOURCE_NAME="%s"' % body['name'] - print 'export AWS_ACCESS_KEY_ID="%s"' % body['userdata']['access-key-id'] - print 'export AWS_SECRET_ACCESS_KEY="%s"' % body['userdata']['secret-access-key'] + print 'export BOSKOS_RESOURCE_NAME="%s";' % body['name'] + print 'export AWS_ACCESS_KEY_ID="%s";' % body['userdata']['access-key-id'] + print 'export AWS_SECRET_ACCESS_KEY="%s";' % body['userdata']['secret-access-key'] diff --git a/hack/e2e-aws-disallowed.txt b/hack/e2e-aws-disallowed.txt new file mode 100644 index 0000000000..d534933d2c --- /dev/null +++ b/hack/e2e-aws-disallowed.txt @@ -0,0 +1,9 @@ +# This file is a line-delimited list of MD5 checksums of AWS keys that should +# not be used with the CAPA e2e tests or any janitorial work that occurs after +# the test execution. +# +# To add a new key to the file, execute: +# +# $ echo "${AWS_ACCESS_KEY_ID}" | { md5sum 2>/dev/null || md5; } | awk '{print $1}' >> hack/e2e-aws-disallowed.txt + +afbf8e5c5622470940b050594f82c47e \ No newline at end of file diff --git a/hack/e2e-aws-janitor.sh b/hack/e2e-aws-janitor.sh new file mode 100755 index 0000000000..145d9479a6 --- /dev/null +++ b/hack/e2e-aws-janitor.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# usage: e2e-aws-janitor.sh [FLAGS] +# This program is a wrapper for running the aws-janitor command with a check +# that prevents disallowed AWS keys from being used. +# +# FLAGS +# To see a full list of flags supported by this program, run "aws-janitor -h" +################################################################################ + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# Require the aws-janitor command. +command -v aws-janitor >/dev/null 2>&1 || \ + { echo "aws-janitor not found" 1>&2; exit 1; } + +# Prevent a disallowed AWS key from being used. +if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" | \ + { md5sum 2>/dev/null || md5; } | \ + awk '{print $1}')" hack/e2e-aws-disallowed.txt; then + echo "The provided AWS key is not allowed" 1>&2 + exit 1 +fi + +exec aws-janitor -all "${@-}" diff --git a/hack/e2e-aws-resources-list.sh b/hack/e2e-aws-resources-list.sh new file mode 100755 index 0000000000..656a8d9ecf --- /dev/null +++ b/hack/e2e-aws-resources-list.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +################################################################################ +# usage: e2e-aws-resources-list.sh [FLAGS] +# This program is a wrapper for running the aws-resources-list command with +# a check that prevents disallowed AWS keys from being used. +# +# FLAGS +# To see a full list of flags supported by this program, run +# "aws-resources-list -h" +################################################################################ + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# Require the aws-resources-list command. +command -v aws-resources-list >/dev/null 2>&1 || \ + { echo "aws-resources-list not found" 1>&2; exit 1; } + +# Prevent a disallowed AWS key from being used. +if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" | \ + { md5sum 2>/dev/null || md5; } | \ + awk '{print $1}')" hack/e2e-aws-disallowed.txt; then + echo "The provided AWS key is not allowed" 1>&2 + exit 1 +fi + +exec aws-resources-list "${@-}" diff --git a/hack/e2e.sh b/hack/e2e.sh new file mode 100755 index 0000000000..7c2eebb790 --- /dev/null +++ b/hack/e2e.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# usage: e2e.sh +# This program runs the e2e tests. +# +# ENVIRONMENT VARIABLES +# JANITOR_ENABLED +# Set to 1 to run the aws-janitor command after running the e2e tests. +################################################################################ + +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# If BOSKOS_HOST is set then acquire an AWS account from Boskos. +if [ -n "${BOSKOS_HOST:-}" ]; then + # Check out the account from Boskos and store the produced environment + # variables in a temporary file. + account_env_var_file="$(mktemp)" + python hack/checkout_account.py 1>"${account_env_var_file}" + checkout_account_status="${?}" + + # If the checkout process was a success then load the account's + # environment variables into this process. + # shellcheck disable=SC1090 + [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}" + + # Always remove the account environment variable file. It contains + # sensitive information. + rm -f "${account_env_var_file}" + + if [ ! "${checkout_account_status}" = "0" ]; then + echo "error getting account from boskos" 1>&2 + exit "${checkout_account_status}" + fi +fi + +# Prevent a disallowed AWS key from being used. +if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" | \ + { md5sum 2>/dev/null || md5; } | \ + awk '{print $1}')" hack/e2e-aws-disallowed.txt; then + echo "The provided AWS key is not allowed" 1>&2 + exit 1 +fi + +bazel test --define='gotags=e2e' --test_output all //test/e2e/... +bazel_status="${?}" + +# If the artifacts environment variable is set then coalesce the test results. +[ -z "${ARTIFACTS:-}" ] || python hack/coalesce.py + +# If Boskos is being used then release the AWS account back to Boskos. +[ -z "${BOSKOS_HOST:-}" ] || hack/checkin_account.py + +# The janitor is typically not run as part of the e2e process, but rather +# in a parallel process via a service on the same cluster that runs Prow and +# Boskos. +# +# However, setting JANITOR_ENABLED=1 tells this program to run the janitor +# after the e2e test is executed. +if [ "${JANITOR_ENABLED:-0}" = "1" ]; then + if ! command -v aws-janitor >/dev/null 2>&1; then + echo "skipping janitor; aws-janitor not found" 1>&2 + else + aws-janitor -all -v 2 + fi +else + echo "skipping janitor; JANITOR_ENABLED=${JANITOR_ENABLED}" 1>&2 +fi + +exit "${bazel_status}" diff --git a/hack/print-workspace-status.sh b/hack/print-workspace-status.sh index db6bccc1fa..150bfc6471 100755 --- a/hack/print-workspace-status.sh +++ b/hack/print-workspace-status.sh @@ -79,8 +79,9 @@ GIT_VERSION ${GIT_VERSION-} GIT_BRANCH ${GIT_BRANCH-} GIT_RELEASE_TAG ${GIT_RELEASE_TAG-} GIT_RELEASE_COMMIT ${GIT_RELEASE_COMMIT-} -AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID-} -AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY-} -AWS_SESSION_TOKEN ${AWS_SESSION_TOKEN-} -AWS_REGION ${AWS_REGION:-us-east-1} +STABLE_AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID-} +STABLE_AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY-} +STABLE_AWS_DEFAULT_REGION ${AWS_DEFAULT_REGION-} +STABLE_AWS_REGION ${AWS_REGION:-${AWS_DEFAULT_REGION-}} +STABLE_AWS_SESSION_TOKEN ${AWS_SESSION_TOKEN-} EOF diff --git a/scripts/ci-bazel-integration.sh b/scripts/ci-bazel-integration.sh index de972be32a..3e93db5169 100755 --- a/scripts/ci-bazel-integration.sh +++ b/scripts/ci-bazel-integration.sh @@ -18,8 +18,8 @@ set -o nounset set -o pipefail REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. - cd "${REPO_ROOT}" || exit 1 + bazel test --define='gotags=integration' --test_output all //test/integration/... bazel_status="${?}" python hack/coalesce.py diff --git a/test/e2e/BUILD b/test/e2e/BUILD index d3a36dcf3b..3fa684ac01 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -38,6 +38,7 @@ go_test( "-credFile=$(location manifests/provider-credentials.profile)", "-clusterYAML=$(location manifests/cluster.yaml)", "-machineYAML=$(location manifests/machines.yaml)", + "-regionFile=$(location region.txt)", ], data = [ "manifests/addons.yaml", @@ -45,6 +46,7 @@ go_test( "manifests/machines.yaml", "manifests/provider-credentials.profile", "provider-components-e2e.yaml", + "region.txt", "//cmd/manager:manager-amd64.tar", "@io_k8s_kubernetes//cmd/kubectl:kubectl", "@io_k8s_sigs_kind//:kind", @@ -119,9 +121,9 @@ genrule( outs = ["manifests/provider-credentials.profile"], cmd = " && ".join([ "touch $@", - "export AWS_ACCESS_KEY_ID=$$(grep ^AWS_ACCESS_KEY_ID bazel-out/volatile-status.txt | cut -f2 -d\" \")", - "export AWS_SECRET_ACCESS_KEY=$$(grep ^AWS_SECRET_ACCESS_KEY bazel-out/volatile-status.txt | cut -f2 -d\" \")", - "export AWS_SESSION_TOKEN=$$(grep ^AWS_SESSION_TOKEN bazel-out/volatile-status.txt | cut -f2 -d\" \")", + "export AWS_ACCESS_KEY_ID=$$(grep ^STABLE_AWS_ACCESS_KEY_ID bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_SECRET_ACCESS_KEY=$$(grep ^STABLE_AWS_SECRET_ACCESS_KEY bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_SESSION_TOKEN=$$(grep ^STABLE_AWS_SESSION_TOKEN bazel-out/stable-status.txt | cut -f2 -d\" \")", "echo '[default]' >> $@", "echo aws_access_key_id = $$AWS_ACCESS_KEY_ID >> $@", "echo aws_secret_access_key = $$AWS_SECRET_ACCESS_KEY >> $@", @@ -136,13 +138,15 @@ genrule( outs = ["manifests/provider-credentials.sh"], cmd = " && ".join([ "touch $@", - "export AWS_ACCESS_KEY_ID=$$(grep ^AWS_ACCESS_KEY_ID bazel-out/volatile-status.txt | cut -f2 -d\" \")", - "export AWS_SECRET_ACCESS_KEY=$$(grep ^AWS_SECRET_ACCESS_KEY bazel-out/volatile-status.txt | cut -f2 -d\" \")", - "export AWS_SESSION_TOKEN=$$(grep ^AWS_SESSION_TOKEN bazel-out/volatile-status.txt | cut -f2 -d\" \")", - "export AWS_REGION=$$(grep ^AWS_REGION bazel-out/volatile-status.txt | cut -f2 -d\" \")", + "export AWS_ACCESS_KEY_ID=$$(grep ^STABLE_AWS_ACCESS_KEY_ID bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_SECRET_ACCESS_KEY=$$(grep ^STABLE_AWS_SECRET_ACCESS_KEY bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_SESSION_TOKEN=$$(grep ^STABLE_AWS_SESSION_TOKEN bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_DEFAULT_REGION=$$(grep ^STABLE_AWS_DEFAULT_REGION bazel-out/stable-status.txt | cut -f2 -d\" \")", + "export AWS_REGION=$$(grep ^STABLE_AWS_REGION bazel-out/stable-status.txt | cut -f2 -d\" \")", "echo export AWS_ACCESS_KEY_ID=$$AWS_ACCESS_KEY_ID >> $@", "echo export AWS_SECRET_ACCESS_KEY=$$AWS_SECRET_ACCESS_KEY >> $@", "echo export AWS_SESSION_TOKEN=$$AWS_SESSION_TOKEN >> $@", + "echo export AWS_DEFAULT_REGION=$$AWS_DEFAULT_REGION >> $@", "echo export AWS_REGION=$$AWS_REGION >> $@", ]), stamp = 1, @@ -164,3 +168,15 @@ genrule( tools = ["@io_k8s_sigs_kustomize//:kustomize"], visibility = ["//visibility:public"], ) + +genrule( + name = "e2e-region", + outs = ["region.txt"], + cmd = " && ".join([ + "touch $@", + "export AWS_DEFAULT_REGION=$$(grep ^STABLE_AWS_DEFAULT_REGION bazel-out/stable-status.txt | cut -f2 -d\" \")", + "echo $$AWS_DEFAULT_REGION >> $@", + ]), + stamp = 1, + visibility = ["//visibility:private"], +) diff --git a/test/e2e/aws_test.go b/test/e2e/aws_test.go index 9555492ac4..0343a05487 100644 --- a/test/e2e/aws_test.go +++ b/test/e2e/aws_test.go @@ -17,9 +17,9 @@ limitations under the License. package e2e_test import ( + "bytes" "flag" "io/ioutil" - "os" "time" . "github.com/onsi/ginkgo" @@ -64,18 +64,20 @@ const ( ) var ( - region string credFile = flag.String("credFile", "", "path to an AWS credentials file") clusterYAML = flag.String("clusterYAML", "", "path to the YAML for the cluster we're creating") machineYAML = flag.String("machineYAML", "", "path to the YAML describing the control plane we're creating") + regionFile = flag.String("regionFile", "", "The path to a text file containing the AWS region") + region string ) -func init() { - if region = os.Getenv("AWS_DEFAULT_REGION"); region == "" { - if region = os.Getenv("AWS_REGION"); region == "" { - region = "us-east-1" - } +func initRegion() error { + data, err := ioutil.ReadFile(*regionFile) + if err != nil { + return fmt.Errorf("error reading AWS region file: %v", err) } + region = string(bytes.TrimSpace(data)) + return nil } var _ = Describe("AWS", func() { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f6ca2530e7..0cf709e4ff 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -24,6 +24,9 @@ import ( ) func TestE2e(t *testing.T) { + if err := initRegion(); err != nil { + t.Fatal(err) + } RegisterFailHandler(Fail) RunSpecs(t, "e2e Suite") }