From 98bacc9d3efad16827b083cc5c3dfcbdf5151d2e Mon Sep 17 00:00:00 2001 From: Nick Turner Date: Fri, 8 Oct 2021 08:16:39 +0000 Subject: [PATCH] Run integration tests with per-test role * Create IAM role for each test run in test-integration.sh. * Use role in integration test. * This is necessary in order to run integration tests in prow because the default AWS credentials are an IAM user, and this avoids relying on any assumptions about the default credentials. --- hack/test-integration.sh | 69 ++++++++++++++++++++--- tests/integration/server/server_test.go | 28 ++++++++- tests/integration/testutils/testserver.go | 14 ++++- 3 files changed, 96 insertions(+), 15 deletions(-) diff --git a/hack/test-integration.sh b/hack/test-integration.sh index 21f8f527e..6e66fbaa0 100755 --- a/hack/test-integration.sh +++ b/hack/test-integration.sh @@ -14,29 +14,80 @@ # See the License for the specific language governing permissions and # limitations under the License. -function role_arn() { - sts=$(aws sts get-caller-identity --query "Arn" --output text) - tmp=${sts#*/} +set -o errexit +set -o pipefail +set -o nounset + +command -v aws || { echo "Command 'aws' not found" && exit 1; } + +function account_from_default_credentials() { + sts_account=$(aws sts get-caller-identity --query "Account" --output text) + echo $sts_account +} + +function role_arn_from_default_credentials() { + sts_arn=$(aws sts get-caller-identity --query "Arn" --output text) + tmp=${sts_arn#*/} role_name=${tmp%/*} - tmp=${sts%:*} + tmp=${sts_arn%:*} account_id=${tmp##*:} echo "arn:aws:iam::${account_id}:role/${role_name}" } +function write-role-policy() { + local account_id=$1 + local file_name=$2 + + cat > ${file_name} < /dev/null && pwd)" TEST_ARTIFACTS="${TEST_ARTIFACTS:-"${REPO_ROOT}/test-artifacts"}" -TEST_ROLE_ARN="${TEST_ROLE_ARN:-$(role_arn)}" +TEST_ROLE_ARN="${TEST_ROLE_ARN:-$(role_arn_from_default_credentials)}" -command -v aws || { echo "Command 'aws' not found" && exit 1; } +function cleanup { + if [[ "${CREATE_TEST_ROLE}" = "true" ]]; then + echo "Cleaning up test role ${GENERATED_TEST_ROLE_NAME}" + aws iam delete-role --role-name "${GENERATED_TEST_ROLE_NAME}" || echo "Failed to clean up test role ${GENERATED_TEST_ROLE_NAME}" + fi +} +trap cleanup EXIT + +if [[ "${CREATE_TEST_ROLE}" = "true" ]]; then + echo "Creating test role ${GENERATED_TEST_ROLE_NAME}" + write-role-policy "$(account_from_default_credentials)" ${GENERATED_TEST_ROLE_POLICY_FILE} + create_role_output=$(aws iam create-role --role-name "${GENERATED_TEST_ROLE_NAME}" --assume-role-policy-document "file://${GENERATED_TEST_ROLE_POLICY_FILE}") + rm ${GENERATED_TEST_ROLE_POLICY_FILE} + TEST_ROLE_ARN="$(echo ${create_role_output} | jq -r '.Role.Arn')" +fi make clean make bin -if [[ ! -d ${TEST_ARTIFACTS}/k8s.io/kubernetes ]]; then - mkdir -p ${TEST_ARTIFACTS}/k8s.io/kubernetes - git clone --branch ${KUBERNETES_TAG} --depth 1 https://github.com/kubernetes/kubernetes.git ${TEST_ARTIFACTS}/k8s.io/kubernetes --depth 1 +if [[ -d ${TEST_ARTIFACTS}/k8s.io/kubernetes ]]; then + rm -rf ${TEST_ARTIFACTS}/k8s.io/kubernetes fi + +mkdir -p ${TEST_ARTIFACTS}/k8s.io/kubernetes +git clone --branch ${KUBERNETES_TAG} --depth 1 https://github.com/kubernetes/kubernetes.git ${TEST_ARTIFACTS}/k8s.io/kubernetes --depth 1 + pushd ${TEST_ARTIFACTS}/k8s.io/kubernetes make generated_files popd diff --git a/tests/integration/server/server_test.go b/tests/integration/server/server_test.go index c2eaf56e1..0050edf62 100644 --- a/tests/integration/server/server_test.go +++ b/tests/integration/server/server_test.go @@ -5,7 +5,8 @@ import ( "fmt" "testing" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/aws-iam-authenticator/pkg/config" "sigs.k8s.io/aws-iam-authenticator/tests/integration/testutils" @@ -20,18 +21,39 @@ func TestServer(t *testing.T) { ModifyAuthenticatorServerConfig: func(*config.Config) {}, AuthenticatorClientBinaryPath: authenticatorBinaryPath, TestArtifacts: testArtifactsDir, + ClusterID: "test-cluster", + BackendMode: []string{"EKSConfigMap"}, + RoleArn: roleARN, }, ) t.Log("Creating aws-auth") - _, err := adminClient.CoreV1().ConfigMaps("kube-system").Create(context.TODO(), &v1.ConfigMap{ + userName := "test-user" + _, err := adminClient.CoreV1().ConfigMaps("kube-system").Create(context.TODO(), &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "aws-auth"}, - Data: map[string]string{"mapRoles": fmt.Sprintf(" - rolearn: %s\n username: system:node:{{EC2PrivateDNSName}}\n groups:\n - system:bootstrappers\n - system:nodes", roleARN)}, + Data: map[string]string{"mapRoles": fmt.Sprintf(" - rolearn: %s\n username: %s\n", roleARN, userName)}, }, metav1.CreateOptions{}) if err != nil { t.Fatalf("error creating aws-auth configmap: %v\n", err) } + _, err = adminClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "test-user-binding"}, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: userName, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "cluster-admin", + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("error creating clusterrolebinding: %v\n", err) + } + t.Log("Testing authentication") _, err = execClient.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{}) if err != nil { diff --git a/tests/integration/testutils/testserver.go b/tests/integration/testutils/testserver.go index 00bc294de..82924869d 100644 --- a/tests/integration/testutils/testserver.go +++ b/tests/integration/testutils/testserver.go @@ -33,6 +33,9 @@ type AuthenticatorTestFrameworkSetup struct { ModifyAuthenticatorServerConfig func(*config.Config) AuthenticatorClientBinaryPath string TestArtifacts string + ClusterID string + BackendMode []string + RoleArn string } func StartAuthenticatorTestFramework(t *testing.T, stopCh <-chan struct{}, setup AuthenticatorTestFrameworkSetup) (client.Interface, client.Interface) { @@ -88,10 +91,15 @@ func StartAuthenticatorTestFramework(t *testing.T, stopCh <-chan struct{}, setup t.Fatal(err) } + args := []string{"token", "-i", setup.ClusterID} + if setup.RoleArn != "" { + args = append(args, "--role", setup.RoleArn) + } + // Create aws-iam-authenticator client kubeAPIServerClientConfig.ExecProvider = &clientcmdapi.ExecConfig{ Command: setup.AuthenticatorClientBinaryPath, - Args: []string{"token", "-i", "test-cluster"}, + Args: args, APIVersion: "client.authentication.k8s.io/v1beta1", InteractiveMode: clientcmdapi.NeverExecInteractiveMode, } @@ -111,14 +119,14 @@ func testConfig(t *testing.T, setup AuthenticatorTestFrameworkSetup) (config.Con cfg := config.Config{ PartitionID: "aws", - ClusterID: "test-cluster", + ClusterID: setup.ClusterID, Hostname: "localhost", HostPort: hardcodedAuthenticatorServerPort, KubeconfigPregenerated: true, Address: "127.0.0.1", Kubeconfig: filepath.Join(testDir, "apiserver.kubeconfig"), GenerateKubeconfigPath: filepath.Join(testDir, "webhook.kubeconfig"), - BackendMode: []string{"EKSConfigMap"}, + BackendMode: setup.BackendMode, StateDir: testDir, }