Add feature: Custom IAM Instance Profile

This way Cluster IAM roles can be managed externally, either manually,
using cloudformation or any other tool.

cmd/kops/create_cluster_integration_test.go

@@ -18,17 +18,19 @@ package main
 
 import (
 	"bytes"
-	"github.com/golang/glog"
 	"io/ioutil"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/diff"
 	"k8s.io/kops/pkg/testutils"
-	"path"
-	"strings"
-	"testing"
-	"time"
+
+	"github.com/golang/glog"
 )
 
 var MagicTimestamp = metav1.Time{Time: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}

cmd/kops/integration_test.go

@@ -22,36 +22,39 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
-	"golang.org/x/crypto/ssh"
 	"io/ioutil"
-	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/diff"
-	"k8s.io/kops/pkg/testutils"
 	"os"
 	"path"
 	"reflect"
 	"sort"
 	"strings"
 	"testing"
 	"time"
+
+	"golang.org/x/crypto/ssh"
+
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/diff"
+	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/testutils"
 )
 
 // TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
 func TestMinimal(t *testing.T) {
-	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha0", false, 1)
-	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha1", false, 1)
-	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha2", false, 1)
+	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha0", false, 1, true)
+	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha1", false, 1, true)
+	runTest(t, "minimal.example.com", "../../tests/integration/minimal", "v1alpha2", false, 1, true)
 }
 
 // TestHA runs the test on a simple HA configuration, similar to kops create cluster minimal.example.com --zones us-west-1a,us-west-1b,us-west-1c --master-count=3
 func TestHA(t *testing.T) {
-	runTest(t, "ha.example.com", "../../tests/integration/ha", "v1alpha1", false, 3)
-	runTest(t, "ha.example.com", "../../tests/integration/ha", "v1alpha2", false, 3)
+	runTest(t, "ha.example.com", "../../tests/integration/ha", "v1alpha1", false, 3, true)
+	runTest(t, "ha.example.com", "../../tests/integration/ha", "v1alpha2", false, 3, true)
 }
 
 // TestComplex runs the test on a more complex configuration, intended to hit more of the edge cases
 func TestComplex(t *testing.T) {
-	runTest(t, "complex.example.com", "../../tests/integration/complex", "v1alpha2", false, 1)
+	runTest(t, "complex.example.com", "../../tests/integration/complex", "v1alpha2", false, 1, true)
 }
 
 // TestMinimalCloudformation runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
@@ -63,49 +66,55 @@ func TestMinimalCloudformation(t *testing.T) {
 
 // TestMinimal_141 runs the test on a configuration from 1.4.1 release
 func TestMinimal_141(t *testing.T) {
-	runTest(t, "minimal-141.example.com", "../../tests/integration/minimal-141", "v1alpha0", false, 1)
+	runTest(t, "minimal-141.example.com", "../../tests/integration/minimal-141", "v1alpha0", false, 1, true)
 }
 
 // TestPrivateWeave runs the test on a configuration with private topology, weave networking
 func TestPrivateWeave(t *testing.T) {
-	runTest(t, "privateweave.example.com", "../../tests/integration/privateweave", "v1alpha1", true, 1)
-	runTest(t, "privateweave.example.com", "../../tests/integration/privateweave", "v1alpha2", true, 1)
+	runTest(t, "privateweave.example.com", "../../tests/integration/privateweave", "v1alpha1", true, 1, true)
+	runTest(t, "privateweave.example.com", "../../tests/integration/privateweave", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateFlannel runs the test on a configuration with private topology, flannel networking
 func TestPrivateFlannel(t *testing.T) {
-	runTest(t, "privateflannel.example.com", "../../tests/integration/privateflannel", "v1alpha1", true, 1)
-	runTest(t, "privateflannel.example.com", "../../tests/integration/privateflannel", "v1alpha2", true, 1)
+	runTest(t, "privateflannel.example.com", "../../tests/integration/privateflannel", "v1alpha1", true, 1, true)
+	runTest(t, "privateflannel.example.com", "../../tests/integration/privateflannel", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateCalico runs the test on a configuration with private topology, calico networking
 func TestPrivateCalico(t *testing.T) {
-	runTest(t, "privatecalico.example.com", "../../tests/integration/privatecalico", "v1alpha1", true, 1)
-	runTest(t, "privatecalico.example.com", "../../tests/integration/privatecalico", "v1alpha2", true, 1)
+	runTest(t, "privatecalico.example.com", "../../tests/integration/privatecalico", "v1alpha1", true, 1, true)
+	runTest(t, "privatecalico.example.com", "../../tests/integration/privatecalico", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateCanal runs the test on a configuration with private topology, canal networking
 func TestPrivateCanal(t *testing.T) {
-	runTest(t, "privatecanal.example.com", "../../tests/integration/privatecanal", "v1alpha1", true, 1)
-	runTest(t, "privatecanal.example.com", "../../tests/integration/privatecanal", "v1alpha2", true, 1)
+	runTest(t, "privatecanal.example.com", "../../tests/integration/privatecanal", "v1alpha1", true, 1, true)
+	runTest(t, "privatecanal.example.com", "../../tests/integration/privatecanal", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateKopeio runs the test on a configuration with private topology, kopeio networking
 func TestPrivateKopeio(t *testing.T) {
-	runTest(t, "privatekopeio.example.com", "../../tests/integration/privatekopeio", "v1alpha2", true, 1)
+	runTest(t, "privatekopeio.example.com", "../../tests/integration/privatekopeio", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateDns runs the test on a configuration with private topology, private dns
 func TestPrivateDns1(t *testing.T) {
-	runTest(t, "privatedns1.example.com", "../../tests/integration/privatedns1", "v1alpha2", true, 1)
+	runTest(t, "privatedns1.example.com", "../../tests/integration/privatedns1", "v1alpha2", true, 1, true)
 }
 
 // TestPrivateDns runs the test on a configuration with private topology, private dns, extant vpc
 func TestPrivateDns2(t *testing.T) {
-	runTest(t, "privatedns2.example.com", "../../tests/integration/privatedns2", "v1alpha2", true, 1)
+	runTest(t, "privatedns2.example.com", "../../tests/integration/privatedns2", "v1alpha2", true, 1, true)
+}
+
+// TestCreateClusterCustomAuthProfile runs kops create cluster custom_iam_role.example.com --zones us-test-1a
+func TestCreateClusterCustomAuthProfile(t *testing.T) {
+	featureflag.ParseFlags("+CustomAuthProfileSupport")
+	runTest(t, "custom-iam-role.example.com", "../../tests/integration/custom_iam_role", "v1alpha2", false, 1, false)
 }
 
-func runTest(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int) {
+func runTest(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, expectPolicies bool) {
 	var stdout bytes.Buffer
 
 	inputYAML := "in-" + version + ".yaml"
@@ -207,13 +216,21 @@ func runTest(t *testing.T, clusterName string, srcDir string, version string, pr
 			actualFilenames = append(actualFilenames, f.Name())
 		}
 
-		expectedFilenames := []string{
-			"aws_iam_role_masters." + clusterName + "_policy",
-			"aws_iam_role_nodes." + clusterName + "_policy",
-			"aws_iam_role_policy_masters." + clusterName + "_policy",
-			"aws_iam_role_policy_nodes." + clusterName + "_policy",
-			"aws_key_pair_kubernetes." + clusterName + "-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key",
-			"aws_launch_configuration_nodes." + clusterName + "_user_data",
+		var expectedFilenames []string
+		if expectPolicies {
+			expectedFilenames = []string{
+				"aws_iam_role_masters." + clusterName + "_policy",
+				"aws_iam_role_nodes." + clusterName + "_policy",
+				"aws_iam_role_policy_masters." + clusterName + "_policy",
+				"aws_iam_role_policy_nodes." + clusterName + "_policy",
+				"aws_key_pair_kubernetes." + clusterName + "-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key",
+				"aws_launch_configuration_nodes." + clusterName + "_user_data",
+			}
+		} else {
+			expectedFilenames = []string{
+				"aws_key_pair_kubernetes." + clusterName + "-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key",
+				"aws_launch_configuration_nodes." + clusterName + "_user_data",
+			}
 		}
 
 		for i := 0; i < zones; i++ {

docs/cluster_spec.md

@@ -19,6 +19,23 @@ spec:
     dns: {}
 ```
 
+### authProfile ALPHA SUPPORT
+
+This configuration allows a cluster to utilize existing IAM instance profiles.  Currently this configuration only supports aws.  
+In order to use this feature you have to have to have the instance profile arn of a pre-existing role, and use the kops feature flag by setting
+`export KOPS_FEATURE_FLAGS=+CustomAuthProfileSupport`.  This feature is in ALPHA release only, and can cause very unusual behavior
+with Kubernetes if use incorrectly.
+
+AuthRole example:
+
+```yaml
+spec:
+  authPofile:
+    master: arn:aws:iam::123417490108:instance-profile/kops-custom-master-role
+    node: arn:aws:iam::123417490108:instance-profile/kops-custom-node-role
+```
+
+### api
 
 When configuring a LoadBalancer, you can also choose to have a public ELB or an internal (VPC only) ELB.  The `type`
 field should be `Public` or `Internal`.

docs/iam_roles.md

@@ -2,9 +2,9 @@
 
 Two IAM roles are created for the cluster: one for the masters, and one for the nodes.
 
-> Work is being done on scoping permissions to the minimum required to setup and maintain cluster. 
+> Work is being done on scoping permissions to the minimum required to setup and maintain cluster.
 > Please note that currently all Pods running on your cluster have access to instance IAM role.
-> Consider using projects such as [kube2iam](https://github.com/jtblin/kube2iam) to prevent that. 
+> Consider using projects such as [kube2iam](https://github.com/jtblin/kube2iam) to prevent that.
 
 Master permissions:
 
@@ -136,3 +136,30 @@ You can have an additional policy for each kops role (node, master, bastion). Fo
         }
       ]
 ```
+
+## Reusing Existing Instance Profile
+
+Sometimes you may need to reuse existing IAM Instance Profiles. You can do this
+through the `authProfile` cluster spec API field.  This setting is highly advanced
+and only enabled via CustomAuthProfileSupport`` feature flag.  Setting the wrong role
+permissions can impact various components inside of Kubernetes, and cause
+unexpected issues.  This feature is in place to support the initial documenting and testing the creation of custom roles. Again, use the existing kops functionality, or reach out
+if you want to help!
+
+At this point, we do not have a full definition of the fine grain roles. Please refer
+[to](https://github.com/kubernetes/kops/issues/1873) for more information.
+
+Please use this feature wisely! Enable the feature flag by:
+
+```console
+$ export KOPS_FEATURE_FLAGS="+CustomAuthProfileSupport"
+```
+Inside the cluster spec define one or two instance profiles specific to the master and
+a node.
+
+```yaml
+spec:
+  authPofile:
+    master: arn:aws:iam::123417490108:instance-profile/kops-custom-master-role
+    node: arn:aws:iam::123417490108:instance-profile/kops-custom-node-role
+```

docs/security.md

@@ -34,7 +34,6 @@ This stores the [config.json](https://docs.docker.com/engine/reference/commandli
 All Pods running on your cluster have access to underlying instance IAM role.
 Currently permission scope is quite broad. See [iam_roles.md](iam_roles.md) for details and ways to mitigate that.
 
-
 ## Kubernetes API
 
 (this section is a work in progress)

pkg/apis/kops/cluster.go

@@ -111,7 +111,13 @@ type ClusterSpec struct {
 	//   'external' do not apply updates automatically - they are applied manually or by an external system
 	//   missing: default policy (currently OS security upgrades that do not require a reboot)
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
+
+	// Use an existing custom cloud security policy for the instances.
+	// Only supported for AWS
+	AuthProfile *AuthProfile `json:"authProfile,omitempty"`
+
 	// Additional policies to add for roles
+	// Map is keyed by: master, node
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 	// EtcdClusters stores the configuration for each cluster
 	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`
@@ -224,6 +230,17 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
+type AuthProfile struct {
+
+	// Name is the name of the instance profile to use for the master
+	// Format expected is arn:aws:iam::123456789012:instance-profile/ExampleMasterRole
+	Master *string `json:"master,omitempty"`
+
+	// Name is the name of the instance profile to use for the node
+	// Format expected is arn:aws:iam::123456789012:instance-profile/ExampleNodeRole
+	Node *string `json:"node,omitempty"`
+}
+
 // EtcdStorageType defined the etcd storage backend
 type EtcdStorageType string
 

pkg/apis/kops/v1alpha1/cluster.go

@@ -131,6 +131,10 @@ type ClusterSpec struct {
 	//   missing: default policy (currently OS security upgrades that do not require a reboot)
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 
+	// Use an existing custom cloud security policy for the instances.  One example is to specify the name
+	// of an AWS IAM role for the master and another for the nodes.
+	AuthProfile *AuthProfile `json:"authProfile,omitempty"`
+
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
@@ -327,13 +331,16 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
-//type MasterConfig struct {
-//	Name string `json:",omitempty"`
-//
-//	Image       string `json:",omitempty"`
-//	Zone        string `json:",omitempty"`
-//	MachineType string `json:",omitempty"`
-//}
+type AuthProfile struct {
+
+	// Name is the name of the instance profile to use for the master
+	// Format expected is arn:aws:iam::123456789012:instance-profile/ExampleMasterRole
+	Master *string `json:"master,omitempty"`
+
+	// Name is the name of the instance profile to use for the node
+	// Format expected is arn:aws:iam::123456789012:instance-profile/ExampleNodeRole
+	Node *string `json:"node,omitempty"`
+}
 
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -41,6 +41,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha1_AlwaysAllowAuthorizationSpec,
 		Convert_v1alpha1_Assets_To_kops_Assets,
 		Convert_kops_Assets_To_v1alpha1_Assets,
+		Convert_v1alpha1_AuthProfile_To_kops_AuthProfile,
+		Convert_kops_AuthProfile_To_v1alpha1_AuthProfile,
 		Convert_v1alpha1_AuthenticationSpec_To_kops_AuthenticationSpec,
 		Convert_kops_AuthenticationSpec_To_v1alpha1_AuthenticationSpec,
 		Convert_v1alpha1_AuthorizationSpec_To_kops_AuthorizationSpec,
@@ -222,6 +224,28 @@ func Convert_kops_Assets_To_v1alpha1_Assets(in *kops.Assets, out *Assets, s conv
 	return autoConvert_kops_Assets_To_v1alpha1_Assets(in, out, s)
 }
 
+func autoConvert_v1alpha1_AuthProfile_To_kops_AuthProfile(in *AuthProfile, out *kops.AuthProfile, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+// Convert_v1alpha1_AuthProfile_To_kops_AuthProfile is an autogenerated conversion function.
+func Convert_v1alpha1_AuthProfile_To_kops_AuthProfile(in *AuthProfile, out *kops.AuthProfile, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AuthProfile_To_kops_AuthProfile(in, out, s)
+}
+
+func autoConvert_kops_AuthProfile_To_v1alpha1_AuthProfile(in *kops.AuthProfile, out *AuthProfile, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+// Convert_kops_AuthProfile_To_v1alpha1_AuthProfile is an autogenerated conversion function.
+func Convert_kops_AuthProfile_To_v1alpha1_AuthProfile(in *kops.AuthProfile, out *AuthProfile, s conversion.Scope) error {
+	return autoConvert_kops_AuthProfile_To_v1alpha1_AuthProfile(in, out, s)
+}
+
 func autoConvert_v1alpha1_AuthenticationSpec_To_kops_AuthenticationSpec(in *AuthenticationSpec, out *kops.AuthenticationSpec, s conversion.Scope) error {
 	if in.Kopeio != nil {
 		in, out := &in.Kopeio, &out.Kopeio
@@ -527,6 +551,15 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	// WARNING: in.AdminAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthProfile != nil {
+		in, out := &in.AuthProfile, &out.AuthProfile
+		*out = new(kops.AuthProfile)
+		if err := Convert_v1alpha1_AuthProfile_To_kops_AuthProfile(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthProfile = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EgressProxy != nil {
 		in, out := &in.EgressProxy, &out.EgressProxy
@@ -730,6 +763,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	// WARNING: in.KubernetesAPIAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthProfile != nil {
+		in, out := &in.AuthProfile, &out.AuthProfile
+		*out = new(AuthProfile)
+		if err := Convert_kops_AuthProfile_To_v1alpha1_AuthProfile(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthProfile = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters