Add feature: Custom IAM Roles

This way Cluster IAM roles can be managed externally, either manually,
using cloudformation or any other tool.

cmd/kops/create_cluster_integration_test.go

@@ -18,16 +18,17 @@ package main
 
 import (
 	"bytes"
-	"github.com/golang/glog"
 	"io/ioutil"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/diff"
 	"path"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/golang/glog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/diff"
 )
 
 var MagicTimestamp = metav1.Time{Time: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}
@@ -38,6 +39,13 @@ func TestCreateClusterMinimal(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal", "v1alpha2")
 }
 
+// TODO: https://github.com/kubernetes/kops/issues/2438
+// TestCreateClusterCustomIamRole runs kops create cluster custom_iam_role.example.com --zones us-test-1a
+// func TestCreateClusterCustomIamRole(t *testing.T) {
+// 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/custom_iam_role", "v1alpha1")
+// 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/custom_iam_role", "v1alpha2")
+// }
+
 // TestCreateClusterHA runs kops create cluster ha.example.com --zones us-test-1a,us-test-1b,us-test-1c --master-zones us-test-1a,us-test-1b,us-test-1c
 func TestCreateClusterHA(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha", "v1alpha1")

docs/iam_roles.md

@@ -2,9 +2,9 @@
 
 Two IAM roles are created for the cluster: one for the masters, and one for the nodes.
 
-> Work is being done on scoping permissions to the minimum required to setup and maintain cluster. 
+> Work is being done on scoping permissions to the minimum required to setup and maintain cluster.
 > Please note that currently all Pods running on your cluster have access to instance IAM role.
-> Consider using projects such as [kube2iam](https://github.com/jtblin/kube2iam) to prevent that. 
+> Consider using projects such as [kube2iam](https://github.com/jtblin/kube2iam) to prevent that.
 
 Master permissions:
 
@@ -136,3 +136,29 @@ You can have an additional policy for each kops role (node, master, bastion). Fo
         }
       ]
 ```
+
+## Reusing Existing Policies
+
+Sometimes you may need to reuse existing IAM roles. You can do this
+through the `customPolicies` cluster spec API field.  This setting is highly advanced
+and only enabled via CustomPoliciesSupport feature flag.  Setting the wrong role
+permissions can impact various components inside of Kubernetes, and cause
+unexpected issues.  This feature is in place to support the initial documenting and testing the creation of custom roles. Again, use the existing kops functionality, or reach out
+if you want to help!
+
+At this point, we do not have a full definition of the fine grain roles. Please refer
+[to](https://github.com/kubernetes/kops/issues/1873) for more information.
+
+Please use this feature wisely! Enable the feature flag by:
+
+```console
+$ export KOPS_FEATURE_FLAGS="+CustomPoliciesSupport"
+```
+Inside the cluster spec define one or two roles specific to the master and
+a node.
+
+```yaml
+  customPolicies:
+    node: "arn:aws:iam::123456789012:role/kops-node"
+    master: "arn:aws:iam::123456789012:role/kops-master"
+``` 

pkg/apis/kops/cluster.go

@@ -132,7 +132,13 @@ type ClusterSpec struct {
 	//   missing: default policy (currently OS security upgrades that do not require a reboot)
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 
+	// Use an existing custom cloud security policy for the instances.  One example is to specify the name
+	// of an AWS IAM role for the master and another for the nodes.
+	// Map is keyed by: master, node
+	AuthPolicySpec *AuthPolicySpec `json:"authorizationSpec,omitempty"`
+
 	// Additional policies to add for roles
+	// Map is keyed by: master, node
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
 	//HairpinMode                   string `json:",omitempty"`
@@ -306,6 +312,13 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
+type AuthPolicySpec struct {
+	// Name is the name of the policy to use for the master
+	Master *string `json:"master,omitempty"`
+	// Name is the name of the policy to use for the node
+	Node *string `json:"node,omitempty"`
+}
+
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)
 	Name string `json:"name,omitempty"`

pkg/apis/kops/v1alpha1/cluster.go

@@ -129,6 +129,11 @@ type ClusterSpec struct {
 	//   missing: default policy (currently OS security upgrades that do not require a reboot)
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 
+	// Use an existing custom cloud security policy for the instances.  One example is to specify the name
+	// of an AWS IAM role for the master and another for the nodes.
+	// Map is keyed by: master, node
+	AuthPolicySpec *AuthPolicySpec `json:"authorizationSpec,omitempty"`
+
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
@@ -308,6 +313,13 @@ type KubeDNSConfig struct {
 //	MachineType string `json:",omitempty"`
 //}
 
+type AuthPolicySpec struct {
+	// Name is the name of the policy to use for the master
+	Master *string `json:"master,omitempty"`
+	// Name is the name of the policy to use for the node
+	Node *string `json:"node,omitempty"`
+}
+
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)
 	Name string `json:"name,omitempty"`

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -39,6 +39,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_AccessSpec_To_v1alpha1_AccessSpec,
 		Convert_v1alpha1_AlwaysAllowAuthorizationSpec_To_kops_AlwaysAllowAuthorizationSpec,
 		Convert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha1_AlwaysAllowAuthorizationSpec,
+		Convert_v1alpha1_AuthPolicySpec_To_kops_AuthPolicySpec,
+		Convert_kops_AuthPolicySpec_To_v1alpha1_AuthPolicySpec,
 		Convert_v1alpha1_AuthorizationSpec_To_kops_AuthorizationSpec,
 		Convert_kops_AuthorizationSpec_To_v1alpha1_AuthorizationSpec,
 		Convert_v1alpha1_CNINetworkingSpec_To_kops_CNINetworkingSpec,
@@ -184,6 +186,26 @@ func Convert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha1_AlwaysAllowAuthorizat
 	return autoConvert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha1_AlwaysAllowAuthorizationSpec(in, out, s)
 }
 
+func autoConvert_v1alpha1_AuthPolicySpec_To_kops_AuthPolicySpec(in *AuthPolicySpec, out *kops.AuthPolicySpec, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+func Convert_v1alpha1_AuthPolicySpec_To_kops_AuthPolicySpec(in *AuthPolicySpec, out *kops.AuthPolicySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AuthPolicySpec_To_kops_AuthPolicySpec(in, out, s)
+}
+
+func autoConvert_kops_AuthPolicySpec_To_v1alpha1_AuthPolicySpec(in *kops.AuthPolicySpec, out *AuthPolicySpec, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+func Convert_kops_AuthPolicySpec_To_v1alpha1_AuthPolicySpec(in *kops.AuthPolicySpec, out *AuthPolicySpec, s conversion.Scope) error {
+	return autoConvert_kops_AuthPolicySpec_To_v1alpha1_AuthPolicySpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_AuthorizationSpec_To_kops_AuthorizationSpec(in *AuthorizationSpec, out *kops.AuthorizationSpec, s conversion.Scope) error {
 	if in.AlwaysAllow != nil {
 		in, out := &in.AlwaysAllow, &out.AlwaysAllow
@@ -435,6 +457,15 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	// WARNING: in.AdminAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthPolicySpec != nil {
+		in, out := &in.AuthPolicySpec, &out.AuthPolicySpec
+		*out = new(kops.AuthPolicySpec)
+		if err := Convert_v1alpha1_AuthPolicySpec_To_kops_AuthPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthPolicySpec = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
@@ -602,6 +633,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	// WARNING: in.KubernetesAPIAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthPolicySpec != nil {
+		in, out := &in.AuthPolicySpec, &out.AuthPolicySpec
+		*out = new(AuthPolicySpec)
+		if err := Convert_kops_AuthPolicySpec_To_v1alpha1_AuthPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthPolicySpec = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters

pkg/apis/kops/v1alpha2/cluster.go

@@ -133,6 +133,11 @@ type ClusterSpec struct {
 	//   missing: default policy (currently OS security upgrades that do not require a reboot)
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 
+	// Use an existing custom cloud security policy for the instances.  One example is to specify the name
+	// of an AWS IAM role for the master and another for the nodes.
+	// Map is keyed by: master, node
+	AuthPolicySpec *AuthPolicySpec `json:"authorizationSpec,omitempty"`
+
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
@@ -268,3 +273,10 @@ type ClusterSubnetSpec struct {
 
 	Type SubnetType `json:"type,omitempty"`
 }
+
+type AuthPolicySpec struct {
+	// Name is the name of the policy to use for the master
+	Master *string `json:"master,omitempty"`
+	// Name is the name of the policy to use for the node
+	Node *string `json:"node,omitempty"`
+}

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -39,6 +39,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_AccessSpec_To_v1alpha2_AccessSpec,
 		Convert_v1alpha2_AlwaysAllowAuthorizationSpec_To_kops_AlwaysAllowAuthorizationSpec,
 		Convert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha2_AlwaysAllowAuthorizationSpec,
+		Convert_v1alpha2_AuthPolicySpec_To_kops_AuthPolicySpec,
+		Convert_kops_AuthPolicySpec_To_v1alpha2_AuthPolicySpec,
 		Convert_v1alpha2_AuthorizationSpec_To_kops_AuthorizationSpec,
 		Convert_kops_AuthorizationSpec_To_v1alpha2_AuthorizationSpec,
 		Convert_v1alpha2_BastionSpec_To_kops_BastionSpec,
@@ -190,6 +192,26 @@ func Convert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha2_AlwaysAllowAuthorizat
 	return autoConvert_kops_AlwaysAllowAuthorizationSpec_To_v1alpha2_AlwaysAllowAuthorizationSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_AuthPolicySpec_To_kops_AuthPolicySpec(in *AuthPolicySpec, out *kops.AuthPolicySpec, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+func Convert_v1alpha2_AuthPolicySpec_To_kops_AuthPolicySpec(in *AuthPolicySpec, out *kops.AuthPolicySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_AuthPolicySpec_To_kops_AuthPolicySpec(in, out, s)
+}
+
+func autoConvert_kops_AuthPolicySpec_To_v1alpha2_AuthPolicySpec(in *kops.AuthPolicySpec, out *AuthPolicySpec, s conversion.Scope) error {
+	out.Master = in.Master
+	out.Node = in.Node
+	return nil
+}
+
+func Convert_kops_AuthPolicySpec_To_v1alpha2_AuthPolicySpec(in *kops.AuthPolicySpec, out *AuthPolicySpec, s conversion.Scope) error {
+	return autoConvert_kops_AuthPolicySpec_To_v1alpha2_AuthPolicySpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_AuthorizationSpec_To_kops_AuthorizationSpec(in *AuthorizationSpec, out *kops.AuthorizationSpec, s conversion.Scope) error {
 	if in.AlwaysAllow != nil {
 		in, out := &in.AlwaysAllow, &out.AlwaysAllow
@@ -471,6 +493,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.KubernetesAPIAccess = in.KubernetesAPIAccess
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthPolicySpec != nil {
+		in, out := &in.AuthPolicySpec, &out.AuthPolicySpec
+		*out = new(kops.AuthPolicySpec)
+		if err := Convert_v1alpha2_AuthPolicySpec_To_kops_AuthPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthPolicySpec = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
@@ -652,6 +683,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	out.KubernetesAPIAccess = in.KubernetesAPIAccess
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.AuthPolicySpec != nil {
+		in, out := &in.AuthPolicySpec, &out.AuthPolicySpec
+		*out = new(AuthPolicySpec)
+		if err := Convert_kops_AuthPolicySpec_To_v1alpha2_AuthPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AuthPolicySpec = nil
+	}
 	out.AdditionalPolicies = in.AdditionalPolicies
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters

pkg/featureflag/featureflag.go

@@ -52,6 +52,9 @@ var VSphereCloudProvider = New("VSphereCloudProvider", Bool(false))
 
 var EnableExternalDNS = New("EnableExternalDNS", Bool(false))
 
+// CustomPoliciesSupport if set will allow for the setting of Custom Security Policies for Node and Master.
+var CustomPoliciesSupport = New("CustomPoliciesSupport", Bool(false))
+
 var flags = make(map[string]*FeatureFlag)
 var flagsMutex sync.Mutex