Add --internal flag for export kubecfg that targets the internal dns …

…name

Kops creates an "api.internal.$clustername" dns A record that points to the master IP(s)

This adds a flag that will use that name and force the CA cert to be included.
This is a workaround for client certificate authentication not working on API ELBs with ACM certificates.
The ELB has a TLS listener rather than TCP, so the client certificate is not passed through to the apiserver.
Using --internal will bypass the API ELB so that the client certificate will be passed directly to the apiserver.
This also requires that the masters' security groups allow 443 access from the client which this does not handle automatically.

cmd/kops/export_kubecfg.go

@@ -46,6 +46,9 @@ var (
 
 	# export using a user already existing in the kubeconfig file
 	kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+
+	# export using the internal DNS name, bypassing the cloud load balancer
+	kops export kubecfg kubernetes-cluster.example.com --internal
 		`))
 
 	exportKubecfgShort = i18n.T(`Export kubecfg.`)
@@ -56,6 +59,7 @@ type ExportKubecfgOptions struct {
 	all            bool
 	admin          time.Duration
 	user           string
+	internal       bool
 }
 
 func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
@@ -80,6 +84,7 @@ func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.admin, "admin", options.admin, "export a cluster admin user credential with the given lifetime and add it to the cluster context")
 	cmd.Flags().Lookup("admin").NoOptDefVal = kubeconfig.DefaultKubecfgAdminLifetime.String()
 	cmd.Flags().StringVar(&options.user, "user", options.user, "add an existing user to the cluster context")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "use the cluster's internal DNS name")
 
 	return cmd
 }
@@ -130,7 +135,7 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio
 			return err
 		}
 
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user)
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user, options.internal)
 		if err != nil {
 			return err
 		}

cmd/kops/update_cluster.go

@@ -70,6 +70,7 @@ type UpdateClusterOptions struct {
 	CreateKubecfg bool
 	admin         time.Duration
 	user          string
+	internal      bool
 
 	Phase string
 
@@ -120,6 +121,7 @@ func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.admin, "admin", options.admin, "Also export a cluster admin user credential with the specified lifetime and add it to the cluster context")
 	cmd.Flags().Lookup("admin").NoOptDefVal = kubeconfig.DefaultKubecfgAdminLifetime.String()
 	cmd.Flags().StringVar(&options.user, "user", options.user, "Existing user to add to the cluster context. Implies --create-kube-config")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "Use the cluster's internal DNS name. Implies --create-kube-config")
 	cmd.Flags().BoolVar(&options.AllowKopsDowngrade, "allow-kops-downgrade", options.AllowKopsDowngrade, "Allow an older version of kops to update the cluster than last used")
 	cmd.Flags().StringVar(&options.Phase, "phase", options.Phase, "Subset of tasks to run: "+strings.Join(cloudup.Phases.List(), ", "))
 	cmd.Flags().StringSliceVar(&options.LifecycleOverrides, "lifecycle-overrides", options.LifecycleOverrides, "comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges")
@@ -161,6 +163,11 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		c.CreateKubecfg = true
 	}
 
+	if c.internal && !c.CreateKubecfg {
+		klog.Info("--internal implies --create-kube-config")
+		c.CreateKubecfg = true
+	}
+
 	// direct requires --yes (others do not, because they don't do anything!)
 	if c.Target == cloudup.TargetDirect {
 		if !c.Yes {
@@ -299,7 +306,7 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		firstRun = !hasKubecfg
 
 		klog.Infof("Exporting kubecfg for cluster")
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user)
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user, c.internal)
 		if err != nil {
 			return nil, err
 		}

docs/cli/kops_export_kubecfg.md

@@ -21,6 +21,9 @@ kops export kubecfg CLUSTERNAME [flags]
   
   # export using a user already existing in the kubeconfig file
   kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+  
+  # export using the internal DNS name, bypassing the cloud load balancer
+  kops export kubecfg kubernetes-cluster.example.com --internal
 ```
 
 ### Options
@@ -29,6 +32,7 @@ kops export kubecfg CLUSTERNAME [flags]
       --admin duration[=18h0m0s]   export a cluster admin user credential with the given lifetime and add it to the cluster context
       --all                        export all clusters from the kops state store
   -h, --help                       help for kubecfg
+      --internal                   use the cluster's internal DNS name
       --kubeconfig string          the location of the kubeconfig file to create.
       --user string                add an existing user to the cluster context
 ```

docs/cli/kops_update_cluster.md

@@ -29,6 +29,7 @@ kops update cluster [flags]
       --allow-kops-downgrade          Allow an older version of kops to update the cluster than last used
       --create-kube-config            Will control automatically creating the kube config file on your local filesystem
   -h, --help                          help for cluster
+      --internal                      Use the cluster's internal DNS name. Implies --create-kube-config
       --lifecycle-overrides strings   comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges
       --out string                    Path to write any local output
       --phase string                  Subset of tasks to run: assets, cluster, network, security

docs/cluster_spec.md

@@ -48,7 +48,9 @@ spec:
       idleTimeoutSeconds: 300
 ```
 
-You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported:
+You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported.
+
+Note that when using `sslCertificate`, client certificate authentication, such as with the credentials generated via `kops export kubecfg`, will not work through the load balancer. A `kubecfg` that bypasses the load balancer may be created with the `--internal` flag to `kops update cluster` or `kops export kubecfg`. Security groups may need to be opened to allow access from the clients to the master instances' port TCP/443, for example by using the `additionalSecurityGroups` field on the master instance groups.
 
 ```yaml
 spec:

pkg/kubeconfig/create_kubecfg.go

@@ -34,12 +34,20 @@ import (
 
 const DefaultKubecfgAdminLifetime = 18 * time.Hour
 
-func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin time.Duration, user string) (*KubeconfigBuilder, error) {
+func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin time.Duration, user string, internal bool) (*KubeconfigBuilder, error) {
 	clusterName := cluster.ObjectMeta.Name
 
-	master := cluster.Spec.MasterPublicName
-	if master == "" {
-		master = "api." + clusterName
+	var master string
+	if internal {
+		master = cluster.Spec.MasterInternalName
+		if master == "" {
+			master = "api.internal." + clusterName
+		}
+	} else {
+		master = cluster.Spec.MasterPublicName
+		if master == "" {
+			master = "api." + clusterName
+		}
 	}
 
 	server := "https://" + master
@@ -94,8 +102,8 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 	b.Context = clusterName
 	b.Server = server
 
-	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB
-	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" {
+	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB or are targeting the internal DNS name
+	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || internal {
 		cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
 		if err != nil {
 			return nil, fmt.Errorf("error fetching CA keypair: %v", err)

pkg/kubeconfig/create_kubecfg_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubeconfig
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -78,6 +79,7 @@ func buildMinimalCluster(clusterName string, masterPublicName string) *kops.Clus
 	}
 
 	c.Spec.MasterPublicName = masterPublicName
+	c.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName)
 	c.Spec.KubernetesAPIAccess = []string{"0.0.0.0/0"}
 	c.Spec.SSHAccess = []string{"0.0.0.0/0"}
 
@@ -128,6 +130,7 @@ func TestBuildKubecfg(t *testing.T) {
 		configAccess clientcmd.ConfigAccess
 		admin        time.Duration
 		user         string
+		internal     bool
 	}
 
 	publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com")
@@ -157,6 +160,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				DefaultKubecfgAdminLifetime,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testcluster",
@@ -183,6 +187,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"myuser",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testcluster",
@@ -209,6 +214,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "emptyMasterPublicNameCluster",
@@ -243,6 +249,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testgossipcluster.k8s.local",
@@ -252,10 +259,39 @@ func TestBuildKubecfg(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"Test Kube Config Data For internal DNS name with admin",
+			args{
+				publiccluster,
+				fakeKeyStore{
+					FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+						return fakeCertificate(),
+							fakePrivateKey(),
+							true,
+							nil
+					},
+				},
+				nil,
+				fakeStatusStore{},
+				nil,
+				DefaultKubecfgAdminLifetime,
+				"",
+				true,
+			},
+			&KubeconfigBuilder{
+				Context:    "testcluster",
+				Server:     "https://internal.testcluster.test.com",
+				CACert:     []byte(certData),
+				ClientCert: []byte(certData),
+				ClientKey:  []byte(privatekeyData),
+				User:       "testcluster",
+			},
+			false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user)
+			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user, tt.args.internal)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("BuildKubecfg() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -271,7 +307,7 @@ func TestBuildKubecfg(t *testing.T) {
 				tt.want.ClientKey = got.ClientKey
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("BuildKubecfg() = %v, want %v", got, tt.want)
+				t.Errorf("BuildKubecfg() = %+v, want %+v", got, tt.want)
 			}
 		})
 	}