From 7edc7fde88e0312c940b97353095f71f100cc705 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Tue, 11 Aug 2020 16:24:55 -0500 Subject: [PATCH] Add --internal flag for export kubecfg that targets the internal dns name Kops creates an "api.internal.$clustername" dns A record that points to the master IP(s) This adds a flag that will use that name and force the CA cert to be included. This is a workaround for client certificate authentication not working on API ELBs with ACM certificates. The ELB has a TLS listener rather than TCP, so the client certificate is not passed through to the apiserver. Using --internal will bypass the API ELB so that the client certificate will be passed directly to the apiserver. This also requires that the masters' security groups allow 443 access from the client which this does not handle automatically. --- cmd/kops/export_kubecfg.go | 7 ++++- cmd/kops/update_cluster.go | 9 +++++- docs/cli/kops_export_kubecfg.md | 4 +++ docs/cli/kops_update_cluster.md | 1 + docs/cluster_spec.md | 4 ++- pkg/kubeconfig/create_kubecfg.go | 20 ++++++++++---- pkg/kubeconfig/create_kubecfg_test.go | 40 +++++++++++++++++++++++++-- 7 files changed, 74 insertions(+), 11 deletions(-) diff --git a/cmd/kops/export_kubecfg.go b/cmd/kops/export_kubecfg.go index fa3dabce4ab66..e44d416d1af34 100644 --- a/cmd/kops/export_kubecfg.go +++ b/cmd/kops/export_kubecfg.go @@ -45,6 +45,9 @@ var ( # export using a user already existing in the kubeconfig file kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user + + # export using the internal DNS name, bypassing the cloud load balancer + kops export kubecfg kubernetes-cluster.example.com --internal `)) exportKubecfgShort = i18n.T(`Export kubecfg.`) @@ -55,6 +58,7 @@ type ExportKubecfgOptions struct { all bool admin bool user string + internal bool } func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command { @@ -78,6 +82,7 @@ func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().BoolVar(&options.all, "all", options.all, "export all clusters from the kops state store") cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "export the cluster admin user and add it to the context") cmd.Flags().StringVar(&options.user, "user", options.user, "add an existing user to the cluster context") + cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "use the cluster's internal DNS name") return cmd } @@ -128,7 +133,7 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio return err } - conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user) + conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user, options.internal) if err != nil { return err } diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go index b31fa3905d2cd..6376bd39da8bc 100644 --- a/cmd/kops/update_cluster.go +++ b/cmd/kops/update_cluster.go @@ -69,6 +69,7 @@ type UpdateClusterOptions struct { CreateKubecfg bool admin bool user string + internal bool Phase string @@ -118,6 +119,7 @@ func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().BoolVar(&options.CreateKubecfg, "create-kube-config", options.CreateKubecfg, "Will control automatically creating the kube config file on your local filesystem") cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "Also export the admin user. Implies --create-kube-config") cmd.Flags().StringVar(&options.user, "user", options.user, "Existing user to add to the cluster context. Implies --create-kube-config") + cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "Use the cluster's internal DNS name. Implies --create-kube-config") cmd.Flags().BoolVar(&options.AllowKopsDowngrade, "allow-kops-downgrade", options.AllowKopsDowngrade, "Allow an older version of kops to update the cluster than last used") cmd.Flags().StringVar(&options.Phase, "phase", options.Phase, "Subset of tasks to run: "+strings.Join(cloudup.Phases.List(), ", ")) cmd.Flags().StringSliceVar(&options.LifecycleOverrides, "lifecycle-overrides", options.LifecycleOverrides, "comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges") @@ -159,6 +161,11 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string, c.CreateKubecfg = true } + if c.internal && !c.CreateKubecfg { + klog.Info("--internal implies --create-kube-config") + c.CreateKubecfg = true + } + // direct requires --yes (others do not, because they don't do anything!) if c.Target == cloudup.TargetDirect { if !c.Yes { @@ -304,7 +311,7 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string, } if kubecfgCert != nil { klog.Infof("Exporting kubecfg for cluster") - conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user) + conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user, c.internal) if err != nil { return nil, err } diff --git a/docs/cli/kops_export_kubecfg.md b/docs/cli/kops_export_kubecfg.md index 7f4fb4eb3d706..b14ef7cc995d8 100644 --- a/docs/cli/kops_export_kubecfg.md +++ b/docs/cli/kops_export_kubecfg.md @@ -21,6 +21,9 @@ kops export kubecfg CLUSTERNAME [flags] # export using a user already existing in the kubeconfig file kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user + + # export using the internal DNS name, bypassing the cloud load balancer + kops export kubecfg kubernetes-cluster.example.com --internal ``` ### Options @@ -29,6 +32,7 @@ kops export kubecfg CLUSTERNAME [flags] --admin export the cluster admin user and add it to the context --all export all clusters from the kops state store -h, --help help for kubecfg + --internal use the cluster's internal DNS name --kubeconfig string the location of the kubeconfig file to create. --user string add an existing user to the cluster context ``` diff --git a/docs/cli/kops_update_cluster.md b/docs/cli/kops_update_cluster.md index 725bcfe6491b1..d63083abace20 100644 --- a/docs/cli/kops_update_cluster.md +++ b/docs/cli/kops_update_cluster.md @@ -29,6 +29,7 @@ kops update cluster [flags] --allow-kops-downgrade Allow an older version of kops to update the cluster than last used --create-kube-config Will control automatically creating the kube config file on your local filesystem -h, --help help for cluster + --internal Use the cluster's internal DNS name. Implies --create-kube-config --lifecycle-overrides strings comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges --out string Path to write any local output --phase string Subset of tasks to run: assets, cluster, network, security diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 8d559bd40cfde..a6a460a6fd9d6 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -48,7 +48,9 @@ spec: idleTimeoutSeconds: 300 ``` -You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported: +You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported. + +Note that when using `sslCertificate`, client certificate authentication, such as with the credentials generated via `kops export kubecfg`, will not work through the load balancer. A `kubecfg` that bypasses the load balancer may be created with the `--internal` flag to `kops update cluster` or `kops export kubecfg`. Security groups may need to be opened to allow access from the clients to the master instances' port TCP/443, for example by using the `additionalSecurityGroups` field on the master instance groups. ```yaml spec: diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go index ea3873ab59f14..16c2eaec28805 100644 --- a/pkg/kubeconfig/create_kubecfg.go +++ b/pkg/kubeconfig/create_kubecfg.go @@ -28,12 +28,20 @@ import ( "k8s.io/kops/upup/pkg/fi" ) -func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin bool, user string) (*KubeconfigBuilder, error) { +func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin bool, user string, internal bool) (*KubeconfigBuilder, error) { clusterName := cluster.ObjectMeta.Name - master := cluster.Spec.MasterPublicName - if master == "" { - master = "api." + clusterName + var master string + if internal { + master = cluster.Spec.MasterInternalName + if master == "" { + master = "api.internal." + clusterName + } + } else { + master = cluster.Spec.MasterPublicName + if master == "" { + master = "api." + clusterName + } } server := "https://" + master @@ -88,8 +96,8 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se b.Context = clusterName b.Server = server - // add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB - if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" { + // add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB or are targeting the internal DNS name + if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || internal { cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) if err != nil { return nil, fmt.Errorf("error fetching CA keypair: %v", err) diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index aa15634e9cb70..57f52da57f610 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -17,6 +17,7 @@ limitations under the License. package kubeconfig import ( + "fmt" "reflect" "testing" @@ -77,6 +78,7 @@ func buildMinimalCluster(clusterName string, masterPublicName string) *kops.Clus } c.Spec.MasterPublicName = masterPublicName + c.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName) c.Spec.KubernetesAPIAccess = []string{"0.0.0.0/0"} c.Spec.SSHAccess = []string{"0.0.0.0/0"} @@ -121,6 +123,7 @@ func TestBuildKubecfg(t *testing.T) { configAccess clientcmd.ConfigAccess admin bool user string + internal bool } publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com") @@ -150,6 +153,7 @@ func TestBuildKubecfg(t *testing.T) { nil, true, "", + false, }, &KubeconfigBuilder{ Context: "testcluster", @@ -178,6 +182,7 @@ func TestBuildKubecfg(t *testing.T) { nil, false, "myuser", + false, }, &KubeconfigBuilder{ Context: "testcluster", @@ -206,6 +211,7 @@ func TestBuildKubecfg(t *testing.T) { nil, true, "", + false, }, &KubeconfigBuilder{ Context: "emptyMasterPublicNameCluster", @@ -242,6 +248,7 @@ func TestBuildKubecfg(t *testing.T) { nil, true, "", + false, }, &KubeconfigBuilder{ Context: "testgossipcluster.k8s.local", @@ -253,16 +260,45 @@ func TestBuildKubecfg(t *testing.T) { }, false, }, + { + "Test Kube Config Data For internal DNS name with admin", + args{ + publiccluster, + fakeKeyStore{ + FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { + return fakeCertificate(), + fakePrivateKey(), + true, + nil + }, + }, + nil, + fakeStatusStore{}, + nil, + true, + "", + true, + }, + &KubeconfigBuilder{ + Context: "testcluster", + Server: "https://internal.testcluster.test.com", + CACert: []byte(certData), + ClientCert: []byte(certData), + ClientKey: []byte(privatekeyData), + User: "testcluster", + }, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user) + got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user, tt.args.internal) if (err != nil) != tt.wantErr { t.Errorf("BuildKubecfg() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("BuildKubecfg() = %v, want %v", got, tt.want) + t.Errorf("BuildKubecfg() = %+v, want %+v", got, tt.want) } }) }