From ce2e83d1ed207ab6048cdbe2c9ed09a6af682c35 Mon Sep 17 00:00:00 2001 From: Chris Hein Date: Tue, 4 May 2021 09:49:32 -0700 Subject: [PATCH] adding NestedControlPlane & NestedCluster controllers Signed-off-by: Chris Hein --- PROJECT | 14 +- .../v1alpha4/nestedapiserver_types.go | 26 +- .../v1alpha4/nestedcontrollermanager_types.go | 26 +- .../v1alpha4/nestedcontrolplane_types.go | 23 +- .../controlplane/v1alpha4/nestedetcd_types.go | 26 +- .../v1alpha4/zz_generated.deepcopy.go | 1 - .../v1alpha4/groupversion_info.go | 36 +++ .../v1alpha4/nestedcluster_types.go | 64 ++++ .../v1alpha4/zz_generated.deepcopy.go | 115 +++++++ .../nested-apiserver-service-template.yaml | 13 +- ...nested-apiserver-statefulset-template.yaml | 90 ++++-- ...ontrollermanager-statefulset-template.yaml | 40 ++- .../nested-etcd-service-template.yaml | 8 +- .../nested-etcd-statefulset-template.yaml | 50 +-- ...ane.cluster.x-k8s.io_nestedapiservers.yaml | 3 + ...er.x-k8s.io_nestedcontrollermanagers.yaml} | 7 +- ....cluster.x-k8s.io_nestedcontrolplanes.yaml | 19 +- ...rolplane.cluster.x-k8s.io_nestedetcds.yaml | 3 + ...cture.cluster.x-k8s.io_nestedclusters.yaml | 79 +++++ config/crd/kustomization.yaml | 27 ++ config/rbac/nestedcluster_editor_role.yaml | 24 ++ config/rbac/nestedcluster_viewer_role.yaml | 20 ++ config/rbac/role.yaml | 26 ++ controllers/controlplane/consts.go | 15 +- controllers/controlplane/controller_util.go | 112 ++----- .../controlplane/controller_util_test.go | 21 +- .../nestedapiserver_controller.go | 98 +++++- .../nestedcontrollermanager_controller.go | 20 +- .../nestedcontrolplane_controller.go | 269 +++++++++++++++- .../controlplane/nestedetcd_controller.go | 97 +++++- .../nestedcluster_controller.go | 99 ++++++ controllers/infrastructure/suite_test.go | 80 +++++ main.go | 13 +- pki/pki.go | 298 ++++++++++++++++++ pki/util/util.go | 127 ++++++++ secret/secret.go | 49 +++ 36 files changed, 1804 insertions(+), 234 deletions(-) create mode 100644 apis/infrastructure/v1alpha4/groupversion_info.go create mode 100644 apis/infrastructure/v1alpha4/nestedcluster_types.go create mode 100644 apis/infrastructure/v1alpha4/zz_generated.deepcopy.go rename config/crd/bases/{controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml => controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml} (97%) create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/rbac/nestedcluster_editor_role.yaml create mode 100644 config/rbac/nestedcluster_viewer_role.yaml create mode 100644 controllers/infrastructure/nestedcluster_controller.go create mode 100644 controllers/infrastructure/suite_test.go create mode 100644 pki/pki.go create mode 100644 pki/util/util.go create mode 100644 secret/secret.go diff --git a/PROJECT b/PROJECT index e742de5d..6c309cf7 100644 --- a/PROJECT +++ b/PROJECT @@ -1,5 +1,6 @@ domain: cluster.x-k8s.io -layout: go.kubebuilder.io/v3 +layout: +- go.kubebuilder.io/v3 multigroup: true projectName: cluster-api-provider-nested repo: sigs.k8s.io/cluster-api-provider-nested @@ -22,4 +23,13 @@ resources: group: controlplane kind: NestedControllerManager version: v1alpha4 -version: 3-alpha +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: infrastructure + kind: NestedCluster + path: sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4 + version: v1alpha4 +version: "3" diff --git a/apis/controlplane/v1alpha4/nestedapiserver_types.go b/apis/controlplane/v1alpha4/nestedapiserver_types.go index ac44e7dd..0245b703 100644 --- a/apis/controlplane/v1alpha4/nestedapiserver_types.go +++ b/apis/controlplane/v1alpha4/nestedapiserver_types.go @@ -41,8 +41,7 @@ type NestedAPIServerStatus struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedapiservers,shortName=nkas -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=nkas,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -68,3 +67,26 @@ type NestedAPIServerList struct { func init() { SchemeBuilder.Register(&NestedAPIServer{}, &NestedAPIServerList{}) } + +var _ addonv1alpha1.CommonObject = &NestedAPIServer{} +var _ addonv1alpha1.Patchable = &NestedAPIServer{} + +func (c *NestedAPIServer) ComponentName() string { + return string(APIServer) +} + +func (c *NestedAPIServer) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +func (c *NestedAPIServer) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +func (c *NestedAPIServer) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +func (c *NestedAPIServer) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go b/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go index 9e5feb42..519ad06b 100644 --- a/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go +++ b/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go @@ -36,8 +36,7 @@ type NestedControllerManagerStatus struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedcontrollermanager,shortName=nkcm -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=nkcm,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -63,3 +62,26 @@ type NestedControllerManagerList struct { func init() { SchemeBuilder.Register(&NestedControllerManager{}, &NestedControllerManagerList{}) } + +var _ addonv1alpha1.CommonObject = &NestedControllerManager{} +var _ addonv1alpha1.Patchable = &NestedControllerManager{} + +func (c *NestedControllerManager) ComponentName() string { + return string(ControllerManager) +} + +func (c *NestedControllerManager) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +func (c *NestedControllerManager) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +func (c *NestedControllerManager) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +func (c *NestedControllerManager) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/nestedcontrolplane_types.go b/apis/controlplane/v1alpha4/nestedcontrolplane_types.go index 96df8beb..045c3875 100644 --- a/apis/controlplane/v1alpha4/nestedcontrolplane_types.go +++ b/apis/controlplane/v1alpha4/nestedcontrolplane_types.go @@ -17,17 +17,21 @@ limitations under the License. package v1alpha4 import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + NestedControlPlaneFinalizer = "nested.controlplane.cluster.x-k8s.io" ) // NestedControlPlaneSpec defines the desired state of NestedControlPlane type NestedControlPlaneSpec struct { - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - // +optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - // EtcdRef is the reference to the NestedEtcd EtcdRef *corev1.ObjectReference `json:"etcd,omitempty"` @@ -87,10 +91,9 @@ type NestedControlPlaneStatusAPIServer struct { ServiceCIDR string `json:"serviceCidr,omitempty"` } -// +kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,shortName=ncp -//+kubebuilder:categories=capi,capn -//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.rady" +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope=Namespaced,shortName=ncp,categories=capi;capn +//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -115,3 +118,7 @@ type NestedControlPlaneList struct { func init() { SchemeBuilder.Register(&NestedControlPlane{}, &NestedControlPlaneList{}) } + +func (r *NestedControlPlane) GetOwnerCluster(ctx context.Context, cli client.Client) (cluster *clusterv1.Cluster, err error) { + return util.GetOwnerCluster(ctx, cli, r.ObjectMeta) +} diff --git a/apis/controlplane/v1alpha4/nestedetcd_types.go b/apis/controlplane/v1alpha4/nestedetcd_types.go index 8b8abc09..7a96848f 100644 --- a/apis/controlplane/v1alpha4/nestedetcd_types.go +++ b/apis/controlplane/v1alpha4/nestedetcd_types.go @@ -53,8 +53,7 @@ type NestedEtcdAddress struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedetcds,shortName=netcd -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=netcd,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -80,3 +79,26 @@ type NestedEtcdList struct { func init() { SchemeBuilder.Register(&NestedEtcd{}, &NestedEtcdList{}) } + +var _ addonv1alpha1.CommonObject = &NestedEtcd{} +var _ addonv1alpha1.Patchable = &NestedEtcd{} + +func (c *NestedEtcd) ComponentName() string { + return string(Etcd) +} + +func (c *NestedEtcd) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +func (c *NestedEtcd) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +func (c *NestedEtcd) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +func (c *NestedEtcd) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/zz_generated.deepcopy.go b/apis/controlplane/v1alpha4/zz_generated.deepcopy.go index 436d1cf6..44fc3434 100644 --- a/apis/controlplane/v1alpha4/zz_generated.deepcopy.go +++ b/apis/controlplane/v1alpha4/zz_generated.deepcopy.go @@ -202,7 +202,6 @@ func (in *NestedControlPlaneList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NestedControlPlaneSpec) DeepCopyInto(out *NestedControlPlaneSpec) { *out = *in - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint if in.EtcdRef != nil { in, out := &in.EtcdRef, &out.EtcdRef *out = new(v1.ObjectReference) diff --git a/apis/infrastructure/v1alpha4/groupversion_info.go b/apis/infrastructure/v1alpha4/groupversion_info.go new file mode 100644 index 00000000..c8afe8d5 --- /dev/null +++ b/apis/infrastructure/v1alpha4/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha4 contains API Schema definitions for the infrastructure v1alpha4 API group +//+kubebuilder:object:generate=true +//+groupName=infrastructure.cluster.x-k8s.io +package v1alpha4 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1alpha4"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/infrastructure/v1alpha4/nestedcluster_types.go b/apis/infrastructure/v1alpha4/nestedcluster_types.go new file mode 100644 index 00000000..6e419776 --- /dev/null +++ b/apis/infrastructure/v1alpha4/nestedcluster_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha4 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" +) + +// NestedClusterSpec defines the desired state of NestedCluster +type NestedClusterSpec struct { + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // +optional + ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` +} + +// NestedClusterStatus defines the observed state of NestedCluster +type NestedClusterStatus struct { + // Ready is when the NestedControlPlane has a API server URL. + // +optional + Ready bool `json:"ready,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope=Namespaced,shortName=nc,categories=capi;capn +//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready" +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +//+kubebuilder:subresource:status + +// NestedCluster is the Schema for the nestedclusters API +type NestedCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NestedClusterSpec `json:"spec,omitempty"` + Status NestedClusterStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// NestedClusterList contains a list of NestedCluster +type NestedClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NestedCluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NestedCluster{}, &NestedClusterList{}) +} diff --git a/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go b/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go new file mode 100644 index 00000000..9506c643 --- /dev/null +++ b/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go @@ -0,0 +1,115 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha4 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedCluster) DeepCopyInto(out *NestedCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedCluster. +func (in *NestedCluster) DeepCopy() *NestedCluster { + if in == nil { + return nil + } + out := new(NestedCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NestedCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterList) DeepCopyInto(out *NestedClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NestedCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterList. +func (in *NestedClusterList) DeepCopy() *NestedClusterList { + if in == nil { + return nil + } + out := new(NestedClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NestedClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterSpec) DeepCopyInto(out *NestedClusterSpec) { + *out = *in + out.ControlPlaneEndpoint = in.ControlPlaneEndpoint +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterSpec. +func (in *NestedClusterSpec) DeepCopy() *NestedClusterSpec { + if in == nil { + return nil + } + out := new(NestedClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterStatus) DeepCopyInto(out *NestedClusterStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterStatus. +func (in *NestedClusterStatus) DeepCopy() *NestedClusterStatus { + if in == nil { + return nil + } + out := new(NestedClusterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml b/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml index c2bccc61..0c7fcf20 100644 --- a/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml +++ b/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml @@ -1,15 +1,16 @@ apiVersion: v1 kind: Service metadata: - name: {{.nestedAPIServerName}} - namespace: {{.nestedAPIServerNamespace}} + name: {{.clusterName}}-apiserver + namespace: {{.componentNamespace}} labels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} spec: selector: - component-name: {{.nestedAPIServerName}} - type: NodePort + component-name: {{.componentName}} + type: ClusterIP ports: - - port: 6443 + - name: api + port: 6443 protocol: TCP targetPort: api diff --git a/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml b/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml index 0ac96c1e..67ade092 100644 --- a/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml +++ b/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml @@ -1,45 +1,52 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedAPIServerName}} - namespace: {{.nestedAPIServerNamespace}} + name: {{.clusterName}}-apiserver + namespace: {{.componentNamespace}} spec: revisionHistoryLimit: 10 - serviceName: {{.nestedAPIServerName}} + serviceName: {{.componentName}} selector: matchLabels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} # apiserver will not be updated, unless it is deleted updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} spec: hostname: apiserver - subdomain: apiserver-svc + subdomain: {{.clusterName}}-apiserver containers: - - name: {{.nestedAPIServerName}} + - name: {{.componentName}} image: virtualcluster/apiserver-v1.16.2 imagePullPolicy: Always command: - kube-apiserver + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - --bind-address=0.0.0.0 - --allow-privileged=true - --anonymous-auth=true - - --client-ca-file=/etc/kubernetes/pki/root/tls.crt + - --client-ca-file=/etc/kubernetes/pki/apiserver/ca/tls.crt - --tls-cert-file=/etc/kubernetes/pki/apiserver/tls.crt - --tls-private-key-file=/etc/kubernetes/pki/apiserver/tls.key - --kubelet-https=true - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver/tls.crt - - --kubelet-client-key=/etc/kubernetes/pki/apiserver/tls.key + - --kubelet-certificate-authority=/etc/kubernetes/pki/apiserver/ca/tls.crt + - --kubelet-client-certificate=/etc/kubernetes/pki/kubelet/tls.crt + - --kubelet-client-key=/etc/kubernetes/pki/kubelet/tls.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP - --enable-bootstrap-token-auth=true - - --etcd-servers=https://{{.nestedEtcdName}}-0.{{.nestedEtcdName}}:2379 - - --etcd-cafile=/etc/kubernetes/pki/root/tls.crt - - --etcd-certfile=/etc/kubernetes/pki/apiserver/tls.crt - - --etcd-keyfile=/etc/kubernetes/pki/apiserver/tls.key + - --etcd-servers=https://{{.clusterName}}-etcd-0.{{.clusterName}}-etcd.$(NAMESPACE):2379 + - --etcd-cafile=/etc/kubernetes/pki/etcd/ca/tls.crt + - --etcd-certfile=/etc/kubernetes/pki/etcd/tls.crt + - --etcd-keyfile=/etc/kubernetes/pki/etcd/tls.key - --service-account-key-file=/etc/kubernetes/pki/service-account/tls.key - --service-cluster-ip-range=10.32.0.0/16 - --service-node-port-range=30000-32767 @@ -71,29 +78,64 @@ spec: periodSeconds: 2 timeoutSeconds: 30 volumeMounts: + - mountPath: /etc/kubernetes/pki/proxy/ca + name: {{.clusterName}}-proxy-ca + readOnly: true + - mountPath: /etc/kubernetes/pki/proxy + name: {{.clusterName}}-proxy-client + readOnly: true + - mountPath: /etc/kubernetes/pki/etcd/ca + name: {{.clusterName}}-etcd-ca + readOnly: true + - mountPath: /etc/kubernetes/pki/etcd + name: {{.clusterName}}-etcd-client + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver/ca + name: {{.clusterName}}-ca + readOnly: true - mountPath: /etc/kubernetes/pki/apiserver - name: {{.nestedControlPlaneName}}-apiserver + name: {{.clusterName}}-apiserver-client readOnly: true - - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + - mountPath: /etc/kubernetes/pki/kubelet + name: {{.clusterName}}-kubelet-client readOnly: true - mountPath: /etc/kubernetes/pki/service-account - name: {{.nestedControlPlaneName}}-sa + name: {{.clusterName}}-sa readOnly: true terminationGracePeriodSeconds: 30 dnsConfig: searches: - cluster.local volumes: - - name: {{.nestedControlPlaneName}}-apiserver + - name: {{.clusterName}}-proxy-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-proxy + - name: {{.clusterName}}-apiserver-client + secret: + defaultMode: 420 + secretName: {{.clusterName}}-apiserver-client + - name: {{.clusterName}}-proxy-client + secret: + defaultMode: 420 + secretName: {{.clusterName}}-proxy-client + - name: {{.clusterName}}-etcd-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd + - name: {{.clusterName}}-etcd-client + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd-client + - name: {{.clusterName}}-kubelet-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-apiserver - - name: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-kubelet-client + - name: {{.clusterName}}-ca secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca - - name: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-ca + - name: {{.clusterName}}-sa secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-sa diff --git a/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml b/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml index 2c972aa4..e8a774f3 100644 --- a/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml +++ b/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml @@ -1,21 +1,21 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedControllerManagerName}} - namespace: {{.nestedControllerManagerNamespace}} + name: {{.clusterName}}-controller-manager + namespace: {{.componentNamespace}} spec: selector: matchLabels: - component-name: {{.nestedControllerManagerName}} + component-name: {{.componentName}} updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedControllerManagerName}} + component-name: {{.componentName}} spec: containers: - - name: {{.nestedControllerManagerName}} + - name: {{.componentName}} image: virtualcluster/controller-manager-v1.16.2 imagePullPolicy: Always command: @@ -30,7 +30,7 @@ spec: - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig # control plane contains only one instance for now - --leader-elect=false - - --root-ca-file=/etc/kubernetes/pki/root/tls.crt + - --root-ca-file=/etc/kubernetes/pki/root/ca/tls.crt - --service-account-private-key-file=/etc/kubernetes/pki/service-account/tls.key - --service-cluster-ip-range=10.32.0.0/24 - --use-service-account-credentials=true @@ -56,25 +56,35 @@ spec: periodSeconds: 2 timeoutSeconds: 15 volumeMounts: + - mountPath: /etc/kubernetes/pki/root/ca + name: {{.clusterName}}-ca + readOnly: true - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + name: {{.clusterName}}-apiserver-client readOnly: true - mountPath: /etc/kubernetes/pki/service-account - name: {{.nestedControlPlaneName}}-sa + name: {{.clusterName}}-sa readOnly: true - mountPath: /etc/kubernetes/kubeconfig - name: {{.nestedControllerManagerName}}-kubeconfig + name: {{.clusterName}}-kubeconfig readOnly: true volumes: - - name: {{.nestedControlPlaneName}}-ca + - name: {{.clusterName}}-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-ca + - name: {{.clusterName}}-apiserver-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca - - name: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-apiserver-client + - name: {{.clusterName}}-sa secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-sa - - name: {{.nestedControllerManagerName}}-kubeconfig + secretName: {{.clusterName}}-sa + - name: {{.clusterName}}-kubeconfig secret: defaultMode: 420 - secretName: {{.nestedControllerManagerName}}-kubeconfig + secretName: {{.clusterName}}-kubeconfig + items: + - key: value + path: controller-manager-kubeconfig diff --git a/config/component-templates/nested-etcd/nested-etcd-service-template.yaml b/config/component-templates/nested-etcd/nested-etcd-service-template.yaml index edca7010..58cd080b 100644 --- a/config/component-templates/nested-etcd/nested-etcd-service-template.yaml +++ b/config/component-templates/nested-etcd/nested-etcd-service-template.yaml @@ -1,12 +1,12 @@ apiVersion: v1 kind: Service metadata: - name: {{.nestedEtcdName}} - namespace: {{.nestedEtcdNamespace}} + name: {{.clusterName}}-etcd + namespace: {{.componentNamespace}} labels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} spec: publishNotReadyAddresses: true clusterIP: None selector: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} diff --git a/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml b/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml index 314af417..95c53009 100644 --- a/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml +++ b/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml @@ -1,25 +1,24 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedEtcdName}} - namespace: {{.nestedEtcdNamespace}} + name: {{.clusterName}}-etcd + namespace: {{.componentNamespace}} spec: revisionHistoryLimit: 10 - serviceName: {{.nestedEtcdName}} + serviceName: {{.clusterName}}-etcd selector: matchLabels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} # etcd will not be updated, unless it is deleted updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} spec: - subdomain: etcd containers: - - name: {{.nestedEtcdName}} + - name: {{.componentName}} image: virtualcluster/etcd-v3.4.0 imagePullPolicy: Always command: @@ -30,21 +29,25 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - --name=$(HOSTNAME) - - --trusted-ca-file=/etc/kubernetes/pki/root/tls.crt + - --trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - --client-cert-auth - --cert-file=/etc/kubernetes/pki/etcd/tls.crt - --key-file=/etc/kubernetes/pki/etcd/tls.key - --peer-client-cert-auth - - --peer-trusted-ca-file=/etc/kubernetes/pki/root/tls.crt + - --peer-trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - --peer-cert-file=/etc/kubernetes/pki/etcd/tls.crt - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key - --listen-peer-urls=https://0.0.0.0:2380 - --listen-client-urls=https://0.0.0.0:2379 - - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2380 + - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2380 # we use a headless service to encapsulate each pod - - --advertise-client-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2379 + - --advertise-client-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2379 - --initial-cluster-state=new - --initial-cluster-token=vc-etcd - --data-dir=/var/lib/etcd/data @@ -54,7 +57,7 @@ spec: command: - sh - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health + - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health failureThreshold: 8 initialDelaySeconds: 60 timeoutSeconds: 15 @@ -63,24 +66,31 @@ spec: command: - sh - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health + - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health failureThreshold: 8 initialDelaySeconds: 15 periodSeconds: 2 timeoutSeconds: 15 volumeMounts: + - mountPath: /etc/kubernetes/pki/ca + name: {{.clusterName}}-etcd-ca + readOnly: true - mountPath: /etc/kubernetes/pki/etcd - name: {{.nestedControlPlaneName}}-etcd + name: {{.clusterName}}-etcd-client readOnly: true - - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + - mountPath: /etc/kubernetes/pki/health + name: {{.clusterName}}-etcd-health-client readOnly: true volumes: - - name: {{.nestedControlPlaneName}}-etcd + - name: {{.clusterName}}-etcd-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd + - name: {{.clusterName}}-etcd-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-etcd - - name: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-etcd-client + - name: {{.clusterName}}-etcd-health-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-etcd-health-client diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml index 1bcf3eef..d6fbdda9 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedAPIServer listKind: NestedAPIServerList plural: nestedapiservers diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml similarity index 97% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml rename to config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml index ee7cc635..4bb4ceb5 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml @@ -6,13 +6,16 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1-0.20201002000720-57250aac17f6 creationTimestamp: null - name: nestedcontrollermanager.controlplane.cluster.x-k8s.io + name: nestedcontrollermanagers.controlplane.cluster.x-k8s.io spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedControllerManager listKind: NestedControllerManagerList - plural: nestedcontrollermanager + plural: nestedcontrollermanagers shortNames: - nkcm singular: nestedcontrollermanager diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml index a3641a0c..549d9ad5 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedControlPlane listKind: NestedControlPlaneList plural: nestedcontrolplanes @@ -19,7 +22,7 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .status.rady + - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .metadata.creationTimestamp @@ -66,20 +69,6 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object controllerManager: description: ContollerManagerRef is the reference to the NestedControllerManager properties: diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml index 2e7893f0..aab3535d 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedEtcd listKind: NestedEtcdList plural: nestedetcds diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml new file mode 100644 index 00000000..d29f95b5 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml @@ -0,0 +1,79 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1-0.20201002000720-57250aac17f6 + creationTimestamp: null + name: nestedclusters.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - capi + - capn + kind: NestedCluster + listKind: NestedClusterList + plural: nestedclusters + shortNames: + - nc + singular: nestedcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha4 + schema: + openAPIV3Schema: + description: NestedCluster is the Schema for the nestedclusters API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NestedClusterSpec defines the desired state of NestedCluster + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: NestedClusterStatus defines the observed state of NestedCluster + properties: + ready: + description: Ready is when the NestedControlPlane has a API server URL. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 00000000..0ba4564c --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,27 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml +- bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml +- bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml +- bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml +- bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +# - kustomizeconfig.yaml + +commonLabels: + cluster.x-k8s.io/v1alpha3: v1alpha4 + cluster.x-k8s.io/v1alpha4: v1alpha4 \ No newline at end of file diff --git a/config/rbac/nestedcluster_editor_role.yaml b/config/rbac/nestedcluster_editor_role.yaml new file mode 100644 index 00000000..289d30c7 --- /dev/null +++ b/config/rbac/nestedcluster_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit nestedclusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nestedcluster-editor-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get diff --git a/config/rbac/nestedcluster_viewer_role.yaml b/config/rbac/nestedcluster_viewer_role.yaml new file mode 100644 index 00000000..4cc730eb --- /dev/null +++ b/config/rbac/nestedcluster_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view nestedclusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nestedcluster-viewer-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b6f997c3..265223b9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -134,3 +134,29 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/finalizers + verbs: + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get + - patch + - update diff --git a/controllers/controlplane/consts.go b/controllers/controlplane/consts.go index 5adb1a8a..d1e57ae4 100644 --- a/controllers/controlplane/consts.go +++ b/controllers/controlplane/consts.go @@ -20,19 +20,14 @@ const ( statefulsetOwnerKeyNEtcd = ".metadata.netcd.controller" statefulsetOwnerKeyNKas = ".metadata.nkas.controller" statefulsetOwnerKeyNKcm = ".metadata.nkcm.controller" - defaultEtcdStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultEtcdStatefulSetURL = "./config/component-templates/" + "nested-etcd/nested-etcd-statefulset-template.yaml" - defaultEtcdServiceURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultEtcdServiceURL = "./config/component-templates/" + "nested-etcd/nested-etcd-service-template.yaml" - defaultKASStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKASStatefulSetURL = "./config/component-templates/" + "nested-apiserver/nested-apiserver-statefulset-template.yaml" - defaultKASServiceURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKASServiceURL = "./config/component-templates/" + "nested-apiserver/nested-apiserver-service-template.yaml" - defaultKCMStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKCMStatefulSetURL = "./config/component-templates/" + "nested-controllermanager/nested-controllermanager-statefulset-template.yaml" ) diff --git a/controllers/controlplane/controller_util.go b/controllers/controlplane/controller_util.go index 4790b9ed..83e690c8 100644 --- a/controllers/controlplane/controller_util.go +++ b/controllers/controlplane/controller_util.go @@ -19,13 +19,12 @@ package controlplane import ( "bytes" "context" - "crypto/tls" - "errors" "fmt" "io/ioutil" - "net/http" "text/template" + openuri "github.com/utahta/go-openuri" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -47,26 +46,31 @@ func createNestedComponentSts(ctx context.Context, cli ctrlcli.Client, ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, ncKind clusterv1.ComponentKind, - clusterName string, log logr.Logger) error { + controlPlaneName, clusterName string, log logr.Logger) error { var ( ncSts appsv1.StatefulSet ncSvc corev1.Service err error ) + // Setup the ownerReferences for all objects + or := metav1.NewControllerRef(&ncMeta, + clusterv1.GroupVersion.WithKind(string(ncKind))) + // 1. Using the template defined by version/channel to create the // StatefulSet and the Service // TODO check the template version/channel, if not set, use the default. if ncSpec.Version == "" && ncSpec.Channel == "" { log.V(4).Info("The Version and Channel are not set, " + "will use the default template.") - ncSts, err = genStatefulSetObject(ncMeta, ncSpec, ncKind, clusterName, cli, log) + ncSts, err = genStatefulSetObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, cli, log) if err != nil { return fmt.Errorf("fail to generate the Statefulset object: %v", err) } if ncKind != clusterv1.ControllerManager { // no need to create the service for the NestedControllerManager - ncSvc, err = genServiceObject(ncMeta, ncSpec, ncKind, clusterName, log) + ncSvc, err = genServiceObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log) + ncSvc.SetOwnerReferences([]metav1.OwnerReference{*or}) if err != nil { return fmt.Errorf("fail to generate the Service object: %v", err) } @@ -81,8 +85,6 @@ func createNestedComponentSts(ctx context.Context, panic("NOT IMPLEMENT YET") } // 2. set the NestedComponent object as the owner of the StatefulSet - or := metav1.NewControllerRef(&ncMeta, - clusterv1.GroupVersion.WithKind(string(ncKind))) ncSts.SetOwnerReferences([]metav1.OwnerReference{*or}) // 4. create the NestedComponent StatefulSet @@ -93,7 +95,7 @@ func createNestedComponentSts(ctx context.Context, // NestedComponent func genServiceObject(ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, ncKind clusterv1.ComponentKind, - clusterName string, log logr.Logger) (ncSvc corev1.Service, retErr error) { + controlPlaneName, clusterName string, log logr.Logger) (ncSvc corev1.Service, retErr error) { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { switch ncKind { @@ -114,21 +116,7 @@ func genServiceObject(ncMeta metav1.ObjectMeta, return } - var templateCtx map[string]string - switch ncKind { - case clusterv1.APIServer: - templateCtx = map[string]string{ - "nestedAPIServerName": ncMeta.GetName(), - "nestedAPIServerNamespace": ncMeta.GetNamespace(), - } - case clusterv1.Etcd: - templateCtx = map[string]string{ - "nestedEtcdName": ncMeta.GetName(), - "nestedEtcdNamespace": ncMeta.GetNamespace(), - } - default: - panic("Unreachable") - } + templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) svcStr, err := substituteTemplate(templateCtx, svcTmpl) if err != nil { @@ -157,7 +145,7 @@ func genServiceObject(ncMeta metav1.ObjectMeta, func genStatefulSetObject( ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, - ncKind clusterv1.ComponentKind, clusterName string, + ncKind clusterv1.ComponentKind, controlPlaneName, clusterName string, cli ctrlcli.Client, log logr.Logger) (ncSts appsv1.StatefulSet, retErr error) { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { @@ -185,35 +173,7 @@ func genStatefulSetObject( return } // 2 substitute the statefulset template - var templateCtx map[string]string - switch ncKind { - case clusterv1.Etcd: - templateCtx = map[string]string{ - "nestedEtcdName": ncMeta.GetName(), - "nestedEtcdNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - } - case clusterv1.APIServer: - etcdName, err := getEtcdName(cli, ncMeta.GetOwnerReferences(), ncMeta.GetNamespace()) - if err != nil { - retErr = err - return - } - templateCtx = map[string]string{ - "nestedAPIServerName": ncMeta.GetName(), - "nestedAPIServerNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - "nestedEtcdName": etcdName, - } - case clusterv1.ControllerManager: - templateCtx = map[string]string{ - "nestedControllerManagerName": ncMeta.GetName(), - "nestedControllerManagerNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - } - default: - panic("Unreachable") - } + templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) stsStr, err := substituteTemplate(templateCtx, stsTmpl) if err != nil { retErr = fmt.Errorf("fail to substitute the default template "+ @@ -249,7 +209,7 @@ func genStatefulSetObject( // 6 set the "--initial-cluster" command line flag for the Etcd container if ncKind == clusterv1.Etcd { - icaVal := genInitialClusterArgs(1, ncMeta.GetName(), ncMeta.GetName()) + icaVal := genInitialClusterArgs(1, clusterName, clusterName, ncMeta.GetNamespace()) stsArgs := append(stsObj.Spec.Template.Spec.Containers[0].Args, "--initial-cluster", icaVal) stsObj.Spec.Template.Spec.Containers[0].Args = stsArgs @@ -261,27 +221,13 @@ func genStatefulSetObject( return } -// getEtcdName gets the name of the NestedEtcd through the NestedControlPlane -func getEtcdName(cli ctrlcli.Client, - ors []metav1.OwnerReference, ns string) (string, error) { - var or *metav1.OwnerReference - for _, orf := range ors { - if orf.Kind == "NestedControlPlane" { - or = &orf - } - break - } - if or == nil { - return "", errors.New("OwnerReference is not set") +func getTemplateArgs(ncMeta metav1.ObjectMeta, controlPlaneName, clusterName string) map[string]string { + return map[string]string{ + "componentName": ncMeta.GetName(), + "componentNamespace": ncMeta.GetNamespace(), + "clusterName": clusterName, + "controlPlaneName": controlPlaneName, } - var ncp clusterv1.NestedControlPlane - if err := cli.Get(context.TODO(), types.NamespacedName{ - Name: or.Name, - Namespace: ns, - }, &ncp); err != nil { - return "", err - } - return ncp.Spec.EtcdRef.Name, nil } // yamlToObject deserialize the yaml to the runtime object @@ -312,19 +258,13 @@ func substituteTemplate(context interface{}, tmpl string) (string, error) { // fetchTemplate fetches the component template through the tmplateURL func fetchTemplate(templateURL string) (string, error) { - // TODO mount host CA to manager pods - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }} - - rep, err := client.Get(templateURL) + rep, err := openuri.Open(templateURL) if err != nil { return "", err } + defer rep.Close() - defer rep.Body.Close() - - bodyBytes, err := ioutil.ReadAll(rep.Body) + bodyBytes, err := ioutil.ReadAll(rep) if err != nil { return "", err } @@ -350,14 +290,14 @@ func getOwner(ncMeta metav1.ObjectMeta) metav1.OwnerReference { // genAPIServerSvcRef generates the ObjectReference that points to the // APISrver service func genAPIServerSvcRef(cli ctrlcli.Client, - nkas clusterv1.NestedAPIServer) (corev1.ObjectReference, error) { + nkas clusterv1.NestedAPIServer, clusterName string) (corev1.ObjectReference, error) { var ( svc corev1.Service objRef corev1.ObjectReference ) if err := cli.Get(context.TODO(), types.NamespacedName{ Namespace: nkas.GetNamespace(), - Name: nkas.GetName(), + Name: fmt.Sprintf("%s-apiserver", clusterName), }, &svc); err != nil { return objRef, err } diff --git a/controllers/controlplane/controller_util_test.go b/controllers/controlplane/controller_util_test.go index d339e47a..85b6c7db 100644 --- a/controllers/controlplane/controller_util_test.go +++ b/controllers/controlplane/controller_util_test.go @@ -153,7 +153,7 @@ func TestGetOwner(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - get := getOwner(st.netcd) + get := getOwner(st.netcd.ObjectMeta) if !reflect.DeepEqual(get, st.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) } @@ -167,25 +167,28 @@ func TestGetOwner(t *testing.T) { func TestGenInitialClusterArgs(t *testing.T) { tests := []struct { - name string - replicas int32 - stsName string - svcName string - expect string + name string + replicas int32 + stsName string + svcName string + svcNamespace string + expect string }{ { "1 replicas", 1, "netcdSts", "netcdSvc", - "netcdSts-0=https://netcdSts-0.netcdSvc:2380", + "default", + "netcdSts-0=https://netcdSts-0.netcdSvc.default.svc:2380", }, { "2 replicas", 2, "netcdSts", "netcdSvc", - "netcdSts-0=https://netcdSts-0.netcdSvc:2380,netcdSts-1=https://netcdSts-1.netcdSvc:2380", + "default", + "netcdSts-0=https://netcdSts-0.netcdSvc.default.svc:2380,netcdSts-1=https://netcdSts-1.netcdSvc.default.svc:2380", }, } for _, tt := range tests { @@ -194,7 +197,7 @@ func TestGenInitialClusterArgs(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - get := genInitialClusterArgs(st.replicas, st.stsName, st.svcName) + get := genInitialClusterArgs(st.replicas, st.stsName, st.svcName, st.svcNamespace) if !reflect.DeepEqual(get, st.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) } diff --git a/controllers/controlplane/nestedapiserver_controller.go b/controllers/controlplane/nestedapiserver_controller.go index a876a747..a2bb10b0 100644 --- a/controllers/controlplane/nestedapiserver_controller.go +++ b/controllers/controlplane/nestedapiserver_controller.go @@ -18,6 +18,7 @@ package controlplane import ( "context" + "fmt" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -25,11 +26,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/pki" + clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" ) // NestedAPIServerReconciler reconciles a NestedAPIServer object @@ -70,11 +76,26 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: nkas.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", nkas.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + // 2. create the NestedAPIServer StatefulSet if not found + nkasName := fmt.Sprintf("%s-apiserver", cluster.GetName()) var nkasSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: nkas.GetNamespace(), - Name: nkas.GetName(), + Name: nkasName, }, &nkasSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedAPIServer as unready @@ -88,10 +109,15 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } } + if err := r.createAPIServerClientCrts(ctx, cluster, &ncp, &nkas); err != nil { + log.Error(err, "fail to create NestedAPIServer Client Certs") + return ctrl.Result{}, err + } + // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkas.ObjectMeta, nkas.Spec.NestedComponentSpec, - clusterv1.APIServer, owner.Name, log); err != nil { + clusterv1.APIServer, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedAPIServer StatefulSet") return ctrl.Result{}, err } @@ -110,7 +136,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // As the NestedAPIServer StatefulSet is ready, update // NestedAPIServer status nkas.Status.Phase = string(clusterv1.Ready) - objRef, err := genAPIServerSvcRef(r.Client, nkas) + objRef, err := genAPIServerSvcRef(r.Client, nkas, cluster.GetName()) if err != nil { log.Error(err, "fail to generate NestedAPIServer Service Reference") return ctrl.Result{}, err @@ -170,3 +196,69 @@ func (r *NestedAPIServerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&appsv1.StatefulSet{}). Complete(r) } + +// createAPIServerClientCrts will find of create client certs for the etcd cluster +func (r *NestedAPIServerReconciler) createAPIServerClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, nkas *clusterv1.NestedAPIServer) error { + certificates := secret.NewCertificatesForInitialControlPlane(nil) + if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { + return err + } + cacert := certificates.GetByPurpose(secret.ClusterCA) + if cacert == nil { + return fmt.Errorf("could not fetch ClusterCA") + } + + cacrt, err := certs.DecodeCertPEM(cacert.KeyPair.Cert) + if err != nil { + return err + } + + cakey, err := certs.DecodePrivateKeyPEM(cacert.KeyPair.Key) + if err != nil { + return err + } + + // TODO(christopherhein) figure out how to get service clusterIPs + apiKeyPair, err := pki.NewAPIServerCrtAndKey(&pki.KeyPair{Cert: cacrt, Key: cakey}, nkas.GetName(), "", cluster.Spec.ControlPlaneEndpoint.Host) + if err != nil { + return err + } + + kubeletKeyPair, err := pki.NewAPIServerKubeletClientCertAndKey(&pki.KeyPair{Cert: cacrt, Key: cakey}) + if err != nil { + return err + } + + fpcert := certificates.GetByPurpose(secret.FrontProxyCA) + if cacert == nil { + return fmt.Errorf("could not fetch FrontProxyCA") + } + + fpcrt, err := certs.DecodeCertPEM(fpcert.KeyPair.Cert) + if err != nil { + return err + } + + fpkey, err := certs.DecodePrivateKeyPEM(fpcert.KeyPair.Key) + if err != nil { + return err + } + + frontProxyKeyPair, err := pki.NewFrontProxyClientCertAndKey(&pki.KeyPair{Cert: fpcrt, Key: fpkey}) + if err != nil { + return err + } + + certs := &pki.KeyPairs{ + apiKeyPair, + kubeletKeyPair, + frontProxyKeyPair, + } + + controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + return err + } + + return nil +} diff --git a/controllers/controlplane/nestedcontrollermanager_controller.go b/controllers/controlplane/nestedcontrollermanager_controller.go index 1f6dac48..ebec066e 100644 --- a/controllers/controlplane/nestedcontrollermanager_controller.go +++ b/controllers/controlplane/nestedcontrollermanager_controller.go @@ -18,6 +18,7 @@ package controlplane import ( "context" + "fmt" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -69,11 +70,26 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: nkcm.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", nkcm.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + // 2. create the NestedControllerManager StatefulSet if not found + nkcmName := fmt.Sprintf("%s-controller-manager", cluster.GetName()) var nkcmSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: nkcm.GetNamespace(), - Name: nkcm.GetName(), + Name: nkcmName, }, &nkcmSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedControllerManager @@ -91,7 +107,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkcm.ObjectMeta, nkcm.Spec.NestedComponentSpec, - clusterv1.ControllerManager, owner.Name, log); err != nil { + clusterv1.ControllerManager, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedControllerManager StatefulSet") return ctrl.Result{}, err } diff --git a/controllers/controlplane/nestedcontrolplane_controller.go b/controllers/controlplane/nestedcontrolplane_controller.go index 731acc3b..9f64ea69 100644 --- a/controllers/controlplane/nestedcontrolplane_controller.go +++ b/controllers/controlplane/nestedcontrolplane_controller.go @@ -18,15 +18,33 @@ package controlplane import ( "context" + "time" "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/kubeconfig" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" ) +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update + // NestedControlPlaneReconciler reconciles a NestedControlPlane object type NestedControlPlaneReconciler struct { client.Client @@ -34,19 +52,248 @@ type NestedControlPlaneReconciler struct { Scheme *runtime.Scheme } -// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch +// SetupWithManager will configure the controller with the manager +func (r *NestedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&controlplanev1.NestedControlPlane{}). + Complete(r) +} + +// Reconcile is ths main process which will handle updating teh NCP +func (r *NestedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("nestedcontrolplane", req.NamespacedName) + log.Info("Reconciling NestedControlPlane...") + // Fetch the NestedControlPlane + ncp := &v1alpha4.NestedControlPlane{} + if err := r.Get(ctx, req.NamespacedName, ncp); err != nil { + // check for not found and don't requeue + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + // If there are errors we should retry + return ctrl.Result{Requeue: true}, nil + } + + // Fetch the cluster object + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the API Server") + return ctrl.Result{}, err + } + log = log.WithValues("cluster", cluster.Name) + + // TODO(christopherhein) add paused handling + // if annotations.IsPaused(cluster, ncp) { + // log.Info("Reconciliation is paused for this object") + // return ctrl.Result{}, nil + // } -func (r *NestedControlPlaneReconciler) Reconcile(_ context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = r.Log.WithValues("nestedcontrolplane", req.NamespacedName) + // Initialize the patch helper. + patchHelper, err := patch.NewHelper(ncp, r.Client) + if err != nil { + log.Error(err, "Failed to configure the patch helper") + return ctrl.Result{Requeue: true}, nil + } - // your logic here + if !controllerutil.ContainsFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) { + controllerutil.AddFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) + // patch and return right away instead of reusing the main defer, + // because the main defer may take too much time to get cluster status + // Patch ObservedGeneration only if the reconciliation completed successfully + patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}} + if err := patchHelper.Patch(ctx, ncp, patchOpts...); err != nil { + log.Error(err, "Failed to patch NestedControlPlane to add finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + // TODO(christopherhein) handle deletion + if !ncp.ObjectMeta.DeletionTimestamp.IsZero() { + // Handle deletion reconciliation loop. + return r.reconcileDelete(ctx, log, cluster, ncp) + } + + // Handle normal reconciliation loop. + return r.reconcile(ctx, log, cluster, ncp) +} + +// reconcileDelete will delete the control plane and all it's nestedcomponents +func (r *NestedControlPlaneReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (ctrl.Result, error) { + + // TODO(christopherhein) delete the controlplane namespace + + patchHelper, err := patch.NewHelper(ncp, r.Client) + if err != nil { + log.Error(err, "Failed to configure the patch helper") + return ctrl.Result{Requeue: true}, nil + } + + if controllerutil.ContainsFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) { + controllerutil.RemoveFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) + + // patch and return right away instead of reusing the main defer, + // because the main defer may take too much time to get cluster status + // Patch ObservedGeneration only if the reconciliation completed successfully + patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}} + if err := patchHelper.Patch(ctx, ncp, patchOpts...); err != nil { + log.Error(err, "Failed to patch NestedControlPlane to remove finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } return ctrl.Result{}, nil } -func (r *NestedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controlplanev1alpha4.NestedControlPlane{}). - Complete(r) +// reconcile will handle all "normal" NCP reconciles this means create/update actions +func (r *NestedControlPlaneReconciler) reconcile(ctx context.Context, log logr.Logger, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (res ctrl.Result, reterr error) { + log.Info("Reconcile NestedControlPlane") + + certificates := secret.NewCertificatesForInitialControlPlane(nil) + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + log.Error(err, "unable to lookup or create cluster certificates") + // TODO(christopherhein) use conditions to mark when failed + // conditions.MarkFalse(ncp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) + return ctrl.Result{}, err + } + // TODO(christopherhein) use conditions to mark when ready + // conditions.MarkTrue(ncp, controlplanev1.CertificatesAvailableCondition) + + // If ControlPlaneEndpoint is not set, return early + if !cluster.Spec.ControlPlaneEndpoint.IsValid() { + log.Info("Cluster does not yet have a ControlPlaneEndpoint defined") + return ctrl.Result{}, nil + } + + if result, err := r.reconcileKubeconfig(ctx, cluster, ncp); !result.IsZero() || err != nil { + if err != nil { + log.Error(err, "failed to reconcile Kubeconfig") + } + return result, err + } + + addOwners := []client.Object{} + isReady := []int{} + nestedComponents := map[client.Object]*corev1.ObjectReference{ + &controlplanev1.NestedEtcd{}: ncp.Spec.EtcdRef, + &controlplanev1.NestedAPIServer{}: ncp.Spec.APIServerRef, + &controlplanev1.NestedControllerManager{}: ncp.Spec.ControllerManagerRef, + } + + // Adopt NestedComponents in the same Namespace + for component, nestedComponent := range nestedComponents { + if nestedComponent != nil { + objectKey := types.NamespacedName{Namespace: ncp.GetNamespace(), Name: nestedComponent.Name} + if err := r.Get(ctx, objectKey, component); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, err + } + } + + if !util.HasOwner(component.GetOwnerReferences(), controlplanev1.GroupVersion.String(), []string{"NestedControlPlane"}) { + log.Info("Component Missing Owner", "component", nestedComponent) + addOwners = append(addOwners, component) + } + + if commonObject, ok := component.(addonv1alpha1.CommonObject); ok { + if IsComponentReady(commonObject.GetCommonStatus()) { + isReady = append(isReady, 1) + } else { + log.Info("Component is not ready", "component", nestedComponent) + } + } + } + } + + // Add Controller Reference + if err := r.reconcileControllerOwners(ctx, ncp, addOwners); err != nil { + return ctrl.Result{Requeue: true}, err + } + + // Set Initialized + if !ncp.Status.Initialized { + ncp.Status.Initialized = true + if err := r.Status().Update(ctx, ncp); err != nil { + return ctrl.Result{}, err + } + } + + // Set Ready + if !ncp.Status.Ready && len(isReady) == 3 { + ncp.Status.Ready = true + if err := r.Status().Update(ctx, ncp); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *NestedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + endpoint := cluster.Spec.ControlPlaneEndpoint + if endpoint.IsZero() { + return ctrl.Result{}, nil + } + + controllerOwnerRef := *metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + clusterName := util.ObjectKey(cluster) + configSecret, err := secret.GetFromNamespacedName(ctx, r.Client, clusterName, secret.Kubeconfig) + switch { + case apierrors.IsNotFound(err): + createErr := kubeconfig.CreateSecretWithOwner( + ctx, + r.Client, + clusterName, + endpoint.String(), + controllerOwnerRef, + ) + if errors.Is(createErr, kubeconfig.ErrDependentCertificateNotFound) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + // always return if we have just created in order to skip rotation checks + return ctrl.Result{}, createErr + case err != nil: + return ctrl.Result{}, errors.Wrap(err, "failed to retrieve kubeconfig Secret") + } + + // only do rotation on owned secrets + if !util.IsControlledBy(configSecret, ncp) { + return ctrl.Result{}, nil + } + + needsRotation, err := kubeconfig.NeedsClientCertRotation(configSecret, certs.ClientCertificateRenewalDuration) + if err != nil { + return ctrl.Result{}, err + } + + if needsRotation { + log.Info("rotating kubeconfig secret") + if err := kubeconfig.RegenerateSecret(ctx, r.Client, configSecret); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig") + } + } + + return ctrl.Result{}, nil +} + +func (r *NestedControlPlaneReconciler) reconcileControllerOwners(ctx context.Context, ncp *controlplanev1.NestedControlPlane, addOwners []client.Object) error { + for _, component := range addOwners { + if err := ctrl.SetControllerReference(ncp, component, r.Scheme); err != nil { + if _, ok := err.(*controllerutil.AlreadyOwnedError); !ok { + continue + } + return err + } + + if err := r.Update(ctx, component); err != nil { + return err + } + } + return nil } diff --git a/controllers/controlplane/nestedetcd_controller.go b/controllers/controlplane/nestedetcd_controller.go index b2922e49..c1c1ed4c 100644 --- a/controllers/controlplane/nestedetcd_controller.go +++ b/controllers/controlplane/nestedetcd_controller.go @@ -28,11 +28,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/pki" + "sigs.k8s.io/cluster-api/util" ) // NestedEtcdReconciler reconciles a NestedEtcd object @@ -71,10 +76,25 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: netcd.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", netcd.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + + etcdName := fmt.Sprintf("%s-etcd", cluster.GetName()) var netcdSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: netcd.GetNamespace(), - Name: netcd.GetName(), + Name: etcdName, }, &netcdSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedEtcd as unready @@ -88,11 +108,17 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } } + + if err := r.createEtcdClientCrts(ctx, cluster, &ncp, &netcd); err != nil { + log.Error(err, "fail to create NestedEtcd Client Certs") + return ctrl.Result{}, err + } + // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, netcd.ObjectMeta, netcd.Spec.NestedComponentSpec, - clusterv1.Etcd, owner.Name, log); err != nil { + clusterv1.Etcd, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedEtcd StatefulSet") return ctrl.Result{}, err } @@ -107,7 +133,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Info("The NestedEtcd StatefulSet is ready") if !IsComponentReady(netcd.Status.CommonStatus) { // As the NestedEtcd StatefulSet is ready, update NestedEtcd status - ip, err := getNestedEtcdSvcClusterIP(ctx, r.Client, netcd) + ip, err := getNestedEtcdSvcClusterIP(ctx, r.Client, cluster.GetName(), &netcd) if err != nil { log.Error(err, "fail to get NestedEtcd Service ClusterIP") return ctrl.Result{}, err @@ -176,25 +202,25 @@ func (r *NestedEtcdReconciler) SetupWithManager(mgr ctrl.Manager) error { } func getNestedEtcdSvcClusterIP(ctx context.Context, cli ctrlcli.Client, - netcd clusterv1.NestedEtcd) (string, error) { + clusterName string, netcd *clusterv1.NestedEtcd) (string, error) { var svc corev1.Service if err := cli.Get(ctx, types.NamespacedName{ Namespace: netcd.GetNamespace(), - Name: netcd.GetName(), + Name: fmt.Sprintf("%s-etcd", clusterName), }, &svc); err != nil { return "", err } return svc.Spec.ClusterIP, nil } -// genInitialClusterArgs generates the values for `--inital-cluster` option of +// genInitialClusterArgs generates the values for `--initial-cluster` option of // etcd based on the number of replicas specified in etcd StatefulSet func genInitialClusterArgs(replicas int32, - stsName, svcName string) (argsVal string) { + stsName, svcName, svcNamespace string) (argsVal string) { for i := int32(0); i < replicas; i++ { // use 2380 as the default port for etcd peer communication - peerAddr := fmt.Sprintf("%s-%d=https://%s-%d.%s:%d", - stsName, i, stsName, i, svcName, 2380) + peerAddr := fmt.Sprintf("%s-etcd-%d=https://%s-etcd-%d.%s-etcd.%s:%d", + stsName, i, stsName, i, svcName, svcNamespace, 2380) if i == replicas-1 { argsVal = argsVal + peerAddr break @@ -204,3 +230,56 @@ func genInitialClusterArgs(replicas int32, return argsVal } + +func getEtcdServers(name, namespace string, replicas int32) (etcdServers []string) { + var i int32 + for ; i < replicas; i++ { + etcdServers = append(etcdServers, fmt.Sprintf("%s-etcd-%d.%s-etcd.%s", name, i, name, namespace)) + } + etcdServers = append(etcdServers, name) + return etcdServers +} + +// createEtcdClientCrts will find of create client certs for the etcd cluster +func (r *NestedEtcdReconciler) createEtcdClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, netcd *clusterv1.NestedEtcd) error { + certificates := secret.NewCertificatesForInitialControlPlane(nil) + if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { + return err + } + cert := certificates.GetByPurpose(secret.EtcdCA) + if cert == nil { + return fmt.Errorf("could not fetch EtcdCA") + } + + crt, err := certs.DecodeCertPEM(cert.KeyPair.Cert) + if err != nil { + return err + } + + key, err := certs.DecodePrivateKeyPEM(cert.KeyPair.Key) + if err != nil { + return err + } + + etcdKeyPair, err := pki.NewEtcdServerCrtAndKey(&pki.KeyPair{Cert: crt, Key: key}, getEtcdServers(cluster.GetName(), cluster.GetNamespace(), netcd.Spec.Replicas)) + if err != nil { + return err + } + + etcdHealthKeyPair, err := pki.NewEtcdHealthcheckClientCertAndKey(&pki.KeyPair{Cert: crt, Key: key}) + if err != nil { + return err + } + + certs := &pki.KeyPairs{ + etcdKeyPair, + etcdHealthKeyPair, + } + + controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + return err + } + + return nil +} diff --git a/controllers/infrastructure/nestedcluster_controller.go b/controllers/infrastructure/nestedcluster_controller.go new file mode 100644 index 00000000..e301ce93 --- /dev/null +++ b/controllers/infrastructure/nestedcluster_controller.go @@ -0,0 +1,99 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package infrastructure + +import ( + "context" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + infrav1 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" +) + +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/finalizers,verbs=update + +// NestedClusterReconciler reconciles a NestedCluster object +type NestedClusterReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NestedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.NestedCluster{}). + Owns(&clusterv1.NestedControlPlane{}). + Complete(r) +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NestedCluster object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile +func (r *NestedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("nestedcluster", req.NamespacedName) + log.Info("Reconciling NestedCluster...") + nc := &infrav1.NestedCluster{} + if err := r.Get(ctx, req.NamespacedName, nc); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + cluster, err := util.GetOwnerCluster(ctx, r.Client, nc.ObjectMeta) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + + objectKey := types.NamespacedName{ + Namespace: cluster.Spec.ControlPlaneRef.Namespace, + Name: cluster.Spec.ControlPlaneRef.Name, + } + ncp := &clusterv1.NestedControlPlane{} + if err := r.Get(ctx, objectKey, ncp); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + + if !nc.Status.Ready && ncp.Status.Ready && ncp.Status.Initialized { + nc.Status.Ready = true + if err := r.Status().Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} diff --git a/controllers/infrastructure/suite_test.go b/controllers/infrastructure/suite_test.go new file mode 100644 index 00000000..bda22002 --- /dev/null +++ b/controllers/infrastructure/suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package infrastructure + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = infrastructurev1alpha4.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/main.go b/main.go index 29c975b2..e11ab47e 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" controlplanecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/controlplane" + infrastructurecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/infrastructure" // +kubebuilder:scaffold:imports ) @@ -63,6 +65,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme)) utilruntime.Must(controlplanev1alpha4.AddToScheme(scheme)) + utilruntime.Must(infrastructurev1alpha4.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -145,7 +148,6 @@ func main() { os.Exit(1) } - // TODO(community): Register controllers and webhooks here. if err = (&controlplanecontrollers.NestedControlPlaneReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControlPlane"), @@ -154,6 +156,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NestedControlPlane") os.Exit(1) } + if err = (&infrastructurecontrollers.NestedClusterReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("infrastructure").WithName("NestedCluster"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedCluster") + os.Exit(1) + } if err = (&controlplanecontrollers.NestedEtcdReconciler{ Client: mgr.GetClient(), @@ -180,6 +190,7 @@ func main() { os.Exit(1) } // +kubebuilder:scaffold:builder + setupLog.Info("Starting manager", "version", version.Get().String()) if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") diff --git a/pki/pki.go b/pki/pki.go new file mode 100644 index 00000000..68a08289 --- /dev/null +++ b/pki/pki.go @@ -0,0 +1,298 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pki + +import ( + "context" + "crypto" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + + "github.com/pkg/errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + + "k8s.io/client-go/util/cert" + "sigs.k8s.io/cluster-api-provider-nested/pki/util" + + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + defaultClusterDomain = "cluster.local" +) + +const ( + // EtcdClient defines the client cert name for etcd + EtcdClient secret.Purpose = "etcd-client" + + // EtcdHealthClient defines the client cert name for etcd + EtcdHealthClient secret.Purpose = "etcd-health-client" + + // APIServerClient defines the client cert name for apiserver + APIServerClient secret.Purpose = "apiserver-client" + + // APIServerEtcdClient mirrors capi APIServerEtcdClient + APIServerEtcdClient secret.Purpose = secret.APIServerEtcdClient + + // KubeletClient defines the client cert name for kubelet + KubeletClient secret.Purpose = "kubelet-client" + + // ProxyClient defines the client cert name for the front proxy + ProxyClient secret.Purpose = "proxy-client" + + // ControllerManagerKubeconfig defines the secret purpose for KCM Kubeconfigs + ControllerManagerKubeconfig secret.Purpose = "controller-manager-kubeconfig" +) + +type KeyPair struct { + Purpose secret.Purpose + Cert *x509.Certificate + Key crypto.Signer + Generated bool + New bool +} + +func (k *KeyPair) AsSecret(clusterName client.ObjectKey, owner metav1.OwnerReference) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterName.Namespace, + Name: secret.Name(clusterName.Name, k.Purpose), + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName.Name, + }, + }, + Data: map[string][]byte{ + secret.TLSKeyDataName: util.EncodePrivateKeyPEM(k.Key.(*rsa.PrivateKey)), + secret.TLSCrtDataName: util.EncodeCertPEM(k.Cert), + }, + Type: clusterv1.ClusterSecretType, + } + + if k.Generated { + s.OwnerReferences = []metav1.OwnerReference{owner} + } + return s +} + +type KeyPairs []*KeyPair + +// NewAPIServerCertAndKey creates crt and key for apiserver using ca. +func NewAPIServerCrtAndKey(ca *KeyPair, clusterName, clusterDomainArg, apiserverDomain string, apiserverIPs ...string) (*KeyPair, error) { + clusterDomain := defaultClusterDomain + if clusterDomainArg != "" { + clusterDomain = clusterDomainArg + } + apiserverIPs = append(apiserverIPs, "127.0.0.1", "0.0.0.0") + + // create AltNames with defaults DNSNames/IPs + altNames := &cert.AltNames{ + DNSNames: []string{ + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + fmt.Sprintf("kubernetes.default.svc.%s", clusterDomain), + apiserverDomain, + // add virtual cluster name (i.e. namespace) for vn-agent + clusterName, + }, + } + + for _, ip := range apiserverIPs { + if ip != "" { + altNames.IPs = append(altNames.IPs, net.ParseIP(ip)) + } + } + + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: clusterName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }, + } + + apiCert, apiKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create apiserver crt and key: %v", err) + } + + rsaKey, ok := apiKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + + return &KeyPair{APIServerClient, apiCert, rsaKey, true, true}, nil +} + +// NewAPIServerKubeletClientCertAndKey creates certificate for the apiservers to connect to the +// kubelets securely, signed by the ca. +func NewAPIServerKubeletClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-apiserver-kubelet-client", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + apiClientCert, apiClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return &KeyPair{}, fmt.Errorf("failure while creating API server kubelet client key and certificate: %v", err) + } + + rsaKey, ok := apiClientKey.(*rsa.PrivateKey) + if !ok { + return &KeyPair{}, errors.New("fail to assert rsa private key") + } + + return &KeyPair{KubeletClient, apiClientCert, rsaKey, true, true}, nil +} + +// NewEtcdServerCrtAndKey creates new crt-key pair using ca for etcd +func NewEtcdServerCrtAndKey(ca *KeyPair, etcdDomains []string) (*KeyPair, error) { + // create AltNames with defaults DNSNames/IPs + altNames := &cert.AltNames{ + DNSNames: etcdDomains, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, + } + + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-etcd", + AltNames: *altNames, + // all peers will use this crt-key pair as well + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }, + } + etcdServerCert, etcdServerKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create etcd crt and key: %v", err) + } + + rsaKey, ok := etcdServerKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + + return &KeyPair{EtcdClient, etcdServerCert, rsaKey, true, true}, nil +} + +// NewEtcdHealthcheckClientCertAndKey creates certificate for liveness probes to healthcheck etcd, +// signed by the given ca. +func NewEtcdHealthcheckClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-etcd-healthcheck-client", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + etcdHealcheckClientCert, etcdHealcheckClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return &KeyPair{}, fmt.Errorf("failure while creating etcd healthcheck client key and certificate: %v", err) + } + + rsaKey, ok := etcdHealcheckClientKey.(*rsa.PrivateKey) + if !ok { + return &KeyPair{}, errors.New("fail to assert rsa private key") + } + + return &KeyPair{EtcdHealthClient, etcdHealcheckClientCert, rsaKey, true, true}, nil +} + +// NewFrontProxyClientCertAndKey creates crt-key pair for proxy client using ca. +func NewFrontProxyClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "front-proxy-client", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + frontProxyClientCert, frontProxyClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create crt and key for front-proxy: %v", err) + } + rsaKey, ok := frontProxyClientKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + return &KeyPair{ProxyClient, frontProxyClientCert, rsaKey, true, true}, nil +} + +// newPrivateKey creates an RSA private key +func newPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, 2048) +} + +// Lookup looks up each certificate from secrets and populates the certificate with the secret data. +func (kp KeyPairs) Lookup(ctx context.Context, cli client.Client, clusterName client.ObjectKey) error { + // Look up each certificate as a secret and populate the certificate/key + for _, certificate := range kp { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: secret.Name(clusterName.Name, certificate.Purpose), + Namespace: clusterName.Namespace, + } + if err := cli.Get(ctx, key, s); err != nil { + if apierrors.IsNotFound(err) { + certificate.New = true + continue + } + return errors.WithStack(err) + } + certificate.New = false + } + return nil +} + +// SaveGenerated will save any certificates that have been generated as Kubernetes secrets. +func (c KeyPairs) SaveGenerated(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { + for _, keyPair := range c { + if !keyPair.Generated && !keyPair.New { + continue + } + s := keyPair.AsSecret(clusterName, owner) + if err := ctrlclient.Create(ctx, s); err != nil { + // We might want to trigger off updates from here. + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + } + } + return nil +} + +// LookupOrGenerate is a convenience function that wraps cluster bootstrap certificate behavior. +func (kp KeyPairs) LookupOrSave(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { + // Find the certificates that exist + if err := kp.Lookup(ctx, ctrlclient, clusterName); err != nil { + return err + } + + // Save any certificates that have been generated + if err := kp.SaveGenerated(ctx, ctrlclient, clusterName, owner); err != nil { + return err + } + + return nil +} diff --git a/pki/util/util.go b/pki/util/util.go new file mode 100644 index 00000000..b5227a3a --- /dev/null +++ b/pki/util/util.go @@ -0,0 +1,127 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math" + "math/big" + "time" + + "github.com/pkg/errors" + certutil "k8s.io/client-go/util/cert" +) + +const ( + // PrivateKeyBlockType is a possible value for pem.Block.Type. + PrivateKeyBlockType = "PRIVATE KEY" + // PublicKeyBlockType is a possible value for pem.Block.Type. + PublicKeyBlockType = "PUBLIC KEY" + // CertificateBlockType is a possible value for pem.Block.Type. + CertificateBlockType = "CERTIFICATE" + // RSAPrivateKeyBlockType is a possible value for pem.Block.Type. + RSAPrivateKeyBlockType = "RSA PRIVATE KEY" + rsaKeySize = 2048 + + // CertificateValidity defines the validity for all the signed certificates generated by this package + CertificateValidity = time.Hour * 24 * 365 +) + +// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm. +type CertConfig struct { + certutil.Config + PublicKeyAlgorithm x509.PublicKeyAlgorithm +} + +// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key +func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) { + key, err := NewPrivateKey(config.PublicKeyAlgorithm) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to create private key") + } + + cert, err := NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to sign certificate") + } + + return cert, key, nil +} + +// NewPrivateKey creates an RSA private key +func NewPrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) { + if keyType == x509.ECDSA { + return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + } + + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(CertificateValidity).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +// EncodeCertPEM returns PEM-endcoded certificate data +func EncodeCertPEM(cert *x509.Certificate) []byte { + block := pem.Block{ + Type: CertificateBlockType, + Bytes: cert.Raw, + } + return pem.EncodeToMemory(&block) +} + +// EncodePrivateKeyPEM returns PEM-encoded private key data +func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { + block := pem.Block{ + Type: RSAPrivateKeyBlockType, + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return pem.EncodeToMemory(&block) +} diff --git a/secret/secret.go b/secret/secret.go new file mode 100644 index 00000000..1a450c3f --- /dev/null +++ b/secret/secret.go @@ -0,0 +1,49 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secret + +import ( + "path/filepath" + + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" + "sigs.k8s.io/cluster-api/util/secret" +) + +func NewClientCertificatesForControlPlane(config *v1beta1.ClusterConfiguration) secret.Certificates { + certificatesDir := secret.DefaultCertificatesDir + if config != nil && config.CertificatesDir != "" { + certificatesDir = config.CertificatesDir + } + + return secret.Certificates{ + &secret.Certificate{ + Purpose: secret.Purpose("etcd-client-cert"), + CertFile: filepath.Join(certificatesDir, "etcd", "tls.crt"), + KeyFile: filepath.Join(certificatesDir, "etcd", "tls.key"), + }, + &secret.Certificate{ + Purpose: secret.Purpose("kubelet-client-cert"), + CertFile: filepath.Join(certificatesDir, "kubelet", "tls.crt"), + KeyFile: filepath.Join(certificatesDir, "kubelet", "ca.key"), + }, + &secret.Certificate{ + Purpose: secret.Purpose("proxy-client-cert"), + CertFile: filepath.Join(certificatesDir, "ca.crt"), + KeyFile: filepath.Join(certificatesDir, "ca.key"), + }, + } +}