diff --git a/Dockerfile b/Dockerfile index 5169a8c0..27faa3c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN go mod download COPY main.go main.go COPY apis/ apis/ COPY controllers/ controllers/ +COPY certificate/ certificate/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go diff --git a/PROJECT b/PROJECT index e742de5d..6c309cf7 100644 --- a/PROJECT +++ b/PROJECT @@ -1,5 +1,6 @@ domain: cluster.x-k8s.io -layout: go.kubebuilder.io/v3 +layout: +- go.kubebuilder.io/v3 multigroup: true projectName: cluster-api-provider-nested repo: sigs.k8s.io/cluster-api-provider-nested @@ -22,4 +23,13 @@ resources: group: controlplane kind: NestedControllerManager version: v1alpha4 -version: 3-alpha +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: infrastructure + kind: NestedCluster + path: sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4 + version: v1alpha4 +version: "3" diff --git a/apis/controlplane/v1alpha4/nestedapiserver_types.go b/apis/controlplane/v1alpha4/nestedapiserver_types.go index ac44e7dd..e1b27ebd 100644 --- a/apis/controlplane/v1alpha4/nestedapiserver_types.go +++ b/apis/controlplane/v1alpha4/nestedapiserver_types.go @@ -41,8 +41,7 @@ type NestedAPIServerStatus struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedapiservers,shortName=nkas -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=nkas,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -68,3 +67,35 @@ type NestedAPIServerList struct { func init() { SchemeBuilder.Register(&NestedAPIServer{}, &NestedAPIServerList{}) } + +var _ addonv1alpha1.CommonObject = &NestedAPIServer{} +var _ addonv1alpha1.Patchable = &NestedAPIServer{} + +// ComponentName returns the name of the component for use with +// addonv1alpha1.CommonObject +func (c *NestedAPIServer) ComponentName() string { + return string(APIServer) +} + +// CommonSpec returns the addons spec of the object allowing common funcs like +// Channel & Version to be usable +func (c *NestedAPIServer) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +// GetCommonStatus will return the common status for checking is a component +// was successfully deployed +func (c *NestedAPIServer) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +// SetCommonStatus will set the status so that abstract representations can set +// Ready and Phases +func (c *NestedAPIServer) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +// PatchSpec returns the patches to be applied +func (c *NestedAPIServer) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go b/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go index 9e5feb42..f8eb46c2 100644 --- a/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go +++ b/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go @@ -36,8 +36,7 @@ type NestedControllerManagerStatus struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedcontrollermanager,shortName=nkcm -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=nkcm,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -63,3 +62,35 @@ type NestedControllerManagerList struct { func init() { SchemeBuilder.Register(&NestedControllerManager{}, &NestedControllerManagerList{}) } + +var _ addonv1alpha1.CommonObject = &NestedControllerManager{} +var _ addonv1alpha1.Patchable = &NestedControllerManager{} + +// ComponentName returns the name of the component for use with +// addonv1alpha1.CommonObject +func (c *NestedControllerManager) ComponentName() string { + return string(ControllerManager) +} + +// CommonSpec returns the addons spec of the object allowing common funcs like +// Channel & Version to be usable +func (c *NestedControllerManager) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +// GetCommonStatus will return the common status for checking is a component +// was successfully deployed +func (c *NestedControllerManager) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +// SetCommonStatus will set the status so that abstract representations can set +// Ready and Phases +func (c *NestedControllerManager) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +// PatchSpec returns the patches to be applied +func (c *NestedControllerManager) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/nestedcontrolplane_types.go b/apis/controlplane/v1alpha4/nestedcontrolplane_types.go index 96df8beb..26201d6d 100644 --- a/apis/controlplane/v1alpha4/nestedcontrolplane_types.go +++ b/apis/controlplane/v1alpha4/nestedcontrolplane_types.go @@ -17,17 +17,23 @@ limitations under the License. package v1alpha4 import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // NestedControlPlaneFinalizer is added to the NestedControlPlane to allow + // nested deletions to happen before the object is cleaned up + NestedControlPlaneFinalizer = "nested.controlplane.cluster.x-k8s.io" ) // NestedControlPlaneSpec defines the desired state of NestedControlPlane type NestedControlPlaneSpec struct { - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - // +optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - // EtcdRef is the reference to the NestedEtcd EtcdRef *corev1.ObjectReference `json:"etcd,omitempty"` @@ -87,10 +93,9 @@ type NestedControlPlaneStatusAPIServer struct { ServiceCIDR string `json:"serviceCidr,omitempty"` } -// +kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,shortName=ncp -//+kubebuilder:categories=capi,capn -//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.rady" +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope=Namespaced,shortName=ncp,categories=capi;capn +//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -115,3 +120,16 @@ type NestedControlPlaneList struct { func init() { SchemeBuilder.Register(&NestedControlPlane{}, &NestedControlPlaneList{}) } + +// GetOwnerCluster is a utility to return the owning clusterv1.Cluster +func (r *NestedControlPlane) GetOwnerCluster(ctx context.Context, cli client.Client) (cluster *clusterv1.Cluster, err error) { + return util.GetOwnerCluster(ctx, cli, r.ObjectMeta) +} + +func (in *NestedControlPlane) GetConditions() clusterv1.Conditions { + return in.Status.Conditions +} + +func (in *NestedControlPlane) SetConditions(conditions clusterv1.Conditions) { + in.Status.Conditions = conditions +} diff --git a/apis/controlplane/v1alpha4/nestedetcd_types.go b/apis/controlplane/v1alpha4/nestedetcd_types.go index 8b8abc09..2cd01dc9 100644 --- a/apis/controlplane/v1alpha4/nestedetcd_types.go +++ b/apis/controlplane/v1alpha4/nestedetcd_types.go @@ -53,8 +53,7 @@ type NestedEtcdAddress struct { } //+kubebuilder:object:root=true -//+kubebuilder:resource:scope=Namespaced,path=nestedetcds,shortName=netcd -//+kubebuilder:categories=capi,capn +//+kubebuilder:resource:scope=Namespaced,shortName=netcd,categories=capi;capn //+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" //+kubebuilder:subresource:status @@ -80,3 +79,35 @@ type NestedEtcdList struct { func init() { SchemeBuilder.Register(&NestedEtcd{}, &NestedEtcdList{}) } + +var _ addonv1alpha1.CommonObject = &NestedEtcd{} +var _ addonv1alpha1.Patchable = &NestedEtcd{} + +// ComponentName returns the name of the component for use with +// addonv1alpha1.CommonObjec +func (c *NestedEtcd) ComponentName() string { + return string(Etcd) +} + +// CommonSpec returns the addons spec of the object allowing common funcs like +// Channel & Version to be usabl +func (c *NestedEtcd) CommonSpec() addonv1alpha1.CommonSpec { + return c.Spec.CommonSpec +} + +// GetCommonStatus will return the common status for checking is a component +// was successfully deployed +func (c *NestedEtcd) GetCommonStatus() addonv1alpha1.CommonStatus { + return c.Status.CommonStatus +} + +// SetCommonStatus will set the status so that abstract representations can set +// Ready and Phases +func (c *NestedEtcd) SetCommonStatus(s addonv1alpha1.CommonStatus) { + c.Status.CommonStatus = s +} + +// PatchSpec returns the patches to be applie +func (c *NestedEtcd) PatchSpec() addonv1alpha1.PatchSpec { + return c.Spec.PatchSpec +} diff --git a/apis/controlplane/v1alpha4/zz_generated.deepcopy.go b/apis/controlplane/v1alpha4/zz_generated.deepcopy.go index 436d1cf6..44fc3434 100644 --- a/apis/controlplane/v1alpha4/zz_generated.deepcopy.go +++ b/apis/controlplane/v1alpha4/zz_generated.deepcopy.go @@ -202,7 +202,6 @@ func (in *NestedControlPlaneList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NestedControlPlaneSpec) DeepCopyInto(out *NestedControlPlaneSpec) { *out = *in - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint if in.EtcdRef != nil { in, out := &in.EtcdRef, &out.EtcdRef *out = new(v1.ObjectReference) diff --git a/apis/infrastructure/v1alpha4/groupversion_info.go b/apis/infrastructure/v1alpha4/groupversion_info.go new file mode 100644 index 00000000..97ff37b3 --- /dev/null +++ b/apis/infrastructure/v1alpha4/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha4 contains API Schema definitions for the infrastructure v1alpha4 API group +//+kubebuilder:object:generate=true +//+groupName=infrastructure.cluster.x-k8s.io +package v1alpha4 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1alpha4"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/infrastructure/v1alpha4/nestedcluster_types.go b/apis/infrastructure/v1alpha4/nestedcluster_types.go new file mode 100644 index 00000000..c814ce1d --- /dev/null +++ b/apis/infrastructure/v1alpha4/nestedcluster_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha4 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" +) + +// NestedClusterSpec defines the desired state of NestedCluster +type NestedClusterSpec struct { + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // +optional + ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` +} + +// NestedClusterStatus defines the observed state of NestedCluster +type NestedClusterStatus struct { + // Ready is when the NestedControlPlane has a API server URL. + // +optional + Ready bool `json:"ready,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope=Namespaced,shortName=nc,categories=capi;capn +//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready" +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +//+kubebuilder:subresource:status + +// NestedCluster is the Schema for the nestedclusters API +type NestedCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NestedClusterSpec `json:"spec,omitempty"` + Status NestedClusterStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// NestedClusterList contains a list of NestedCluster +type NestedClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NestedCluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NestedCluster{}, &NestedClusterList{}) +} diff --git a/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go b/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go new file mode 100644 index 00000000..9506c643 --- /dev/null +++ b/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go @@ -0,0 +1,115 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha4 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedCluster) DeepCopyInto(out *NestedCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedCluster. +func (in *NestedCluster) DeepCopy() *NestedCluster { + if in == nil { + return nil + } + out := new(NestedCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NestedCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterList) DeepCopyInto(out *NestedClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NestedCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterList. +func (in *NestedClusterList) DeepCopy() *NestedClusterList { + if in == nil { + return nil + } + out := new(NestedClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NestedClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterSpec) DeepCopyInto(out *NestedClusterSpec) { + *out = *in + out.ControlPlaneEndpoint = in.ControlPlaneEndpoint +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterSpec. +func (in *NestedClusterSpec) DeepCopy() *NestedClusterSpec { + if in == nil { + return nil + } + out := new(NestedClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NestedClusterStatus) DeepCopyInto(out *NestedClusterStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NestedClusterStatus. +func (in *NestedClusterStatus) DeepCopy() *NestedClusterStatus { + if in == nil { + return nil + } + out := new(NestedClusterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/certificate/consts.go b/certificate/consts.go new file mode 100644 index 00000000..db676f05 --- /dev/null +++ b/certificate/consts.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import "sigs.k8s.io/cluster-api/util/secret" + +const ( + // defaultClusterDomain defines the default that all control planes are + // provisioned with + defaultClusterDomain = "cluster.local" + + // EtcdClient defines the client cert name for etcd + EtcdClient secret.Purpose = "etcd-client" + + // EtcdHealthClient defines the client cert name for etcd + EtcdHealthClient secret.Purpose = "etcd-health-client" + + // APIServerClient defines the client cert name for apiserver + APIServerClient secret.Purpose = "apiserver-client" + + // APIServerEtcdClient mirrors capi APIServerEtcdClient + APIServerEtcdClient secret.Purpose = secret.APIServerEtcdClient + + // KubeletClient defines the client cert name for kubelet + KubeletClient secret.Purpose = "kubelet-client" + + // ProxyClient defines the client cert name for the front proxy + ProxyClient secret.Purpose = "proxy-client" + + // ControllerManagerKubeconfig defines the secret purpose for KCM Kubeconfigs + ControllerManagerKubeconfig secret.Purpose = "controller-manager-kubeconfig" +) diff --git a/certificate/helpers.go b/certificate/helpers.go new file mode 100644 index 00000000..4ceec803 --- /dev/null +++ b/certificate/helpers.go @@ -0,0 +1,174 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + + "github.com/pkg/errors" + + "k8s.io/client-go/util/cert" + "sigs.k8s.io/cluster-api-provider-nested/certificate/util" +) + +// NewAPIServerCertAndKey creates crt and key for apiserver using ca. +func NewAPIServerCrtAndKey(ca *KeyPair, clusterName, clusterDomainArg, apiserverDomain string, apiserverIPs ...string) (*KeyPair, error) { + clusterDomain := defaultClusterDomain + if clusterDomainArg != "" { + clusterDomain = clusterDomainArg + } + apiserverIPs = append(apiserverIPs, "127.0.0.1", "0.0.0.0") + + // create AltNames with defaults DNSNames/IPs + altNames := &cert.AltNames{ + DNSNames: []string{ + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + fmt.Sprintf("kubernetes.default.svc.%s", clusterDomain), + apiserverDomain, + // add virtual cluster name (i.e. namespace) for vn-agent + clusterName, + }, + } + + for _, ip := range apiserverIPs { + if ip != "" { + altNames.IPs = append(altNames.IPs, net.ParseIP(ip)) + } + } + + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: clusterName, + AltNames: *altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }, + } + + apiCert, apiKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create apiserver crt and key: %v", err) + } + + rsaKey, ok := apiKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + + return &KeyPair{APIServerClient, apiCert, rsaKey, true, true}, nil +} + +// NewAPIServerKubeletClientCertAndKey creates certificate for the apiservers to connect to the +// kubelets securely, signed by the ca. +func NewAPIServerKubeletClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-apiserver-kubelet-client", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + apiClientCert, apiClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return &KeyPair{}, fmt.Errorf("failure while creating API server kubelet client key and certificate: %v", err) + } + + rsaKey, ok := apiClientKey.(*rsa.PrivateKey) + if !ok { + return &KeyPair{}, errors.New("fail to assert rsa private key") + } + + return &KeyPair{KubeletClient, apiClientCert, rsaKey, true, true}, nil +} + +// NewEtcdServerCrtAndKey creates new crt-key pair using ca for etcd +func NewEtcdServerCrtAndKey(ca *KeyPair, etcdDomains []string) (*KeyPair, error) { + // create AltNames with defaults DNSNames/IPs + altNames := &cert.AltNames{ + DNSNames: etcdDomains, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, + } + + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-etcd", + AltNames: *altNames, + // all peers will use this crt-key pair as well + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }, + } + etcdServerCert, etcdServerKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create etcd crt and key: %v", err) + } + + rsaKey, ok := etcdServerKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + + return &KeyPair{EtcdClient, etcdServerCert, rsaKey, true, true}, nil +} + +// NewEtcdHealthcheckClientCertAndKey creates certificate for liveness probes to healthcheck etcd, +// signed by the given ca. +func NewEtcdHealthcheckClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "kube-etcd-healthcheck-client", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + etcdHealcheckClientCert, etcdHealcheckClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return &KeyPair{}, fmt.Errorf("failure while creating etcd healthcheck client key and certificate: %v", err) + } + + rsaKey, ok := etcdHealcheckClientKey.(*rsa.PrivateKey) + if !ok { + return &KeyPair{}, errors.New("fail to assert rsa private key") + } + + return &KeyPair{EtcdHealthClient, etcdHealcheckClientCert, rsaKey, true, true}, nil +} + +// NewFrontProxyClientCertAndKey creates crt-key pair for proxy client using ca. +func NewFrontProxyClientCertAndKey(ca *KeyPair) (*KeyPair, error) { + config := &util.CertConfig{ + Config: cert.Config{ + CommonName: "front-proxy-client", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + frontProxyClientCert, frontProxyClientKey, err := util.NewCertAndKey(ca.Cert, ca.Key, config) + if err != nil { + return nil, fmt.Errorf("fail to create crt and key for front-proxy: %v", err) + } + rsaKey, ok := frontProxyClientKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("fail to assert rsa private key") + } + return &KeyPair{ProxyClient, frontProxyClientCert, rsaKey, true, true}, nil +} + +// newPrivateKey creates an RSA private key +func newPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, 2048) +} diff --git a/certificate/keypair.go b/certificate/keypair.go new file mode 100644 index 00000000..47bbfcfa --- /dev/null +++ b/certificate/keypair.go @@ -0,0 +1,49 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/rsa" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + + "sigs.k8s.io/cluster-api-provider-nested/certificate/util" + + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (k *KeyPair) AsSecret(clusterName client.ObjectKey, owner metav1.OwnerReference) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterName.Namespace, + Name: secret.Name(clusterName.Name, k.Purpose), + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName.Name, + }, + }, + Data: map[string][]byte{ + secret.TLSKeyDataName: util.EncodePrivateKeyPEM(k.Key.(*rsa.PrivateKey)), + secret.TLSCrtDataName: util.EncodeCertPEM(k.Cert), + }, + Type: clusterv1.ClusterSecretType, + } + + if k.Generated { + s.OwnerReferences = []metav1.OwnerReference{owner} + } + return s +} diff --git a/certificate/keypair_test.go b/certificate/keypair_test.go new file mode 100644 index 00000000..3a8bcf7b --- /dev/null +++ b/certificate/keypair_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func newCA() *KeyPair { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Company, INC."}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"Golden Gate Bridge"}, + PostalCode: []string{"94016"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, _ := rsa.GenerateKey(rand.Reader, 4096) + return &KeyPair{ + Purpose: secret.EtcdCA, + Cert: ca, + Key: caPrivKey, + Generated: true, + New: false, + } +} + +func TestKeyPair_AsSecret(t *testing.T) { + clusterName := client.ObjectKey{Name: "test-cluster", Namespace: "default"} + ncp := &controlplanev1.NestedControlPlane{ObjectMeta: metav1.ObjectMeta{Name: "ncp", Namespace: "default"}} + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + type args struct { + clusterName client.ObjectKey + owner metav1.OwnerReference + } + tests := []struct { + name string + keyGen func() *KeyPair + args args + wantOwnerRef bool + }{ + { + "TestCreateGeneratedSecret", + func() *KeyPair { + kp, _ := NewFrontProxyClientCertAndKey(newCA()) + return kp + }, + args{ + clusterName, + *controllerRef, + }, + true, + }, + { + "TestCreateExistingSecret", + func() *KeyPair { + kp, _ := NewFrontProxyClientCertAndKey(newCA()) + kp.Generated = false + return kp + }, + args{ + clusterName, + *controllerRef, + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k := tt.keyGen() + got := k.AsSecret(tt.args.clusterName, tt.args.owner) + if tt.wantOwnerRef && len(got.OwnerReferences) == 0 { + t.Errorf("KeyPair.AsSecret().OwnerReferences = %v", got.OwnerReferences) + } + }) + } +} diff --git a/certificate/keypairs.go b/certificate/keypairs.go new file mode 100644 index 00000000..2eb7f7b9 --- /dev/null +++ b/certificate/keypairs.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "context" + + "github.com/pkg/errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Lookup looks up each certificate from secrets and populates the certificate with the secret data. +func (kp KeyPairs) Lookup(ctx context.Context, cli client.Client, clusterName client.ObjectKey) error { + // Look up each certificate as a secret and populate the certificate/key + for _, certificate := range kp { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: secret.Name(clusterName.Name, certificate.Purpose), + Namespace: clusterName.Namespace, + } + if err := cli.Get(ctx, key, s); err != nil { + if apierrors.IsNotFound(err) { + certificate.New = true + continue + } + return errors.WithStack(err) + } + certificate.New = false + } + return nil +} + +// SaveGenerated will save any certificates that have been generated as Kubernetes secrets. +func (c KeyPairs) SaveGenerated(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { + for _, keyPair := range c { + if !keyPair.Generated && !keyPair.New { + continue + } + s := keyPair.AsSecret(clusterName, owner) + if err := ctrlclient.Create(ctx, s); err != nil { + // We might want to trigger off updates from here. + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + } + } + return nil +} + +// LookupOrGenerate is a convenience function that wraps cluster bootstrap certificate behavior. +func (kp KeyPairs) LookupOrSave(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { + // Find the certificates that exist + if err := kp.Lookup(ctx, ctrlclient, clusterName); err != nil { + return err + } + + // Save any certificates that have been generated + if err := kp.SaveGenerated(ctx, ctrlclient, clusterName, owner); err != nil { + return err + } + + return nil +} diff --git a/certificate/keypairs_test.go b/certificate/keypairs_test.go new file mode 100644 index 00000000..1b58634f --- /dev/null +++ b/certificate/keypairs_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "context" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newSecret(fns ...func(*v1.Secret)) *v1.Secret { + secret := &v1.Secret{} + for _, fn := range fns { + fn(secret) + } + return secret +} + +func TestKeyPairs_Lookup(t *testing.T) { + ctx := context.TODO() + clusterName := client.ObjectKey{Name: "test-cluster", Namespace: "default"} + type args struct { + ctx context.Context + cli client.Client + clusterName client.ObjectKey + } + tests := []struct { + name string + kp KeyPairs + args args + wantErr bool + wantNew bool + }{ + { + "TestNotFoundCertificateTrue", + KeyPairs{&KeyPair{Purpose: EtcdClient, New: false}}, + args{ctx, fake.NewFakeClient(newSecret()), clusterName}, + false, + true, + }, + { + "TestFoundCertificateFalse", + KeyPairs{&KeyPair{Purpose: EtcdClient, New: true}}, + args{ + ctx, + fake.NewFakeClient(newSecret(func(s *v1.Secret) { + s.Name = "test-cluster-etcd-client" + s.Namespace = "default" + })), + clusterName, + }, + false, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.kp.Lookup(tt.args.ctx, tt.args.cli, tt.args.clusterName); (err != nil) != tt.wantErr { + t.Errorf("KeyPairs.Lookup() error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.wantNew != tt.kp[0].New { + t.Errorf("KeyPairs.Lookup() new = %v, wantNew %v", tt.kp[0].New, tt.wantNew) + } + }) + } +} + +func TestKeyPairs_SaveGenerated(t *testing.T) { + ctx := context.TODO() + clusterName := client.ObjectKey{Name: "test-cluster", Namespace: "default"} + ncp := &controlplanev1.NestedControlPlane{ObjectMeta: metav1.ObjectMeta{Name: "ncp", Namespace: "default"}} + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + type args struct { + ctx context.Context + ctrlclient client.Client + clusterName client.ObjectKey + owner metav1.OwnerReference + } + tests := []struct { + name string + keypairsFunc func() KeyPairs + args args + wantErr bool + wantCount int + }{ + { + "TestCreatingNewSecret", + func() KeyPairs { + kp, _ := NewFrontProxyClientCertAndKey(newCA()) + return KeyPairs{kp} + }, + args{ + ctx, + fake.NewFakeClient(), + clusterName, + *controllerRef, + }, + false, + 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + keypairs := tt.keypairsFunc() + if err := keypairs.SaveGenerated(tt.args.ctx, tt.args.ctrlclient, tt.args.clusterName, tt.args.owner); (err != nil) != tt.wantErr { + t.Errorf("KeyPairs.SaveGenerated() error = %v, wantErr %v", err, tt.wantErr) + } + + secrets := &v1.SecretList{} + tt.args.ctrlclient.List(tt.args.ctx, secrets) + if len(secrets.Items) != tt.wantCount { + t.Errorf("KeyPairs.SaveGenerated().Count expected = %v, got %v", len(secrets.Items), tt.wantCount) + } + }) + } +} diff --git a/certificate/types.go b/certificate/types.go new file mode 100644 index 00000000..65965155 --- /dev/null +++ b/certificate/types.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto" + "crypto/x509" + + "sigs.k8s.io/cluster-api/util/secret" +) + +// KeyPair defines a cert/key pair that is used for the Kubernetes clients +// this was inspired by CAPI's KCP and how it manages CAs +type KeyPair struct { + Purpose secret.Purpose + Cert *x509.Certificate + Key crypto.Signer + Generated bool + New bool +} + +// KeyPairs defines a set of keypairs to act on, this is useful in providing +// helpers to operate on many keypairs +type KeyPairs []*KeyPair diff --git a/certificate/util/util.go b/certificate/util/util.go new file mode 100644 index 00000000..a829d59a --- /dev/null +++ b/certificate/util/util.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math" + "math/big" + "time" + + "github.com/pkg/errors" + certutil "k8s.io/client-go/util/cert" +) + +const ( + // certificateBlockType is a possible value for pem.Block.Type. + certificateBlockType = "CERTIFICATE" + // rsaPrivateKeyBlockType is a possible value for pem.Block.Type. + rsaPrivateKeyBlockType = "RSA PRIVATE KEY" + rsaKeySize = 2048 + + // certificateValidity defines the validity for all the signed certificates generated by this package + certificateValidity = time.Hour * 24 * 365 +) + +// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm. +type CertConfig struct { + certutil.Config + PublicKeyAlgorithm x509.PublicKeyAlgorithm +} + +// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key +func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) { + key, err := NewPrivateKey(config.PublicKeyAlgorithm) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to create private key") + } + + cert, err := NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to sign certificate") + } + + return cert, key, nil +} + +// NewPrivateKey creates an RSA private key +func NewPrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) { + if keyType == x509.ECDSA { + return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + } + + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(certificateValidity).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +// EncodeCertPEM returns PEM-endcoded certificate data +func EncodeCertPEM(cert *x509.Certificate) []byte { + block := pem.Block{ + Type: certificateBlockType, + Bytes: cert.Raw, + } + return pem.EncodeToMemory(&block) +} + +// EncodePrivateKeyPEM returns PEM-encoded private key data +func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { + block := pem.Block{ + Type: rsaPrivateKeyBlockType, + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return pem.EncodeToMemory(&block) +} diff --git a/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml b/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml index c2bccc61..0c7fcf20 100644 --- a/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml +++ b/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml @@ -1,15 +1,16 @@ apiVersion: v1 kind: Service metadata: - name: {{.nestedAPIServerName}} - namespace: {{.nestedAPIServerNamespace}} + name: {{.clusterName}}-apiserver + namespace: {{.componentNamespace}} labels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} spec: selector: - component-name: {{.nestedAPIServerName}} - type: NodePort + component-name: {{.componentName}} + type: ClusterIP ports: - - port: 6443 + - name: api + port: 6443 protocol: TCP targetPort: api diff --git a/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml b/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml index 0ac96c1e..d616e5a0 100644 --- a/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml +++ b/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml @@ -1,45 +1,52 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedAPIServerName}} - namespace: {{.nestedAPIServerNamespace}} + name: {{.clusterName}}-apiserver + namespace: {{.componentNamespace}} spec: revisionHistoryLimit: 10 - serviceName: {{.nestedAPIServerName}} + serviceName: {{.componentName}} selector: matchLabels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} # apiserver will not be updated, unless it is deleted updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedAPIServerName}} + component-name: {{.componentName}} spec: hostname: apiserver - subdomain: apiserver-svc + subdomain: {{.clusterName}}-apiserver containers: - - name: {{.nestedAPIServerName}} + - name: {{.componentName}} image: virtualcluster/apiserver-v1.16.2 imagePullPolicy: Always command: - kube-apiserver + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - --bind-address=0.0.0.0 - --allow-privileged=true - --anonymous-auth=true - - --client-ca-file=/etc/kubernetes/pki/root/tls.crt + - --client-ca-file=/etc/kubernetes/pki/apiserver/ca/tls.crt - --tls-cert-file=/etc/kubernetes/pki/apiserver/tls.crt - --tls-private-key-file=/etc/kubernetes/pki/apiserver/tls.key - --kubelet-https=true - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver/tls.crt - - --kubelet-client-key=/etc/kubernetes/pki/apiserver/tls.key + - --kubelet-certificate-authority=/etc/kubernetes/pki/apiserver/ca/tls.crt + - --kubelet-client-certificate=/etc/kubernetes/pki/kubelet/tls.crt + - --kubelet-client-key=/etc/kubernetes/pki/kubelet/tls.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP - --enable-bootstrap-token-auth=true - - --etcd-servers=https://{{.nestedEtcdName}}-0.{{.nestedEtcdName}}:2379 - - --etcd-cafile=/etc/kubernetes/pki/root/tls.crt - - --etcd-certfile=/etc/kubernetes/pki/apiserver/tls.crt - - --etcd-keyfile=/etc/kubernetes/pki/apiserver/tls.key + - --etcd-servers=https://{{.clusterName}}-etcd-0.{{.clusterName}}-etcd.$(NAMESPACE):2379 + - --etcd-cafile=/etc/kubernetes/pki/etcd/ca/tls.crt + - --etcd-certfile=/etc/kubernetes/pki/etcd/tls.crt + - --etcd-keyfile=/etc/kubernetes/pki/etcd/tls.key - --service-account-key-file=/etc/kubernetes/pki/service-account/tls.key - --service-cluster-ip-range=10.32.0.0/16 - --service-node-port-range=30000-32767 @@ -71,29 +78,50 @@ spec: periodSeconds: 2 timeoutSeconds: 30 volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd/ca + name: {{.clusterName}}-etcd-ca + readOnly: true + - mountPath: /etc/kubernetes/pki/etcd + name: {{.clusterName}}-etcd-client + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver/ca + name: {{.clusterName}}-ca + readOnly: true - mountPath: /etc/kubernetes/pki/apiserver - name: {{.nestedControlPlaneName}}-apiserver + name: {{.clusterName}}-apiserver-client readOnly: true - - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + - mountPath: /etc/kubernetes/pki/kubelet + name: {{.clusterName}}-kubelet-client readOnly: true - mountPath: /etc/kubernetes/pki/service-account - name: {{.nestedControlPlaneName}}-sa + name: {{.clusterName}}-sa readOnly: true terminationGracePeriodSeconds: 30 dnsConfig: searches: - cluster.local volumes: - - name: {{.nestedControlPlaneName}}-apiserver + - name: {{.clusterName}}-apiserver-client + secret: + defaultMode: 420 + secretName: {{.clusterName}}-apiserver-client + - name: {{.clusterName}}-etcd-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd + - name: {{.clusterName}}-etcd-client + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd-client + - name: {{.clusterName}}-kubelet-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-apiserver - - name: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-kubelet-client + - name: {{.clusterName}}-ca secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca - - name: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-ca + - name: {{.clusterName}}-sa secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-sa diff --git a/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml b/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml index 2c972aa4..e8a774f3 100644 --- a/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml +++ b/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml @@ -1,21 +1,21 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedControllerManagerName}} - namespace: {{.nestedControllerManagerNamespace}} + name: {{.clusterName}}-controller-manager + namespace: {{.componentNamespace}} spec: selector: matchLabels: - component-name: {{.nestedControllerManagerName}} + component-name: {{.componentName}} updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedControllerManagerName}} + component-name: {{.componentName}} spec: containers: - - name: {{.nestedControllerManagerName}} + - name: {{.componentName}} image: virtualcluster/controller-manager-v1.16.2 imagePullPolicy: Always command: @@ -30,7 +30,7 @@ spec: - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig # control plane contains only one instance for now - --leader-elect=false - - --root-ca-file=/etc/kubernetes/pki/root/tls.crt + - --root-ca-file=/etc/kubernetes/pki/root/ca/tls.crt - --service-account-private-key-file=/etc/kubernetes/pki/service-account/tls.key - --service-cluster-ip-range=10.32.0.0/24 - --use-service-account-credentials=true @@ -56,25 +56,35 @@ spec: periodSeconds: 2 timeoutSeconds: 15 volumeMounts: + - mountPath: /etc/kubernetes/pki/root/ca + name: {{.clusterName}}-ca + readOnly: true - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + name: {{.clusterName}}-apiserver-client readOnly: true - mountPath: /etc/kubernetes/pki/service-account - name: {{.nestedControlPlaneName}}-sa + name: {{.clusterName}}-sa readOnly: true - mountPath: /etc/kubernetes/kubeconfig - name: {{.nestedControllerManagerName}}-kubeconfig + name: {{.clusterName}}-kubeconfig readOnly: true volumes: - - name: {{.nestedControlPlaneName}}-ca + - name: {{.clusterName}}-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-ca + - name: {{.clusterName}}-apiserver-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca - - name: {{.nestedControlPlaneName}}-sa + secretName: {{.clusterName}}-apiserver-client + - name: {{.clusterName}}-sa secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-sa - - name: {{.nestedControllerManagerName}}-kubeconfig + secretName: {{.clusterName}}-sa + - name: {{.clusterName}}-kubeconfig secret: defaultMode: 420 - secretName: {{.nestedControllerManagerName}}-kubeconfig + secretName: {{.clusterName}}-kubeconfig + items: + - key: value + path: controller-manager-kubeconfig diff --git a/config/component-templates/nested-etcd/nested-etcd-service-template.yaml b/config/component-templates/nested-etcd/nested-etcd-service-template.yaml index edca7010..58cd080b 100644 --- a/config/component-templates/nested-etcd/nested-etcd-service-template.yaml +++ b/config/component-templates/nested-etcd/nested-etcd-service-template.yaml @@ -1,12 +1,12 @@ apiVersion: v1 kind: Service metadata: - name: {{.nestedEtcdName}} - namespace: {{.nestedEtcdNamespace}} + name: {{.clusterName}}-etcd + namespace: {{.componentNamespace}} labels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} spec: publishNotReadyAddresses: true clusterIP: None selector: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} diff --git a/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml b/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml index 314af417..95c53009 100644 --- a/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml +++ b/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml @@ -1,25 +1,24 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{.nestedEtcdName}} - namespace: {{.nestedEtcdNamespace}} + name: {{.clusterName}}-etcd + namespace: {{.componentNamespace}} spec: revisionHistoryLimit: 10 - serviceName: {{.nestedEtcdName}} + serviceName: {{.clusterName}}-etcd selector: matchLabels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} # etcd will not be updated, unless it is deleted updateStrategy: type: OnDelete template: metadata: labels: - component-name: {{.nestedEtcdName}} + component-name: {{.componentName}} spec: - subdomain: etcd containers: - - name: {{.nestedEtcdName}} + - name: {{.componentName}} image: virtualcluster/etcd-v3.4.0 imagePullPolicy: Always command: @@ -30,21 +29,25 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - --name=$(HOSTNAME) - - --trusted-ca-file=/etc/kubernetes/pki/root/tls.crt + - --trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - --client-cert-auth - --cert-file=/etc/kubernetes/pki/etcd/tls.crt - --key-file=/etc/kubernetes/pki/etcd/tls.key - --peer-client-cert-auth - - --peer-trusted-ca-file=/etc/kubernetes/pki/root/tls.crt + - --peer-trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - --peer-cert-file=/etc/kubernetes/pki/etcd/tls.crt - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key - --listen-peer-urls=https://0.0.0.0:2380 - --listen-client-urls=https://0.0.0.0:2379 - - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2380 + - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2380 # we use a headless service to encapsulate each pod - - --advertise-client-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2379 + - --advertise-client-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2379 - --initial-cluster-state=new - --initial-cluster-token=vc-etcd - --data-dir=/var/lib/etcd/data @@ -54,7 +57,7 @@ spec: command: - sh - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health + - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health failureThreshold: 8 initialDelaySeconds: 60 timeoutSeconds: 15 @@ -63,24 +66,31 @@ spec: command: - sh - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health + - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health failureThreshold: 8 initialDelaySeconds: 15 periodSeconds: 2 timeoutSeconds: 15 volumeMounts: + - mountPath: /etc/kubernetes/pki/ca + name: {{.clusterName}}-etcd-ca + readOnly: true - mountPath: /etc/kubernetes/pki/etcd - name: {{.nestedControlPlaneName}}-etcd + name: {{.clusterName}}-etcd-client readOnly: true - - mountPath: /etc/kubernetes/pki/root - name: {{.nestedControlPlaneName}}-ca + - mountPath: /etc/kubernetes/pki/health + name: {{.clusterName}}-etcd-health-client readOnly: true volumes: - - name: {{.nestedControlPlaneName}}-etcd + - name: {{.clusterName}}-etcd-ca + secret: + defaultMode: 420 + secretName: {{.clusterName}}-etcd + - name: {{.clusterName}}-etcd-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-etcd - - name: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-etcd-client + - name: {{.clusterName}}-etcd-health-client secret: defaultMode: 420 - secretName: {{.nestedControlPlaneName}}-ca + secretName: {{.clusterName}}-etcd-health-client diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml index 1bcf3eef..d6fbdda9 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedAPIServer listKind: NestedAPIServerList plural: nestedapiservers diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml similarity index 97% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml rename to config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml index ee7cc635..4bb4ceb5 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanager.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml @@ -6,13 +6,16 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1-0.20201002000720-57250aac17f6 creationTimestamp: null - name: nestedcontrollermanager.controlplane.cluster.x-k8s.io + name: nestedcontrollermanagers.controlplane.cluster.x-k8s.io spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedControllerManager listKind: NestedControllerManagerList - plural: nestedcontrollermanager + plural: nestedcontrollermanagers shortNames: - nkcm singular: nestedcontrollermanager diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml index a3641a0c..549d9ad5 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedControlPlane listKind: NestedControlPlaneList plural: nestedcontrolplanes @@ -19,7 +22,7 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .status.rady + - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .metadata.creationTimestamp @@ -66,20 +69,6 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object controllerManager: description: ContollerManagerRef is the reference to the NestedControllerManager properties: diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml index 2e7893f0..aab3535d 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml @@ -10,6 +10,9 @@ metadata: spec: group: controlplane.cluster.x-k8s.io names: + categories: + - capi + - capn kind: NestedEtcd listKind: NestedEtcdList plural: nestedetcds diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml new file mode 100644 index 00000000..d29f95b5 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml @@ -0,0 +1,79 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1-0.20201002000720-57250aac17f6 + creationTimestamp: null + name: nestedclusters.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - capi + - capn + kind: NestedCluster + listKind: NestedClusterList + plural: nestedclusters + shortNames: + - nc + singular: nestedcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha4 + schema: + openAPIV3Schema: + description: NestedCluster is the Schema for the nestedclusters API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NestedClusterSpec defines the desired state of NestedCluster + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: NestedClusterStatus defines the observed state of NestedCluster + properties: + ready: + description: Ready is when the NestedControlPlane has a API server URL. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 00000000..899557ad --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,31 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml +- bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml +- bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml +- bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml +- bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +# - kustomizeconfig.yaml + + +commonLabels: + # When using CAPI we need to define the contract version labels so that the + # capi system can cross reference the proper controlplane & infra refs + # https://cluster-api.sigs.k8s.io/developer/providers/v1alpha2-to-v1alpha3.html#apply-the-contract-version-label-clusterx-k8sioversion-version1_version2_version3-to-your-crds + cluster.x-k8s.io/v1alpha3: v1alpha4 + cluster.x-k8s.io/v1alpha4: v1alpha4 \ No newline at end of file diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 1096a7aa..bf803201 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,16 +1,12 @@ -# Adds namespace to all resources. +namePrefix: capn- namespace: capn-system -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -namePrefix: capn- # Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +commonLabels: + # Label to denote name of the infra provider + # https://cluster-api.sigs.k8s.io/clusterctl/provider-contract.html#labels + cluster.x-k8s.io/provider: "infrastructure-aws" bases: - ../crd diff --git a/config/rbac/nestedcluster_editor_role.yaml b/config/rbac/nestedcluster_editor_role.yaml new file mode 100644 index 00000000..289d30c7 --- /dev/null +++ b/config/rbac/nestedcluster_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit nestedclusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nestedcluster-editor-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get diff --git a/config/rbac/nestedcluster_viewer_role.yaml b/config/rbac/nestedcluster_viewer_role.yaml new file mode 100644 index 00000000..4cc730eb --- /dev/null +++ b/config/rbac/nestedcluster_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view nestedclusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nestedcluster-viewer-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b6f997c3..265223b9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -134,3 +134,29 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/finalizers + verbs: + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - nestedclusters/status + verbs: + - get + - patch + - update diff --git a/controllers/controlplane/consts.go b/controllers/controlplane/consts.go index 5adb1a8a..d1e57ae4 100644 --- a/controllers/controlplane/consts.go +++ b/controllers/controlplane/consts.go @@ -20,19 +20,14 @@ const ( statefulsetOwnerKeyNEtcd = ".metadata.netcd.controller" statefulsetOwnerKeyNKas = ".metadata.nkas.controller" statefulsetOwnerKeyNKcm = ".metadata.nkcm.controller" - defaultEtcdStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultEtcdStatefulSetURL = "./config/component-templates/" + "nested-etcd/nested-etcd-statefulset-template.yaml" - defaultEtcdServiceURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultEtcdServiceURL = "./config/component-templates/" + "nested-etcd/nested-etcd-service-template.yaml" - defaultKASStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKASStatefulSetURL = "./config/component-templates/" + "nested-apiserver/nested-apiserver-statefulset-template.yaml" - defaultKASServiceURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKASServiceURL = "./config/component-templates/" + "nested-apiserver/nested-apiserver-service-template.yaml" - defaultKCMStatefulSetURL = "https://raw.githubusercontent.com/kubernetes-sigs/" + - "cluster-api-provider-nested/master/config/component-templates/" + + defaultKCMStatefulSetURL = "./config/component-templates/" + "nested-controllermanager/nested-controllermanager-statefulset-template.yaml" ) diff --git a/controllers/controlplane/controller_util.go b/controllers/controlplane/controller_util.go index 4790b9ed..83e690c8 100644 --- a/controllers/controlplane/controller_util.go +++ b/controllers/controlplane/controller_util.go @@ -19,13 +19,12 @@ package controlplane import ( "bytes" "context" - "crypto/tls" - "errors" "fmt" "io/ioutil" - "net/http" "text/template" + openuri "github.com/utahta/go-openuri" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -47,26 +46,31 @@ func createNestedComponentSts(ctx context.Context, cli ctrlcli.Client, ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, ncKind clusterv1.ComponentKind, - clusterName string, log logr.Logger) error { + controlPlaneName, clusterName string, log logr.Logger) error { var ( ncSts appsv1.StatefulSet ncSvc corev1.Service err error ) + // Setup the ownerReferences for all objects + or := metav1.NewControllerRef(&ncMeta, + clusterv1.GroupVersion.WithKind(string(ncKind))) + // 1. Using the template defined by version/channel to create the // StatefulSet and the Service // TODO check the template version/channel, if not set, use the default. if ncSpec.Version == "" && ncSpec.Channel == "" { log.V(4).Info("The Version and Channel are not set, " + "will use the default template.") - ncSts, err = genStatefulSetObject(ncMeta, ncSpec, ncKind, clusterName, cli, log) + ncSts, err = genStatefulSetObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, cli, log) if err != nil { return fmt.Errorf("fail to generate the Statefulset object: %v", err) } if ncKind != clusterv1.ControllerManager { // no need to create the service for the NestedControllerManager - ncSvc, err = genServiceObject(ncMeta, ncSpec, ncKind, clusterName, log) + ncSvc, err = genServiceObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log) + ncSvc.SetOwnerReferences([]metav1.OwnerReference{*or}) if err != nil { return fmt.Errorf("fail to generate the Service object: %v", err) } @@ -81,8 +85,6 @@ func createNestedComponentSts(ctx context.Context, panic("NOT IMPLEMENT YET") } // 2. set the NestedComponent object as the owner of the StatefulSet - or := metav1.NewControllerRef(&ncMeta, - clusterv1.GroupVersion.WithKind(string(ncKind))) ncSts.SetOwnerReferences([]metav1.OwnerReference{*or}) // 4. create the NestedComponent StatefulSet @@ -93,7 +95,7 @@ func createNestedComponentSts(ctx context.Context, // NestedComponent func genServiceObject(ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, ncKind clusterv1.ComponentKind, - clusterName string, log logr.Logger) (ncSvc corev1.Service, retErr error) { + controlPlaneName, clusterName string, log logr.Logger) (ncSvc corev1.Service, retErr error) { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { switch ncKind { @@ -114,21 +116,7 @@ func genServiceObject(ncMeta metav1.ObjectMeta, return } - var templateCtx map[string]string - switch ncKind { - case clusterv1.APIServer: - templateCtx = map[string]string{ - "nestedAPIServerName": ncMeta.GetName(), - "nestedAPIServerNamespace": ncMeta.GetNamespace(), - } - case clusterv1.Etcd: - templateCtx = map[string]string{ - "nestedEtcdName": ncMeta.GetName(), - "nestedEtcdNamespace": ncMeta.GetNamespace(), - } - default: - panic("Unreachable") - } + templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) svcStr, err := substituteTemplate(templateCtx, svcTmpl) if err != nil { @@ -157,7 +145,7 @@ func genServiceObject(ncMeta metav1.ObjectMeta, func genStatefulSetObject( ncMeta metav1.ObjectMeta, ncSpec clusterv1.NestedComponentSpec, - ncKind clusterv1.ComponentKind, clusterName string, + ncKind clusterv1.ComponentKind, controlPlaneName, clusterName string, cli ctrlcli.Client, log logr.Logger) (ncSts appsv1.StatefulSet, retErr error) { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { @@ -185,35 +173,7 @@ func genStatefulSetObject( return } // 2 substitute the statefulset template - var templateCtx map[string]string - switch ncKind { - case clusterv1.Etcd: - templateCtx = map[string]string{ - "nestedEtcdName": ncMeta.GetName(), - "nestedEtcdNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - } - case clusterv1.APIServer: - etcdName, err := getEtcdName(cli, ncMeta.GetOwnerReferences(), ncMeta.GetNamespace()) - if err != nil { - retErr = err - return - } - templateCtx = map[string]string{ - "nestedAPIServerName": ncMeta.GetName(), - "nestedAPIServerNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - "nestedEtcdName": etcdName, - } - case clusterv1.ControllerManager: - templateCtx = map[string]string{ - "nestedControllerManagerName": ncMeta.GetName(), - "nestedControllerManagerNamespace": ncMeta.GetNamespace(), - "nestedControlPlaneName": clusterName, - } - default: - panic("Unreachable") - } + templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) stsStr, err := substituteTemplate(templateCtx, stsTmpl) if err != nil { retErr = fmt.Errorf("fail to substitute the default template "+ @@ -249,7 +209,7 @@ func genStatefulSetObject( // 6 set the "--initial-cluster" command line flag for the Etcd container if ncKind == clusterv1.Etcd { - icaVal := genInitialClusterArgs(1, ncMeta.GetName(), ncMeta.GetName()) + icaVal := genInitialClusterArgs(1, clusterName, clusterName, ncMeta.GetNamespace()) stsArgs := append(stsObj.Spec.Template.Spec.Containers[0].Args, "--initial-cluster", icaVal) stsObj.Spec.Template.Spec.Containers[0].Args = stsArgs @@ -261,27 +221,13 @@ func genStatefulSetObject( return } -// getEtcdName gets the name of the NestedEtcd through the NestedControlPlane -func getEtcdName(cli ctrlcli.Client, - ors []metav1.OwnerReference, ns string) (string, error) { - var or *metav1.OwnerReference - for _, orf := range ors { - if orf.Kind == "NestedControlPlane" { - or = &orf - } - break - } - if or == nil { - return "", errors.New("OwnerReference is not set") +func getTemplateArgs(ncMeta metav1.ObjectMeta, controlPlaneName, clusterName string) map[string]string { + return map[string]string{ + "componentName": ncMeta.GetName(), + "componentNamespace": ncMeta.GetNamespace(), + "clusterName": clusterName, + "controlPlaneName": controlPlaneName, } - var ncp clusterv1.NestedControlPlane - if err := cli.Get(context.TODO(), types.NamespacedName{ - Name: or.Name, - Namespace: ns, - }, &ncp); err != nil { - return "", err - } - return ncp.Spec.EtcdRef.Name, nil } // yamlToObject deserialize the yaml to the runtime object @@ -312,19 +258,13 @@ func substituteTemplate(context interface{}, tmpl string) (string, error) { // fetchTemplate fetches the component template through the tmplateURL func fetchTemplate(templateURL string) (string, error) { - // TODO mount host CA to manager pods - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }} - - rep, err := client.Get(templateURL) + rep, err := openuri.Open(templateURL) if err != nil { return "", err } + defer rep.Close() - defer rep.Body.Close() - - bodyBytes, err := ioutil.ReadAll(rep.Body) + bodyBytes, err := ioutil.ReadAll(rep) if err != nil { return "", err } @@ -350,14 +290,14 @@ func getOwner(ncMeta metav1.ObjectMeta) metav1.OwnerReference { // genAPIServerSvcRef generates the ObjectReference that points to the // APISrver service func genAPIServerSvcRef(cli ctrlcli.Client, - nkas clusterv1.NestedAPIServer) (corev1.ObjectReference, error) { + nkas clusterv1.NestedAPIServer, clusterName string) (corev1.ObjectReference, error) { var ( svc corev1.Service objRef corev1.ObjectReference ) if err := cli.Get(context.TODO(), types.NamespacedName{ Namespace: nkas.GetNamespace(), - Name: nkas.GetName(), + Name: fmt.Sprintf("%s-apiserver", clusterName), }, &svc); err != nil { return objRef, err } diff --git a/controllers/controlplane/controller_util_test.go b/controllers/controlplane/controller_util_test.go index d339e47a..4c7c1716 100644 --- a/controllers/controlplane/controller_util_test.go +++ b/controllers/controlplane/controller_util_test.go @@ -153,7 +153,7 @@ func TestGetOwner(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - get := getOwner(st.netcd) + get := getOwner(st.netcd.ObjectMeta) if !reflect.DeepEqual(get, st.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) } @@ -167,25 +167,28 @@ func TestGetOwner(t *testing.T) { func TestGenInitialClusterArgs(t *testing.T) { tests := []struct { - name string - replicas int32 - stsName string - svcName string - expect string + name string + replicas int32 + stsName string + svcName string + svcNamespace string + expect string }{ { "1 replicas", 1, "netcdSts", "netcdSvc", - "netcdSts-0=https://netcdSts-0.netcdSvc:2380", + "default", + "netcdSts-etcd-0=https://netcdSts-etcd-0.netcdSvc-etcd.default.svc:2380", }, { "2 replicas", 2, "netcdSts", "netcdSvc", - "netcdSts-0=https://netcdSts-0.netcdSvc:2380,netcdSts-1=https://netcdSts-1.netcdSvc:2380", + "default", + "netcdSts-etcd-0=https://netcdSts-etcd-0.netcdSvc-etcd.default.svc:2380,netcdSts-etcd-1=https://netcdSts-etcd-1.netcdSvc-etcd.default.svc:2380", }, } for _, tt := range tests { @@ -194,7 +197,7 @@ func TestGenInitialClusterArgs(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - get := genInitialClusterArgs(st.replicas, st.stsName, st.svcName) + get := genInitialClusterArgs(st.replicas, st.stsName, st.svcName, st.svcNamespace) if !reflect.DeepEqual(get, st.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) } diff --git a/controllers/controlplane/nestedapiserver_controller.go b/controllers/controlplane/nestedapiserver_controller.go index a876a747..1afe4a3c 100644 --- a/controllers/controlplane/nestedapiserver_controller.go +++ b/controllers/controlplane/nestedapiserver_controller.go @@ -18,6 +18,7 @@ package controlplane import ( "context" + "fmt" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -25,11 +26,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/certificate" + clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" ) // NestedAPIServerReconciler reconciles a NestedAPIServer object @@ -70,11 +76,26 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: nkas.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", nkas.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + // 2. create the NestedAPIServer StatefulSet if not found + nkasName := fmt.Sprintf("%s-apiserver", cluster.GetName()) var nkasSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: nkas.GetNamespace(), - Name: nkas.GetName(), + Name: nkasName, }, &nkasSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedAPIServer as unready @@ -88,10 +109,15 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } } + if err := r.createAPIServerClientCrts(ctx, cluster, &ncp, &nkas); err != nil { + log.Error(err, "fail to create NestedAPIServer Client Certs") + return ctrl.Result{}, err + } + // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkas.ObjectMeta, nkas.Spec.NestedComponentSpec, - clusterv1.APIServer, owner.Name, log); err != nil { + clusterv1.APIServer, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedAPIServer StatefulSet") return ctrl.Result{}, err } @@ -110,7 +136,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // As the NestedAPIServer StatefulSet is ready, update // NestedAPIServer status nkas.Status.Phase = string(clusterv1.Ready) - objRef, err := genAPIServerSvcRef(r.Client, nkas) + objRef, err := genAPIServerSvcRef(r.Client, nkas, cluster.GetName()) if err != nil { log.Error(err, "fail to generate NestedAPIServer Service Reference") return ctrl.Result{}, err @@ -170,3 +196,69 @@ func (r *NestedAPIServerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&appsv1.StatefulSet{}). Complete(r) } + +// createAPIServerClientCrts will find of create client certs for the etcd cluster +func (r *NestedAPIServerReconciler) createAPIServerClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, nkas *clusterv1.NestedAPIServer) error { + certificates := secret.NewCertificatesForInitialControlPlane(nil) + if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { + return err + } + cacert := certificates.GetByPurpose(secret.ClusterCA) + if cacert == nil { + return fmt.Errorf("could not fetch ClusterCA") + } + + cacrt, err := certs.DecodeCertPEM(cacert.KeyPair.Cert) + if err != nil { + return err + } + + cakey, err := certs.DecodePrivateKeyPEM(cacert.KeyPair.Key) + if err != nil { + return err + } + + // TODO(christopherhein) figure out how to get service clusterIPs + apiKeyPair, err := certificate.NewAPIServerCrtAndKey(&certificate.KeyPair{Cert: cacrt, Key: cakey}, nkas.GetName(), "", cluster.Spec.ControlPlaneEndpoint.Host) + if err != nil { + return err + } + + kubeletKeyPair, err := certificate.NewAPIServerKubeletClientCertAndKey(&certificate.KeyPair{Cert: cacrt, Key: cakey}) + if err != nil { + return err + } + + fpcert := certificates.GetByPurpose(secret.FrontProxyCA) + if cacert == nil { + return fmt.Errorf("could not fetch FrontProxyCA") + } + + fpcrt, err := certs.DecodeCertPEM(fpcert.KeyPair.Cert) + if err != nil { + return err + } + + fpkey, err := certs.DecodePrivateKeyPEM(fpcert.KeyPair.Key) + if err != nil { + return err + } + + frontProxyKeyPair, err := certificate.NewFrontProxyClientCertAndKey(&certificate.KeyPair{Cert: fpcrt, Key: fpkey}) + if err != nil { + return err + } + + certs := &certificate.KeyPairs{ + apiKeyPair, + kubeletKeyPair, + frontProxyKeyPair, + } + + controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + return err + } + + return nil +} diff --git a/controllers/controlplane/nestedcontrollermanager_controller.go b/controllers/controlplane/nestedcontrollermanager_controller.go index 1f6dac48..ebec066e 100644 --- a/controllers/controlplane/nestedcontrollermanager_controller.go +++ b/controllers/controlplane/nestedcontrollermanager_controller.go @@ -18,6 +18,7 @@ package controlplane import ( "context" + "fmt" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -69,11 +70,26 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: nkcm.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", nkcm.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + // 2. create the NestedControllerManager StatefulSet if not found + nkcmName := fmt.Sprintf("%s-controller-manager", cluster.GetName()) var nkcmSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: nkcm.GetNamespace(), - Name: nkcm.GetName(), + Name: nkcmName, }, &nkcmSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedControllerManager @@ -91,7 +107,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkcm.ObjectMeta, nkcm.Spec.NestedComponentSpec, - clusterv1.ControllerManager, owner.Name, log); err != nil { + clusterv1.ControllerManager, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedControllerManager StatefulSet") return ctrl.Result{}, err } diff --git a/controllers/controlplane/nestedcontrolplane_controller.go b/controllers/controlplane/nestedcontrolplane_controller.go index 731acc3b..4f2b6254 100644 --- a/controllers/controlplane/nestedcontrolplane_controller.go +++ b/controllers/controlplane/nestedcontrolplane_controller.go @@ -18,15 +18,36 @@ package controlplane import ( "context" + "time" "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/kubeconfig" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" ) +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update + // NestedControlPlaneReconciler reconciles a NestedControlPlane object type NestedControlPlaneReconciler struct { client.Client @@ -34,19 +55,280 @@ type NestedControlPlaneReconciler struct { Scheme *runtime.Scheme } -// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch +// SetupWithManager will configure the controller with the manager +func (r *NestedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&controlplanev1.NestedControlPlane{}). + Owns(&controlplanev1.NestedEtcd{}). + Owns(&controlplanev1.NestedAPIServer{}). + Owns(&controlplanev1.NestedControllerManager{}). + Complete(r) +} + +// Reconcile is ths main process which will handle updating teh NCP +func (r *NestedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("nestedcontrolplane", req.NamespacedName) + log.Info("Reconciling NestedControlPlane...") + // Fetch the NestedControlPlane + ncp := &v1alpha4.NestedControlPlane{} + if err := r.Get(ctx, req.NamespacedName, ncp); err != nil { + // check for not found and don't requeue + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + // If there are errors we should retry + return ctrl.Result{Requeue: true}, nil + } + + // Fetch the cluster object + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the API Server") + return ctrl.Result{}, err + } + log = log.WithValues("cluster", cluster.Name) + + if annotations.IsPaused(cluster, ncp) { + log.Info("Reconciliation is paused for this object") + return ctrl.Result{}, nil + } -func (r *NestedControlPlaneReconciler) Reconcile(_ context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = r.Log.WithValues("nestedcontrolplane", req.NamespacedName) + // Initialize the patch helper. + patchHelper, err := patch.NewHelper(ncp, r.Client) + if err != nil { + log.Error(err, "Failed to configure the patch helper") + return ctrl.Result{Requeue: true}, nil + } - // your logic here + if !controllerutil.ContainsFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) { + controllerutil.AddFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) + // patch and return right away instead of reusing the main defer, + // because the main defer may take too much time to get cluster status + // Patch ObservedGeneration only if the reconciliation completed successfully + patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}} + if err := patchHelper.Patch(ctx, ncp, patchOpts...); err != nil { + log.Error(err, "Failed to patch NestedControlPlane to add finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + // TODO(christopherhein) handle deletion + if !ncp.ObjectMeta.DeletionTimestamp.IsZero() { + // Handle deletion reconciliation loop. + return r.reconcileDelete(ctx, log, cluster, ncp) + } + + defer func() { + if err := patchControlPlane(ctx, patchHelper, ncp); err != nil { + log.Error(err, "Failed to patch KubeadmControlPlane") + } + }() + + // Handle normal reconciliation loop. + return r.reconcile(ctx, log, cluster, ncp) +} + +// reconcileDelete will delete the control plane and all it's nestedcomponents +func (r *NestedControlPlaneReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (ctrl.Result, error) { + patchHelper, err := patch.NewHelper(ncp, r.Client) + if err != nil { + log.Error(err, "Failed to configure the patch helper") + return ctrl.Result{Requeue: true}, nil + } + + if controllerutil.ContainsFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) { + controllerutil.RemoveFinalizer(ncp, controlplanev1.NestedControlPlaneFinalizer) + + // patch and return right away instead of reusing the main defer, + // because the main defer may take too much time to get cluster status + // Patch ObservedGeneration only if the reconciliation completed successfully + patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}} + if err := patchHelper.Patch(ctx, ncp, patchOpts...); err != nil { + log.Error(err, "Failed to patch NestedControlPlane to remove finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } return ctrl.Result{}, nil } -func (r *NestedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controlplanev1alpha4.NestedControlPlane{}). - Complete(r) +func patchControlPlane(ctx context.Context, patchHelper *patch.Helper, ncp *controlplanev1.NestedControlPlane) error { + // Always update the readyCondition by summarizing the state of other conditions. + conditions.SetSummary(ncp, + conditions.WithConditions( + kcpv1.AvailableCondition, + kcpv1.CertificatesAvailableCondition, + ), + ) + + // Patch the object, ignoring conflicts on the conditions owned by this controller. + return patchHelper.Patch( + ctx, + ncp, + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + clusterv1.ReadyCondition, + kcpv1.AvailableCondition, + kcpv1.CertificatesAvailableCondition, + }}, + patch.WithStatusObservedGeneration{}, + ) +} + +// reconcile will handle all "normal" NCP reconciles this means create/update actions +func (r *NestedControlPlaneReconciler) reconcile(ctx context.Context, log logr.Logger, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (res ctrl.Result, reterr error) { + log.Info("Reconcile NestedControlPlane") + + certificates := secret.NewCertificatesForInitialControlPlane(nil) + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + log.Error(err, "unable to lookup or create cluster certificates") + conditions.MarkFalse(ncp, kcpv1.CertificatesAvailableCondition, kcpv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) + return ctrl.Result{}, err + } + // TODO(christopherhein) use conditions to mark when ready + conditions.MarkTrue(ncp, kcpv1.CertificatesAvailableCondition) + + // If ControlPlaneEndpoint is not set, return early + if !cluster.Spec.ControlPlaneEndpoint.IsValid() { + log.Info("Cluster does not yet have a ControlPlaneEndpoint defined") + return ctrl.Result{}, nil + } + + if result, err := r.reconcileKubeconfig(ctx, cluster, ncp); !result.IsZero() || err != nil { + if err != nil { + log.Error(err, "failed to reconcile Kubeconfig") + } + return result, err + } + + addOwners := []client.Object{} + isReady := []int{} + nestedComponents := map[client.Object]*corev1.ObjectReference{ + &controlplanev1.NestedEtcd{}: ncp.Spec.EtcdRef, + &controlplanev1.NestedAPIServer{}: ncp.Spec.APIServerRef, + &controlplanev1.NestedControllerManager{}: ncp.Spec.ControllerManagerRef, + } + + // Adopt NestedComponents in the same Namespace + for component, nestedComponent := range nestedComponents { + if nestedComponent != nil { + objectKey := types.NamespacedName{Namespace: ncp.GetNamespace(), Name: nestedComponent.Name} + if err := r.Get(ctx, objectKey, component); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, err + } + } + + if !util.HasOwner(component.GetOwnerReferences(), controlplanev1.GroupVersion.String(), []string{"NestedControlPlane"}) { + log.Info("Component Missing Owner", "component", nestedComponent) + addOwners = append(addOwners, component) + } + + if commonObject, ok := component.(addonv1alpha1.CommonObject); ok { + if IsComponentReady(commonObject.GetCommonStatus()) { + isReady = append(isReady, 1) + } else { + log.Info("Component is not ready", "component", nestedComponent) + } + } + } + } + + // Add Controller Reference + if err := r.reconcileControllerOwners(ctx, ncp, addOwners); err != nil { + return ctrl.Result{Requeue: true}, err + } + + // Set Initialized + if !ncp.Status.Initialized { + conditions.MarkTrue(ncp, kcpv1.AvailableCondition) + ncp.Status.Initialized = true + if err := r.Status().Update(ctx, ncp); err != nil { + return ctrl.Result{}, err + } + } + + // Set Ready + if !ncp.Status.Ready && len(isReady) == 3 { + conditions.MarkTrue(ncp, clusterv1.ReadyCondition) + ncp.Status.Ready = true + if err := r.Status().Update(ctx, ncp); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +// reconcileKubeconfig will check if the control plane endpoint has been set +// and if so it will generate the KUBECONFIG or regenerate if it's expired. +func (r *NestedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + endpoint := cluster.Spec.ControlPlaneEndpoint + if endpoint.IsZero() { + return ctrl.Result{}, nil + } + + controllerOwnerRef := *metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) + clusterName := util.ObjectKey(cluster) + configSecret, err := secret.GetFromNamespacedName(ctx, r.Client, clusterName, secret.Kubeconfig) + switch { + case apierrors.IsNotFound(err): + createErr := kubeconfig.CreateSecretWithOwner( + ctx, + r.Client, + clusterName, + endpoint.String(), + controllerOwnerRef, + ) + if errors.Is(createErr, kubeconfig.ErrDependentCertificateNotFound) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + // always return if we have just created in order to skip rotation checks + return ctrl.Result{}, createErr + case err != nil: + return ctrl.Result{}, errors.Wrap(err, "failed to retrieve kubeconfig Secret") + } + + // only do rotation on owned secrets + if !util.IsControlledBy(configSecret, ncp) { + return ctrl.Result{}, nil + } + + needsRotation, err := kubeconfig.NeedsClientCertRotation(configSecret, certs.ClientCertificateRenewalDuration) + if err != nil { + return ctrl.Result{}, err + } + + if needsRotation { + log.Info("rotating kubeconfig secret") + if err := kubeconfig.RegenerateSecret(ctx, r.Client, configSecret); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig") + } + } + + return ctrl.Result{}, nil +} + +// reconcileControllerOwners will loop through any known nested components that +// aren't owned by a control plane yet and associate them +func (r *NestedControlPlaneReconciler) reconcileControllerOwners(ctx context.Context, ncp *controlplanev1.NestedControlPlane, addOwners []client.Object) error { + for _, component := range addOwners { + if err := ctrl.SetControllerReference(ncp, component, r.Scheme); err != nil { + if _, ok := err.(*controllerutil.AlreadyOwnedError); !ok { + continue + } + return err + } + + if err := r.Update(ctx, component); err != nil { + return err + } + } + return nil } diff --git a/controllers/controlplane/nestedetcd_controller.go b/controllers/controlplane/nestedetcd_controller.go index b2922e49..1c0c67c3 100644 --- a/controllers/controlplane/nestedetcd_controller.go +++ b/controllers/controlplane/nestedetcd_controller.go @@ -28,11 +28,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/util/certs" + "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/certificate" + "sigs.k8s.io/cluster-api/util" ) // NestedEtcdReconciler reconciles a NestedEtcd object @@ -71,10 +76,25 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{Requeue: true}, nil } + var ncp clusterv1.NestedControlPlane + if err := r.Get(ctx, types.NamespacedName{Namespace: netcd.GetNamespace(), Name: owner.Name}, &ncp); err != nil { + log.Info("the owner could not be found, will retry later", + "namespace", netcd.GetNamespace(), + "name", owner.Name) + return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) + } + + cluster, err := ncp.GetOwnerCluster(ctx, r.Client) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + + etcdName := fmt.Sprintf("%s-etcd", cluster.GetName()) var netcdSts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{ Namespace: netcd.GetNamespace(), - Name: netcd.GetName(), + Name: etcdName, }, &netcdSts); err != nil { if apierrors.IsNotFound(err) { // as the statefulset is not found, mark the NestedEtcd as unready @@ -88,11 +108,17 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } } + + if err := r.createEtcdClientCrts(ctx, cluster, &ncp, &netcd); err != nil { + log.Error(err, "fail to create NestedEtcd Client Certs") + return ctrl.Result{}, err + } + // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, netcd.ObjectMeta, netcd.Spec.NestedComponentSpec, - clusterv1.Etcd, owner.Name, log); err != nil { + clusterv1.Etcd, owner.Name, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedEtcd StatefulSet") return ctrl.Result{}, err } @@ -107,7 +133,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Info("The NestedEtcd StatefulSet is ready") if !IsComponentReady(netcd.Status.CommonStatus) { // As the NestedEtcd StatefulSet is ready, update NestedEtcd status - ip, err := getNestedEtcdSvcClusterIP(ctx, r.Client, netcd) + ip, err := getNestedEtcdSvcClusterIP(ctx, r.Client, cluster.GetName(), &netcd) if err != nil { log.Error(err, "fail to get NestedEtcd Service ClusterIP") return ctrl.Result{}, err @@ -176,25 +202,25 @@ func (r *NestedEtcdReconciler) SetupWithManager(mgr ctrl.Manager) error { } func getNestedEtcdSvcClusterIP(ctx context.Context, cli ctrlcli.Client, - netcd clusterv1.NestedEtcd) (string, error) { + clusterName string, netcd *clusterv1.NestedEtcd) (string, error) { var svc corev1.Service if err := cli.Get(ctx, types.NamespacedName{ Namespace: netcd.GetNamespace(), - Name: netcd.GetName(), + Name: fmt.Sprintf("%s-etcd", clusterName), }, &svc); err != nil { return "", err } return svc.Spec.ClusterIP, nil } -// genInitialClusterArgs generates the values for `--inital-cluster` option of +// genInitialClusterArgs generates the values for `--initial-cluster` option of // etcd based on the number of replicas specified in etcd StatefulSet func genInitialClusterArgs(replicas int32, - stsName, svcName string) (argsVal string) { + stsName, svcName, svcNamespace string) (argsVal string) { for i := int32(0); i < replicas; i++ { // use 2380 as the default port for etcd peer communication - peerAddr := fmt.Sprintf("%s-%d=https://%s-%d.%s:%d", - stsName, i, stsName, i, svcName, 2380) + peerAddr := fmt.Sprintf("%s-etcd-%d=https://%s-etcd-%d.%s-etcd.%s.svc:%d", + stsName, i, stsName, i, svcName, svcNamespace, 2380) if i == replicas-1 { argsVal = argsVal + peerAddr break @@ -204,3 +230,56 @@ func genInitialClusterArgs(replicas int32, return argsVal } + +func getEtcdServers(name, namespace string, replicas int32) (etcdServers []string) { + var i int32 + for ; i < replicas; i++ { + etcdServers = append(etcdServers, fmt.Sprintf("%s-etcd-%d.%s-etcd.%s", name, i, name, namespace)) + } + etcdServers = append(etcdServers, name) + return etcdServers +} + +// createEtcdClientCrts will find of create client certs for the etcd cluster +func (r *NestedEtcdReconciler) createEtcdClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, netcd *clusterv1.NestedEtcd) error { + certificates := secret.NewCertificatesForInitialControlPlane(nil) + if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { + return err + } + cert := certificates.GetByPurpose(secret.EtcdCA) + if cert == nil { + return fmt.Errorf("could not fetch EtcdCA") + } + + crt, err := certs.DecodeCertPEM(cert.KeyPair.Cert) + if err != nil { + return err + } + + key, err := certs.DecodePrivateKeyPEM(cert.KeyPair.Key) + if err != nil { + return err + } + + etcdKeyPair, err := certificate.NewEtcdServerCrtAndKey(&certificate.KeyPair{Cert: crt, Key: key}, getEtcdServers(cluster.GetName(), cluster.GetNamespace(), netcd.Spec.Replicas)) + if err != nil { + return err + } + + etcdHealthKeyPair, err := certificate.NewEtcdHealthcheckClientCertAndKey(&certificate.KeyPair{Cert: crt, Key: key}) + if err != nil { + return err + } + + certs := &certificate.KeyPairs{ + etcdKeyPair, + etcdHealthKeyPair, + } + + controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + return err + } + + return nil +} diff --git a/controllers/controlplane/suite_test.go b/controllers/controlplane/suite_test.go index 4c69e06c..50a0118d 100644 --- a/controllers/controlplane/suite_test.go +++ b/controllers/controlplane/suite_test.go @@ -54,7 +54,7 @@ var _ = BeforeSuite(func(done Done) { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, } var err error diff --git a/controllers/infrastructure/nestedcluster_controller.go b/controllers/infrastructure/nestedcluster_controller.go new file mode 100644 index 00000000..e668ca03 --- /dev/null +++ b/controllers/infrastructure/nestedcluster_controller.go @@ -0,0 +1,99 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package infrastructure + +import ( + "context" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + infrav1 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" +) + +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/finalizers,verbs=update + +// NestedClusterReconciler reconciles a NestedCluster object +type NestedClusterReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NestedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.NestedCluster{}). + Owns(&clusterv1.NestedControlPlane{}). + Complete(r) +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NestedCluster object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile +func (r *NestedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("nestedcluster", req.NamespacedName) + log.Info("Reconciling NestedCluster...") + nc := &infrav1.NestedCluster{} + if err := r.Get(ctx, req.NamespacedName, nc); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + cluster, err := util.GetOwnerCluster(ctx, r.Client, nc.ObjectMeta) + if err != nil || cluster == nil { + log.Error(err, "Failed to retrieve owner Cluster from the control plane") + return ctrl.Result{}, err + } + + objectKey := types.NamespacedName{ + Namespace: cluster.Spec.ControlPlaneRef.Namespace, + Name: cluster.Spec.ControlPlaneRef.Name, + } + ncp := &clusterv1.NestedControlPlane{} + if err := r.Get(ctx, objectKey, ncp); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + + if !nc.Status.Ready && ncp.Status.Ready && ncp.Status.Initialized { + nc.Status.Ready = true + if err := r.Status().Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} diff --git a/controllers/infrastructure/suite_test.go b/controllers/infrastructure/suite_test.go new file mode 100644 index 00000000..be252e53 --- /dev/null +++ b/controllers/infrastructure/suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package infrastructure + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = infrastructurev1alpha4.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 84dedf85..daba3a17 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/main.go b/main.go index 29c975b2..e11ab47e 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" controlplanecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/controlplane" + infrastructurecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/infrastructure" // +kubebuilder:scaffold:imports ) @@ -63,6 +65,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme)) utilruntime.Must(controlplanev1alpha4.AddToScheme(scheme)) + utilruntime.Must(infrastructurev1alpha4.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -145,7 +148,6 @@ func main() { os.Exit(1) } - // TODO(community): Register controllers and webhooks here. if err = (&controlplanecontrollers.NestedControlPlaneReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControlPlane"), @@ -154,6 +156,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NestedControlPlane") os.Exit(1) } + if err = (&infrastructurecontrollers.NestedClusterReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("infrastructure").WithName("NestedCluster"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedCluster") + os.Exit(1) + } if err = (&controlplanecontrollers.NestedEtcdReconciler{ Client: mgr.GetClient(), @@ -180,6 +190,7 @@ func main() { os.Exit(1) } // +kubebuilder:scaffold:builder + setupLog.Info("Starting manager", "version", version.Get().String()) if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager")