Skip to content

Commit

Permalink
Adapt clusterctl to the new manifests format
Browse files Browse the repository at this point in the history
  • Loading branch information
@fabriziopandini
fabriziopandini committed Dec 3, 2020
1 parent ca2cf1a commit b8dc33e
Show file tree
Hide file tree
Showing 12 changed files with 55 additions and 664 deletions.
15 changes: 0 additions & 15 deletions cmd/clusterctl/api/v1alpha3/labels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,10 @@ const (
// ClusterctlCoreLabelName is applied to all the core objects managed by clusterctl.
ClusterctlCoreLabelName = "clusterctl.cluster.x-k8s.io/core"

// ClusterctlResourceLifecyleLabelName describes the lifecyle for a specific resource.
//
// Example: resources shared between instances of the same provider: CRDs,
// ValidatingWebhookConfiguration, MutatingWebhookConfiguration, and so on.
ClusterctlResourceLifecyleLabelName = "clusterctl.cluster.x-k8s.io/lifecycle"

// ClusterctlMoveLabelName can be set on CRDs that providers wish to move that are not part of a cluster
ClusterctlMoveLabelName = "clusterctl.cluster.x-k8s.io/move"
)

// ResourceLifecycle configures the lifecycle of a resource
type ResourceLifecycle string

const (
// ResourceLifecycleShared is used to indicate that a resource is shared between
// multiple instances of a provider.
ResourceLifecycleShared = ResourceLifecycle("shared")
)

// ManifestLabel returns the cluster.x-k8s.io/provider label value for a provider/type.
//
// Note: the label uniquely describes the provider type and its kind (e.g. bootstrap-kubeadm);
Expand Down
4 changes: 2 additions & 2 deletions cmd/clusterctl/client/cluster/cert_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (cm *certManagerClient) setManifestVersion() error {

for i := range objs {
o := objs[i]
if o.GetKind() == "CustomResourceDefinition" {
if o.GetKind() == customResourceDefinitionKind {
labels := o.GetLabels()
version, ok := labels[certmanagerVersionLabel]
if ok {
Expand Down Expand Up @@ -271,7 +271,7 @@ func (cm *certManagerClient) deleteObjs(objs []unstructured.Unstructured) error

// CRDs, and namespace are preserved in order to avoid deletion of user objects;
// web-hooks are preserved to avoid a user attempting to CREATE a cert-manager resource while the upgrade is in progress.
if obj.GetKind() == "CustomResourceDefinition" ||
if obj.GetKind() == customResourceDefinitionKind ||
obj.GetKind() == "Namespace" ||
obj.GetKind() == "MutatingWebhookConfiguration" ||
obj.GetKind() == "ValidatingWebhookConfiguration" {
Expand Down
35 changes: 5 additions & 30 deletions cmd/clusterctl/client/cluster/components.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,19 @@ limitations under the License.
package cluster

import (
"fmt"
"strings"

"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util"
logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const customResourceDefinitionKind = "CustomResourceDefinition"

type DeleteOptions struct {
Provider clusterctlv1.Provider
IncludeNamespace bool
Expand Down Expand Up @@ -117,21 +114,14 @@ func (p *providerComponents) Delete(options DeleteOptions) error {

// Fetch all the components belonging to a provider.
// We want that the delete operation is able to clean-up everything in a the most common use case that is
// single-tenant management clusters. However, the downside of this is that this operation might be destructive
// in multi-tenant scenario, because a single operation could delete both instance specific and shared CRDs/web-hook components.
// This is considered acceptable because we are considering the multi-tenant scenario an advanced use case, and the assumption
// is that user in this case understand the potential impacts of this operation.
// single-tenant management clusters.
// TODO: in future we can eventually block delete --IncludeCRDs in case more than one instance of a provider exists
labels := map[string]string{
clusterctlv1.ClusterctlLabelName: "",
clusterv1.ProviderLabelName: options.Provider.ManifestLabel(),
}

namespaces := []string{options.Provider.Namespace}
if options.IncludeCRDs {
namespaces = append(namespaces, repository.WebhookNamespaceName)
}

resources, err := p.proxy.ListResources(labels, namespaces...)
if err != nil {
return err
Expand All @@ -140,12 +130,10 @@ func (p *providerComponents) Delete(options DeleteOptions) error {
// Filter the resources according to the delete options
resourcesToDelete := []unstructured.Unstructured{}
namespacesToDelete := sets.NewString()
instanceNamespacePrefix := fmt.Sprintf("%s-", options.Provider.Namespace)
for _, obj := range resources {
// If the CRDs (and by extensions, all the shared resources) should NOT be deleted, skip it;
// If the CRDs should NOT be deleted, skip it;
// NB. Skipping CRDs deletion ensures that also the objects of Kind defined in the CRDs Kind are not deleted.
isSharedResource := util.IsSharedResource(obj)
if !options.IncludeCRDs && isSharedResource {
if !options.IncludeCRDs && obj.GetKind() == customResourceDefinitionKind {
continue
}

Expand All @@ -164,19 +152,6 @@ func (p *providerComponents) Delete(options DeleteOptions) error {
namespacesToDelete.Insert(obj.GetName())
}

// If not a shared resource or not a namespace
if !isSharedResource && !isNamespace {
// If the resource is a cluster resource, skip it if the resource name does not start with the instance prefix.
// This is required because there are cluster resources like e.g. ClusterRoles and ClusterRoleBinding, which are instance specific;
// During the installation, clusterctl adds the instance namespace prefix to such resources (see fixRBAC), and so we can rely
// on that for deleting only the global resources belonging the the instance we are processing.
if util.IsClusterResource(obj.GetKind()) {
if !strings.HasPrefix(obj.GetName(), instanceNamespacePrefix) {
continue
}
}
}

resourcesToDelete = append(resourcesToDelete, obj)
}

Expand Down
75 changes: 20 additions & 55 deletions cmd/clusterctl/client/cluster/components_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand All @@ -37,22 +36,18 @@ func Test_providerComponents_Delete(t *testing.T) {
labels := map[string]string{
clusterv1.ProviderLabelName: "infrastructure-infra",
}
sharedLabels := map[string]string{
clusterv1.ProviderLabelName: "infrastructure-infra",
clusterctlv1.ClusterctlResourceLifecyleLabelName: string(clusterctlv1.ResourceLifecycleShared),
}

crd := unstructured.Unstructured{}
crd.SetAPIVersion("apiextensions.k8s.io/v1beta1")
crd.SetKind("CustomResourceDefinition")
crd.SetName("crd1")
crd.SetLabels(sharedLabels)
crd.SetLabels(labels)

mutatingWebhook := unstructured.Unstructured{}
mutatingWebhook.SetAPIVersion("admissionregistration.k8s.io/v1beta1")
mutatingWebhook.SetKind("MutatingWebhookConfiguration")
mutatingWebhook.SetName("mwh1")
mutatingWebhook.SetLabels(sharedLabels)
mutatingWebhook.SetLabels(labels)

initObjs := []client.Object{
// Namespace (should be deleted only if includeNamespace)
Expand Down Expand Up @@ -88,29 +83,8 @@ func Test_providerComponents_Delete(t *testing.T) {
},
// CRDs (should be deleted only if includeCRD)
&crd,
// CRDs (should always be deleted)
&mutatingWebhook,
&corev1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
},
ObjectMeta: metav1.ObjectMeta{
Name: repository.WebhookNamespaceName,
Labels: map[string]string{
clusterctlv1.ClusterctlResourceLifecyleLabelName: string(clusterctlv1.ResourceLifecycleShared),
//NB. the capi-webhook-system namespace doe not have a provider label (see fixSharedLabels)
},
},
},
&corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: repository.WebhookNamespaceName,
Name: "podx",
Labels: sharedLabels,
},
},
// A cluster-wide provider component (should always be deleted)
&rbacv1.ClusterRole{
TypeMeta: metav1.TypeMeta{
Expand All @@ -127,8 +101,7 @@ func Test_providerComponents_Delete(t *testing.T) {
Kind: "ClusterRole",
},
ObjectMeta: metav1.ObjectMeta{
Name: "some-cluster-role",
Labels: labels,
Name: "some-cluster-role",
},
},
// Another object out of the provider namespace (should never be deleted)
Expand Down Expand Up @@ -168,16 +141,14 @@ func Test_providerComponents_Delete(t *testing.T) {
includeCRD: false,
},
wantDiff: []wantDiff{
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: false}, // namespace should be preserved
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: false}, // crd should be preserved
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: false}, // MutatingWebhookConfiguration should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: repository.WebhookNamespaceName}, deleted: false}, // capi-webhook-system namespace should never be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: repository.WebhookNamespaceName, Name: "podx"}, deleted: false}, // provider objects in the capi-webhook-system namespace should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: false}, // other objects in the namespace should not be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "ns1-cluster-role"}, deleted: true}, // cluster-wide provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "some-cluster-role"}, deleted: false}, // other cluster-wide objects should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: false}, // namespace should be preserved
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: false}, // crd should be preserved
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: true}, // MutatingWebhookConfiguration should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: false}, // other objects in the namespace should not be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "ns1-cluster-role"}, deleted: true}, // cluster-wide provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "some-cluster-role"}, deleted: false}, // other cluster-wide objects should be preserved
},
wantErr: false,
},
Expand All @@ -189,16 +160,14 @@ func Test_providerComponents_Delete(t *testing.T) {
includeCRD: false,
},
wantDiff: []wantDiff{
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: true}, // namespace should be deleted
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: false}, // crd should be preserved
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: false}, // MutatingWebhookConfiguration should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: repository.WebhookNamespaceName}, deleted: false}, // capi-webhook-system namespace should never be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: repository.WebhookNamespaceName, Name: "podx"}, deleted: false}, // provider objects in the capi-webhook-system namespace should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: true}, // other objects in the namespace goes away when deleting the namespace
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "ns1-cluster-role"}, deleted: true}, // cluster-wide provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "some-cluster-role"}, deleted: false}, // other cluster-wide objects should be preserved
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: true}, // namespace should be deleted
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: false}, // crd should be preserved
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: true}, // MutatingWebhookConfiguration should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: true}, // other objects in the namespace goes away when deleting the namespace
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "ns1-cluster-role"}, deleted: true}, // cluster-wide provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Name: "some-cluster-role"}, deleted: false}, // other cluster-wide objects should be preserved
},
wantErr: false,
},
Expand All @@ -213,8 +182,6 @@ func Test_providerComponents_Delete(t *testing.T) {
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: false}, // namespace should be preserved
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: true}, // crd should be deleted
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: true}, // MutatingWebhookConfiguration should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: repository.WebhookNamespaceName}, deleted: false}, // capi-webhook-system namespace should never be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: repository.WebhookNamespaceName, Name: "podx"}, deleted: true}, // provider objects in the capi-webhook-system namespace should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: false}, // other objects in the namespace should not be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
Expand All @@ -234,8 +201,6 @@ func Test_providerComponents_Delete(t *testing.T) {
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: "ns1"}, deleted: true}, // namespace should be deleted
{object: corev1.ObjectReference{APIVersion: "apiextensions.k8s.io/v1beta1", Kind: "CustomResourceDefinition", Name: "crd1"}, deleted: true}, // crd should be deleted
{object: corev1.ObjectReference{APIVersion: "admissionregistration.k8s.io/v1beta1", Kind: "MutatingWebhookConfiguration", Name: "mwh1"}, deleted: true}, // MutatingWebhookConfiguration should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Namespace", Name: repository.WebhookNamespaceName}, deleted: false}, // capi-webhook-namespace should never be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: repository.WebhookNamespaceName, Name: "podx"}, deleted: true}, // provider objects in the capi-webhook-namespace should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod1"}, deleted: true}, // provider components should be deleted
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns1", Name: "pod2"}, deleted: true}, // other objects in the namespace goes away when deleting the namespace
{object: corev1.ObjectReference{APIVersion: "v1", Kind: "Pod", Namespace: "ns2", Name: "pod3"}, deleted: false}, // this object is in another namespace, and should never be touched by delete
Expand Down
Loading

0 comments on commit b8dc33e

Please sign in to comment.