Skip to content

Commit

Permalink
tmp
Browse files Browse the repository at this point in the history
  • Loading branch information
@fabriziopandini
fabriziopandini committed Aug 7, 2020
1 parent ad631d7 commit 2c1f290
Show file tree
Hide file tree
Showing 3 changed files with 513 additions and 58 deletions.
226 changes: 173 additions & 53 deletions cmd/clusterctl/client/cluster/cert_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/version"
"sigs.k8s.io/controller-runtime/pkg/client"

clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
Expand All @@ -33,20 +35,6 @@ import (
utilyaml "sigs.k8s.io/cluster-api/util/yaml"
)

var (
// This is declared as a variable mapping a version number to the hash of the
// embedded cert-manager.yaml file.
// The hash is used to ensure that when the cert-manager.yaml file is updated,
// the version number marker here is _also_ updated.
// If the cert-manager.yaml asset is modified, this line **MUST** be updated
// accordingly else future upgrades of the cert-manager component will not
// be possible, as there'll be no record of the version installed.
// You can either generate the SHA256 hash of the file, or alternatively
// run `go test` against this package. THe Test_VersionMarkerUpToDate will output
// the expected hash if it does not match the hash here.
embeddedCertManagerManifestVersion = map[string]string{"v0.16.0": "5770f5f01c10a902355b3522b8ce44508ebb6ec88955efde9a443afe5b3969d7"}
)

const (
embeddedCertManagerManifestPath = "cmd/clusterctl/config/assets/cert-manager.yaml"
embeddedCertManagerTestResourcesManifestPath = "cmd/clusterctl/config/assets/cert-manager-test-resources.yaml"
Expand All @@ -56,6 +44,21 @@ const (

certManagerImageComponent = "cert-manager"
timeoutConfigKey = "cert-manager-timeout"

certmanagerVersionAnnotations = "certmanager.clusterctl.cluster.x-k8s.io/version"
certmanagerHashAnnotations = "certmanager.clusterctl.cluster.x-k8s.io/hash"

embeddedCertManagerManifestVersion = "v0.16.0"

// The hash is used to ensure that when the cert-manager.yaml file is updated,
// the version number marker here is _also_ updated.
// If the cert-manager.yaml asset is modified, this line **MUST** be updated
// accordingly else future upgrades of the cert-manager component will not
// be possible, as there'll be no record of the version installed.
// You can either generate the SHA256 hash of the file, or alternatively
// run `go test` against this package. THe Test_VersionMarkerUpToDate will output
// the expected hash if it does not match the hash here.
embeddedCertManagerManifestHash = "5770f5f01c10a902355b3522b8ce44508ebb6ec88955efde9a443afe5b3969d7"
)

// CertManagerClient has methods to work with cert-manager components in the cluster.
Expand All @@ -64,6 +67,10 @@ type CertManagerClient interface {
// This is required to install a new provider.
EnsureInstalled() error

// EnsureLatestVersion checks the cert-manager version currently installed, and if it is
// older than the version currently embedded in clusterctl, upgrades it.
EnsureLatestVersion() error

// Images return the list of images required for installing the cert-manager.
Images() ([]string, error)
}
Expand Down Expand Up @@ -115,21 +122,22 @@ func (cm *certManagerClient) EnsureInstalled() error {
return nil
}

log.Info("Installing cert-manager")
return cm.install()
}

func (cm *certManagerClient) install() error {
// Gets the cert-manager objects from the embedded assets.
objs, err := cm.getManifestObjs()
if err != nil {
return err
}

log.Info("Installing cert-manager")

// Install all cert-manager manifests
createCertManagerBackoff := newWriteBackoff()
objs = utilresource.SortForCreate(objs)
for i := range objs {
o := objs[i]
log.V(5).Info("Creating", logf.UnstructuredToValues(o)...)

// Create the Kubernetes object.
// Nb. The operation is wrapped in a retry loop to make ensureCerts more resilient to unexpected conditions.
if err := retryWithExponentialBackoff(createCertManagerBackoff, func() error {
Expand All @@ -147,6 +155,108 @@ func (cm *certManagerClient) EnsureInstalled() error {
return nil
}

// EnsureLatestVersion checks the cert-manager version currently installed, and if it is
// older than the version currently embedded in clusterctl, upgrades it.
func (cm *certManagerClient) EnsureLatestVersion() error {
log := logf.Log
log.Info("Checking cert-manager version...")

objs, err := cm.proxy.ListResources(map[string]string{clusterctlv1.ClusterctlCoreLabelName: "cert-manager"})
if err != nil {
return errors.Wrap(err, "failed get cert manager components")
}

shouldUpgrade, err := shouldUpgrade(objs)
if err != nil {
return err
}

if !shouldUpgrade {
log.Info("Cert-manager is already up to date")
return nil
}
log.Info("Upgrading cert-manager")

// delete the cert-manager version currently installed (because it should be upgraded);
// NOTE: CRDs, and namespace are preserved in order to avoid deletion of user objects;
// web-hooks are preserved to avoid a user attempting to CREATE a cert-manager resource while the upgrade is in progress.
if err := cm.deleteObjs(objs); err != nil {
return err
}

// install the cert-manager version embedded in clusterctl
return cm.install()
}

func (cm *certManagerClient) deleteObjs(objs []unstructured.Unstructured) error {
for i := range objs {
obj := objs[i]

// CRDs, and namespace are preserved in order to avoid deletion of user objects;
// web-hooks are preserved to avoid a user attempting to CREATE a cert-manager resource while the upgrade is in progress.
if obj.GetKind() == "CustomResourceDefinition" ||
obj.GetKind() == "Namespace" ||
obj.GetKind() == "MutatingWebhookConfiguration" ||
obj.GetKind() == "ValidatingWebhookConfiguration" {
continue
}

if err := cm.deleteObj(obj); err != nil {
// tolerate NotFound errors when deleting the resources
if apierrors.IsNotFound(err) {
continue
}
return err
}
}
return nil
}

func shouldUpgrade(objs []unstructured.Unstructured) (bool, error) {
needUpgrade := false
for i := range objs {
obj := objs[i]

// if no version then upgrade (v0.11.0)
objVersion, ok := obj.GetAnnotations()[certmanagerVersionAnnotations]
if !ok {
// if there is no version annotation, this means the obj is cert-manager v0.11.0 (installed with older version of clusterctl)
needUpgrade = true
break
}

objSemVersion, err := version.ParseSemantic(objVersion)
if err != nil {
return false, errors.Wrapf(err, "failed to parse version for cert-manager component %s/%s", obj.GetKind(), obj.GetName())
}

c, err := objSemVersion.Compare(embeddedCertManagerManifestVersion)
if err != nil {
return false, errors.Wrapf(err, "failed to compare version for cert-manager component %s/%s", obj.GetKind(), obj.GetName())
}

switch {
case c < 0:
// if version < current, then upgrade
needUpgrade = true
break
case c == 0:
// if version == current, check the manifest hash; if it does not exists or if it is different, then upgrade
objHash, ok := obj.GetAnnotations()[certmanagerHashAnnotations]
if !ok || objHash != embeddedCertManagerManifestHash {
needUpgrade = true
break
}
// otherwise we are already at the latest version
continue
case c > 0:
// the installed version is higher than the one embedded in clusterctl, so we are ok
continue
}
}
return needUpgrade, nil
}

func (cm *certManagerClient) getWaitTimeout() time.Duration {
log := logf.Log

Expand Down Expand Up @@ -201,37 +311,62 @@ func getTestResourcesManifestObjs() ([]unstructured.Unstructured, error) {
return objs, nil
}

func (cm *certManagerClient) createObj(o unstructured.Unstructured) error {
c, err := cm.proxy.NewClient()
if err != nil {
return err
}
func (cm *certManagerClient) createObj(obj unstructured.Unstructured) error {
log := logf.Log

labels := o.GetLabels()
labels := obj.GetLabels()
if labels == nil {
labels = map[string]string{}
}
labels[clusterctlv1.ClusterctlCoreLabelName] = "cert-manager"
o.SetLabels(labels)
obj.SetLabels(labels)

// persist version marker information as annotations to avoid character and length
// restrictions on label values.
annotations := o.GetAnnotations()
annotations := obj.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
// persist the version number of stored resources to make a
// future enhancement to add upgrade support possible.
version, hash := embeddedCertManagerVersion()
annotations["certmanager.clusterctl.cluster.x-k8s.io/version"] = version
annotations["certmanager.clusterctl.cluster.x-k8s.io/hash"] = hash
o.SetAnnotations(annotations)

if err = c.Create(ctx, &o); err != nil {
if apierrors.IsAlreadyExists(err) {
return nil
annotations[certmanagerVersionAnnotations] = embeddedCertManagerManifestVersion
annotations[certmanagerHashAnnotations] = embeddedCertManagerManifestHash
obj.SetAnnotations(annotations)

c, err := cm.proxy.NewClient()
if err != nil {
return err
}

// check if the component already exists, and eventually update it; otherwise create it
// NOTE: This is required because this func is used also for upgrading cert-manager and during upgrades
// some objects of the previews release are preserved in order to avoid to delete user data (e.g. CRDs).
currentR := &unstructured.Unstructured{}
currentR.SetGroupVersionKind(obj.GroupVersionKind())

key := client.ObjectKey{
Namespace: obj.GetNamespace(),
Name: obj.GetName(),
}
if err := c.Get(ctx, key, currentR); err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to get cert-manager object %s, %s/%s", obj.GroupVersionKind(), obj.GetNamespace(), obj.GetName())
}
return errors.Wrapf(err, "failed to create cert-manager component: %s, %s/%s", o.GroupVersionKind(), o.GetNamespace(), o.GetName())

// if it does not exists, create the component
log.V(5).Info("Creating", logf.UnstructuredToValues(obj)...)
if err := c.Create(ctx, &obj); err != nil {
return errors.Wrapf(err, "failed to create cert-manager component %s, %s/%s", obj.GroupVersionKind(), obj.GetNamespace(), obj.GetName())
}
return nil
}

// otherwise update the component
// NB. we are using client.Merge PatchOption so the new objects gets compared with the current one server side
log.V(5).Info("Patching", logf.UnstructuredToValues(obj)...)
obj.SetResourceVersion(currentR.GetResourceVersion())
if err := c.Patch(ctx, &obj, client.Merge); err != nil {
return errors.Wrapf(err, "failed to patch cert-manager component %s, %s/%s", obj.GroupVersionKind(), obj.GetNamespace(), obj.GetName())
}
return nil
}
Expand Down Expand Up @@ -261,7 +396,9 @@ func (cm *certManagerClient) deleteObj(obj unstructured.Unstructured) error {
func (cm *certManagerClient) waitForAPIReady(ctx context.Context, retry bool) error {
log := logf.Log
// Waits for for the cert-manager web-hook to be available.
log.Info("Waiting for cert-manager to be available...")
if retry {
log.Info("Waiting for cert-manager to be available...")
}

testObjs, err := getTestResourcesManifestObjs()
if err != nil {
Expand All @@ -270,20 +407,16 @@ func (cm *certManagerClient) waitForAPIReady(ctx context.Context, retry bool) er

for i := range testObjs {
o := testObjs[i]
log.V(5).Info("Creating", logf.UnstructuredToValues(o)...)

// Create the Kubernetes object.
// This is wrapped with a retry as the cert-manager API may not be available
// yet, so we need to keep retrying until it is.
if err := cm.pollImmediateWaiter(waitCertManagerInterval, cm.getWaitTimeout(), func() (bool, error) {
if err := cm.createObj(o); err != nil {
log.V(5).Info("Failed to create cert-manager test resource", logf.UnstructuredToValues(o)...)

// If retrying is disabled, return the error here.
if !retry {
return false, err
}

return false, nil
}
return true, nil
Expand All @@ -305,16 +438,3 @@ func (cm *certManagerClient) waitForAPIReady(ctx context.Context, retry bool) er

return nil
}

// returns the version number and hash of the cert-manager manifest embedded
// into the clusterctl binary
func embeddedCertManagerVersion() (version, hash string) {
if len(embeddedCertManagerManifestVersion) != 1 {
panic("embeddedCertManagerManifestVersion MUST only have one entry")
}
for version, hash := range embeddedCertManagerManifestVersion {
return version, hash
}
// unreachable
return "", ""
}
Loading

0 comments on commit 2c1f290

Please sign in to comment.