Skip to content

Commit

Permalink
feat: refresher interface
Browse files Browse the repository at this point in the history
  • Loading branch information
duffney committed Jul 10, 2024
1 parent 37546b9 commit 78af54e
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 60 deletions.
71 changes: 11 additions & 60 deletions pkg/controllers/clusterresource/keymanagementprovider_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,18 @@ import (
"context"
"encoding/json"
"fmt"
"maps"

"github.com/ratify-project/ratify/internal/constants"
_ "github.com/ratify-project/ratify/pkg/keymanagementprovider/azurekeyvault" // register azure key vault key management provider
_ "github.com/ratify-project/ratify/pkg/keymanagementprovider/inline" // register inline key management provider
apierrors "k8s.io/apimachinery/pkg/api/errors"
"github.com/ratify-project/ratify/pkg/keymanagementprovider/refresh" // register inline key management provider
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/predicate"

configv1beta1 "github.com/ratify-project/ratify/api/v1beta1"
cutils "github.com/ratify-project/ratify/pkg/controllers/utils"
kmp "github.com/ratify-project/ratify/pkg/keymanagementprovider"
"github.com/sirupsen/logrus"
)
Expand All @@ -48,74 +46,27 @@ type KeyManagementProviderReconciler struct {
// +kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=keymanagementproviders/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=keymanagementproviders/finalizers,verbs=update
func (r *KeyManagementProviderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := logrus.WithContext(ctx)

var resource = req.Name
var keyManagementProvider configv1beta1.KeyManagementProvider

logger.Infof("reconciling cluster key management provider '%v'", resource)

if err := r.Get(ctx, req.NamespacedName, &keyManagementProvider); err != nil {
if apierrors.IsNotFound(err) {
logger.Infof("deletion detected, removing key management provider %v", resource)
kmp.DeleteCertificatesFromMap(resource)
kmp.DeleteKeysFromMap(resource)
} else {
logger.Error(err, "unable to fetch key management provider")
}

return ctrl.Result{}, client.IgnoreNotFound(err)
kr := refresh.KubeRefresher{
Client: r.Client,
Request: req,
}

lastFetchedTime := metav1.Now()
isFetchSuccessful := false

// get certificate store list to check if certificate store is configured
// TODO: remove check in v2.0.0+
var certificateStoreList configv1beta1.CertificateStoreList
if err := r.List(ctx, &certificateStoreList); err != nil {
logger.Error(err, "unable to list certificate stores")
return ctrl.Result{}, err
}
// if certificate store is configured, return error. Only one of certificate store and key management provider can be configured
if len(certificateStoreList.Items) > 0 {
// Note: for backwards compatibility in upgrade scenarios, Ratify will only log a warning statement.
logger.Warn("Certificate Store already exists. Key management provider and certificate store should not be configured together. Please migrate to key management provider and delete certificate store.")
// check if kr.client is nil
if kr.Client == nil {
return ctrl.Result{}, fmt.Errorf("client is nil")
}

provider, err := cutils.SpecToKeyManagementProvider(keyManagementProvider.Spec.Parameters.Raw, keyManagementProvider.Spec.Type)
err := kr.Refresh(ctx)
if err != nil {
writeKMProviderStatus(ctx, r, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
return ctrl.Result{}, err
}

// fetch certificates and store in map
certificates, certAttributes, err := provider.GetCertificates(ctx)
if err != nil {
writeKMProviderStatus(ctx, r, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
return ctrl.Result{}, fmt.Errorf("Error fetching certificates in KMProvider %v with %v provider, error: %w", resource, keyManagementProvider.Spec.Type, err)
}

// fetch keys and store in map
keys, keyAttributes, err := provider.GetKeys(ctx)
if err != nil {
writeKMProviderStatus(ctx, r, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
return ctrl.Result{}, fmt.Errorf("Error fetching keys in KMProvider %v with %v provider, error: %w", resource, keyManagementProvider.Spec.Type, err)
}
kmp.SetCertificatesInMap(resource, certificates)
kmp.SetKeysInMap(resource, keyManagementProvider.Spec.Type, keys)
// merge certificates and keys status into one
maps.Copy(keyAttributes, certAttributes)
isFetchSuccessful = true
emptyErrorString := ""
writeKMProviderStatus(ctx, r, &keyManagementProvider, logger, isFetchSuccessful, emptyErrorString, lastFetchedTime, keyAttributes)

logger.Infof("%v certificate(s) & %v key(s) fetched for key management provider %v", len(certificates), len(keys), resource)

// returning empty result and no error to indicate we’ve successfully reconciled this object
return ctrl.Result{}, nil
return kr.Result, nil
}


//TODO: delete helpers, moved to kubeRefresh.go
// SetupWithManager sets up the controller with the Manager.
func (r *KeyManagementProviderReconciler) SetupWithManager(mgr ctrl.Manager) error {
pred := predicate.GenerationChangedPredicate{}
Expand Down
175 changes: 175 additions & 0 deletions pkg/keymanagementprovider/refresh/kubeRefresh.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
package refresh

import (
"context"
"encoding/json"
"fmt"
"maps"
"time"

configv1beta1 "github.com/ratify-project/ratify/api/v1beta1"
"github.com/ratify-project/ratify/internal/constants"
cutils "github.com/ratify-project/ratify/pkg/controllers/utils"
kmp "github.com/ratify-project/ratify/pkg/keymanagementprovider"
"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type KubeRefresher struct {
client.Client
Request ctrl.Request
Result ctrl.Result
}

func (kr *KubeRefresher) Refresh(ctx context.Context) error {
logger := logrus.WithContext(ctx)

var resource = kr.Request.Name
var keyManagementProvider configv1beta1.KeyManagementProvider

logger.Infof("reconciling cluster key management provider '%v'", resource)


if err := kr.Get(ctx, kr.Request.NamespacedName, &keyManagementProvider); err != nil {
if apierrors.IsNotFound(err) {
logger.Infof("deletion detected, removing key management provider %v", resource)
kmp.DeleteCertificatesFromMap(resource)
kmp.DeleteKeysFromMap(resource)
} else {
logger.Error(err, "unable to fetch key management provider")
}

kr.Result = ctrl.Result{}

return client.IgnoreNotFound(err)
}

lastFetchedTime := metav1.Now()
isFetchSuccessful := false

// get certificate store list to check if certificate store is configured
// TODO: remove check in v2.0.0+
var certificateStoreList configv1beta1.CertificateStoreList
if err := kr.List(ctx, &certificateStoreList); err != nil {
logger.Error(err, "unable to list certificate stores")
kr.Result = ctrl.Result{}
return err
}

if len(certificateStoreList.Items) > 0 {
// Note: for backwards compatibility in upgrade scenarios, Ratify will only log a warning statement.
logger.Warn("Certificate Store already exists. Key management provider and certificate store should not be configured together. Please migrate to key management provider and delete certificate store.")
}


provider, err := cutils.SpecToKeyManagementProvider(keyManagementProvider.Spec.Parameters.Raw, keyManagementProvider.Spec.Type)
if err != nil {
writeKMProviderStatus(ctx, kr.Client, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
kr.Request = ctrl.Request{}
return err
}

// fetch certificates and store in map
certificates, certAttributes, err := provider.GetCertificates(ctx)
if err != nil {
writeKMProviderStatus(ctx, kr, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
kr.Request = ctrl.Request{}
return fmt.Errorf("error fetching certificates in KMProvider %v with %v provider, error: %w", resource, keyManagementProvider.Spec.Type, err)
}

// fetch keys and store in map
keys, keyAttributes, err := provider.GetKeys(ctx)
if err != nil {
writeKMProviderStatus(ctx, kr, &keyManagementProvider, logger, isFetchSuccessful, err.Error(), lastFetchedTime, nil)
kr.Request = ctrl.Request{}
return fmt.Errorf("error fetching keys in KMProvider %v with %v provider, error: %w", resource, keyManagementProvider.Spec.Type, err)
}
kmp.SetCertificatesInMap(resource, certificates)
kmp.SetKeysInMap(resource, keyManagementProvider.Spec.Type, keys)
// merge certificates and keys status into one
maps.Copy(keyAttributes, certAttributes)
isFetchSuccessful = true
emptyErrorString := ""
writeKMProviderStatus(ctx, kr, &keyManagementProvider, logger, isFetchSuccessful, emptyErrorString, lastFetchedTime, keyAttributes)

logger.Infof("%v certificate(s) & %v key(s) fetched for key management provider %v", len(certificates), len(keys), resource)

// returning empty result and no error to indicate we’ve successfully reconciled this object
// will not reconcile again unless resource is recreated
if !keyManagementProvider.Spec.Refreshable {
// resource is not refreshable, no need to requeue
kr.Request = ctrl.Request{}
return nil
}

// resource is refreshable, requeue after interval
intervalDuration := time.Duration(keyManagementProvider.Spec.Interval) * time.Minute
logger.Info("Reconciled KeyManagementProvider", "intervalDuration", intervalDuration)
kr.Result = ctrl.Result{RequeueAfter: intervalDuration}

return nil
}

/*
reconcile method logic
"path/to/your/project/refresher"
lastRefresh, err := c.Refresher.Refresh()
func (r *KeyManagementProviderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
refreshResult := r.KubeRefresher.Refresh(ctx)
if refreshResult.Error != nil {
// Handle error
return ctrl.Result{}, refreshResult.Error
}
// Continue with reconcile logic using refreshResult.Result if necessary
return refreshResult.Result, nil
}
*/

func writeKMProviderStatus(ctx context.Context, r client.StatusClient, keyManagementProvider *configv1beta1.KeyManagementProvider, logger *logrus.Entry, isSuccess bool, errorString string, operationTime metav1.Time, kmProviderStatus kmp.KeyManagementProviderStatus) {
if isSuccess {
updateKMProviderSuccessStatus(keyManagementProvider, &operationTime, kmProviderStatus)
} else {
updateKMProviderErrorStatus(keyManagementProvider, errorString, &operationTime)
}
if statusErr := r.Status().Update(ctx, keyManagementProvider); statusErr != nil {
logger.Error(statusErr, ",unable to update key management provider error status")
}
}

// updateKMProviderErrorStatus updates the key management provider status with error, brief error and last fetched time
func updateKMProviderErrorStatus(keyManagementProvider *configv1beta1.KeyManagementProvider, errorString string, operationTime *metav1.Time) {
// truncate brief error string to maxBriefErrLength
briefErr := errorString
if len(errorString) > constants.MaxBriefErrLength {
briefErr = fmt.Sprintf("%s...", errorString[:constants.MaxBriefErrLength])
}
keyManagementProvider.Status.IsSuccess = false
keyManagementProvider.Status.Error = errorString
keyManagementProvider.Status.BriefError = briefErr
keyManagementProvider.Status.LastFetchedTime = operationTime
}

// updateKMProviderSuccessStatus updates the key management provider status if status argument is non nil
// Success status includes last fetched time and other provider-specific properties
func updateKMProviderSuccessStatus(keyManagementProvider *configv1beta1.KeyManagementProvider, lastOperationTime *metav1.Time, kmProviderStatus kmp.KeyManagementProviderStatus) {
keyManagementProvider.Status.IsSuccess = true
keyManagementProvider.Status.Error = ""
keyManagementProvider.Status.BriefError = ""
keyManagementProvider.Status.LastFetchedTime = lastOperationTime

if kmProviderStatus != nil {
jsonString, _ := json.Marshal(kmProviderStatus)

raw := runtime.RawExtension{
Raw: jsonString,
}
keyManagementProvider.Status.Properties = raw
}
}
9 changes: 9 additions & 0 deletions pkg/keymanagementprovider/refresh/refresh.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package refresh

import(
"context"
)

type Refresher interface {
Refresh(ctx context.Context) error
}

0 comments on commit 78af54e

Please sign in to comment.