Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Update status when reconciler failed #1127

Closed
wants to merge 2 commits into from
Closed

feat: Update status when reconciler failed #1127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f0d7f56
feat: Update status when reconciler failed
tolusha Oct 6, 2021
0215abb
Fix
tolusha Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 72 additions & 80 deletions controllers/che/checluster_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/runtime"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/discovery"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -54,11 +54,6 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
)

var (
// CheServiceAccountName - service account name for che-server.
CheServiceAccountName = "che"
)

const (
failedValidationReason = "InstallOrUpdateFailed"
failedNoOpenshiftUser = "NoOpenshiftUsers"
Expand All @@ -77,7 +72,7 @@ const (
// CheClusterReconciler reconciles a CheCluster object
type CheClusterReconciler struct {
Log logr.Logger
Scheme *runtime.Scheme
Scheme *k8sruntime.Scheme

// This client, initialized using mgr.Client() above, is a split client
// that reads objects from the cache and writes to the apiserver
Expand Down Expand Up @@ -236,9 +231,9 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
DiscoveryClient: r.discoveryClient,
Scheme: r.Scheme,
}

// Fetch the CheCluster instance
tests := r.tests
instance, err := r.GetCR(req)
checluster, err := r.GetCR(req)

if err != nil {
if errors.IsNotFound(err) {
Expand All @@ -253,10 +248,32 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)

deployContext := &deploy.DeployContext{
ClusterAPI: clusterAPI,
CheCluster: instance,
CheCluster: checluster,
}

result, err := r.doReconcile(deployContext)
if err != nil {
logrus.Errorf("Reconciliation failed, cause: %v", err)
if err := deploy.SetStatusDetails(deployContext, failedValidationReason, err.Error(), ""); err != nil {
return ctrl.Result{}, err
}
} else {
logrus.Info("Reconciliation completed.")
mmorhun marked this conversation as resolved.
Show resolved Hide resolved
if checluster.Status.Reason != "" {
if err := deploy.SetStatusDetails(deployContext, "", "", ""); err != nil {
return ctrl.Result{}, err
}
}
}

if isCheGoingToBeUpdated(instance) {
return result, err
}

func (r *CheClusterReconciler) doReconcile(deployContext *deploy.DeployContext) (ctrl.Result, error) {
tests := r.tests
checluster := deployContext.CheCluster

if isCheGoingToBeUpdated(checluster) {
// Current operator is newer than deployed Che
backupCR, err := getBackupCRForUpdate(deployContext)
if err != nil {
Expand Down Expand Up @@ -293,27 +310,21 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)

// Check Che CR correctness
if !util.IsTestMode() {
if err := ValidateCheCR(instance); err != nil {
// Che cannot be deployed with current configuration.
// Print error message in logs and wait until the configuration is changed.
logrus.Error(err)
if err := deploy.SetStatusDetails(deployContext, failedValidationReason, err.Error(), ""); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
if err := ValidateCheCR(checluster); err != nil {
return reconcile.Result{}, err
}
}

if util.IsOpenShift4 && util.IsDeleteOAuthInitialUser(instance) {
if util.IsOpenShift4 && util.IsDeleteOAuthInitialUser(checluster) {
if err := r.userHandler.DeleteOAuthInitialUser(deployContext); err != nil {
logrus.Errorf("Unable to delete initial OpenShift OAuth user from a cluster. Cause: %s", err.Error())
instance.Spec.Auth.InitialOpenShiftOAuthUser = nil
checluster.Spec.Auth.InitialOpenShiftOAuthUser = nil
err := deploy.UpdateCheCRSpec(deployContext, "initialOpenShiftOAuthUser", "nil")
return reconcile.Result{}, err
}

instance.Spec.Auth.OpenShiftoAuth = nil
instance.Spec.Auth.InitialOpenShiftOAuthUser = nil
checluster.Spec.Auth.OpenShiftoAuth = nil
checluster.Spec.Auth.InitialOpenShiftOAuthUser = nil
updateFields := map[string]string{
"openShiftoAuth": "nil",
"initialOpenShiftOAuthUser": "nil",
Expand All @@ -327,30 +338,30 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}

// Update status if OpenShift initial user is deleted (in the previous step)
if instance.Spec.Auth.InitialOpenShiftOAuthUser == nil && instance.Status.OpenShiftOAuthUserCredentialsSecret != "" {
if checluster.Spec.Auth.InitialOpenShiftOAuthUser == nil && checluster.Status.OpenShiftOAuthUserCredentialsSecret != "" {
secret := &corev1.Secret{}
exists, err := getOpenShiftOAuthUserCredentialsSecret(deployContext, secret)
if err != nil {
// We should `Requeue` since we deal with cluster scope objects
return ctrl.Result{RequeueAfter: time.Second}, err
} else if !exists {
instance.Status.OpenShiftOAuthUserCredentialsSecret = ""
checluster.Status.OpenShiftOAuthUserCredentialsSecret = ""
if err := deploy.UpdateCheCRStatus(deployContext, "openShiftOAuthUserCredentialsSecret", ""); err != nil {
return reconcile.Result{}, err
}
}
}

if util.IsOpenShift && instance.Spec.DevWorkspace.Enable && instance.Spec.Auth.NativeUserMode == nil {
if util.IsOpenShift && checluster.Spec.DevWorkspace.Enable && checluster.Spec.Auth.NativeUserMode == nil {
newNativeUserModeValue := util.NewBoolPointer(true)
instance.Spec.Auth.NativeUserMode = newNativeUserModeValue
checluster.Spec.Auth.NativeUserMode = newNativeUserModeValue
if err := deploy.UpdateCheCRSpec(deployContext, "nativeUserMode", strconv.FormatBool(*newNativeUserModeValue)); err != nil {
return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 1}, err
}
}

if util.IsOpenShift && instance.Spec.Auth.OpenShiftoAuth == nil {
if reconcileResult, err := r.autoEnableOAuth(deployContext, req, util.IsOpenShift4); err != nil {
if util.IsOpenShift && checluster.Spec.Auth.OpenShiftoAuth == nil {
if reconcileResult, err := r.autoEnableOAuth(deployContext, util.IsOpenShift4); err != nil {
return reconcileResult, err
}
}
Expand All @@ -377,7 +388,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if proxy.TrustedCAMapName != "" {
provisioned, err := r.putOpenShiftCertsIntoConfigMap(deployContext)
if !provisioned {
configMapName := instance.Spec.Server.ServerTrustStoreConfigMapName
configMapName := checluster.Spec.Server.ServerTrustStoreConfigMapName
if err != nil {
r.Log.Error(err, "Error on provisioning", "config map", configMapName)
} else {
Expand All @@ -400,13 +411,13 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// To use Openshift v4 OAuth, the OAuth endpoints are served from a namespace
// and NOT from the Openshift API Master URL (as in v3)
// So we also need the self-signed certificate to access them (same as the Che server)
(util.IsOpenShift4 && util.IsOAuthEnabled(instance) && !instance.Spec.Server.TlsSupport) {
(util.IsOpenShift4 && util.IsOAuthEnabled(checluster) && !checluster.Spec.Server.TlsSupport) {
if err := deploy.CreateTLSSecretFromEndpoint(deployContext, "", deploy.CheTLSSelfSignedCertificateSecretName); err != nil {
return ctrl.Result{}, err
}
}

if util.IsOAuthEnabled(instance) {
if util.IsOAuthEnabled(checluster) {
// create a secret with OpenShift API crt to be added to keystore that RH SSO will consume
apiUrl, apiInternalUrl, err := util.GetOpenShiftAPIUrls()
if err != nil {
Expand All @@ -420,8 +431,8 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
} else {
// Handle Che TLS certificates on Kubernetes infrastructure
if instance.Spec.Server.TlsSupport {
if instance.Spec.K8s.TlsSecretName != "" {
if checluster.Spec.Server.TlsSupport {
if checluster.Spec.K8s.TlsSecretName != "" {
// Self-signed certificate should be created to secure Che ingresses
result, err := deploy.K8sHandleCheTLSSecrets(deployContext)
if result.Requeue || result.RequeueAfter > 0 {
Expand Down Expand Up @@ -453,14 +464,10 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}

if err := deploy.SetStatusDetails(deployContext, "", "", ""); err != nil {
return ctrl.Result{}, err
}

// Create service account "che" for che-server component.
// "che" is the one which token is used to create workspace objects.
// Notice: Also we have on more "che-workspace" SA used by plugins like exec, terminal, metrics with limited privileges.
done, err = deploy.SyncServiceAccountToCluster(deployContext, CheServiceAccountName)
done, err = deploy.SyncServiceAccountToCluster(deployContext, deploy.CheServiceAccountName)
if !done {
if err != nil {
logrus.Error(err)
Expand All @@ -485,12 +492,12 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{RequeueAfter: time.Second}, err
}

if len(instance.Spec.Server.CheClusterRoles) > 0 {
cheClusterRoles := strings.Split(instance.Spec.Server.CheClusterRoles, ",")
if len(checluster.Spec.Server.CheClusterRoles) > 0 {
cheClusterRoles := strings.Split(checluster.Spec.Server.CheClusterRoles, ",")
for _, cheClusterRole := range cheClusterRoles {
cheClusterRole := strings.TrimSpace(cheClusterRole)
cheClusterRoleBindingName := cheClusterRole
done, err := deploy.SyncClusterRoleBindingAndAddFinalizerToCluster(deployContext, cheClusterRoleBindingName, CheServiceAccountName, cheClusterRole)
done, err := deploy.SyncClusterRoleBindingAndAddFinalizerToCluster(deployContext, cheClusterRoleBindingName, deploy.CheServiceAccountName, cheClusterRole)
if !tests {
if !done {
logrus.Infof("Waiting on cluster role binding '%s' to be created", cheClusterRoleBindingName)
Expand All @@ -505,7 +512,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)

// If the user specified an additional cluster role to use for the Che workspace, create a role binding for it
// Use a role binding instead of a cluster role binding to keep the additional access scoped to the workspace's namespace
workspaceClusterRole := instance.Spec.Server.CheWorkspaceClusterRole
workspaceClusterRole := checluster.Spec.Server.CheWorkspaceClusterRole
if workspaceClusterRole != "" {
done, err := deploy.SyncRoleBindingToCluster(deployContext, "che-workspace-custom", "view", workspaceClusterRole, "ClusterRole")
if !done {
Expand All @@ -516,8 +523,8 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
}

if err := r.GenerateAndSaveFields(deployContext, req); err != nil {
instance, _ = r.GetCR(req)
if err := r.GenerateAndSaveFields(deployContext); err != nil {
_ = deploy.ReloadCheClusterCR(deployContext)
return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 1}, err
}

Expand All @@ -544,7 +551,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}

// create and provision Keycloak related objects
if !instance.Spec.Auth.ExternalIdentityProvider {
if !checluster.Spec.Auth.ExternalIdentityProvider {
provisioned, err := identity_provider.SyncIdentityProviderToCluster(deployContext)
if !provisioned {
if err != nil {
Expand All @@ -553,17 +560,17 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}
} else {
keycloakURL := instance.Spec.Auth.IdentityProviderURL
if instance.Status.KeycloakURL != keycloakURL {
instance.Status.KeycloakURL = keycloakURL
keycloakURL := checluster.Spec.Auth.IdentityProviderURL
if checluster.Status.KeycloakURL != keycloakURL {
checluster.Status.KeycloakURL = keycloakURL
if err := deploy.UpdateCheCRStatus(deployContext, "status: Keycloak URL", keycloakURL); err != nil {
return reconcile.Result{}, err
}
}
}

devfileRegistry := devfileregistry.NewDevfileRegistry(deployContext)
if !instance.Spec.Server.ExternalDevfileRegistry {
if !checluster.Spec.Server.ExternalDevfileRegistry {
done, err := devfileRegistry.SyncAll()
if !done {
if err != nil {
Expand All @@ -573,7 +580,7 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
}

if !instance.Spec.Server.ExternalPluginRegistry {
if !checluster.Spec.Server.ExternalPluginRegistry {
pluginRegistry := pluginregistry.NewPluginRegistry(deployContext)
done, err := pluginRegistry.SyncAll()
if !done {
Expand All @@ -583,9 +590,9 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}
} else {
if instance.Spec.Server.PluginRegistryUrl != instance.Status.PluginRegistryURL {
instance.Status.PluginRegistryURL = instance.Spec.Server.PluginRegistryUrl
if err := deploy.UpdateCheCRStatus(deployContext, "status: Plugin Registry URL", instance.Spec.Server.PluginRegistryUrl); err != nil {
if checluster.Spec.Server.PluginRegistryUrl != checluster.Status.PluginRegistryURL {
checluster.Status.PluginRegistryURL = checluster.Spec.Server.PluginRegistryUrl
if err := deploy.UpdateCheCRStatus(deployContext, "status: Plugin Registry URL", checluster.Spec.Server.PluginRegistryUrl); err != nil {
return reconcile.Result{}, err
}
}
Expand Down Expand Up @@ -631,20 +638,20 @@ func (r *CheClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// ignore error
deploy.DeleteFinalizer(deployContext, deploy.OAuthFinalizerName)
for {
instance.Status.OpenShiftoAuthProvisioned = false
checluster.Status.OpenShiftoAuthProvisioned = false
if err := deploy.UpdateCheCRStatus(deployContext, "status: provisioned with OpenShift identity provider", "false"); err != nil &&
errors.IsConflict(err) {
instance, _ = r.GetCR(req)
_ = deploy.ReloadCheClusterCR(deployContext)
continue
}
break
}
for {
instance.Spec.Auth.OAuthSecret = ""
instance.Spec.Auth.OAuthClientName = ""
checluster.Spec.Auth.OAuthSecret = ""
checluster.Spec.Auth.OAuthClientName = ""
if err := deploy.UpdateCheCRStatus(deployContext, "clean oAuth secret name and client name", ""); err != nil &&
errors.IsConflict(err) {
instance, _ = r.GetCR(req)
_ = deploy.ReloadCheClusterCR(deployContext)
continue
}
break
Expand Down Expand Up @@ -691,16 +698,13 @@ func isTrustedBundleConfigMap(mgr ctrl.Manager, obj client.Object) (bool, ctrl.R
}
}

func (r *CheClusterReconciler) autoEnableOAuth(deployContext *deploy.DeployContext, request ctrl.Request, isOpenShift4 bool) (reconcile.Result, error) {
var message, reason string
func (r *CheClusterReconciler) autoEnableOAuth(deployContext *deploy.DeployContext, isOpenShift4 bool) (reconcile.Result, error) {
oauth := false
cr := deployContext.CheCluster
if isOpenShift4 {
openshitOAuth, err := GetOpenshiftOAuth(deployContext.ClusterAPI.NonCachedClient)
if err != nil {
message = "Unable to get Openshift oAuth. Cause: " + err.Error()
logrus.Error(message)
reason = failedUnableToGetOAuth
logrus.Error("Unable to get Openshift oAuth. Cause: " + err.Error())
} else {
if len(openshitOAuth.Spec.IdentityProviders) > 0 {
oauth = true
Expand All @@ -711,10 +715,8 @@ func (r *CheClusterReconciler) autoEnableOAuth(deployContext *deploy.DeployConte
} else if util.IsInitialOpenShiftOAuthUserEnabled(cr) {
provisioned, err := r.userHandler.SyncOAuthInitialUser(openshitOAuth, deployContext)
if err != nil {
message = warningNoIdentityProvidersMessage + " Operator tried to create initial OpenShift OAuth user for HTPasswd identity provider, but failed. Cause: " + err.Error()
logrus.Error(message)
logrus.Error(warningNoIdentityProvidersMessage + " Operator tried to create initial OpenShift OAuth user for HTPasswd identity provider, but failed. Cause: " + err.Error())
logrus.Info("To enable OpenShift OAuth, please add identity provider first: " + howToAddIdentityProviderLinkOS4)
reason = failedNoIdentityProviders
// Don't try to create initial user any more, che-operator shouldn't hang on this step.
cr.Spec.Auth.InitialOpenShiftOAuthUser = nil
if err := deploy.UpdateCheCRStatus(deployContext, "initialOpenShiftOAuthUser", ""); err != nil {
Expand All @@ -739,15 +741,11 @@ func (r *CheClusterReconciler) autoEnableOAuth(deployContext *deploy.DeployConte
users := &userv1.UserList{}
listOptions := &client.ListOptions{}
if err := r.nonCachedClient.List(context.TODO(), users, listOptions); err != nil {
message = failedUnableToGetOpenshiftUsers + " Cause: " + err.Error()
logrus.Error(message)
reason = failedNoOpenshiftUser
logrus.Error(failedUnableToGetOpenshiftUsers + " Cause: " + err.Error())
} else {
oauth = len(users.Items) >= 1
if !oauth {
message = warningNoRealUsersMessage + " " + howToConfigureOAuthLinkOS3
logrus.Warn(message)
reason = failedNoOpenshiftUser
logrus.Warn(warningNoRealUsersMessage + " " + howToConfigureOAuthLinkOS3)
}
}
}
Expand All @@ -760,12 +758,6 @@ func (r *CheClusterReconciler) autoEnableOAuth(deployContext *deploy.DeployConte
}
}

if message != "" && reason != "" {
if err := deploy.SetStatusDetails(deployContext, message, reason, ""); err != nil {
return reconcile.Result{}, err
}
}

return reconcile.Result{}, nil
}

Expand Down Expand Up @@ -838,7 +830,7 @@ func (r *CheClusterReconciler) reconcileFinalizers(deployContext *deploy.DeployC
}

// Removes any legacy CRB https://github.com/eclipse/che/issues/19506
cheClusterRoleBindingName = deploy.GetLegacyUniqueClusterRoleBindingName(deployContext, CheServiceAccountName, cheClusterRole)
cheClusterRoleBindingName = deploy.GetLegacyUniqueClusterRoleBindingName(deployContext, deploy.CheServiceAccountName, cheClusterRole)
if err := deploy.ReconcileLegacyClusterRoleBindingFinalizer(deployContext, cheClusterRoleBindingName); err != nil {
logrus.Error(err)
}
Expand Down
3 changes: 1 addition & 2 deletions controllers/che/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@ import (
"github.com/eclipse-che/che-operator/pkg/util"
"github.com/sirupsen/logrus"
appsv1 "k8s.io/api/apps/v1"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

func (r *CheClusterReconciler) GenerateAndSaveFields(deployContext *deploy.DeployContext, request reconcile.Request) (err error) {
func (r *CheClusterReconciler) GenerateAndSaveFields(deployContext *deploy.DeployContext) (err error) {
cheFlavor := deploy.DefaultCheFlavor(deployContext.CheCluster)
cheNamespace := deployContext.CheCluster.Namespace
if len(deployContext.CheCluster.Spec.Server.CheFlavor) < 1 {
Expand Down
Loading