From 4e52eb3f9d7025d448b49d8883e40fb9d22d4023 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Thu, 14 May 2020 15:21:40 +0530 Subject: [PATCH] Intermediate Commit for transition from Pod to Deployment based Jenkins Controller Allows for using the annotation jenkins.io/use-deployment and setting the value to true makes the operator use a Deployment instead of Pod for serving Jenkins. This is part of a temporary feature and has to be committed to avoid bigger PRs. --- pkg/apis/apis.go | 2 + .../backuprestore/backuprestore.go | 4 +- .../jenkins/configuration/base/deployment.go | 56 ++++++++++++++++++ .../jenkins/configuration/base/reconcile.go | 29 ++++++++- .../base/resources/deployment.go | 59 +++++++++++++++++++ .../configuration/base/resources/pod.go | 4 +- .../configuration/base/validate_test.go | 4 +- .../jenkins/configuration/configuration.go | 51 ++++++++++------ test/e2e/jenkins.go | 4 +- 9 files changed, 183 insertions(+), 30 deletions(-) create mode 100644 pkg/controller/jenkins/configuration/base/deployment.go create mode 100644 pkg/controller/jenkins/configuration/base/resources/deployment.go diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index 515f2f6ff..bb5ee814e 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -3,6 +3,7 @@ package apis import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" routev1 "github.com/openshift/api/route/v1" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -19,4 +20,5 @@ func init() { // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back AddToSchemes = append(AddToSchemes, v1alpha2.SchemeBuilder.AddToScheme) AddToSchemes = append(AddToSchemes, routev1.AddToScheme) + AddToSchemes = append(AddToSchemes, appsv1.AddToScheme) } diff --git a/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go b/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go index d9bc1602a..27a200f21 100644 --- a/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go +++ b/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go @@ -137,7 +137,7 @@ func (bar *BackupAndRestore) Restore(jenkinsClient jenkinsclient.Jenkins) error backupNumber = jenkins.Status.LastBackup } bar.logger.Info(fmt.Sprintf("Restoring backup '%d'", backupNumber)) - podName := resources.GetJenkinsMasterPodName(*jenkins) + podName := resources.GetJenkinsMasterPodName(jenkins) command := jenkins.Spec.Restore.Action.Exec.Command command = append(command, fmt.Sprintf("%d", backupNumber)) _, _, err := bar.Exec(podName, jenkins.Spec.Restore.ContainerName, command) @@ -170,7 +170,7 @@ func (bar *BackupAndRestore) Backup(setBackupDoneBeforePodDeletion bool) error { } backupNumber := jenkins.Status.PendingBackup bar.logger.Info(fmt.Sprintf("Performing backup '%d'", backupNumber)) - podName := resources.GetJenkinsMasterPodName(*jenkins) + podName := resources.GetJenkinsMasterPodName(jenkins) command := jenkins.Spec.Backup.Action.Exec.Command command = append(command, fmt.Sprintf("%d", backupNumber)) _, _, err := bar.Exec(podName, jenkins.Spec.Backup.ContainerName, command) diff --git a/pkg/controller/jenkins/configuration/base/deployment.go b/pkg/controller/jenkins/configuration/base/deployment.go new file mode 100644 index 000000000..07a9dacf4 --- /dev/null +++ b/pkg/controller/jenkins/configuration/base/deployment.go @@ -0,0 +1,56 @@ +package base + +import ( + "context" + "fmt" + + "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/event" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/reason" + "github.com/jenkinsci/kubernetes-operator/version" + stackerr "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsDeployment(meta metav1.ObjectMeta) (reconcile.Result, error) { + userAndPasswordHash, err := r.calculateUserAndPasswordHash() + if err != nil { + return reconcile.Result{}, err + } + + jenkinsDeployment, err := r.GetJenkinsDeployment() + if apierrors.IsNotFound(err) { + jenkinsDeployment = resources.NewJenkinsDeployment(meta, r.Configuration.Jenkins) + *r.Notifications <- event.Event{ + Jenkins: *r.Configuration.Jenkins, + Phase: event.PhaseBase, + Level: v1alpha2.NotificationLevelInfo, + Reason: reason.NewPodCreation(reason.OperatorSource, []string{"Creating a Jenkins Deployment"}), + } + + r.logger.Info(fmt.Sprintf("Creating a new Jenkins Deployment %s/%s", jenkinsDeployment.Namespace, jenkinsDeployment.Name)) + err := r.CreateResource(jenkinsDeployment) + if err != nil { + return reconcile.Result{}, stackerr.WithStack(err) + } + + now := metav1.Now() + r.Configuration.Jenkins.Status = v1alpha2.JenkinsStatus{ + OperatorVersion: version.Version, + ProvisionStartTime: &now, + LastBackup: r.Configuration.Jenkins.Status.LastBackup, + PendingBackup: r.Configuration.Jenkins.Status.LastBackup, + UserAndPasswordHash: userAndPasswordHash, + } + return reconcile.Result{Requeue: true}, r.Client.Update(context.TODO(), r.Configuration.Jenkins) + } else if err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, stackerr.WithStack(err) + } + + // TODO (waveywaves): replace with a cleaner solution + _ = jenkinsDeployment // This is to escape the variable is never used golint err + return reconcile.Result{}, nil +} diff --git a/pkg/controller/jenkins/configuration/base/reconcile.go b/pkg/controller/jenkins/configuration/base/reconcile.go index 1b0029d93..de904ca00 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile.go +++ b/pkg/controller/jenkins/configuration/base/reconcile.go @@ -31,7 +31,7 @@ const ( fetchAllPlugins = 1 ) -// ReconcileJenkinsBaseConfiguration defines values required for Jenkins base configuration +// ReconcileJenkinsBaseConfiguration defines values required for Jenkins base configuration. type ReconcileJenkinsBaseConfiguration struct { configuration.Configuration logger logr.Logger @@ -47,16 +47,30 @@ func New(config configuration.Configuration, jenkinsAPIConnectionSettings jenkin } } -// Reconcile takes care of base configuration +// Reconcile takes care of base configuration. func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenkinsclient.Jenkins, error) { metaObject := resources.NewResourceObjectMeta(r.Configuration.Jenkins) + // Create Necessary Resources err := r.ensureResourcesRequiredForJenkinsPod(metaObject) if err != nil { return reconcile.Result{}, nil, err } r.logger.V(log.VDebug).Info("Kubernetes resources are present") + if useDeploymentForJenkinsMaster(r.Configuration.Jenkins) { + result, err := r.ensureJenkinsDeployment(metaObject) + if err != nil { + return reconcile.Result{}, nil, err + } + if result.Requeue { + return result, nil, nil + } + r.logger.V(log.VDebug).Info("Jenkins Deployment is present") + + return result, nil, err + } + result, err := r.ensureJenkinsMasterPod(metaObject) if err != nil { return reconcile.Result{}, nil, err @@ -110,6 +124,15 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki return result, jenkinsClient, err } +func useDeploymentForJenkinsMaster(jenkins *v1alpha2.Jenkins) bool { + if val, ok := jenkins.Annotations["jenkins.io/use-deployment"]; ok { + if val == "true" { + return true + } + } + return false +} + func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod(metaObject metav1.ObjectMeta) error { if err := r.createOperatorCredentialsSecret(metaObject); err != nil { return err @@ -245,7 +268,7 @@ func compareEnv(expected, actual []corev1.EnvVar) bool { return reflect.DeepEqual(expected, actualEnv) } -// CompareContainerVolumeMounts returns true if two containers volume mounts are the same +// CompareContainerVolumeMounts returns true if two containers volume mounts are the same. func CompareContainerVolumeMounts(expected corev1.Container, actual corev1.Container) bool { var withoutServiceAccount []corev1.VolumeMount for _, volumeMount := range actual.VolumeMounts { diff --git a/pkg/controller/jenkins/configuration/base/resources/deployment.go b/pkg/controller/jenkins/configuration/base/resources/deployment.go new file mode 100644 index 000000000..942e040ea --- /dev/null +++ b/pkg/controller/jenkins/configuration/base/resources/deployment.go @@ -0,0 +1,59 @@ +package resources + +import ( + "fmt" + + "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +// NewJenkinsMasterPod builds Jenkins Master Kubernetes Pod resource. +func NewJenkinsDeployment(objectMeta metav1.ObjectMeta, jenkins *v1alpha2.Jenkins) *appsv1.Deployment { + serviceAccountName := objectMeta.Name + objectMeta.Annotations = jenkins.Spec.Master.Annotations + objectMeta.Name = GetJenkinsDeploymentName(jenkins) + selector := &metav1.LabelSelector{MatchLabels: objectMeta.Labels} + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectMeta.Name, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + OwnerReferences: []metav1.OwnerReference{ + { + BlockOwnerDeletion: pointer.BoolPtr(true), + Controller: pointer.BoolPtr(true), + Kind: jenkins.Kind, + Name: jenkins.Name, + APIVersion: jenkins.APIVersion, + UID: jenkins.UID, + }, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32Ptr(1), + Strategy: appsv1.DeploymentStrategy{Type: appsv1.RollingUpdateDeploymentStrategyType}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: objectMeta, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, + NodeSelector: jenkins.Spec.Master.NodeSelector, + Containers: newContainers(jenkins), + Volumes: append(GetJenkinsMasterPodBaseVolumes(jenkins), jenkins.Spec.Master.Volumes...), + SecurityContext: jenkins.Spec.Master.SecurityContext, + ImagePullSecrets: jenkins.Spec.Master.ImagePullSecrets, + Tolerations: jenkins.Spec.Master.Tolerations, + PriorityClassName: jenkins.Spec.Master.PriorityClassName, + }, + }, + Selector: selector, + }, + } +} + +// GetJenkinsMasterPodName returns Jenkins pod name for given CR +func GetJenkinsDeploymentName(jenkins *v1alpha2.Jenkins) string { + return fmt.Sprintf("jenkins-%s", jenkins.Name) +} diff --git a/pkg/controller/jenkins/configuration/base/resources/pod.go b/pkg/controller/jenkins/configuration/base/resources/pod.go index dde4609bb..ae8d8ab77 100644 --- a/pkg/controller/jenkins/configuration/base/resources/pod.go +++ b/pkg/controller/jenkins/configuration/base/resources/pod.go @@ -291,7 +291,7 @@ func newContainers(jenkins *v1alpha2.Jenkins) (containers []corev1.Container) { } // GetJenkinsMasterPodName returns Jenkins pod name for given CR -func GetJenkinsMasterPodName(jenkins v1alpha2.Jenkins) string { +func GetJenkinsMasterPodName(jenkins *v1alpha2.Jenkins) string { return fmt.Sprintf("jenkins-%s", jenkins.Name) } @@ -313,7 +313,7 @@ func GetJenkinsMasterPodLabels(jenkins v1alpha2.Jenkins) map[string]string { func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha2.Jenkins) *corev1.Pod { serviceAccountName := objectMeta.Name objectMeta.Annotations = jenkins.Spec.Master.Annotations - objectMeta.Name = GetJenkinsMasterPodName(*jenkins) + objectMeta.Name = GetJenkinsMasterPodName(jenkins) objectMeta.Labels = GetJenkinsMasterPodLabels(*jenkins) return &corev1.Pod{ diff --git a/pkg/controller/jenkins/configuration/base/validate_test.go b/pkg/controller/jenkins/configuration/base/validate_test.go index 2f455f8e1..c08b38366 100644 --- a/pkg/controller/jenkins/configuration/base/validate_test.go +++ b/pkg/controller/jenkins/configuration/base/validate_test.go @@ -573,7 +573,7 @@ func TestValidateConfigMapVolume(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: &v1alpha2.Jenkins{ObjectMeta: metav1.ObjectMeta{Name: "example"}}, - Client: fakeClient, + Client: fakeClient, }, client.JenkinsAPIConnectionSettings{}) got, err := baseReconcileLoop.validateConfigMapVolume(volume) @@ -652,7 +652,7 @@ func TestValidateSecretVolume(t *testing.T) { fakeClient := fake.NewFakeClient() baseReconcileLoop := New(configuration.Configuration{ Jenkins: &v1alpha2.Jenkins{ObjectMeta: metav1.ObjectMeta{Name: "example"}}, - Client: fakeClient, + Client: fakeClient, }, client.JenkinsAPIConnectionSettings{}) got, err := baseReconcileLoop.validateSecretVolume(volume) diff --git a/pkg/controller/jenkins/configuration/configuration.go b/pkg/controller/jenkins/configuration/configuration.go index 26b9c70a8..949c40c84 100644 --- a/pkg/controller/jenkins/configuration/configuration.go +++ b/pkg/controller/jenkins/configuration/configuration.go @@ -6,12 +6,8 @@ import ( "strings" "time" - "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" - jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" - "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" - "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/event" - "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/reason" stackerr "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,9 +19,15 @@ import ( "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" + jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/event" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/reason" ) -// Configuration holds required for Jenkins configuration +// Configuration holds required for Jenkins configuration. type Configuration struct { Client client.Client ClientSet kubernetes.Clientset @@ -36,7 +38,7 @@ type Configuration struct { JenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings } -// RestartJenkinsMasterPod terminate Jenkins master pod and notifies about it +// RestartJenkinsMasterPod terminate Jenkins master pod and notifies about it. func (c *Configuration) RestartJenkinsMasterPod(reason reason.Reason) error { currentJenkinsMasterPod, err := c.GetJenkinsMasterPod() if err != nil { @@ -57,9 +59,9 @@ func (c *Configuration) RestartJenkinsMasterPod(reason reason.Reason) error { return stackerr.WithStack(c.Client.Delete(context.TODO(), currentJenkinsMasterPod)) } -// GetJenkinsMasterPod gets the jenkins master pod +// GetJenkinsMasterPod gets the jenkins master pod. func (c *Configuration) GetJenkinsMasterPod() (*corev1.Pod, error) { - jenkinsMasterPodName := resources.GetJenkinsMasterPodName(*c.Jenkins) + jenkinsMasterPodName := resources.GetJenkinsMasterPodName(c.Jenkins) currentJenkinsMasterPod := &corev1.Pod{} err := c.Client.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: c.Jenkins.Namespace}, currentJenkinsMasterPod) if err != nil { @@ -68,7 +70,18 @@ func (c *Configuration) GetJenkinsMasterPod() (*corev1.Pod, error) { return currentJenkinsMasterPod, nil } -// IsJenkinsTerminating returns true if the Jenkins pod is terminating +// GetJenkinsMasterPod gets the jenkins master pod. +func (c *Configuration) GetJenkinsDeployment() (*appsv1.Deployment, error) { + jenkinsDeploymentName := resources.GetJenkinsDeploymentName(c.Jenkins) + currentJenkinsDeployment := &appsv1.Deployment{} + err := c.Client.Get(context.TODO(), types.NamespacedName{Name: jenkinsDeploymentName, Namespace: c.Jenkins.Namespace}, currentJenkinsDeployment) + if err != nil { + return nil, err // don't wrap error + } + return currentJenkinsDeployment, nil +} + +// IsJenkinsTerminating returns true if the Jenkins pod is terminating. func (c *Configuration) IsJenkinsTerminating(pod corev1.Pod) bool { return pod.ObjectMeta.DeletionTimestamp != nil } @@ -80,7 +93,7 @@ func (c *Configuration) CreateResource(obj metav1.Object) error { return stackerr.Errorf("is not a %T a runtime.Object", obj) } - // Set Jenkins instance as the owner and controller + // Set Jenkins instance as the owner and controller. if err := controllerutil.SetControllerReference(c.Jenkins, obj, c.Scheme); err != nil { return stackerr.WithStack(err) } @@ -88,7 +101,7 @@ func (c *Configuration) CreateResource(obj metav1.Object) error { return c.Client.Create(context.TODO(), runtimeObj) // don't wrap error } -// UpdateResource is updating kubernetes resource and references it to Jenkins CR +// UpdateResource is updating kubernetes resource and references it to Jenkins CR. func (c *Configuration) UpdateResource(obj metav1.Object) error { runtimeObj, ok := obj.(runtime.Object) if !ok { @@ -101,7 +114,7 @@ func (c *Configuration) UpdateResource(obj metav1.Object) error { return c.Client.Update(context.TODO(), runtimeObj) // don't wrap error } -// CreateOrUpdateResource is creating or updating kubernetes resource and references it to Jenkins CR +// CreateOrUpdateResource is creating or updating kubernetes resource and references it to Jenkins CR. func (c *Configuration) CreateOrUpdateResource(obj metav1.Object) error { runtimeObj, ok := obj.(runtime.Object) if !ok { @@ -121,7 +134,7 @@ func (c *Configuration) CreateOrUpdateResource(obj metav1.Object) error { return nil } -// Exec executes command in the given pod and it's container +// Exec executes command in the given pod and it's container. func (c *Configuration) Exec(podName, containerName string, command []string) (stdout, stderr bytes.Buffer, err error) { req := c.ClientSet.CoreV1().RESTClient().Post(). Resource("pods"). @@ -155,7 +168,7 @@ func (c *Configuration) Exec(podName, containerName string, command []string) (s return } -// GetJenkinsMasterContainer returns the Jenkins master container from the CR +// GetJenkinsMasterContainer returns the Jenkins master container from the CR. func (c *Configuration) GetJenkinsMasterContainer() *v1alpha2.Container { if len(c.Jenkins.Spec.Master.Containers) > 0 { // the first container is the Jenkins master, it is forced jenkins_controller.go @@ -164,7 +177,7 @@ func (c *Configuration) GetJenkinsMasterContainer() *v1alpha2.Container { return nil } -// GetJenkinsClient gets jenkins client from a configuration +// GetJenkinsClient gets jenkins client from a configuration. func (c *Configuration) GetJenkinsClient() (jenkinsclient.Jenkins, error) { switch c.Jenkins.Spec.JenkinsAPISettings.AuthorizationStrategy { case v1alpha2.ServiceAccountAuthorizationStrategy: @@ -194,14 +207,14 @@ func (c *Configuration) getJenkinsAPIUrl() (string, error) { return jenkinsURL, nil } -// GetJenkinsClientFromServiceAccount gets jenkins client from a serviceAccount +// GetJenkinsClientFromServiceAccount gets jenkins client from a serviceAccount. func (c *Configuration) GetJenkinsClientFromServiceAccount() (jenkinsclient.Jenkins, error) { jenkinsAPIUrl, err := c.getJenkinsAPIUrl() if err != nil { return nil, err } - podName := resources.GetJenkinsMasterPodName(*c.Jenkins) + podName := resources.GetJenkinsMasterPodName(c.Jenkins) token, _, err := c.Exec(podName, resources.JenkinsMasterContainerName, []string{"cat", "/var/run/secrets/kubernetes.io/serviceaccount/token"}) if err != nil { return nil, err @@ -210,7 +223,7 @@ func (c *Configuration) GetJenkinsClientFromServiceAccount() (jenkinsclient.Jenk return jenkinsclient.NewBearerTokenAuthorization(jenkinsAPIUrl, token.String()) } -// GetJenkinsClientFromSecret gets jenkins client from a secret +// GetJenkinsClientFromSecret gets jenkins client from a secret. func (c *Configuration) GetJenkinsClientFromSecret() (jenkinsclient.Jenkins, error) { jenkinsURL, err := c.getJenkinsAPIUrl() if err != nil { diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index 5b42738b8..80631d526 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -156,7 +156,7 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S func createJenkinsAPIClientFromServiceAccount(t *testing.T, jenkins *v1alpha2.Jenkins, jenkinsAPIURL string) (jenkinsclient.Jenkins, error) { t.Log("Creating Jenkins API client from service account") - podName := resources.GetJenkinsMasterPodName(*jenkins) + podName := resources.GetJenkinsMasterPodName(jenkins) clientSet, err := kubernetes.NewForConfig(framework.Global.KubeConfig) if err != nil { @@ -197,7 +197,7 @@ func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, namespa }, &service) require.NoError(t, err) - podName := resources.GetJenkinsMasterPodName(*jenkins) + podName := resources.GetJenkinsMasterPodName(jenkins) port, cleanUpFunc, waitFunc, portForwardFunc, err := setupPortForwardToPod(t, namespace, podName, int(constants.DefaultHTTPPortInt32)) if err != nil { t.Fatal(err)