From 7a390ece455057b3c7f22dc81d5b1ac26d9fe62c Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 6 Sep 2017 10:31:18 +0200 Subject: [PATCH] Add OpenShift's recycler templates to Kubernetes controller config When user did not specify any recycler template files, create OpenShift one and let Kubernetes use it. The template file is created in /tmp and is deleted after controller initialization (which is safe). --- .../server/kubernetes/master/master_config.go | 59 +++++++++++++++++++ .../start/start_kube_controller_manager.go | 28 --------- pkg/cmd/server/start/start_master.go | 21 ++++++- 3 files changed, 78 insertions(+), 30 deletions(-) delete mode 100644 pkg/cmd/server/start/start_kube_controller_manager.go diff --git a/pkg/cmd/server/kubernetes/master/master_config.go b/pkg/cmd/server/kubernetes/master/master_config.go index 27033b0ecd13..7651cd3f7e9e 100644 --- a/pkg/cmd/server/kubernetes/master/master_config.go +++ b/pkg/cmd/server/kubernetes/master/master_config.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/url" + "os" "regexp" "strconv" "strings" @@ -43,6 +44,7 @@ import ( kapiserveroptions "k8s.io/kubernetes/cmd/kube-apiserver/app/options" cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" kapi "k8s.io/kubernetes/pkg/api" + kapiv1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/autoscaling" "k8s.io/kubernetes/pkg/apis/batch" @@ -57,15 +59,18 @@ import ( endpointsstorage "k8s.io/kubernetes/pkg/registry/core/endpoint/storage" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" kversion "k8s.io/kubernetes/pkg/version" + "k8s.io/kubernetes/pkg/volume" "github.com/openshift/origin/pkg/api" "github.com/openshift/origin/pkg/authorization/authorizer/scope" "github.com/openshift/origin/pkg/cmd/flagtypes" configapi "github.com/openshift/origin/pkg/cmd/server/api" + "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" "github.com/openshift/origin/pkg/cmd/server/cm" "github.com/openshift/origin/pkg/cmd/server/crypto" "github.com/openshift/origin/pkg/cmd/server/election" cmdflags "github.com/openshift/origin/pkg/cmd/util/flags" + "github.com/openshift/origin/pkg/cmd/util/variable" oauthutil "github.com/openshift/origin/pkg/oauth/util" openapigenerated "github.com/openshift/origin/pkg/openapi" securityapi "github.com/openshift/origin/pkg/security/apis/security" @@ -369,6 +374,60 @@ func BuildControllerManagerServer(masterConfig configapi.MasterConfig) (*cmapp.C return cmserver, cloud, nil } +// AddRecyclerTemplates creates a file with default recycled pod template and +// adds it into PersistentVolumeRecyclerConfiguration if it is not set. It +// returns name of the template file that should be deleted after controllers +// start. +func AddRecyclerTemplates(masterConfig configapi.MasterConfig, cmserver *cmapp.CMServer) (string, error) { + if cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath != "" && cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS != "" { + // Both templates are set by user config, there is nothing to configure. + return "", nil + } + + // At least one template is needed, fill it + imageTemplate := variable.NewDefaultImageTemplate() + imageTemplate.Format = masterConfig.ImageConfig.Format + imageTemplate.Latest = masterConfig.ImageConfig.Latest + uid := int64(0) + + template := volume.NewPersistentVolumeRecyclerPodTemplate() + template.Namespace = "openshift-infra" + template.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName + template.Spec.Containers[0].Image = imageTemplate.ExpandOrDie("recycler") + template.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} + template.Spec.Containers[0].Args = []string{"/scrub"} + template.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} + template.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent + + templateBytes, err := runtime.Encode(kapi.Codecs.LegacyCodec(kapiv1.SchemeGroupVersion), template) + if err != nil { + return "", err + } + + f, err := ioutil.TempFile("", "openshift-recycler-template-") + if err != nil { + return "", err + } + filename := f.Name() + + _, err = f.Write(templateBytes) + if err != nil { + f.Close() + os.Remove(filename) + return "", err + } + f.Close() + + if cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath == "" { + cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath = filename + } + if cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS == "" { + cmserver.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS = filename + } + glog.V(4).Infof("Using %q as recycler template", filename) + return filename, nil +} + func buildProxyClientCerts(masterConfig configapi.MasterConfig) ([]tls.Certificate, error) { var proxyClientCerts []tls.Certificate if len(masterConfig.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 { diff --git a/pkg/cmd/server/start/start_kube_controller_manager.go b/pkg/cmd/server/start/start_kube_controller_manager.go deleted file mode 100644 index 7aa40f2695b7..000000000000 --- a/pkg/cmd/server/start/start_kube_controller_manager.go +++ /dev/null @@ -1,28 +0,0 @@ -package start - -import ( - "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" - "k8s.io/kubernetes/pkg/api/v1" - kapiv1 "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/volume" -) - -// newPersistentVolumeRecyclerPodTemplate provides a function which makes our recycler pod template for use in the kube-controller-manager -// this is a stop-gap until the kube-controller-manager take a pod manifest -func newPersistentVolumeRecyclerPodTemplate(recyclerImageName string) func() *v1.Pod { - oldTemplateFunc := volume.NewPersistentVolumeRecyclerPodTemplate - return func() *v1.Pod { - uid := int64(0) - defaultScrubPod := oldTemplateFunc() - // TODO: Move the recycler pods to dedicated namespace instead of polluting openshift-infra. - defaultScrubPod.Namespace = "openshift-infra" - defaultScrubPod.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName - defaultScrubPod.Spec.Containers[0].Image = recyclerImageName - defaultScrubPod.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} - defaultScrubPod.Spec.Containers[0].Args = []string{"/scrub"} - defaultScrubPod.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} - defaultScrubPod.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent - - return defaultScrubPod - } -} diff --git a/pkg/cmd/server/start/start_master.go b/pkg/cmd/server/start/start_master.go index 9a21087f3079..a73fbb5c1934 100644 --- a/pkg/cmd/server/start/start_master.go +++ b/pkg/cmd/server/start/start_master.go @@ -35,7 +35,6 @@ import ( kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/master" - "k8s.io/kubernetes/pkg/volume" kutilerrors "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/errors" assetapiserver "github.com/openshift/origin/pkg/assets/apiserver" @@ -410,7 +409,6 @@ func (m *Master) Start() error { imageTemplate := variable.NewDefaultImageTemplate() imageTemplate.Format = m.config.ImageConfig.Format imageTemplate.Latest = m.config.ImageConfig.Latest - volume.NewPersistentVolumeRecyclerPodTemplate = newPersistentVolumeRecyclerPodTemplate(imageTemplate.ExpandOrDie("recycler")) if !m.api { // you can't double run healthz, so only do this next bit if we aren't starting the API @@ -478,6 +476,25 @@ func (m *Master) Start() error { // continuously run the scheduler while we have the primary lease go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments) + // OpenShift uses a different default volume recycler template than + // Kubernetes. This default template is hardcoded in Kubernetes and + // it isn't possible to pass it via ControllerContext. Crate a + // temporary file with OpenShift's template and let's pretend it was + // set by user as --recycler-pod-template-filepath-hostpath and + // --pv-recycler-pod-template-filepath-nfs arguments. + recyclerPodTemplatePath, err := kubernetes.AddRecyclerTemplates(*m.config, kubeControllerManagerConfig) + if err != nil { + glog.Fatal(err) + } + if recyclerPodTemplatePath != "" { + defer func() { + // Recycler template is loaded synchronously in startControllers + // and can be safely deleted when the function finishes. + glog.V(4).Infof("Deleting recycler template %s", recyclerPodTemplatePath) + os.Remove(recyclerPodTemplatePath) + }() + } + controllerContext, err := getControllerContext(*m.config, kubeControllerManagerConfig, cloudProvider, informers, utilwait.NeverStop) if err != nil { glog.Fatal(err)