From f1e6a4588ecad6c07963b8050d2d7f9d46489875 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 18 Sep 2017 12:53:34 +0200 Subject: [PATCH 1/2] Add OpenShift's recycler templates to Kubernetes controller config When user did not specify any recycler template files, create OpenShift one and let Kubernetes use it. The template file is created in /tmp and is deleted after controller initialization (which is safe). --- .../start/start_kube_controller_manager.go | 109 +++++++++++++----- 1 file changed, 83 insertions(+), 26 deletions(-) diff --git a/pkg/cmd/server/start/start_kube_controller_manager.go b/pkg/cmd/server/start/start_kube_controller_manager.go index 419b21a068ea..ca3473d1e815 100644 --- a/pkg/cmd/server/start/start_kube_controller_manager.go +++ b/pkg/cmd/server/start/start_kube_controller_manager.go @@ -1,6 +1,8 @@ package start import ( + "io/ioutil" + "os" "strconv" "github.com/golang/glog" @@ -11,7 +13,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" controllerapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" controlleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/pkg/api/v1" + kapi "k8s.io/kubernetes/pkg/api" kapiv1 "k8s.io/kubernetes/pkg/api/v1" kexternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" kinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" @@ -24,26 +26,6 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// newPersistentVolumeRecyclerPodTemplate provides a function which makes our recycler pod template for use in the kube-controller-manager -// this is a stop-gap until the kube-controller-manager take a pod manifest -func newPersistentVolumeRecyclerPodTemplate(recyclerImageName string) func() *v1.Pod { - oldTemplateFunc := volume.NewPersistentVolumeRecyclerPodTemplate - return func() *v1.Pod { - uid := int64(0) - defaultScrubPod := oldTemplateFunc() - // TODO: Move the recycler pods to dedicated namespace instead of polluting openshift-infra. - defaultScrubPod.Namespace = "openshift-infra" - defaultScrubPod.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName - defaultScrubPod.Spec.Containers[0].Image = recyclerImageName - defaultScrubPod.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} - defaultScrubPod.Spec.Containers[0].Args = []string{"/scrub"} - defaultScrubPod.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} - defaultScrubPod.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent - - return defaultScrubPod - } -} - // newControllerContext provides a function which overrides the default and plugs a different set of informers in func newControllerContext(informers *informers) func(s *controlleroptions.CMServer, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (controllerapp.ControllerContext, error) { oldContextFunc := controllerapp.CreateControllerContext @@ -67,10 +49,11 @@ func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(fl } } -func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string) (*controlleroptions.CMServer, error) { +func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string) (*controlleroptions.CMServer, []func(), error) { if cmdLineArgs == nil { cmdLineArgs = map[string][]string{} } + cleanupFunctions := []func(){} if _, ok := cmdLineArgs["controllers"]; !ok { cmdLineArgs["controllers"] = []string{ @@ -126,10 +109,43 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po cmdLineArgs["leader-elect-resource-lock"] = []string{"configmaps"} } + _, hostPathTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] + _, nfsTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] + if !hostPathTemplateSet || !nfsTemplateSet { + // OpenShift uses a different default volume recycler template than + // Kubernetes. This default template is hardcoded in Kubernetes and it + // isn't possible to pass it via ControllerContext. Crate a temporary + // file with OpenShift's template and let's pretend it was set by user + // as --recycler-pod-template-filepath-hostpath and + // --pv-recycler-pod-template-filepath-nfs arguments. + // This template then needs to be deleted by caller! + templateFilename, err := createRecylerTemplate(recyclerImage) + if err != nil { + return nil, nil, err + } + + cleanupFunctions = append(cleanupFunctions, func() { + // Remove the template when it's not needed. This is called aftet + // controller is initialized + glog.V(4).Infof("Removing temporary file %s", templateFilename) + err := os.Remove(templateFilename) + if err != nil { + glog.Warningf("Failed to remove %s: %v", templateFilename, err) + } + }) + + if !hostPathTemplateSet { + cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] = []string{templateFilename} + } + if !nfsTemplateSet { + cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] = []string{templateFilename} + } + } + // resolve arguments controllerManager := controlleroptions.NewCMServer() if err := cmdflags.Resolve(cmdLineArgs, kubeControllerManagerAddFlags(controllerManager)); len(err) > 0 { - return nil, kerrors.NewAggregate(err) + return nil, cleanupFunctions, kerrors.NewAggregate(err) } // TODO make this configurable or discoverable. This is going to prevent us from running the stock GC controller @@ -164,19 +180,60 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po componentconfig.GroupResource{Group: "", Resource: "securitycontextconstraints"}, ) - return controllerManager, nil + return controllerManager, cleanupFunctions, nil +} + +func createRecylerTemplate(recyclerImage string) (string, error) { + uid := int64(0) + template := volume.NewPersistentVolumeRecyclerPodTemplate() + template.Namespace = "openshift-infra" + template.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName + template.Spec.Containers[0].Image = recyclerImage + template.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} + template.Spec.Containers[0].Args = []string{"/scrub"} + template.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} + template.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent + + templateBytes, err := runtime.Encode(kapi.Codecs.LegacyCodec(kapiv1.SchemeGroupVersion), template) + if err != nil { + return "", err + } + + f, err := ioutil.TempFile("", "openshift-recycler-template-") + if err != nil { + return "", err + } + filename := f.Name() + glog.V(4).Infof("Creating file %s with recycler templates", filename) + + _, err = f.Write(templateBytes) + if err != nil { + f.Close() + os.Remove(filename) + return "", err + } + f.Close() + return filename, nil } func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string, recyclerImage string, informers *informers) { - volume.NewPersistentVolumeRecyclerPodTemplate = newPersistentVolumeRecyclerPodTemplate(recyclerImage) controllerapp.CreateControllerContext = newControllerContext(informers) controllerapp.StartInformers = func(stop <-chan struct{}) { informers.Start(stop) } // TODO we need a real identity for this. Right now it's just using the loopback connection like it used to. - controllerManager, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, dynamicProvisioningEnabled, cmdLineArgs) + controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs) + defer func() { + // Clean up any temporary files and similar stuff. + // TODO: Make sure this defer is actually called - controllerapp.Run() + // below never returns -> defer is not called. + for _, f := range cleanupFunctions { + f() + } + }() + if err != nil { glog.Fatal(err) } From d2d3175ca377b8f499627a71f24e835109a00529 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 5 Sep 2017 15:16:33 +0200 Subject: [PATCH 2/2] UPSTREAM: revert: bc8249cc6f35fabffff5ccf4e13978ff9a6a2e32: "UPSTREAM: : allow PV controller recycler template override" This reverts bc8249cc6f35fabffff5ccf4e13978ff9a6a2e32 --- vendor/k8s.io/kubernetes/pkg/volume/patch.go | 7 ------- vendor/k8s.io/kubernetes/pkg/volume/plugins.go | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 vendor/k8s.io/kubernetes/pkg/volume/patch.go diff --git a/vendor/k8s.io/kubernetes/pkg/volume/patch.go b/vendor/k8s.io/kubernetes/pkg/volume/patch.go deleted file mode 100644 index b9fffad6657a..000000000000 --- a/vendor/k8s.io/kubernetes/pkg/volume/patch.go +++ /dev/null @@ -1,7 +0,0 @@ -package volume - -import "k8s.io/kubernetes/pkg/api/v1" - -// we carry a patch which allows us to reset this function selectively when we run openshift start, but *not* -// when we run `kube-controller-manager`. We have it in this separate file to avoid conflicts during a rebase -var NewPersistentVolumeRecyclerPodTemplate func() *v1.Pod = newPersistentVolumeRecyclerPodTemplate diff --git a/vendor/k8s.io/kubernetes/pkg/volume/plugins.go b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go index fed0c14bf381..3f061fb601a0 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/plugins.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go @@ -560,7 +560,7 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo // before failing. Recommended. Default is 60 seconds. // // See HostPath and NFS for working recycler examples -func newPersistentVolumeRecyclerPodTemplate() *v1.Pod { +func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod { timeout := int64(60) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{