diff --git a/pkg/cmd/server/start/start_kube_controller_manager.go b/pkg/cmd/server/start/start_kube_controller_manager.go index 419b21a068ea..ca3473d1e815 100644 --- a/pkg/cmd/server/start/start_kube_controller_manager.go +++ b/pkg/cmd/server/start/start_kube_controller_manager.go @@ -1,6 +1,8 @@ package start import ( + "io/ioutil" + "os" "strconv" "github.com/golang/glog" @@ -11,7 +13,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" controllerapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" controlleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/pkg/api/v1" + kapi "k8s.io/kubernetes/pkg/api" kapiv1 "k8s.io/kubernetes/pkg/api/v1" kexternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" kinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" @@ -24,26 +26,6 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// newPersistentVolumeRecyclerPodTemplate provides a function which makes our recycler pod template for use in the kube-controller-manager -// this is a stop-gap until the kube-controller-manager take a pod manifest -func newPersistentVolumeRecyclerPodTemplate(recyclerImageName string) func() *v1.Pod { - oldTemplateFunc := volume.NewPersistentVolumeRecyclerPodTemplate - return func() *v1.Pod { - uid := int64(0) - defaultScrubPod := oldTemplateFunc() - // TODO: Move the recycler pods to dedicated namespace instead of polluting openshift-infra. - defaultScrubPod.Namespace = "openshift-infra" - defaultScrubPod.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName - defaultScrubPod.Spec.Containers[0].Image = recyclerImageName - defaultScrubPod.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} - defaultScrubPod.Spec.Containers[0].Args = []string{"/scrub"} - defaultScrubPod.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} - defaultScrubPod.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent - - return defaultScrubPod - } -} - // newControllerContext provides a function which overrides the default and plugs a different set of informers in func newControllerContext(informers *informers) func(s *controlleroptions.CMServer, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (controllerapp.ControllerContext, error) { oldContextFunc := controllerapp.CreateControllerContext @@ -67,10 +49,11 @@ func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(fl } } -func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string) (*controlleroptions.CMServer, error) { +func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string) (*controlleroptions.CMServer, []func(), error) { if cmdLineArgs == nil { cmdLineArgs = map[string][]string{} } + cleanupFunctions := []func(){} if _, ok := cmdLineArgs["controllers"]; !ok { cmdLineArgs["controllers"] = []string{ @@ -126,10 +109,43 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po cmdLineArgs["leader-elect-resource-lock"] = []string{"configmaps"} } + _, hostPathTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] + _, nfsTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] + if !hostPathTemplateSet || !nfsTemplateSet { + // OpenShift uses a different default volume recycler template than + // Kubernetes. This default template is hardcoded in Kubernetes and it + // isn't possible to pass it via ControllerContext. Crate a temporary + // file with OpenShift's template and let's pretend it was set by user + // as --recycler-pod-template-filepath-hostpath and + // --pv-recycler-pod-template-filepath-nfs arguments. + // This template then needs to be deleted by caller! + templateFilename, err := createRecylerTemplate(recyclerImage) + if err != nil { + return nil, nil, err + } + + cleanupFunctions = append(cleanupFunctions, func() { + // Remove the template when it's not needed. This is called aftet + // controller is initialized + glog.V(4).Infof("Removing temporary file %s", templateFilename) + err := os.Remove(templateFilename) + if err != nil { + glog.Warningf("Failed to remove %s: %v", templateFilename, err) + } + }) + + if !hostPathTemplateSet { + cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] = []string{templateFilename} + } + if !nfsTemplateSet { + cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] = []string{templateFilename} + } + } + // resolve arguments controllerManager := controlleroptions.NewCMServer() if err := cmdflags.Resolve(cmdLineArgs, kubeControllerManagerAddFlags(controllerManager)); len(err) > 0 { - return nil, kerrors.NewAggregate(err) + return nil, cleanupFunctions, kerrors.NewAggregate(err) } // TODO make this configurable or discoverable. This is going to prevent us from running the stock GC controller @@ -164,19 +180,60 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po componentconfig.GroupResource{Group: "", Resource: "securitycontextconstraints"}, ) - return controllerManager, nil + return controllerManager, cleanupFunctions, nil +} + +func createRecylerTemplate(recyclerImage string) (string, error) { + uid := int64(0) + template := volume.NewPersistentVolumeRecyclerPodTemplate() + template.Namespace = "openshift-infra" + template.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName + template.Spec.Containers[0].Image = recyclerImage + template.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} + template.Spec.Containers[0].Args = []string{"/scrub"} + template.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} + template.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent + + templateBytes, err := runtime.Encode(kapi.Codecs.LegacyCodec(kapiv1.SchemeGroupVersion), template) + if err != nil { + return "", err + } + + f, err := ioutil.TempFile("", "openshift-recycler-template-") + if err != nil { + return "", err + } + filename := f.Name() + glog.V(4).Infof("Creating file %s with recycler templates", filename) + + _, err = f.Write(templateBytes) + if err != nil { + f.Close() + os.Remove(filename) + return "", err + } + f.Close() + return filename, nil } func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string, recyclerImage string, informers *informers) { - volume.NewPersistentVolumeRecyclerPodTemplate = newPersistentVolumeRecyclerPodTemplate(recyclerImage) controllerapp.CreateControllerContext = newControllerContext(informers) controllerapp.StartInformers = func(stop <-chan struct{}) { informers.Start(stop) } // TODO we need a real identity for this. Right now it's just using the loopback connection like it used to. - controllerManager, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, dynamicProvisioningEnabled, cmdLineArgs) + controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs) + defer func() { + // Clean up any temporary files and similar stuff. + // TODO: Make sure this defer is actually called - controllerapp.Run() + // below never returns -> defer is not called. + for _, f := range cleanupFunctions { + f() + } + }() + if err != nil { glog.Fatal(err) } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/patch.go b/vendor/k8s.io/kubernetes/pkg/volume/patch.go deleted file mode 100644 index b9fffad6657a..000000000000 --- a/vendor/k8s.io/kubernetes/pkg/volume/patch.go +++ /dev/null @@ -1,7 +0,0 @@ -package volume - -import "k8s.io/kubernetes/pkg/api/v1" - -// we carry a patch which allows us to reset this function selectively when we run openshift start, but *not* -// when we run `kube-controller-manager`. We have it in this separate file to avoid conflicts during a rebase -var NewPersistentVolumeRecyclerPodTemplate func() *v1.Pod = newPersistentVolumeRecyclerPodTemplate diff --git a/vendor/k8s.io/kubernetes/pkg/volume/plugins.go b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go index fed0c14bf381..3f061fb601a0 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/plugins.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/plugins.go @@ -560,7 +560,7 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo // before failing. Recommended. Default is 60 seconds. // // See HostPath and NFS for working recycler examples -func newPersistentVolumeRecyclerPodTemplate() *v1.Pod { +func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod { timeout := int64(60) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{