diff --git a/contrib/completions/bash/openshift b/contrib/completions/bash/openshift index e20f56800662..db7bbe987d44 100644 --- a/contrib/completions/bash/openshift +++ b/contrib/completions/bash/openshift @@ -35301,6 +35301,8 @@ _openshift_start_master_controllers() local_nonpersistent_flags+=("--config=") flags+=("--listen=") local_nonpersistent_flags+=("--listen=") + flags+=("--lock-service-name=") + local_nonpersistent_flags+=("--lock-service-name=") flags+=("--azure-container-registry-config=") flags+=("--google-json-key=") flags+=("--log-flush-frequency=") diff --git a/contrib/completions/zsh/openshift b/contrib/completions/zsh/openshift index 14889f7a237e..589c9d635f41 100644 --- a/contrib/completions/zsh/openshift +++ b/contrib/completions/zsh/openshift @@ -35450,6 +35450,8 @@ _openshift_start_master_controllers() local_nonpersistent_flags+=("--config=") flags+=("--listen=") local_nonpersistent_flags+=("--listen=") + flags+=("--lock-service-name=") + local_nonpersistent_flags+=("--lock-service-name=") flags+=("--azure-container-registry-config=") flags+=("--google-json-key=") flags+=("--log-flush-frequency=") diff --git a/contrib/kubernetes/controllers.yaml b/contrib/kubernetes/controllers.yaml new file mode 100644 index 000000000000..378d9d8893e5 --- /dev/null +++ b/contrib/kubernetes/controllers.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + name: controllers + labels: + master.openshift.io/controllers: 'true' +spec: + containers: + - name: controller + image: openshift/origin:latest + args: + - start + - master + - controllers + - --listen=0.0.0.0:8444 + - --config=/etc/origin/master/master-config.yaml + volumeMounts: + - name: config + mountPath: /etc/origin/master + ports: + - containerPort: 8444 + name: https + volumes: + - hostPath: + path: /data/src/github.com/openshift/origin/openshift.local.test/master + name: config diff --git a/pkg/cmd/server/api/serialization_test.go b/pkg/cmd/server/api/serialization_test.go index 0fce8ef094ee..4bf2f8ce9c2b 100644 --- a/pkg/cmd/server/api/serialization_test.go +++ b/pkg/cmd/server/api/serialization_test.go @@ -43,6 +43,14 @@ func fuzzInternalObject(t *testing.T, forVersion schema.GroupVersion, item runti if len(obj.Controllers) == 0 { obj.Controllers = configapi.ControllersAll } + if election := obj.ControllerConfig.Election; election != nil { + if len(election.LockNamespace) == 0 { + election.LockNamespace = "kube-system" + } + if len(election.LockResource.Group) == 0 && len(election.LockResource.Resource) == 0 { + election.LockResource.Resource = "endpoints" + } + } if obj.ServingInfo.RequestTimeoutSeconds == 0 { obj.ServingInfo.RequestTimeoutSeconds = 60 * 60 } diff --git a/pkg/cmd/server/api/types.go b/pkg/cmd/server/api/types.go index 12057717f7de..426ec6b5a04d 100644 --- a/pkg/cmd/server/api/types.go +++ b/pkg/cmd/server/api/types.go @@ -334,12 +334,14 @@ type MasterConfig struct { Controllers string // PauseControllers instructs the master to not automatically start controllers, but instead // to wait until a notification to the server is received before launching them. - // TODO: will be disabled in function for 1.1. + // Deprecated: Will be removed in 3.7. PauseControllers bool - // ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire - // a lease before controllers start and renewing it within a number of seconds defined by this value. - // Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or + // ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to + // acquire a lease before controllers start and renewing it within a number of seconds defined by this + // value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or // omitted) and controller election can be disabled with -1. + // Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the + // appropriate leader election flags in controllerArguments. Will be removed in 3.9. ControllerLeaseTTL int // TODO: the next field added to controllers must be added to a new controllers struct @@ -1395,11 +1397,37 @@ type AdmissionConfig struct { // ControllerConfig holds configuration values for controllers type ControllerConfig struct { + // Election defines the configuration for electing a controller instance to make changes to + // the cluster. If unspecified, the ControllerTTL value is checked to determine whether the + // legacy direct etcd election code will be used. + Election *ControllerElectionConfig // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for // pods fulfilling a service to serve with. ServiceServingCert ServiceServingCert } +// ControllerElectionConfig contains configuration values for deciding how a controller +// will be elected to act as leader. +type ControllerElectionConfig struct { + // LockName is the resource name used to act as the lock for determining which controller + // instance should lead. + LockName string + // LockNamespace is the resource namespace used to act as the lock for determining which + // controller instance should lead. It defaults to "kube-system" + LockNamespace string + // LockResource is the group and resource name to use to coordinate for the controller lock. + // If unset, defaults to "endpoints". + LockResource GroupResource +} + +// GroupResource points to a resource by its name and API group. +type GroupResource struct { + // Group is the name of an API group + Group string + // Resource is the name of a resource. + Resource string +} + // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for // pods fulfilling a service to serve with. type ServiceServingCert struct { diff --git a/pkg/cmd/server/api/v1/conversions.go b/pkg/cmd/server/api/v1/conversions.go index ebb9a7b17600..c53049488d3c 100644 --- a/pkg/cmd/server/api/v1/conversions.go +++ b/pkg/cmd/server/api/v1/conversions.go @@ -20,6 +20,14 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error { if len(obj.Controllers) == 0 { obj.Controllers = ControllersAll } + if election := obj.ControllerConfig.Election; election != nil { + if len(election.LockNamespace) == 0 { + election.LockNamespace = "kube-system" + } + if len(election.LockResource.Group) == 0 && len(election.LockResource.Resource) == 0 { + election.LockResource.Resource = "endpoints" + } + } if obj.ServingInfo.RequestTimeoutSeconds == 0 { obj.ServingInfo.RequestTimeoutSeconds = 60 * 60 } diff --git a/pkg/cmd/server/api/v1/swagger_doc.go b/pkg/cmd/server/api/v1/swagger_doc.go index 5e78fab098f9..7ce8d4622baa 100644 --- a/pkg/cmd/server/api/v1/swagger_doc.go +++ b/pkg/cmd/server/api/v1/swagger_doc.go @@ -133,6 +133,7 @@ func (ClientConnectionOverrides) SwaggerDoc() map[string]string { var map_ControllerConfig = map[string]string{ "": "ControllerConfig holds configuration values for controllers", + "election": "Election defines the configuration for electing a controller instance to make changes to the cluster. If unspecified, the ControllerTTL value is checked to determine whether the legacy direct etcd election code will be used.", "serviceServingCert": "ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for pods fulfilling a service to serve with.", } @@ -140,6 +141,17 @@ func (ControllerConfig) SwaggerDoc() map[string]string { return map_ControllerConfig } +var map_ControllerElectionConfig = map[string]string{ + "": "ControllerElectionConfig contains configuration values for deciding how a controller will be elected to act as leader.", + "lockName": "LockName is the resource name used to act as the lock for determining which controller instance should lead.", + "lockNamespace": "LockNamespace is the resource namespace used to act as the lock for determining which controller instance should lead. It defaults to \"kube-system\"", + "lockResource": "LockResource is the group and resource name to use to coordinate for the controller lock. If unset, defaults to \"endpoints\".", +} + +func (ControllerElectionConfig) SwaggerDoc() map[string]string { + return map_ControllerElectionConfig +} + var map_DNSConfig = map[string]string{ "": "DNSConfig holds the necessary configuration options for DNS", "bindAddress": "BindAddress is the ip:port to serve DNS on", @@ -259,6 +271,16 @@ func (GrantConfig) SwaggerDoc() map[string]string { return map_GrantConfig } +var map_GroupResource = map[string]string{ + "": "GroupResource points to a resource by its name and API group.", + "group": "Group is the name of an API group", + "resource": "Resource is the name of a resource.", +} + +func (GroupResource) SwaggerDoc() map[string]string { + return map_GroupResource +} + var map_HTPasswdPasswordIdentityProvider = map[string]string{ "": "HTPasswdPasswordIdentityProvider provides identities for users authenticating using htpasswd credentials", "file": "File is a reference to your htpasswd file", @@ -463,8 +485,8 @@ var map_MasterConfig = map[string]string{ "apiLevels": "APILevels is a list of API levels that should be enabled on startup: v1 as examples", "masterPublicURL": "MasterPublicURL is how clients can access the OpenShift API server", "controllers": "Controllers is a list of the controllers that should be started. If set to \"none\", no controllers will start automatically. The default value is \"*\" which will start all controllers. When using \"*\", you may exclude controllers by prepending a \"-\" in front of their name. No other values are recognized at this time.", - "pauseControllers": "PauseControllers instructs the master to not automatically start controllers, but instead to wait until a notification to the server is received before launching them.", - "controllerLeaseTTL": "ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire a lease before controllers start and renewing it within a number of seconds defined by this value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or omitted) and controller election can be disabled with -1.", + "pauseControllers": "PauseControllers instructs the master to not automatically start controllers, but instead to wait until a notification to the server is received before launching them. This field is ignored if controllerConfig.lockServiceName is specified. Deprecated: Will be removed in 3.7.", + "controllerLeaseTTL": "ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to acquire a lease before controllers start and renewing it within a number of seconds defined by this value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or omitted) and controller election can be disabled with -1. This field is ignored if controllerConfig.lockServiceName is specified. Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the\n appropriate leader election flags in controllerArguments. Will be removed in 3.9.", "admissionConfig": "AdmissionConfig contains admission control plugin configuration.", "controllerConfig": "ControllerConfig holds configuration values for controllers", "disabledFeatures": "DisabledFeatures is a list of features that should not be started. We omitempty here because its very unlikely that anyone will want to manually disable features and we don't want to encourage it.", diff --git a/pkg/cmd/server/api/v1/types.go b/pkg/cmd/server/api/v1/types.go index 0360c433d97d..9bd76a292f7c 100644 --- a/pkg/cmd/server/api/v1/types.go +++ b/pkg/cmd/server/api/v1/types.go @@ -197,12 +197,17 @@ type MasterConfig struct { // values are recognized at this time. Controllers string `json:"controllers"` // PauseControllers instructs the master to not automatically start controllers, but instead - // to wait until a notification to the server is received before launching them. + // to wait until a notification to the server is received before launching them. This field is + // ignored if controllerConfig.lockServiceName is specified. + // Deprecated: Will be removed in 3.7. PauseControllers bool `json:"pauseControllers"` - // ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire - // a lease before controllers start and renewing it within a number of seconds defined by this value. - // Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or - // omitted) and controller election can be disabled with -1. + // ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to + // acquire a lease before controllers start and renewing it within a number of seconds defined by this + // value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or + // omitted) and controller election can be disabled with -1. This field is ignored if + // controllerConfig.lockServiceName is specified. + // Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the + // appropriate leader election flags in controllerArguments. Will be removed in 3.9. ControllerLeaseTTL int `json:"controllerLeaseTTL"` // AdmissionConfig contains admission control plugin configuration. @@ -1329,11 +1334,37 @@ type AdmissionConfig struct { // ControllerConfig holds configuration values for controllers type ControllerConfig struct { + // Election defines the configuration for electing a controller instance to make changes to + // the cluster. If unspecified, the ControllerTTL value is checked to determine whether the + // legacy direct etcd election code will be used. + Election *ControllerElectionConfig `json:"election"` // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for // pods fulfilling a service to serve with. ServiceServingCert ServiceServingCert `json:"serviceServingCert"` } +// ControllerElectionConfig contains configuration values for deciding how a controller +// will be elected to act as leader. +type ControllerElectionConfig struct { + // LockName is the resource name used to act as the lock for determining which controller + // instance should lead. + LockName string `json:"lockName"` + // LockNamespace is the resource namespace used to act as the lock for determining which + // controller instance should lead. It defaults to "kube-system" + LockNamespace string `json:"lockNamespace"` + // LockResource is the group and resource name to use to coordinate for the controller lock. + // If unset, defaults to "endpoints". + LockResource GroupResource `json:"lockResource"` +} + +// GroupResource points to a resource by its name and API group. +type GroupResource struct { + // Group is the name of an API group + Group string `json:"group"` + // Resource is the name of a resource. + Resource string `json:"resource"` +} + // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for // pods fulfilling a service to serve with. type ServiceServingCert struct { diff --git a/pkg/cmd/server/api/v1/types_test.go b/pkg/cmd/server/api/v1/types_test.go index ed543deca6c5..ed0be34afd43 100644 --- a/pkg/cmd/server/api/v1/types_test.go +++ b/pkg/cmd/server/api/v1/types_test.go @@ -118,6 +118,7 @@ auditConfig: authConfig: requestHeader: null controllerConfig: + election: null serviceServingCert: signer: null controllerLeaseTTL: 0 diff --git a/pkg/cmd/server/api/validation/master.go b/pkg/cmd/server/api/validation/master.go index 650af2b63efc..7508518b3054 100644 --- a/pkg/cmd/server/api/validation/master.go +++ b/pkg/cmd/server/api/validation/master.go @@ -249,6 +249,23 @@ func ValidateAuditConfig(config api.AuditConfig, fldPath *field.Path) Validation func ValidateControllerConfig(config api.ControllerConfig, fldPath *field.Path) ValidationResults { validationResults := ValidationResults{} + if election := config.Election; election != nil { + if len(election.LockName) == 0 { + validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockName"), election.LockName, "may not be empty")) + } + for _, msg := range kvalidation.ValidateServiceName(election.LockName, false) { + validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockName"), election.LockName, msg)) + } + if len(election.LockNamespace) == 0 { + validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockNamespace"), election.LockNamespace, "may not be empty")) + } + for _, msg := range kvalidation.ValidateNamespaceName(election.LockNamespace, false) { + validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockNamespace"), election.LockNamespace, msg)) + } + if len(election.LockResource.Resource) == 0 { + validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockResource", "resource"), election.LockResource.Resource, "may not be empty")) + } + } if config.ServiceServingCert.Signer != nil { validationResults.AddErrors(ValidateCertInfo(*config.ServiceServingCert.Signer, true, fldPath.Child("serviceServingCert.signer"))...) } diff --git a/pkg/cmd/server/origin/leaderelection.go b/pkg/cmd/server/origin/leaderelection.go new file mode 100644 index 000000000000..da2e574eb7bd --- /dev/null +++ b/pkg/cmd/server/origin/leaderelection.go @@ -0,0 +1,140 @@ +package origin + +import ( + "fmt" + "path" + "time" + + "github.com/golang/glog" + + kapierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kutilrand "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + v1corev1 "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/tools/record" + kapi "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/componentconfig" + kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/client/leaderelection" + rl "k8s.io/kubernetes/pkg/client/leaderelection/resourcelock" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" + "github.com/openshift/origin/pkg/cmd/server/etcd" + "github.com/openshift/origin/pkg/cmd/util/plug" + "github.com/openshift/origin/pkg/util/leaderlease" +) + +// NewLeaderElection returns a plug that blocks controller startup until the lease is acquired +// and a function that will start the process to attain the lease. There are two modes for +// lease operation - a legacy mode that directly connects to etcd, and the preferred mode which +// coordinates on a service endpoints object in the kube-system namespace. The legacy mode will +// periodically poll to see if the endpoints object exists, and if so will stand down, allowing +// newer controllers to take over. +func NewLeaderElection(options configapi.MasterConfig, leader componentconfig.LeaderElectionConfiguration, kc kclientsetexternal.Interface) (plug.Plug, func(), error) { + id := fmt.Sprintf("master-%s", kutilrand.String(8)) + name := "openshift-controller-manager" + namespace := "kube-system" + useEndpoints := false + if election := options.ControllerConfig.Election; election != nil { + if election.LockResource.Resource != "endpoints" || election.LockResource.Group != "" { + return nil, nil, fmt.Errorf("only the \"endpoints\" resource is supported for election") + } + name = election.LockName + namespace = election.LockNamespace + useEndpoints = true + } + + lock := &rl.EndpointsLock{ + EndpointsMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Client: kc, + LockConfig: rl.ResourceLockConfig{ + Identity: id, + }, + } + + // legacy path, for native etcd leases. Will periodically check for the controller service to exist and + // release any held lease if one is detected + if !useEndpoints { + ttl := time.Duration(options.ControllerLeaseTTL) * time.Second + if ttl == 0 { + return plug.New(!options.PauseControllers), func() {}, nil + } + + client, err := etcd.MakeEtcdClient(options.EtcdClientInfo) + if err != nil { + return nil, nil, err + } + + leaser := leaderlease.NewEtcd( + client, + path.Join(options.EtcdStorageConfig.OpenShiftStoragePrefix, "leases/controllers"), + id, + uint64(options.ControllerLeaseTTL), + ) + + leased := plug.NewLeased(leaser) + return leased, legacyLeaderElectionStart(id, name, leased, lock, ttl), nil + } + + // use the endpoints leader election path. + plug := plug.New(false) + events := record.NewBroadcaster() + events.StartLogging(glog.Infof) + events.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kc.Core().RESTClient()).Events("")}) + lock.LockConfig.EventRecorder = events.NewRecorder(kapi.Scheme, v1corev1.EventSource{Component: name}) + elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leader.LeaseDuration.Duration, + RenewDeadline: leader.RenewDeadline.Duration, + RetryPeriod: leader.RetryPeriod.Duration, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(stop <-chan struct{}) { + plug.Start() + }, + OnStoppedLeading: func() { + plug.Stop(fmt.Errorf("%s %s lost election, stepping down", name, id)) + }, + }, + }) + if err != nil { + return nil, nil, err + } + return plug, func() { + glog.V(2).Infof("Attempting to acquire %s lease as %s, renewing every %s, holding for %s, and giving up after %s", name, id, leader.RetryPeriod.Duration, leader.LeaseDuration.Duration, leader.RenewDeadline.Duration) + go elector.Run() + }, nil +} + +// legacyLeaderElectionStart waits to verify lock has not been taken, then attempts to acquire and hold +// the legacy lease. If it detects the lock is acquired it will stop immediately. +func legacyLeaderElectionStart(id, name string, leased *plug.Leased, lock rl.Interface, ttl time.Duration) func() { + return func() { + glog.V(2).Infof("Verifying no controller manager is running for %s", id) + wait.PollInfinite(ttl/2, func() (bool, error) { + _, err := lock.Get() + if err == nil { + return false, nil + } + if kapierrors.IsNotFound(err) { + return true, nil + } + utilruntime.HandleError(fmt.Errorf("unable to confirm %s lease exists: %v", name, err)) + return false, nil + }) + glog.V(2).Infof("Attempting to acquire controller lease as %s, renewing every %s", id, ttl) + go leased.Run() + go wait.PollInfinite(ttl/2, func() (bool, error) { + _, err := lock.Get() + if err == nil { + glog.V(2).Infof("%s lease has been taken, %s is exiting", name, id) + leased.Stop(nil) + return true, nil + } + utilruntime.HandleError(fmt.Errorf("unable to confirm %s lease exists: %v", name, err)) + return false, nil + }) + } +} diff --git a/pkg/cmd/server/origin/master_config.go b/pkg/cmd/server/origin/master_config.go index c89ab4951b3e..7b7f267e9184 100644 --- a/pkg/cmd/server/origin/master_config.go +++ b/pkg/cmd/server/origin/master_config.go @@ -10,7 +10,6 @@ import ( "strings" "time" - etcdclient "github.com/coreos/etcd/client" "github.com/golang/glog" kapierrors "k8s.io/apimachinery/pkg/api/errors" @@ -19,7 +18,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - kutilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/admission" @@ -73,7 +71,6 @@ import ( oadmission "github.com/openshift/origin/pkg/cmd/server/admission" configapi "github.com/openshift/origin/pkg/cmd/server/api" "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" - "github.com/openshift/origin/pkg/cmd/server/etcd" kubernetes "github.com/openshift/origin/pkg/cmd/server/kubernetes/master" originrest "github.com/openshift/origin/pkg/cmd/server/origin/rest" "github.com/openshift/origin/pkg/cmd/util/clientcmd" @@ -102,7 +99,6 @@ import ( groupstorage "github.com/openshift/origin/pkg/user/registry/group/etcd" userregistry "github.com/openshift/origin/pkg/user/registry/user" useretcd "github.com/openshift/origin/pkg/user/registry/user/etcd" - "github.com/openshift/origin/pkg/util/leaderlease" "github.com/openshift/origin/pkg/util/restoptions" ) @@ -190,11 +186,6 @@ type MasterConfig struct { // BuildMasterConfig builds and returns the OpenShift master configuration based on the // provided options func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) { - client, err := etcd.MakeEtcdClient(options.EtcdClientInfo) - if err != nil { - return nil, err - } - restOptsGetter, err := originrest.StorageOptions(options) if err != nil { return nil, err @@ -301,8 +292,6 @@ func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) { return nil, err } - plug, plugStart := newControllerPlug(options, client) - config := &MasterConfig{ Options: options, @@ -326,9 +315,6 @@ func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) { TLS: configapi.UseTLS(options.ServingInfo.ServingInfo), - ControllerPlug: plug, - ControllerPlugStart: plugStart, - ImageFor: imageTemplate.ExpandOrDie, RegistryNameFn: imageapi.DefaultRegistryFunc(defaultRegistryFunc), @@ -634,27 +620,6 @@ func newAdmissionChain(pluginNames []string, admissionConfigFilename string, plu return admission.NewChainHandler(plugins...), nil } -func newControllerPlug(options configapi.MasterConfig, client etcdclient.Client) (plug.Plug, func()) { - switch { - case options.ControllerLeaseTTL > 0: - // TODO: replace with future API for leasing from Kube - id := fmt.Sprintf("master-%s", kutilrand.String(8)) - leaser := leaderlease.NewEtcd( - client, - path.Join(options.EtcdStorageConfig.OpenShiftStoragePrefix, "leases/controllers"), - id, - uint64(options.ControllerLeaseTTL), - ) - leased := plug.NewLeased(leaser) - return leased, func() { - glog.V(2).Infof("Attempting to acquire controller lease as %s, renewing every %d seconds", id, options.ControllerLeaseTTL) - go leased.Run() - } - default: - return plug.New(!options.PauseControllers), func() {} - } -} - func newServiceAccountTokenGetter(options configapi.MasterConfig) (serviceaccount.ServiceAccountTokenGetter, error) { if options.KubernetesMasterConfig == nil { // When we're running against an external Kubernetes, use the external kubernetes client to validate service account tokens diff --git a/pkg/cmd/server/start/start_controllers.go b/pkg/cmd/server/start/start_controllers.go index cdc904517856..583af81bf5ab 100644 --- a/pkg/cmd/server/start/start_controllers.go +++ b/pkg/cmd/server/start/start_controllers.go @@ -78,10 +78,20 @@ func NewCommandStartMasterControllers(name, basename string, out, errout io.Writ }.Default(), } + var lockServiceName string options.MasterArgs = NewDefaultMasterArgs() options.MasterArgs.StartControllers = true options.MasterArgs.OverrideConfig = func(config *configapi.MasterConfig) error { config.ServingInfo.BindAddress = listenArg.ListenAddr.URL.Host + if len(lockServiceName) > 0 { + config.ControllerConfig.Election = &configapi.ControllerElectionConfig{ + LockName: lockServiceName, + LockNamespace: "kube-system", + LockResource: configapi.GroupResource{ + Resource: "endpoints", + }, + } + } return nil } @@ -89,6 +99,7 @@ func NewCommandStartMasterControllers(name, basename string, out, errout io.Writ // This command only supports reading from config and the listen argument flags.StringVar(&options.ConfigFile, "config", "", "Location of the master configuration file to run from. Required") cmd.MarkFlagFilename("config", "yaml", "yml") + flags.StringVar(&lockServiceName, "lock-service-name", "", "Name of a service in the kube-system namespace to use as a lock, overrides config.") BindListenArg(listenArg, flags, "") return cmd, options diff --git a/pkg/cmd/server/start/start_master.go b/pkg/cmd/server/start/start_master.go index ec5b4093656c..e87c1f1d29fe 100644 --- a/pkg/cmd/server/start/start_master.go +++ b/pkg/cmd/server/start/start_master.go @@ -406,6 +406,18 @@ func (m *Master) Start() error { return err } + // initialize the election module if the controllers will start + if m.controllers { + openshiftConfig.ControllerPlug, openshiftConfig.ControllerPlugStart, err = origin.NewLeaderElection( + *m.config, + kubeMasterConfig.ControllerManager.KubeControllerManagerConfiguration.LeaderElection, + openshiftConfig.PrivilegedLoopbackKubernetesClientsetExternal, + ) + if err != nil { + return err + } + } + // any controller that uses a core informer must be initialized *before* the API server starts core informers // the API server adds its controllers at the correct time, but if the controllers are running, they need to be // kicked separately