From 9e705b2fff804ef5526d0fe78e4d2a88f76c5015 Mon Sep 17 00:00:00 2001 From: marshallbrekka Date: Thu, 23 Mar 2017 10:32:50 -0700 Subject: [PATCH] Problem: Etcd3 locks are not released if shutdown improperly. Solution: Write the lock item with the lease. --- physical/etcd3.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/physical/etcd3.go b/physical/etcd3.go index 420d8028bcd9..8076826aece3 100644 --- a/physical/etcd3.go +++ b/physical/etcd3.go @@ -32,6 +32,9 @@ type EtcdBackend struct { etcd *clientv3.Client } +// etcd default lease duration is 60s. set to 15s for faster recovery. +const etcd3LockTimeoutInSeconds = 15 + // newEtcd3Backend constructs a etcd3 backend. func newEtcd3Backend(conf map[string]string, logger log.Logger) (Backend, error) { // Get the etcd path form the configuration. @@ -228,7 +231,7 @@ type EtcdLock struct { // Lock is used for mutual exclusion based on the given key. func (c *EtcdBackend) LockWith(key, value string) (Lock, error) { - session, err := concurrency.NewSession(c.etcd) + session, err := concurrency.NewSession(c.etcd, concurrency.WithTTL(etcd3LockTimeoutInSeconds)) if err != nil { return nil, err } @@ -262,7 +265,7 @@ func (c *EtcdLock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) { } return nil, err } - if _, err := c.etcd.Put(ctx, c.etcdMu.Key(), c.value); err != nil { + if _, err := c.etcd.Put(ctx, c.etcdMu.Key(), c.value, clientv3.WithLease(c.etcdSession.Lease())); err != nil { return nil, err }