From 3633127ce137e9fb3b95d904a17a71908db7812c Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 25 May 2019 15:29:00 -0400
Subject: [PATCH 1/2] Add debug log for retention leases

---
 .../elasticsearch/index/seqno/ReplicationTracker.java    | 9 +++++++++
 .../org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index 892056674019f..ede534e50cdfa 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -227,6 +227,9 @@ public synchronized Tuple<Boolean, RetentionLeases> getRetentionLeases(final boo
         }
         final Collection<RetentionLease> nonExpiredLeases =
                 partitionByExpiration.get(false) != null ? partitionByExpiration.get(false) : Collections.emptyList();
+        if (logger.isDebugEnabled()) {
+            logger.debug("has expired retention leases: current leases [{}]; non-expired leases [{}]", retentionLeases, nonExpiredLeases);
+        }
         retentionLeases = new RetentionLeases(operationPrimaryTerm, retentionLeases.version() + 1, nonExpiredLeases);
         return Tuple.tuple(true, retentionLeases);
     }
@@ -255,6 +258,9 @@ public RetentionLease addRetentionLease(
                 throw new RetentionLeaseAlreadyExistsException(id);
             }
             retentionLease = new RetentionLease(id, retainingSequenceNumber, currentTimeMillisSupplier.getAsLong(), source);
+            if (logger.isDebugEnabled()) {
+                logger.debug("add new retention lease [{}]; current retention leases [{}]", retentionLease, retentionLeases);
+            }
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,
@@ -312,6 +318,9 @@ public void removeRetentionLease(final String id, final ActionListener<Replicati
             if (retentionLeases.contains(id) == false) {
                 throw new RetentionLeaseNotFoundException(id);
             }
+            if (logger.isDebugEnabled()) {
+                logger.debug("remove retention lease [{}]; current retention leases [{}]", id, retentionLeases);
+            }
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
index 9595485e2721c..eb4f4be84233b 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
@@ -88,7 +88,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 
-@TestLogging(value = "org.elasticsearch.xpack.ccr:trace,org.elasticsearch.indices.recovery:trace")
+@TestLogging(value = "org.elasticsearch.xpack.ccr:trace,org.elasticsearch.indices.recovery:trace,org.elasticsearch.index.seqno:debug")
 public class CcrRetentionLeaseIT extends CcrIntegTestCase {
 
     public static final class RetentionLeaseRenewIntervalSettingPlugin extends Plugin {

From a43006015a8b5fb8f3419598ee26b7a1b54d04bc Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sun, 26 May 2019 10:06:00 -0400
Subject: [PATCH 2/2] Iteration

---
 .../index/seqno/ReplicationTracker.java          | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index ede534e50cdfa..cf0fe6a5d25e1 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -221,15 +221,15 @@ public synchronized Tuple<Boolean, RetentionLeases> getRetentionLeases(final boo
                 .leases()
                 .stream()
                 .collect(Collectors.groupingBy(lease -> currentTimeMillis - lease.timestamp() > retentionLeaseMillis));
-        if (partitionByExpiration.get(true) == null) {
+        final Collection<RetentionLease> expiredLeases = partitionByExpiration.get(true);
+        if (expiredLeases == null) {
             // early out as no retention leases have expired
+            logger.debug("no retention leases are expired from current retention leases [{}]", retentionLeases);
             return Tuple.tuple(false, retentionLeases);
         }
         final Collection<RetentionLease> nonExpiredLeases =
                 partitionByExpiration.get(false) != null ? partitionByExpiration.get(false) : Collections.emptyList();
-        if (logger.isDebugEnabled()) {
-            logger.debug("has expired retention leases: current leases [{}]; non-expired leases [{}]", retentionLeases, nonExpiredLeases);
-        }
+        logger.debug("expiring retention leases [{}] from current retention leases [{}]", expiredLeases, retentionLeases);
         retentionLeases = new RetentionLeases(operationPrimaryTerm, retentionLeases.version() + 1, nonExpiredLeases);
         return Tuple.tuple(true, retentionLeases);
     }
@@ -258,9 +258,7 @@ public RetentionLease addRetentionLease(
                 throw new RetentionLeaseAlreadyExistsException(id);
             }
             retentionLease = new RetentionLease(id, retainingSequenceNumber, currentTimeMillisSupplier.getAsLong(), source);
-            if (logger.isDebugEnabled()) {
-                logger.debug("add new retention lease [{}]; current retention leases [{}]", retentionLease, retentionLeases);
-            }
+            logger.debug("adding new retention lease [{}] to current retention leases [{}]", retentionLease, retentionLeases);
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,
@@ -318,9 +316,7 @@ public void removeRetentionLease(final String id, final ActionListener<Replicati
             if (retentionLeases.contains(id) == false) {
                 throw new RetentionLeaseNotFoundException(id);
             }
-            if (logger.isDebugEnabled()) {
-                logger.debug("remove retention lease [{}]; current retention leases [{}]", id, retentionLeases);
-            }
+            logger.debug("removing retention lease [{}] from current retention leases [{}]", id, retentionLeases);
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,