Add eviction cycle metrics

docs/metrics.md

@@ -156,6 +156,8 @@ scp.timing.self-to-others-externalize-lag | timer     | delay between local node
 scp.value.invalid                         | meter     | SCP value is invalid
 scp.value.valid                           | meter     | SCP value is valid
 scp.slot.values-referenced                | histogram | number of values referenced per consensus round
+state-archival.eviction.age               | counter   | the average of the delta between an entry's liveUntilLedger and the ledger when it is evicted
 state-archival.eviction.bytes-scanned     | counter   | number of bytes that eviction scan has read
 state-archival.eviction.entries-evicted   | counter   | number of entries that have been evicted
 state-archival.eviction.incomplete-scan   | counter   | number of buckets that were too large to be fully scanned for eviction
+state-archival.eviction.period            | counter   | number of ledgers to complete an eviction scan

src/bucket/Bucket.cpp

@@ -841,7 +841,8 @@ Bucket::scanForEviction(AbstractLedgerTxn& ltx, EvictionIterator& iter,
                         uint64_t& bytesToScan, uint32_t& maxEntriesToEvict,
                         uint32_t ledgerSeq,
                         medida::Counter& entriesEvictedCounter,
-                        medida::Counter& bytesScannedForEvictionCounter)
+                        medida::Counter& bytesScannedForEvictionCounter,
+                        std::optional<EvictionMetrics>& metrics)
 {
     ZoneScoped;
     if (isEmpty())
@@ -876,6 +877,7 @@ Bucket::scanForEviction(AbstractLedgerTxn& ltx, EvictionIterator& iter,
                 auto initialStreamPos = stream.pos();
 
                 auto ttlKey = getTTLKey(le);
+                uint32_t liveUntilLedger = 0;
                 auto shouldEvict = [&] {
                     auto entryLtxe = ltx.loadWithoutRecord(LedgerEntryKey(le));
                     auto ttlLtxe = ltx.loadWithoutRecord(ttlKey);
@@ -888,13 +890,21 @@ Bucket::scanForEviction(AbstractLedgerTxn& ltx, EvictionIterator& iter,
                     }
 
                     releaseAssert(ttlLtxe);
-
+                    liveUntilLedger =
+                        ttlLtxe.current().data.ttl().liveUntilLedgerSeq;
                     return !isLive(ttlLtxe.current(), ledgerSeq);
                 };
 
                 if (shouldEvict())
                 {
                     ZoneNamedN(evict, "evict entry", true);
+                    if (metrics.has_value())
+                    {
+                        ++metrics->numEntriesEvicted;
+                        metrics->evictedEntriesAgeSum +=
+                            ledgerSeq - liveUntilLedger;
+                    }
+
                     ltx.erase(ttlKey);
                     ltx.erase(LedgerEntryKey(le));
                     entriesEvictedCounter.inc();

src/bucket/Bucket.h

@@ -38,6 +38,7 @@ namespace stellar
 class AbstractLedgerTxn;
 class Application;
 class BucketManager;
+struct EvictionMetrics;
 
 class Bucket : public std::enable_shared_from_this<Bucket>,
                public NonMovableOrCopyable
@@ -145,7 +146,8 @@ class Bucket : public std::enable_shared_from_this<Bucket>,
                          uint64_t& bytesToScan, uint32_t& maxEntriesToEvict,
                          uint32_t ledgerSeq,
                          medida::Counter& entriesEvictedCounter,
-                         medida::Counter& bytesScannedForEvictionCounter);
+                         medida::Counter& bytesScannedForEvictionCounter,
+                         std::optional<EvictionMetrics>& metrics);
 
 #ifdef BUILD_TESTS
     // "Applies" the bucket to the database. For each entry in the bucket,

src/bucket/BucketList.cpp

@@ -831,7 +831,9 @@ BucketList::scanForEviction(Application& app, AbstractLedgerTxn& ltx,
                             uint32_t ledgerSeq,
                             medida::Counter& entriesEvictedCounter,
                             medida::Counter& bytesScannedForEvictionCounter,
-                            medida::Counter& incompleteBucketScanCounter)
+                            medida::Counter& incompleteBucketScanCounter,
+                            medida::Counter& evictionCyclePeriodCounter,
+                            medida::Counter& averageEvictedEntryAgeCounter)
 {
     auto getBucketFromIter = [&levels = mLevels](EvictionIterator const& iter) {
         auto& level = levels.at(iter.bucketListLevel);
@@ -894,7 +896,8 @@ BucketList::scanForEviction(Application& app, AbstractLedgerTxn& ltx,
         auto b = getBucketFromIter(evictionIter);
         while (!b->scanForEviction(
             ltx, evictionIter, scanSize, maxEntriesToEvict, ledgerSeq,
-            entriesEvictedCounter, bytesScannedForEvictionCounter))
+            entriesEvictedCounter, bytesScannedForEvictionCounter,
+            mEvictionMetrics))
         {
             // If we reached eof in curr bucket, start scanning snap.
             // Last level has no snap so cycle back to the initial level.
@@ -916,6 +919,26 @@ BucketList::scanForEviction(Application& app, AbstractLedgerTxn& ltx,
                 if (evictionIter.bucketListLevel == kNumLevels)
                 {
                     evictionIter.bucketListLevel = firstScanLevel;
+
+                    // If eviction metrics are not null, we have accounted for a
+                    // complete cycle and should log the metrics
+                    if (mEvictionMetrics)
+                    {
+                        evictionCyclePeriodCounter.set_count(
+                            ledgerSeq -
+                            mEvictionMetrics->evictionCycleStartLedger);
+
+                        auto averageAge =
+                            mEvictionMetrics->numEntriesEvicted == 0
+                                ? 0
+                                : mEvictionMetrics->evictedEntriesAgeSum /
+                                      mEvictionMetrics->numEntriesEvicted;
+                        averageEvictedEntryAgeCounter.set_count(averageAge);
+                    }
+
+                    // Reset metrics at beginning of new eviction cycle
+                    mEvictionMetrics = std::make_optional<EvictionMetrics>();
+                    mEvictionMetrics->evictionCycleStartLedger = ledgerSeq;
                 }
             }
 

src/bucket/BucketList.h

@@ -402,10 +402,24 @@ class BucketListDepth
     friend class testutil::BucketListDepthModifier;
 };
 
+struct EvictionMetrics
+{
+    // Evicted entry "age" is the delta between its liveUntilLedger and the
+    // ledger when the entry is actually evicted
+    uint64_t evictedEntriesAgeSum;
+    uint64_t numEntriesEvicted;
+    uint32_t evictionCycleStartLedger;
+};
+
 class BucketList
 {
     std::vector<BucketLevel> mLevels;
 
+    // To avoid noisy data, only count metrics that encompass a complete
+    // eviction cycle. If a node joins the network mid cycle, metrics will be
+    // nullopt and be initialized at the start of the next cycle.
+    std::optional<EvictionMetrics> mEvictionMetrics;
+
     // Loops through all buckets, starting with curr at level 0, then snap at
     // level 0, etc. Calls f on each bucket. Exits early if function
     // returns true
@@ -528,6 +542,8 @@ class BucketList
                          uint32_t ledgerSeq,
                          medida::Counter& entriesEvictedCounter,
                          medida::Counter& bytesScannedForEvictionCounter,
-                         medida::Counter& incompleteBucketScanCounter);
+                         medida::Counter& incompleteBucketScanCounter,
+                         medida::Counter& evictionCyclePeriodCounter,
+                         medida::Counter& averageEvictedEntryAgeCounter);
 };
 }

src/bucket/BucketManagerImpl.cpp

@@ -130,6 +130,10 @@ BucketManagerImpl::BucketManagerImpl(Application& app)
           {"state-archival", "eviction", "incomplete-scan"}))
     , mBucketListSizeCounter(
           app.getMetrics().NewCounter({"bucketlist", "size", "bytes"}))
+    , mEvictionCyclePeriod(
+          app.getMetrics().NewCounter({"state-archival", "eviction", "period"}))
+    , mAverageEvictedEntryAge(
+          app.getMetrics().NewCounter({"state-archival", "eviction", "age"}))
     // Minimal DB is stored in the buckets dir, so delete it only when
     // mode does not use minimal DB
     , mDeleteEntireBucketDirInDtor(
@@ -904,9 +908,10 @@ BucketManagerImpl::scanForEviction(AbstractLedgerTxn& ltx, uint32_t ledgerSeq)
     if (protocolVersionStartsFrom(ltx.getHeader().ledgerVersion,
                                   SOROBAN_PROTOCOL_VERSION))
     {
-        mBucketList->scanForEviction(mApp, ltx, ledgerSeq, mEntriesEvicted,
-                                     mBytesScannedForEviction,
-                                     mIncompleteBucketScans);
+        mBucketList->scanForEviction(
+            mApp, ltx, ledgerSeq, mEntriesEvicted, mBytesScannedForEviction,
+            mIncompleteBucketScans, mEvictionCyclePeriod,
+            mAverageEvictedEntryAge);
     }
 }
 

src/bucket/BucketManagerImpl.h

@@ -54,6 +54,8 @@ class BucketManagerImpl : public BucketManager
     medida::Counter& mBytesScannedForEviction;
     medida::Counter& mIncompleteBucketScans;
     medida::Counter& mBucketListSizeCounter;
+    medida::Counter& mEvictionCyclePeriod;
+    medida::Counter& mAverageEvictedEntryAge;
     mutable UnorderedMap<LedgerEntryType, medida::Timer&>
         mBucketListDBPointTimers{};
     mutable UnorderedMap<std::string, medida::Timer&> mBucketListDBBulkTimers{};