diff --git a/flow-data/src/main/java/com/vaadin/flow/data/provider/hierarchy/HierarchicalCommunicationController.java b/flow-data/src/main/java/com/vaadin/flow/data/provider/hierarchy/HierarchicalCommunicationController.java
index 3efb9354128..5833669a482 100644
--- a/flow-data/src/main/java/com/vaadin/flow/data/provider/hierarchy/HierarchicalCommunicationController.java
+++ b/flow-data/src/main/java/com/vaadin/flow/data/provider/hierarchy/HierarchicalCommunicationController.java
@@ -287,7 +287,14 @@ private void passivateInactiveKeys(Set<String> oldActive,
             }
 
             // Finally clear any passivated items that have now been confirmed
-            oldActive.removeAll(newActiveKeyOrder);
+            /*
+             * Due to the implementation of AbstractSet#removeAll, if the size
+             * of newActiveKeyOrder is bigger than oldActive's size, then
+             * calling oldActive.removeAll(newActiveKeyOrder) would end up
+             * calling contains method for all of newActiveKeyOrder items which
+             * is a slow operation on lists. The following avoids that:
+             */
+            newActiveKeyOrder.forEach(oldActive::remove);
             if (!oldActive.isEmpty()) {
                 passivatedByUpdate.put(Integer.valueOf(updateId), oldActive);
             }