From e9f8847c1ea9543c63d90806bbf9e7fb41fbcd85 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 3 Sep 2024 14:23:35 +0530 Subject: [PATCH 1/3] Changed batch size back to 200k. --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index ae9fc913e5..0ca26aa911 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -189,7 +189,7 @@ public class EntityGraphMapper { private static final boolean RESTRICT_PROPAGATION_THROUGH_LINEAGE_DEFAULT = false; private static final boolean RESTRICT_PROPAGATION_THROUGH_HIERARCHY_DEFAULT = false; - public static final int CLEANUP_BATCH_SIZE = 2000; + public static final int CLEANUP_BATCH_SIZE = 200000; private boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); private boolean DIFFERENTIAL_AUDITS = STORE_DIFFERENTIAL_AUDITS.getBoolean(); From 6822a7e12e4cdeb44ca7ca8597edf9fd92fd8edd Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 3 Sep 2024 15:03:33 +0530 Subject: [PATCH 2/3] Fixed PR comment --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 0ca26aa911..c4469fc4ed 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3103,6 +3103,7 @@ public void cleanUpClassificationPropagation(String classificationName, int batc cleanedUpCount += currentAssetsBatchSize; currentAssetVerticesBatch.clear(); + tagVerticesProcessed.clear(); } tagVertices = GraphHelper.getClassificationVertices(graph, classificationName, CLEANUP_BATCH_SIZE); } From 24e7c907dbd4757459f7fcd04e6c3aa920d37f34 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Thu, 5 Sep 2024 22:31:14 +0530 Subject: [PATCH 3/3] Added another stop condition for cleanup task --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index c4469fc4ed..865d58a2b2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3041,7 +3041,7 @@ public void cleanUpClassificationPropagation(String classificationName, int batc List tagVerticesProcessed = new ArrayList<>(0); List currentAssetVerticesBatch = new ArrayList<>(0); - while (tagVertices != null) { + while (tagVertices != null && tagVertices.hasNext()) { if (cleanedUpCount >= CLEANUP_MAX){ return; }