Merge pull request #3797 from atlanhq/dev/janusprefetchwith0.6

PLT-2782 | Janus optimisation
atlanhq · Dec 2, 2024 · d395a6d · d395a6d
2 parents 8d40bb3 + 4fc976d
commit d395a6d
Show file tree

Hide file tree

Showing 7 changed files with 181 additions and 180 deletions.
diff --git a/...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
@@ -334,6 +334,7 @@ private void setPolicyItems(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPo
 
         List<String> users = (List<String>) atlasPolicy.getAttribute("policyUsers");
         List<String> groups = (List<String>) atlasPolicy.getAttribute("policyGroups");
+
         List<String> roles = (List<String>) atlasPolicy.getAttribute("policyRoles");
 
         List<RangerPolicyItemAccess> accesses = new ArrayList<>();

diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java
@@ -50,6 +50,24 @@ public void recordMetric(MetricRecorder recorder) {
         }
     }
 
+    public void recordMetricWithInvocations(MetricRecorder recorder, long count) {
+        if (recorder != null) {
+            final String name = recorder.name;
+            final long timeTaken = recorder.getElapsedTime();
+
+            Metric metric = metrics.get(name);
+
+            if (metric == null) {
+                metric = new Metric(name);
+
+                metrics.put(name, metric);
+            }
+
+            metric.invocations += count;
+            metric.totalTimeMSecs += timeTaken;
+        }
+    }
+
     public void clear() {
         metrics.clear();
     }
@@ -129,7 +147,6 @@ public String getName() {
         public long getInvocations() {
             return invocations;
         }
-
         public void setTotalTimeMSecs(long totalTimeMSecs) {
             this.totalTimeMSecs = totalTimeMSecs;
         }

diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -133,7 +133,9 @@ public enum AtlasConfiguration {
     ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000),
     ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""),
     ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false),
-
+    ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false),
+    ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false),
+    ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT("atlas.indexsearch.attributes.min.limit", 8),
     ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false),
 
     ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100);

diff --git a/...sitory/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/...sitory/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java
@@ -59,6 +59,11 @@ public final class AccessControlUtils {
     public static final String ATTR_PERSONA_USERS   = "personaUsers";
     public static final String ATTR_PERSONA_GROUPS  = "personaGroups";
 
+    public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType";
+    public static final String ATTR_SERVICE_TAG_SERVICE  = "tagService";
+    public static final String ATTR_SERVICE_IS_ENABLED   = "authServiceIsEnabled";
+    public static final String ATTR_SERVICE_LAST_SYNC    = "authServicePolicyLastSync";
+
     public static final String ATTR_PURPOSE_CLASSIFICATIONS  = "purposeClassifications";
 
     public static final String ATTR_POLICY_TYPE  = "policyType";

diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -657,6 +657,13 @@ public void endMetricRecord(MetricRecorder recorder) {
         }
     }
 
+    public void endMetricRecordWithInvocations(MetricRecorder recorder, long invocationCount) {
+        if (metrics != null && recorder != null) {
+            metrics.recordMetricWithInvocations(recorder, invocationCount);
+        }
+    }
+
+
     public void recordEntityGuidUpdate(AtlasEntity entity, String guidInRequest) {
         recordEntityGuidUpdate(new EntityGuidPair(entity, guidInRequest));
     }

diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -401,6 +401,8 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest,
                 perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")");
             }
 
+            RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("dslSize"), parameters.getQuerySize());
+            RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("attributeSize"), parameters.getAttributes().size());
             if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_API_LIMIT.getBoolean() && parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong()) {
                 if(CollectionUtils.isEmpty(parameters.getUtmTags())) {
                     throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong()));