Skip to content

Commit

Permalink
RANGER-4100: Efficient computation of the smallest set of evaluators …
Browse files Browse the repository at this point in the history 
…returned by search of multiple Trie trees - #2

(cherry picked from commit 93e888f)
  • Loading branch information
@mneethiraj
mneethiraj committed Feb 23, 2023
1 parent 63ba221 commit 0f930c9
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 111 deletions.
153 changes: 47 additions & 106 deletions ...ts-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.RangerPolicyResourceEvaluator;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.slf4j.Logger;
Expand Down Expand Up @@ -72,30 +73,30 @@ enum AuditModeEnum {
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
}

private final String serviceName;
private final String zoneName;
private final String appId;
private final RangerPolicyEngineOptions options;
private final RangerPluginContext pluginContext;
private final RangerServiceDef serviceDef;
private /*final*/ List<RangerPolicy> policies;
private final long policyVersion;
private /*final*/ List<RangerContextEnricher> contextEnrichers;
private final AuditModeEnum auditModeEnum;
private final Map<String, AuditInfo> accessAuditCache;
private final String componentServiceName;
private final RangerServiceDef componentServiceDef;
private final Map<String, RangerResourceTrie> policyResourceTrie;
private final Map<String, RangerResourceTrie> dataMaskResourceTrie;
private final Map<String, RangerResourceTrie> rowFilterResourceTrie;
private final Map<String, RangerResourceTrie> auditFilterResourceTrie;
private List<RangerPolicyEvaluator> policyEvaluators;
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
private boolean isContextEnrichersShared = false;
private boolean isPreCleaned = false;
private final String serviceName;
private final String zoneName;
private final String appId;
private final RangerPolicyEngineOptions options;
private final RangerPluginContext pluginContext;
private final RangerServiceDef serviceDef;
private /*final*/ List<RangerPolicy> policies;
private final long policyVersion;
private /*final*/ List<RangerContextEnricher> contextEnrichers;
private final AuditModeEnum auditModeEnum;
private final Map<String, AuditInfo> accessAuditCache;
private final String componentServiceName;
private final RangerServiceDef componentServiceDef;
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> policyResourceTrie;
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> dataMaskResourceTrie;
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> rowFilterResourceTrie;
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> auditFilterResourceTrie;
private List<RangerPolicyEvaluator> policyEvaluators;
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
private boolean isContextEnrichersShared = false;
private boolean isPreCleaned = false;

RangerPolicyRepository(final RangerPolicyRepository other, final List<RangerPolicyDelta> deltas, long policyVersion) {
this.serviceName = other.serviceName;
Expand All @@ -117,8 +118,8 @@ enum AuditModeEnum {
if (other.policyResourceTrie != null) {
this.policyResourceTrie = new HashMap<>();

for (Map.Entry<String, RangerResourceTrie> entry : other.policyResourceTrie.entrySet()) {
policyResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.policyResourceTrie.entrySet()) {
policyResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.policyResourceTrie = null;
Expand All @@ -127,8 +128,8 @@ enum AuditModeEnum {
if (other.dataMaskResourceTrie != null) {
this.dataMaskResourceTrie = new HashMap<>();

for (Map.Entry<String, RangerResourceTrie> entry : other.dataMaskResourceTrie.entrySet()) {
dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.dataMaskResourceTrie.entrySet()) {
dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.dataMaskResourceTrie = null;
Expand All @@ -137,8 +138,8 @@ enum AuditModeEnum {
if (other.rowFilterResourceTrie != null) {
this.rowFilterResourceTrie = new HashMap<>();

for (Map.Entry<String, RangerResourceTrie> entry : other.rowFilterResourceTrie.entrySet()) {
rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.rowFilterResourceTrie.entrySet()) {
rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.rowFilterResourceTrie = null;
Expand All @@ -147,8 +148,8 @@ enum AuditModeEnum {
if (other.auditFilterResourceTrie != null) {
this.auditFilterResourceTrie = new HashMap<>();

for (Map.Entry<String, RangerResourceTrie> entry : other.auditFilterResourceTrie.entrySet()) {
auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.auditFilterResourceTrie.entrySet()) {
auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.auditFilterResourceTrie = null;
Expand All @@ -157,7 +158,7 @@ enum AuditModeEnum {
if (other.accessAuditCache != null) {
int auditResultCacheSize = other.accessAuditCache.size();

this.accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
this.accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
} else {
this.accessAuditCache = null;
}
Expand Down Expand Up @@ -217,7 +218,7 @@ public RangerPolicyRepository(ServicePolicies servicePolicies, RangerPluginConte
final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024;

int auditResultCacheSize = pluginContext.getConfig().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
} else {
accessAuditCache = null;
}
Expand Down Expand Up @@ -587,7 +588,7 @@ List<PolicyEvaluatorForTag> getLikelyMatchPolicyEvaluators(RangerAccessRequest r

if (CollectionUtils.isNotEmpty(tags) && getServiceDef() != null) {

ret = new ArrayList<PolicyEvaluatorForTag>();
ret = new ArrayList<>();

for (RangerTagForEval tag : tags) {
if (tag.isApplicable(accessTime)) {
Expand Down Expand Up @@ -691,7 +692,7 @@ List<RangerPolicyEvaluator> getLikelyMatchAuditPolicyEvaluators(RangerAccessRequ
return auditFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getAuditPolicyEvaluators() : getLikelyMatchPolicyEvaluators(auditFilterResourceTrie, request);
}

private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie> resourceTrie, RangerAccessRequest request) {
private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> resourceTrie, RangerAccessRequest request) {
List<RangerPolicyEvaluator> ret = Collections.EMPTY_LIST;

RangerAccessResource resource = request.getResource();
Expand All @@ -702,67 +703,7 @@ private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, R
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")");
}

List<String> resourceKeys = resource == null ? null : options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys());
Set<RangerPolicyResourceEvaluator> smallestList = null;

if (CollectionUtils.isNotEmpty(resourceKeys)) {

for (String resourceName : resourceKeys) {
RangerResourceTrie<RangerPolicyResourceEvaluator> trie = resourceTrie.get(resourceName);

if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level
continue;
}

Set<RangerPolicyResourceEvaluator> serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope());
Set<RangerPolicyResourceEvaluator> inheritedResourceMatchers = trie.getInheritedEvaluators();

if (smallestList != null) {
if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
smallestList = null;
} else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) {
smallestList.retainAll(serviceResourceMatchersForResource);
} else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
smallestList.retainAll(inheritedResourceMatchers);
} else {
Set<RangerPolicyResourceEvaluator> smaller, bigger;
if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) {
smaller = serviceResourceMatchersForResource;
bigger = inheritedResourceMatchers;
} else {
smaller = inheritedResourceMatchers;
bigger = serviceResourceMatchersForResource;
}
Set<RangerPolicyResourceEvaluator> tmp = new HashSet<>();
if (smallestList.size() < smaller.size()) {
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
} else {
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
if (smallestList.size() < bigger.size()) {
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
} else {
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
}
}
smallestList = tmp;
}
} else {
if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
Set<RangerPolicyResourceEvaluator> tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers;
smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
} else {
smallestList = new HashSet<>(serviceResourceMatchersForResource);
smallestList.addAll(inheritedResourceMatchers);
}
}

if (CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out
smallestList = null;
break;
}
}
}
Collection<RangerPolicyResourceEvaluator> smallestList = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTrie, resource.getAsMap(), request.getResourceMatchingScope());

if (smallestList != null) {
if (smallestList.size() == 0) {
Expand Down Expand Up @@ -1224,8 +1165,8 @@ private List<RangerPolicyEvaluator> getReorderedPolicyEvaluators(List<RangerPoli
return ret;
}

private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
final Map<String, RangerResourceTrie> ret;
private Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;

if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
Expand All @@ -1240,7 +1181,7 @@ private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends Ran
return ret;
}

private void updateTrie(Map<String, RangerResourceTrie> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
private void updateTrie(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): ");
}
Expand Down Expand Up @@ -1436,7 +1377,7 @@ private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final Ranger
break;
}

Map<String, RangerResourceTrie> trieMap = getTrie(policyType);
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap = getTrie(policyType);

if (trieMap != null) {
updateTrie(trieMap, changeType, currentEvaluator, newEvaluator);
Expand All @@ -1457,8 +1398,8 @@ private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final Ranger
return ret;
}

Map<String, RangerResourceTrie> getTrie(final int policyType) {
final Map<String, RangerResourceTrie> ret;
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> getTrie(final int policyType) {
final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;
switch (policyType) {
case RangerPolicy.POLICY_TYPE_ACCESS:
ret = policyResourceTrie;
Expand Down Expand Up @@ -1583,18 +1524,18 @@ private void updateResourceTrie(List<RangerPolicyDelta> deltas) {

for (int policyType = 0; policyType < flags.length; policyType++) {
if (flags[policyType]) {
Map<String, RangerResourceTrie> trie = getTrie(policyType);
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trie = getTrie(policyType);

if (trie != null) {
for (Map.Entry<String, RangerResourceTrie> entry : trie.entrySet()) {
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : trie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
}
}

if (auditFilterResourceTrie != null) {
for (Map.Entry<String, RangerResourceTrie> entry : auditFilterResourceTrie.entrySet()) {
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : auditFilterResourceTrie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
Expand Down
4 changes: 2 additions & 2 deletions agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boo
this(resourceDef, evaluators, isOptimizedForRetrieval, false, pluginContext);
}

public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
public <T extends RangerResourceEvaluator, E> RangerResourceTrie(RangerResourceDef resourceDef, List<E> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ", isOptimizedForSpace=" + isOptimizedForSpace + ")");
}
Expand Down Expand Up @@ -154,7 +154,7 @@ public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boo
this.isOptimizedForRetrieval = !isOptimizedForSpace && isOptimizedForRetrieval; // isOptimizedForSpace takes precedence
this.separatorChar = ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR);

TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
final TrieNode tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);

if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread
this.root = buildTrie(resourceDef, evaluators, 1);
Expand Down
6 changes: 3 additions & 3 deletions agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -995,13 +995,13 @@ public static boolean compare(RangerPolicyRepository me, RangerPolicyRepository
public static boolean compareTrie(final int policyType, RangerPolicyRepository me, RangerPolicyRepository other) {
boolean ret;

Map<String, RangerResourceTrie> myTrie = me.getTrie(policyType);
Map<String, RangerResourceTrie> otherTrie = other.getTrie(policyType);
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> myTrie = me.getTrie(policyType);
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> otherTrie = other.getTrie(policyType);

ret = myTrie.size() == otherTrie.size();

if (ret) {
for (Map.Entry<String, RangerResourceTrie> entry : myTrie.entrySet()) {
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : myTrie.entrySet()) {
RangerResourceTrie myResourceTrie = entry.getValue();
RangerResourceTrie otherResourceTrie = otherTrie.get(entry.getKey());

Expand Down

0 comments on commit 0f930c9

Please sign in to comment.