Optimized Privilege Evaluation (opensearch-project#4380)

Signed-off-by: Nils Bandener <[email protected]>

build.gradle

@@ -588,6 +588,10 @@ dependencies {
     implementation 'com.nimbusds:nimbus-jose-jwt:9.46'
     implementation 'com.rfksystems:blake2b:2.0.0'
     implementation 'com.password4j:password4j:1.8.2'
+
+    // Action privileges: check tables and compact collections
+    implementation 'com.selectivem.collections:special-collections-complete:1.4.0'
+
     //JWT
     implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}"
     implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}"
@@ -742,6 +746,7 @@ dependencies {
     integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
     integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
     integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
+    integrationTestImplementation "org.mockito:mockito-core:5.14.2"
 
     //spotless
     implementation('com.google.googlejavaformat:google-java-format:1.24.0') {

src/integrationTest/java/org/opensearch/security/privileges/ClusterStateMetadataDependentPrivilegesTest.java

@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+package org.opensearch.security.privileges;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.awaitility.Awaitility;
+import org.junit.Test;
+
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.node.Node;
+import org.opensearch.threadpool.ThreadPool;
+
+import org.mockito.Mockito;
+import org.mockito.stubbing.Answer;
+
+public class ClusterStateMetadataDependentPrivilegesTest {
+
+    @Test
+    public void simpleUpdate() {
+        ThreadPool threadPool = threadPool();
+        try {
+            ConcreteTestSubject subject = new ConcreteTestSubject();
+            ClusterState clusterState = clusterState(metadata(1));
+            ClusterService clusterService = Mockito.mock(ClusterService.class);
+            Mockito.when(clusterService.state()).thenReturn(clusterState);
+
+            subject.updateClusterStateMetadataAsync(clusterService, threadPool);
+            Awaitility.await().until(() -> subject.getCurrentlyUsedMetadataVersion() == 1);
+            subject.shutdown();
+        } finally {
+            threadPool.shutdown();
+        }
+    }
+
+    @Test
+    public void frequentUpdates() throws Exception {
+        ThreadPool threadPool = threadPool();
+        try {
+            ConcreteTestSubject subject = new ConcreteTestSubject();
+            AtomicReference<ClusterState> clusterStateReference = new AtomicReference<>(clusterState(metadata(1)));
+            ClusterService clusterService = Mockito.mock(ClusterService.class);
+            Mockito.when(clusterService.state()).thenAnswer((Answer<ClusterState>) invocationOnMock -> clusterStateReference.get());
+            subject.updateClusterStateMetadataAsync(clusterService, threadPool);
+            subject.updateClusterStateMetadataAsync(clusterService, threadPool);
+
+            for (int i = 2; i <= 100; i++) {
+                clusterStateReference.set(clusterState(metadata(i)));
+                subject.updateClusterStateMetadataAsync(clusterService, threadPool);
+                Thread.sleep(10);
+            }
+
+            Awaitility.await().until(() -> subject.getCurrentlyUsedMetadataVersion() == 100);
+            subject.shutdown();
+        } finally {
+            threadPool.shutdown();
+        }
+    }
+
+    @Test
+    public void shutdown() {
+        ThreadPool threadPool = threadPool();
+        try {
+            ConcreteTestSubject subject = new ConcreteTestSubject();
+            ClusterState clusterState = clusterState(metadata(1));
+            ClusterService clusterService = Mockito.mock(ClusterService.class);
+            Mockito.when(clusterService.state()).thenReturn(clusterState);
+            subject.updateClusterStateMetadataAsync(clusterService, threadPool);
+            subject.shutdown();
+        } finally {
+            threadPool.shutdown();
+        }
+    }
+
+    static Metadata metadata(long version) {
+        return Metadata.builder().version(version).build();
+    }
+
+    static ClusterState clusterState(Metadata metadata) {
+        return ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build();
+    }
+
+    static ThreadPool threadPool() {
+        return new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "name").build());
+    }
+
+    static class ConcreteTestSubject extends ClusterStateMetadataDependentPrivileges {
+
+        private long currentMetadataVersion;
+
+        @Override
+        protected void updateClusterStateMetadata(Metadata metadata) {
+            // We need to be slow with updates to test the debounce-functionality
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {}
+
+            this.currentMetadataVersion = metadata.version();
+        }
+
+        @Override
+        protected long getCurrentlyUsedMetadataVersion() {
+            return this.currentMetadataVersion;
+        }
+    }
+}

src/integrationTest/java/org/opensearch/security/privileges/IndexPatternTest.java

@@ -0,0 +1,252 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+package org.opensearch.security.privileges;
+
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoField;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Test;
+
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.security.resolver.IndexResolverReplacer;
+import org.opensearch.security.support.WildcardMatcher;
+import org.opensearch.security.user.User;
+
+import static org.opensearch.security.util.MockIndexMetadataBuilder.indices;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class IndexPatternTest {
+    final static int CURRENT_YEAR = ZonedDateTime.now().get(ChronoField.YEAR);
+    final static int NEXT_YEAR = CURRENT_YEAR + 1;
+
+    final static Metadata INDEX_METADATA = //
+        indices("index_a11", "index_a12", "index_a21", "index_a22", "index_b1", "index_b2")//
+            .alias("alias_a")
+            .of("index_a11", "index_a12", "index_a21", "index_a22")//
+            .alias("alias_b")
+            .of("index_b1", "index_b2")//
+            .dataStream("data_stream_a1")//
+            .dataStream("data_stream_b1")//
+            .index("index_year_" + CURRENT_YEAR)//
+            .index("index_year_" + NEXT_YEAR)//
+            .alias("alias_year_" + CURRENT_YEAR)
+            .of("index_current_year")//
+            .alias("alias_year_" + NEXT_YEAR)
+            .of("index_next_year")//
+            .build();
+    final static ClusterState CLUSTER_STATE = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(INDEX_METADATA).build();
+
+    @Test
+    public void constantIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("index_a11");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+        assertFalse(indexPattern.isEmpty());
+        assertTrue(indexPattern.dynamicOnly().isEmpty());
+        assertEquals("index_a11", indexPattern.toString());
+
+        assertTrue(indexPattern.matches("index_a11", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_a12", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void constantAlias() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("alias_a");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches("alias_a", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("alias_a1", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void constantAlias_onIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("alias_a");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches("index_a11", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_b1", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void constantDataStream_onIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("data_stream_a1");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches(".ds-data_stream_a1-000001", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches(".ds-data_stream_a2-000001", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void patternIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("index_a1*");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches("index_a11", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_a21", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void patternAlias() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("alias_a*");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches("alias_a", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("alias_b", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void patternAlias_onIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("alias_a*");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches("index_a11", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_b1", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void patternDataStream_onIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("data_stream_a*");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+
+        assertTrue(indexPattern.matches(".ds-data_stream_a1-000001", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches(".ds-data_stream_b1-000001", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    /**
+     * Static invalid regular expressions are just ignored
+     */
+    @Test
+    public void regex_invalid() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("/index_x\\/");
+        assertFalse(indexPattern.hasStaticPattern());
+        assertFalse(indexPattern.hasDynamicPattern());
+        assertTrue(indexPattern.isEmpty());
+    }
+
+    @Test
+    public void dateMathIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("<index_year_{now/y{yyyy}}>");
+        assertFalse(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+        assertEquals("<index_year_{now/y{yyyy}}>", indexPattern.toString());
+
+        assertTrue(indexPattern.matches("index_year_" + CURRENT_YEAR, ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_year_" + NEXT_YEAR, ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test
+    public void dateMathAlias_onIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("<alias_year_{now/y{yyyy}}>");
+        assertFalse(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+        assertFalse(indexPattern.isEmpty());
+
+        assertTrue(indexPattern.matches("index_current_year", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_next_year", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test(expected = PrivilegesEvaluationException.class)
+    public void dateMathIndex_invalid() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("<index_year_{now/y{yyyy}>");
+        indexPattern.matches("index_year_" + CURRENT_YEAR, ctx(), INDEX_METADATA.getIndicesLookup());
+    }
+
+    @Test
+    public void templatedIndex() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("index_${attrs.a11}");
+        assertFalse(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+        assertEquals(indexPattern, indexPattern.dynamicOnly());
+
+        assertTrue(indexPattern.matches("index_a11", ctx(), INDEX_METADATA.getIndicesLookup()));
+        assertFalse(indexPattern.matches("index_a12", ctx(), INDEX_METADATA.getIndicesLookup()));
+    }
+
+    @Test(expected = PrivilegesEvaluationException.class)
+    public void templatedIndex_invalid() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("/index_${attrs.a11}\\/");
+        assertFalse(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+
+        indexPattern.matches("whatever", ctx(), INDEX_METADATA.getIndicesLookup());
+    }
+
+    @Test
+    public void mixed() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("index_${attrs.a11}", "index_a12");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+        assertFalse(indexPattern.isEmpty());
+
+        assertEquals(WildcardMatcher.from("index_a12"), indexPattern.getStaticPattern());
+        assertEquals(IndexPattern.from("index_${attrs.a11}"), indexPattern.dynamicOnly());
+        assertEquals("index_a12 index_${attrs.a11}", indexPattern.toString());
+    }
+
+    @Test
+    public void mixed2() throws Exception {
+        IndexPattern indexPattern = IndexPattern.from("<index_year_{now/y{yyyy}}>", "index_a12");
+        assertTrue(indexPattern.hasStaticPattern());
+        assertTrue(indexPattern.hasDynamicPattern());
+        assertFalse(indexPattern.isEmpty());
+
+        assertEquals(WildcardMatcher.from("index_a12"), indexPattern.getStaticPattern());
+        assertEquals(IndexPattern.from("<index_year_{now/y{yyyy}}>"), indexPattern.dynamicOnly());
+        assertEquals("index_a12 <index_year_{now/y{yyyy}}>", indexPattern.toString());
+    }
+
+    @Test
+    public void equals() {
+        IndexPattern a1 = IndexPattern.from("data_stream_a*");
+        IndexPattern a2 = IndexPattern.from("data_stream_a*");
+        IndexPattern b = IndexPattern.from("<index_year_{now/y{yyyy}}>", "data_stream_a*");
+
+        assertEquals(a1, a1);
+        assertEquals(a1, a2);
+        assertNotEquals(a1, b);
+        assertFalse(a1.equals(a1.toString()));
+    }
+
+    private static PrivilegesEvaluationContext ctx() {
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
+        IndexResolverReplacer indexResolverReplacer = new IndexResolverReplacer(indexNameExpressionResolver, () -> CLUSTER_STATE, null);
+        User user = new User("test_user");
+        user.addAttributes(ImmutableMap.of("attrs.a11", "a11"));
+        user.addAttributes(ImmutableMap.of("attrs.year", "year"));
+
+        return new PrivilegesEvaluationContext(
+            user,
+            ImmutableSet.of(),
+            "indices:action/test",
+            null,
+            null,
+            indexResolverReplacer,
+            indexNameExpressionResolver,
+            () -> CLUSTER_STATE
+        );
+    }
+}