Upgrade to opensearch 2.0.0 alpha1

.github/workflows/ci.yml

@@ -79,29 +79,34 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - run: ./gradlew clean assemble
-
-    - id: security-plugin-version
-      uses: madhead/read-java-properties@66cc8c88f5c6f6069ebfb42586024dd6ffe2e451
-      with:
-         file: gradle.properties
-         property: security-plugin.version
-
     - uses: actions/setup-java@v1
       with:
         java-version: 11
 
-    - run: ./gradlew clean assemble
-    - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-SNAPSHOT.jar
+    - run: |
+        security_plugin_version=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}')
+        security_plugin_version_no_snapshot=$(echo $security_plugin_version | sed 's/-SNAPSHOT//g')
+        security_plugin_version_only_number=$(echo $security_plugin_version_no_snapshot | cut -d- -f1)
+        test_qualifier=alpha2
+
+        echo "SECURITY_PLUGIN_VERSION=$security_plugin_version" >> $GITHUB_ENV
+        echo "SECURITY_PLUGIN_VERSION_NO_SNAPSHOT=$security_plugin_version_no_snapshot" >> $GITHUB_ENV
+        echo "SECURITY_PLUGIN_VERSION_ONLY_NUMBER=$security_plugin_version_only_number" >> $GITHUB_ENV
+        echo "TEST_QUALIFIER=$test_qualifier" >> $GITHUB_ENV
+
+    - run: |
+        echo ${{ env.SECURITY_PLUGIN_VERSION }}
+        echo ${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }}
+        echo ${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}
+        echo ${{ env.TEST_QUALIFIER }}
+
+    - run: ./gradlew clean assemble && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION }}.jar
 
-    - run: ./gradlew clean assemble -Dbuild.snapshot=false
-    - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}.jar
+    - run: ./gradlew clean assemble -Dbuild.snapshot=false && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }}.jar
 
-    - run: ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=alpha1
-    - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-alpha1.jar
+    - run: ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}.jar
 
-    - run: ./gradlew clean assemble -Dbuild.version_qualifier=alpha1
-    - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-alpha1-SNAPSHOT.jar
+    - run: ./gradlew clean assemble -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}-SNAPSHOT.jar
 
     - name: List files in the build directory if there was an error
       run: ls -al ./build/

.github/workflows/plugin_install.yml

@@ -9,12 +9,6 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - id: opensearch-version
-      uses: madhead/read-java-properties@66cc8c88f5c6f6069ebfb42586024dd6ffe2e451
-      with:
-         file: gradle.properties
-         property: opensearch-core.version
-
     - name: Set up JDK
       uses: actions/setup-java@v1
       with:
@@ -25,7 +19,8 @@ jobs:
 
     - name: Download OpenSearch Core
       run: |
-        wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ steps.opensearch-version.outputs.value }}/latest/linux/x64/builds/opensearch/dist/opensearch-min-${{ steps.opensearch-version.outputs.value }}-linux-x64.tar.gz
+        opensearch_version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}' | sed 's/-SNAPSHOT//g'`
+        wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$opensearch_version/latest/linux/x64/builds/opensearch/dist/opensearch-min-$opensearch_version-linux-x64.tar.gz
         tar -xzf opensearch-*.tar.gz
         rm -f opensearch-*.tar.gz
 

build.gradle

@@ -55,9 +55,19 @@ repositories {
 }
 
 ext {
-    default_opensearch_version = property("opensearch-core.version") + "-SNAPSHOT"
-    opensearch_version = System.getProperty("opensearch.version", default_opensearch_version)
-    buildVersionQualifier = System.getProperty("build.version_qualifier")
+    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
+    opensearch_version = System.getProperty("opensearch.version", "2.0.0-alpha1-SNAPSHOT")
+    buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1")
+    // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT
+    version_tokens = opensearch_version.tokenize('-')
+    opensearch_build = version_tokens[0] + '.0'
+    if (buildVersionQualifier) {
+        opensearch_build += "-${buildVersionQualifier}"
+        opensearch_build_nosnapshot = opensearch_build
+    }
+    if (isSnapshot) {
+        opensearch_build += "-SNAPSHOT"
+    }
 }
 
 configurations.all {
@@ -124,19 +134,8 @@ dependencies {
     compileOnly "org.opensearch:opensearch:${opensearch_version}"
 }
 
-ext {
-    securityPluginVersion = property('security-plugin.version')
-    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-}
-
 group = 'org.opensearch'
-version = securityPluginVersion
-if (buildVersionQualifier) {
-    version += "-${buildVersionQualifier}"
-}
-if (isSnapshot) {
-    version += "-SNAPSHOT"
-}
+version = opensearch_build
 
 description = 'OpenSearch Security'
 
@@ -326,16 +325,13 @@ task bundleSecurityAdminStandaloneTarGz(dependsOn: jar, type: Tar) {
 }
 
 task createPluginDescriptor() {
-    if (opensearch_version.contains("-SNAPSHOT")) {
-        opensearch_version=opensearch_version.substring(0, opensearch_version.length() - 9)
-    }
     List<String> descriptorProperties = [
         "description=Provide access control related features for OpenSearch",
         "version=${version}",
         "name=opensearch-security",
         "classname=org.opensearch.security.OpenSearchSecurityPlugin",
         "java.version=${java.targetCompatibility}",
-        "opensearch.version=${opensearch_version}",
+        "opensearch.version=${version_tokens[0]}",
     ]
 
     new File("plugin-descriptor.properties").text = descriptorProperties.join ("\n")

src/main/java/org/opensearch/security/compliance/ComplianceIndexingOperationListenerImpl.java

@@ -149,12 +149,12 @@ public void postIndex(ShardId shardId, Index index, IndexResult result) {
             if(previousContent == null) {
                 //no previous content
                 if(!result.isCreated()) {
-                    log.warn("No previous content and not created (its an update but do not find orig source) for {}/{}/{}/{}", index.startTime(), shardId, index.type(), index.id());
+                    log.warn("No previous content and not created (its an update but do not find orig source) for {}/{}/{}", index.startTime(), shardId, index.id());
                 }
                 assert result.isCreated():"No previous content and not created";
             } else {
                 if(result.isCreated()) {
-                    log.warn("Previous content and created for {}/{}/{}/{}", index.startTime(), shardId, index.type(), index.id());
+                    log.warn("Previous content and created for {}/{}/{}", index.startTime(), shardId, index.id());
                 }
                 assert !result.isCreated():"Previous content and created";
             }

src/main/java/org/opensearch/security/compliance/FieldReadCallback.java

@@ -120,12 +120,12 @@ public void binaryFieldRead(final FieldInfo fieldInfo, byte[] fieldValue) {
         }
     }
 
-    public void stringFieldRead(final FieldInfo fieldInfo, final byte[] fieldValue) {
+    public void stringFieldRead(final FieldInfo fieldInfo, final String fieldValue) {
         try {
             if(!recordField(fieldInfo.name, true)) {
                 return;
             }
-            fieldRead0(fieldInfo.name, new String(fieldValue, StandardCharsets.UTF_8));
+            fieldRead0(fieldInfo.name, fieldValue);
         } catch (Exception e) {
             log.error("Unexpected error reading string field '{}' in index '{}'", fieldInfo.name, index.getName());
         }

src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java

@@ -421,11 +421,6 @@ public void checkIntegrity() throws IOException {
         public void close() throws IOException {
             in.close();
         }
-
-        @Override
-        public long ramBytesUsed() {
-            return in.ramBytesUsed();
-        }
     }
 
     @Override
@@ -521,7 +516,7 @@ public int hashCode() {
         }
 
         @Override
-        public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException {
+        public void stringField(final FieldInfo fieldInfo, final String value) throws IOException {
             fieldReadCallback.stringFieldRead(fieldInfo, value);
             delegate.stringField(fieldInfo, value);
         }
@@ -610,7 +605,7 @@ public int hashCode() {
         }
 
         @Override
-        public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException {
+        public void stringField(final FieldInfo fieldInfo, final String value) throws IOException {
             delegate.stringField(fieldInfo, value);
         }
 
@@ -681,7 +676,7 @@ public int hashCode() {
         }
 
         @Override
-        public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException {
+        public void stringField(final FieldInfo fieldInfo, final String value) throws IOException {
             final Optional<MaskedField> mf = maskedFieldsMap.getMaskedField(fieldInfo.name);
 
             if(mf.isPresent()) {
@@ -868,11 +863,6 @@ private SortedDocValues wrapSortedDocValues(final String field, final SortedDocV
             if (mf != null) {
                 return new SortedDocValues() {
 
-                    @Override
-                    public BytesRef binaryValue() throws IOException {
-                        return mf.mask(sortedDocValues.binaryValue());
-                    }
-
                     @Override
                     public int lookupTerm(BytesRef key) throws IOException {
                         return sortedDocValues.lookupTerm(key);

src/test/java/org/opensearch/security/TracingTests.java

@@ -62,14 +62,10 @@ public void testAdvancedMapping() throws Exception {
         setup(Settings.EMPTY, new DynamicSecurityConfig(), Settings.EMPTY, true, ClusterConfiguration.DEFAULT);
 
         try (Client tc = getClient()) {
-            tc.admin().indices().create(new CreateIndexRequest("myindex1")
-            .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex2")
-            .mapping("mytype2", FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex3")
-            .mapping("mytype3", FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex4")
-            .mapping("mytype4", FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex3").mapping(FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex4").mapping(FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet();
         }
 
         RestHelper rh = nonSslRestHelper();

src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java

@@ -338,14 +338,10 @@ public void testAdvancedMapping() throws Exception {
         rh.executePutRequest("_opendistro/_security/api/audit/config", AuditTestUtils.createAuditPayload(settings), encodeBasicHeader("admin", "admin"));
 
         try (Client tc = getClient()) {
-            tc.admin().indices().create(new CreateIndexRequest("myindex1")
-            .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex2")
-            .mapping("mytype2", FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex3")
-            .mapping("mytype3", FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex4")
-            .mapping("mytype4", FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex3").mapping(FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex4").mapping(FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet();
         }
 
         System.out.println("############ write into mapping 1");
@@ -391,10 +387,8 @@ public void testImmutableIndex() throws Exception {
         rh.executePutRequest("_opendistro/_security/api/audit/config", AuditTestUtils.createAuditPayload(Settings.EMPTY), encodeBasicHeader("admin", "admin"));
 
         try (Client tc = getClient()) {
-            tc.admin().indices().create(new CreateIndexRequest("myindex1")
-            .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
-            tc.admin().indices().create(new CreateIndexRequest("myindex2")
-            .mapping("mytype2", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet();
         }
 
         System.out.println("############ immutable 1");

src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java

@@ -36,7 +36,7 @@ public class CustomFieldMaskedComplexMappingTest extends AbstractDlsFlsTest{
     protected void populateData(Client tc) {
 
         try {
-            tc.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet();
+            tc.admin().indices().create(new CreateIndexRequest("logs").mapping(FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet();
 
 
             byte[] data = FileHelper.loadFile("dlsfls/logs_bulk_data.json").getBytes(StandardCharsets.UTF_8);

src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java

@@ -44,8 +44,9 @@ protected void populateData(Client tc) {
                 "";
 
         tc.admin().indices().create(new CreateIndexRequest("deals")
-        .settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build())
-        .mapping("mytype", mapping, XContentType.JSON)).actionGet();
+                .simpleMapping("amount", "type=integer", "owner", "type=text", "my_nested_object", "type=nested")
+                .settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build()))
+                .actionGet();
 
         //tc.index(new IndexRequest("deals").id("3").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
         //        .source("{\"amount\": 7,\"owner\": \"a\", \"my_nested_object\" : {\"name\": \"spock\"}}", XContentType.JSON)).actionGet();

src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java

@@ -273,7 +273,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception {
         setup();
 
         try (Client client = getClient()) {
-            client.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", ImmutableMap.of("properties", ImmutableMap.of("termX", ImmutableMap.of("type", "keyword"))))).actionGet();
+            client.admin().indices().create(new CreateIndexRequest("logs").simpleMapping("termX", "type=keyword")).actionGet();
 
             for (int i = 0; i < 3; i++) {
                 client.index(new IndexRequest("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("amount", i, "termX", "A", "timestamp", "2022-01-06T09:05:00Z")).actionGet();

src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java

@@ -30,12 +30,8 @@ public class FlsExistsFieldsTest extends AbstractDlsFlsTest {
 
     protected void populateData(Client tc) {
 
-        tc.admin().indices().create(new CreateIndexRequest("data").mapping("doc",
-                "@timestamp", "type=date",
-                "host", "type=text,norms=false",
-                "response", "type=text,norms=false",
-                "non-existing", "type=text,norms=false"
-        ))
+        tc.admin().indices().create(new CreateIndexRequest("data")
+                .simpleMapping("@timestamp", "type=date", "host", "type=text,norms=false", "response", "type=text,norms=false", "non-existing", "type=text,norms=false"))
                 .actionGet();
 
         for (int i = 0; i < 1; i++) {

src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java

@@ -34,8 +34,7 @@ public class FlsFieldsTest extends AbstractDlsFlsTest{
 
     protected void populateData(Client tc) {
 
-        tc.admin().indices().create(new CreateIndexRequest("deals")
-        .mapping("deals", "timestamp","type=date","@timestamp","type=date")).actionGet();
+        tc.admin().indices().create(new CreateIndexRequest("deals").simpleMapping("timestamp", "type=date", "@timestamp", "type=date")).actionGet();
 
         try {
             String doc = FileHelper.loadFile("dlsfls/doc1.json");

src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java

@@ -34,8 +34,7 @@ public class FlsFieldsWcTest extends AbstractDlsFlsTest{
 
     protected void populateData(Client tc) {
 
-        tc.admin().indices().create(new CreateIndexRequest("deals")
-        .mapping("deals", "timestamp","type=date","@timestamp","type=date")).actionGet();
+        tc.admin().indices().create(new CreateIndexRequest("deals").simpleMapping("timestamp", "type=date", "@timestamp", "type=date")).actionGet();
 
         try {
             String doc = FileHelper.loadFile("dlsfls/doc1.json");