From 3ce65c8ac86145fecb3269689760440511ec8dac Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Fri, 1 Mar 2024 13:20:13 +1100 Subject: [PATCH 01/24] Added support for Azure Managed Identity in repository-azure Signed-off-by: Chengwu Shi --- CHANGELOG.md | 1 + .../gradle/precommit/LicenseAnalyzer.java | 2 +- plugins/repository-azure/build.gradle | 4 + .../licenses/azure-identity-1.11.2.jar.sha1 | 1 + .../licenses/json-smart-2.5.0.jar.sha1 | 1 + .../licenses/json-smart-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/json-smart-NOTICE.txt | 0 .../licenses/msal4j-1.14.2.jar.sha1 | 1 + .../licenses/msal4j-LICENSE.txt | 21 ++ .../licenses/msal4j-NOTICE.txt | 0 .../licenses/oauth2-oidc-sdk-11.10.jar.sha1 | 1 + .../licenses/oauth2-oidc-sdk-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/oauth2-oidc-sdk-NOTICE.txt | 0 .../azure/AzureRepositoryPlugin.java | 1 + .../azure/AzureStorageService.java | 34 ++- .../azure/AzureStorageSettings.java | 70 ++++-- .../azure/TokenCredentialType.java | 14 ++ .../azure/AzureStorageServiceTests.java | 170 +++++++++++++++ 18 files changed, 707 insertions(+), 18 deletions(-) create mode 100644 plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/json-smart-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/json-smart-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/msal4j-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt create mode 100644 plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b858ede4b78cf..8d08100728ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Added support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423)) - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478)) - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293)) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java index 4c63516126566..c3acd12e5a1cf 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java @@ -145,7 +145,7 @@ public class LicenseAnalyzer { + "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n" + "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n" + "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n" - + "SOFTWARE\\.\n").replaceAll("\\s+", "\\\\s*"), + + "SOFTWARE\\.?\n").replaceAll("\\s+", "\\\\s*"), Pattern.DOTALL ) ), diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index c7836170d658f..e521a1f853ba2 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -56,6 +56,10 @@ dependencies { api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') api 'com.azure:azure-storage-blob:12.23.0' + api 'com.azure:azure-identity:1.11.2' + api 'com.microsoft.azure:msal4j:1.14.2' + api 'com.nimbusds:oauth2-oidc-sdk:11.10' + api 'net.minidev:json-smart:2.5.0' api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}" api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}" api "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 new file mode 100644 index 0000000000000..fc459b573417c --- /dev/null +++ b/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 @@ -0,0 +1 @@ +31a85e3a591a2513736bf5cf787eeab9cd542590 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 new file mode 100644 index 0000000000000..3ec055efa1255 --- /dev/null +++ b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 @@ -0,0 +1 @@ +57a64f421b472849c40e77d2e7cce3a141b41e99 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/json-smart-LICENSE.txt b/plugins/repository-azure/licenses/json-smart-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/json-smart-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/json-smart-NOTICE.txt b/plugins/repository-azure/licenses/json-smart-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 new file mode 100644 index 0000000000000..39c7e6691ccca --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 @@ -0,0 +1 @@ +bbf1c786b4ef555ad0111e59c35938da4271f592 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-LICENSE.txt new file mode 100644 index 0000000000000..21071075c2459 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-LICENSE.txt @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/plugins/repository-azure/licenses/msal4j-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 new file mode 100644 index 0000000000000..102458c2aa7e6 --- /dev/null +++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 @@ -0,0 +1 @@ +385b80a438f7678eb430681b362284a37db0c18d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java index 78db7cb2d0ea7..aca213f9fed79 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java @@ -91,6 +91,7 @@ public List> getSettings() { AzureStorageSettings.ACCOUNT_SETTING, AzureStorageSettings.KEY_SETTING, AzureStorageSettings.SAS_TOKEN_SETTING, + AzureStorageSettings.TOKEN_CREDENTIAL_TYPE_SETTING, AzureStorageSettings.ENDPOINT_SUFFIX_SETTING, AzureStorageSettings.TIMEOUT_SETTING, AzureStorageSettings.MAX_RETRIES_SETTING, diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 74edd4f3eb23c..ecda191957ea1 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -43,6 +43,7 @@ import com.azure.core.util.Configuration; import com.azure.core.util.Context; import com.azure.core.util.logging.ClientLogger; +import com.azure.identity.ManagedIdentityCredentialBuilder; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.ParallelTransferOptions; @@ -56,11 +57,13 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import java.net.Authenticator; import java.net.PasswordAuthentication; +import java.net.URI; import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.time.Duration; @@ -159,10 +162,27 @@ public Tuple> client(String clientName, BiC return new Tuple<>(state.getClient(), () -> buildOperationContext(azureStorageSettings)); } + public StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { + try { + String endpointSuffix = settings.getEndpointSuffix(); + if (!Strings.hasText(endpointSuffix)) { + endpointSuffix = "core.windows.net"; + } + final URI primaryBlobEndpoint = new URI(String.format("https://%s.blob.%s", settings.getAccount(), endpointSuffix)); + final URI secondaryBlobEndpoint = new URI(String.format("https://%s-secondary.blob.%s", settings.getAccount(), endpointSuffix)); + return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); + } catch (URISyntaxException var14) { + throw logger.logExceptionAsError(new RuntimeException(var14)); + } + } + private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiConsumer statsCollector) throws InvalidKeyException, URISyntaxException { final BlobServiceClientBuilder builder = createClientBuilder(azureStorageSettings); - + // When managed identity is enabled, no connection string will be generated, need to declare the primary uri + if (azureStorageSettings.useManagedIdentityCredential()) { + builder.endpoint(getStorageBlobEndpoint(azureStorageSettings).getPrimaryUri()); + } final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(new NioThreadFactory()); final NettyAsyncHttpClientBuilder clientBuilder = new NettyAsyncHttpClientBuilder().eventLoopGroup(eventLoopGroup); @@ -216,8 +236,13 @@ protected PasswordAuthentication getPasswordAuthentication() { * migration guide for mode details: */ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) { - final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger); - final StorageEndpoint endpoint = storageConnectionString.getBlobEndpoint(); + final StorageEndpoint endpoint; + if (settings.useManagedIdentityCredential()) { + endpoint = getStorageBlobEndpoint(settings); + } else { + final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger); + endpoint = storageConnectionString.getBlobEndpoint(); + } if (endpoint == null || endpoint.getPrimaryUri() == null) { throw new IllegalArgumentException("connectionString missing required settings to derive blob service primary endpoint."); @@ -249,6 +274,9 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException { + if (settings.useManagedIdentityCredential()) { + return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().credential(new ManagedIdentityCredentialBuilder().build())); + } return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString())); } diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index e73ded679cf2b..59f8e92c79c6d 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -50,6 +50,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; final class AzureStorageSettings { @@ -77,6 +78,14 @@ final class AzureStorageSettings { key -> SecureSetting.secureString(key, null) ); + /** Azure token credentials such as Managed Identity */ + public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( + AZURE_CLIENT_PREFIX_KEY, + "token_credential_type", + key -> Setting.simpleString(key, Property.NodeScope), + () -> ACCOUNT_SETTING + ); + /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */ public static final AffixSetting MAX_RETRIES_SETTING = Setting.affixKeySetting( AZURE_CLIENT_PREFIX_KEY, @@ -196,6 +205,7 @@ final class AzureStorageSettings { private final String account; private final String connectString; private final String endpointSuffix; + private final String tokenCredentialType; private final TimeValue timeout; private final int maxRetries; private final LocationMode locationMode; @@ -209,6 +219,7 @@ final class AzureStorageSettings { private AzureStorageSettings( String account, String connectString, + String tokenCredentialType, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -221,6 +232,7 @@ private AzureStorageSettings( ) { this.account = account; this.connectString = connectString; + this.tokenCredentialType = tokenCredentialType; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -233,20 +245,22 @@ private AzureStorageSettings( } private AzureStorageSettings( - String account, - String key, - String sasToken, - String endpointSuffix, - TimeValue timeout, - int maxRetries, - TimeValue connectTimeout, - TimeValue writeTimeout, - TimeValue readTimeout, - TimeValue responseTimeout, - ProxySettings proxySettings + String account, + String key, + String sasToken, + String tokenCredentialType, + String endpointSuffix, + TimeValue timeout, + int maxRetries, + TimeValue connectTimeout, + TimeValue writeTimeout, + TimeValue readTimeout, + TimeValue responseTimeout, + ProxySettings proxySettings ) { this.account = account; - this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); + this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, tokenCredentialType); + this.tokenCredentialType = validateTokenCredentialType(tokenCredentialType); this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -257,7 +271,9 @@ private AzureStorageSettings( this.responseTimeout = responseTimeout; this.proxySettings = proxySettings; } - + public boolean useManagedIdentityCredential() { + return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY.name()); + } public String getEndpointSuffix() { return endpointSuffix; } @@ -278,9 +294,32 @@ public String getConnectString() { return connectString; } - private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) { + public String getAccount() { + return account; + } + + private static String validateTokenCredentialType(String tokenCredentialType){ + final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); + boolean isValidTokenCredentialType = false; + for (TokenCredentialType type : TokenCredentialType.values()) { + if (Objects.equals(type.name(), tokenCredentialType)) { + isValidTokenCredentialType = true; + } + } + if (hasTokenCredentialType && !isValidTokenCredentialType) { + throw new SettingsException(String.format("'%s' is currently not supported.", tokenCredentialType)); + } + return tokenCredentialType; + } + + private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix, String tokenCredentialType) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); + final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); + // When a token credential type is declared, we are no longer using connection string + if (hasTokenCredentialType) { + return ""; + } if (hasSasToken == false && hasKey == false) { throw new SettingsException("Neither a secret key nor a shared access token was set."); } @@ -325,6 +364,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("AzureStorageSettings{"); sb.append("account='").append(account).append('\''); sb.append(", timeout=").append(timeout); + sb.append(", tokenCredentialType=").append(tokenCredentialType).append('\''); sb.append(", endpointSuffix='").append(endpointSuffix).append('\''); sb.append(", maxRetries=").append(maxRetries); sb.append(", proxySettings=").append(proxySettings != ProxySettings.NO_PROXY_SETTINGS ? "PROXY_SET" : "PROXY_NOT_SET"); @@ -370,6 +410,7 @@ private static AzureStorageSettings getClientSettings(Settings settings, String account.toString(), key.toString(), sasToken.toString(), + getValue(settings, clientName, TOKEN_CREDENTIAL_TYPE_SETTING), getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING), @@ -431,6 +472,7 @@ static Map overrideLocationMode( new AzureStorageSettings( entry.getValue().account, entry.getValue().connectString, + entry.getValue().tokenCredentialType, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java new file mode 100644 index 0000000000000..9048873053e49 --- /dev/null +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.azure; + +// Type of token credentials that the plugin supports +public enum TokenCredentialType { + MANAGED_IDENTITY +} diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index bb0eafc7d1d4a..c8f85aed70c6e 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -35,6 +35,7 @@ import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.common.policy.RequestRetryPolicy; +import com.azure.identity.CredentialUnavailableException; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; @@ -114,6 +115,77 @@ public void testCreateClientWithEndpointSuffix() throws IOException { } } + public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() throws IOException { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure clients without account key and sas token. + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + // Enabled managed identity for both clients + .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .put("azure.client.azure2.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + // Defined an endpoint suffic for azure client 1 only. + .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix") + .build(); + try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { + final AzureStorageService azureStorageService = plugin.azureStoreService; + + // Expect azure client 1 to use the custom endpoint suffix + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); + + // Expect azure client 2 to use the default endpoint suffix + final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); + assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); + } + } + + public void testGetStorageBlobEndpoint() throws IOException { + final Settings settings = Settings.builder() + .setSecureSettings(buildSecureSettings()) + .build(); + + try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { + final AzureStorageService azureStorageService = plugin.azureStoreService; + final AzureStorageSettings storageSettings= azureStorageService.storageSettings.get("azure1"); + String primaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getPrimaryUri(); + String secondaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getSecondaryUri(); + assertThat(primaryUri, equalTo("https://myaccount1.blob.core.windows.net")); + assertThat(secondaryUri, equalTo("https://myaccount1-secondary.blob.core.windows.net")); + } + } + + public void testGetStorageBlobEndpointWithInvalidValues() throws IOException { + final Settings settings = Settings.builder() + .setSecureSettings(buildSecureSettings()) + .put("azure.client.azure1.endpoint_suffix", "my endpoint suffix") + .build(); + + try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { + final AzureStorageService azureStorageService = plugin.azureStoreService; + final AzureStorageSettings storageSettings= azureStorageService.storageSettings.get("azure1"); + final RuntimeException e = expectThrows(RuntimeException.class, () -> azureStorageService.getStorageBlobEndpoint(storageSettings)); + } + } + + public void testClientUsingManagedIdentity() throws IOException { + // Enabled managed identity + final Settings settings = Settings.builder() + .setSecureSettings(buildSecureSettings()) + .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .build(); + try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { + final AzureStorageService azureStorageService = plugin.azureStoreService; + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + + // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is not setup in the test + final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo()); + assertThat(e.getMessage(), is("Managed Identity authentication is not available.")); + } + } + public void testReinitClientSettings() throws IOException { final MockSecureSettings secureSettings1 = new MockSecureSettings(); secureSettings1.setString("azure.client.azure1.account", "myaccount11"); @@ -420,6 +492,104 @@ public void testBlobNameFromUri() throws URISyntaxException { assertThat(name, is("path/to/myfile")); } + public void testSettingUnsupportedTokenCredentialForAuthentication() { + final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_THAT_DOES_NOT_EXIST"; + final MockSecureSettings secureSettings = new MockSecureSettings(); + + // Azure client without account key or sas token. + secureSettings.setString("azure.client.azure.account", "myaccount"); + + // Enable the unsupported token credential type + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure.token_credential_type", unsupported_token_credential_type) + .build(); + + final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); + assertEquals('\''+ unsupported_token_credential_type + "' is currently not supported.", e.getMessage()); + } + + public void testBuildConnectStringWhenATokenCredentialIsEnabled() { + final String token_credential_type = TokenCredentialType.MANAGED_IDENTITY.name(); + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure1 with account key + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure1.key", encodeKey("mykey")); + + // Azure 2 with sas token + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.sas_token", encodeKey("mysastoken")); + + // Azure 3 with account key and sas token + secureSettings.setString("azure.client.azure3.account", "myaccount3"); + secureSettings.setString("azure.client.azure3.key", encodeKey("mykey")); + secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken")); + + // Azure 4 without sas token and account key + secureSettings.setString("azure.client.azure4.account", "myaccount4"); + + // Enable Managed Identity in all azure clients + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure1.token_credential_type", token_credential_type) + .put("azure.client.azure2.token_credential_type", token_credential_type) + .put("azure.client.azure3.token_credential_type", token_credential_type) + .put("azure.client.azure4.token_credential_type", token_credential_type) + .build(); + final AzureStorageService mock = storageServiceWithSettingsValidation(settings); + + // Expect connection string to be empty when managed identity is selected for authentication + assertEquals("", mock.storageSettings.get("azure1").getConnectString()); + assertEquals("", mock.storageSettings.get("azure2").getConnectString()); + assertEquals("", mock.storageSettings.get("azure3").getConnectString()); + assertEquals("", mock.storageSettings.get("azure4").getConnectString()); + } + + public void testTokenCredentialWhenAccountIsNotProvided() { + // Setting with an account specified + final MockSecureSettings secureSettings = new MockSecureSettings(); + + // Enabled Managed Identity in the settings + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .build(); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> storageServiceWithSettingsValidation(settings)); + + // Expect failure due to missing account name + assertEquals("missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]", e.getMessage()); + } + + public void testAuthenticationMethodNotProvided() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure client without account key and sas token. + secureSettings.setString("azure.client.azure.account", "myaccount"); + + // Disabled Managed Identity in the settings by default + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .build(); + final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); + + // Expect fall back to authentication via sas token or account key when token credential is not specified. + assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage()); + } + + public void testManagedIdentityIsEnabled() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure client without account key or sas token. + secureSettings.setString("azure.client.azure.account", "myaccount"); + + // Enabled Managed Identity in the settings. + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .build(); + + final AzureStorageService mock = storageServiceWithSettingsValidation(settings); + assertEquals(true, mock.storageSettings.get("azure").useManagedIdentityCredential()); + } + private static MockSecureSettings buildSecureSettings() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.azure1.account", "myaccount1"); From 5e253ed5e61e8632c2100d4c4478847006e9f724 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 7 Mar 2024 11:56:14 +1100 Subject: [PATCH 02/24] Refactor tokenCredentialType as an enum when constructing AzureStorageSetting Signed-off-by: Chengwu Shi --- .../azure/AzureStorageService.java | 6 ++-- .../azure/AzureStorageSettings.java | 36 ++++++------------- .../azure/TokenCredentialType.java | 3 +- .../azure/AzureStorageServiceTests.java | 23 +++++++++--- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index ecda191957ea1..263ba4ee2dbc3 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -180,7 +180,7 @@ private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiCon throws InvalidKeyException, URISyntaxException { final BlobServiceClientBuilder builder = createClientBuilder(azureStorageSettings); // When managed identity is enabled, no connection string will be generated, need to declare the primary uri - if (azureStorageSettings.useManagedIdentityCredential()) { + if (azureStorageSettings.usesManagedIdentityCredential()) { builder.endpoint(getStorageBlobEndpoint(azureStorageSettings).getPrimaryUri()); } final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(new NioThreadFactory()); @@ -237,7 +237,7 @@ protected PasswordAuthentication getPasswordAuthentication() { */ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) { final StorageEndpoint endpoint; - if (settings.useManagedIdentityCredential()) { + if (settings.usesManagedIdentityCredential()) { endpoint = getStorageBlobEndpoint(settings); } else { final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger); @@ -274,7 +274,7 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException { - if (settings.useManagedIdentityCredential()) { + if (settings.usesManagedIdentityCredential()) { return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().credential(new ManagedIdentityCredentialBuilder().build())); } return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString())); diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 59f8e92c79c6d..797c15020f775 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -79,10 +79,10 @@ final class AzureStorageSettings { ); /** Azure token credentials such as Managed Identity */ - public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( + public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( AZURE_CLIENT_PREFIX_KEY, "token_credential_type", - key -> Setting.simpleString(key, Property.NodeScope), + (key) -> new Setting<>(key, "not_applicable", s -> TokenCredentialType.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope), () -> ACCOUNT_SETTING ); @@ -205,7 +205,7 @@ final class AzureStorageSettings { private final String account; private final String connectString; private final String endpointSuffix; - private final String tokenCredentialType; + private final TokenCredentialType tokenCredentialType; private final TimeValue timeout; private final int maxRetries; private final LocationMode locationMode; @@ -219,7 +219,7 @@ final class AzureStorageSettings { private AzureStorageSettings( String account, String connectString, - String tokenCredentialType, + TokenCredentialType tokenCredentialType, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -248,7 +248,7 @@ private AzureStorageSettings( String account, String key, String sasToken, - String tokenCredentialType, + TokenCredentialType tokenCredentialType, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -260,7 +260,7 @@ private AzureStorageSettings( ) { this.account = account; this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, tokenCredentialType); - this.tokenCredentialType = validateTokenCredentialType(tokenCredentialType); + this.tokenCredentialType = tokenCredentialType; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -271,8 +271,8 @@ private AzureStorageSettings( this.responseTimeout = responseTimeout; this.proxySettings = proxySettings; } - public boolean useManagedIdentityCredential() { - return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY.name()); + public boolean usesManagedIdentityCredential() { + return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY); } public String getEndpointSuffix() { return endpointSuffix; @@ -298,25 +298,11 @@ public String getAccount() { return account; } - private static String validateTokenCredentialType(String tokenCredentialType){ - final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); - boolean isValidTokenCredentialType = false; - for (TokenCredentialType type : TokenCredentialType.values()) { - if (Objects.equals(type.name(), tokenCredentialType)) { - isValidTokenCredentialType = true; - } - } - if (hasTokenCredentialType && !isValidTokenCredentialType) { - throw new SettingsException(String.format("'%s' is currently not supported.", tokenCredentialType)); - } - return tokenCredentialType; - } - - private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix, String tokenCredentialType) { + private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix, TokenCredentialType tokenCredentialType) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); - final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); - // When a token credential type is declared, we are no longer using connection string + final boolean hasTokenCredentialType = !tokenCredentialType.equals(TokenCredentialType.NOT_APPLICABLE); + // When a valid token credential type is declared, we are no longer using connection string if (hasTokenCredentialType) { return ""; } diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java index 9048873053e49..75346617c91ac 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java @@ -10,5 +10,6 @@ // Type of token credentials that the plugin supports public enum TokenCredentialType { - MANAGED_IDENTITY + MANAGED_IDENTITY, + NOT_APPLICABLE } diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index c8f85aed70c6e..88d43450ab82b 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -492,6 +492,21 @@ public void testBlobNameFromUri() throws URISyntaxException { assertThat(name, is("path/to/myfile")); } + public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure client without account key or sas token. + secureSettings.setString("azure.client.azure.account", "myaccount"); + + // Enabled Managed Identity in the settings using lower case + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure.token_credential_type", "managed_identity") + .build(); + + final AzureStorageService mock = storageServiceWithSettingsValidation(settings); + assertEquals(true, mock.storageSettings.get("azure").usesManagedIdentityCredential()); + } + public void testSettingUnsupportedTokenCredentialForAuthentication() { final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_THAT_DOES_NOT_EXIST"; final MockSecureSettings secureSettings = new MockSecureSettings(); @@ -505,8 +520,8 @@ public void testSettingUnsupportedTokenCredentialForAuthentication() { .put("azure.client.azure.token_credential_type", unsupported_token_credential_type) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); - assertEquals('\''+ unsupported_token_credential_type + "' is currently not supported.", e.getMessage()); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> storageServiceWithSettingsValidation(settings)); + assertEquals("No enum constant org.opensearch.repositories.azure.TokenCredentialType."+ unsupported_token_credential_type, e.getMessage()); } public void testBuildConnectStringWhenATokenCredentialIsEnabled() { @@ -554,7 +569,7 @@ public void testTokenCredentialWhenAccountIsNotProvided() { .setSecureSettings(secureSettings) .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .build(); - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> storageServiceWithSettingsValidation(settings)); + final Exception e = expectThrows(Exception.class, () -> storageServiceWithSettingsValidation(settings)); // Expect failure due to missing account name assertEquals("missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]", e.getMessage()); @@ -587,7 +602,7 @@ public void testManagedIdentityIsEnabled() { .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(true, mock.storageSettings.get("azure").useManagedIdentityCredential()); + assertEquals(true, mock.storageSettings.get("azure").usesManagedIdentityCredential()); } private static MockSecureSettings buildSecureSettings() { From 117d0589689dfa4a3b540fd7fc4f0d672a465af6 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 7 Mar 2024 12:08:19 +1100 Subject: [PATCH 03/24] fixed indentation Signed-off-by: Chengwu Shi --- .../azure/AzureStorageSettings.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 797c15020f775..487f5504b67f8 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -245,18 +245,18 @@ private AzureStorageSettings( } private AzureStorageSettings( - String account, - String key, - String sasToken, - TokenCredentialType tokenCredentialType, - String endpointSuffix, - TimeValue timeout, - int maxRetries, - TimeValue connectTimeout, - TimeValue writeTimeout, - TimeValue readTimeout, - TimeValue responseTimeout, - ProxySettings proxySettings + String account, + String key, + String sasToken, + TokenCredentialType tokenCredentialType, + String endpointSuffix, + TimeValue timeout, + int maxRetries, + TimeValue connectTimeout, + TimeValue writeTimeout, + TimeValue readTimeout, + TimeValue responseTimeout, + ProxySettings proxySettings ) { this.account = account; this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, tokenCredentialType); From e145f3f1273769003656348258c65cca9c31986e Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 7 Mar 2024 13:16:49 +1100 Subject: [PATCH 04/24] fixed syntax Signed-off-by: Chengwu Shi --- .../org/opensearch/repositories/azure/AzureStorageSettings.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 487f5504b67f8..fc3b23bcbe9f6 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -271,9 +271,11 @@ private AzureStorageSettings( this.responseTimeout = responseTimeout; this.proxySettings = proxySettings; } + public boolean usesManagedIdentityCredential() { return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY); } + public String getEndpointSuffix() { return endpointSuffix; } From 26598991ae9afccea42b0d49447e5b6d0e79eab2 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 7 Mar 2024 13:20:17 +1100 Subject: [PATCH 05/24] removed unused imports Signed-off-by: Chengwu Shi --- .../org/opensearch/repositories/azure/AzureStorageSettings.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index fc3b23bcbe9f6..e3c2b4678efa7 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -50,7 +50,6 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.Objects; final class AzureStorageSettings { From d89075199c33802c11acff4ddddacab188d0f419 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 7 Mar 2024 15:47:50 +1100 Subject: [PATCH 06/24] applied changes after running :plugins:repository-azure:spotlessApply Signed-off-by: Chengwu Shi --- .../azure/AzureStorageService.java | 8 ++-- .../azure/AzureStorageSettings.java | 8 +++- .../azure/AzureStorageServiceTests.java | 37 ++++++++++++------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 263ba4ee2dbc3..8fdd6e4740286 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -168,8 +168,8 @@ public StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings setting if (!Strings.hasText(endpointSuffix)) { endpointSuffix = "core.windows.net"; } - final URI primaryBlobEndpoint = new URI(String.format("https://%s.blob.%s", settings.getAccount(), endpointSuffix)); - final URI secondaryBlobEndpoint = new URI(String.format("https://%s-secondary.blob.%s", settings.getAccount(), endpointSuffix)); + final URI primaryBlobEndpoint = new URI("https://" + settings.getAccount() + ".blob." + endpointSuffix); + final URI secondaryBlobEndpoint = new URI("https://" + settings.getAccount() + "-secondary.blob." + endpointSuffix); return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); } catch (URISyntaxException var14) { throw logger.logExceptionAsError(new RuntimeException(var14)); @@ -275,7 +275,9 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException { if (settings.usesManagedIdentityCredential()) { - return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().credential(new ManagedIdentityCredentialBuilder().build())); + return SocketAccess.doPrivilegedException( + () -> new BlobServiceClientBuilder().credential(new ManagedIdentityCredentialBuilder().build()) + ); } return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString())); } diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index e3c2b4678efa7..5e42c541ab06f 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -299,7 +299,13 @@ public String getAccount() { return account; } - private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix, TokenCredentialType tokenCredentialType) { + private static String buildConnectString( + String account, + @Nullable String key, + @Nullable String sasToken, + String endpointSuffix, + TokenCredentialType tokenCredentialType + ) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); final boolean hasTokenCredentialType = !tokenCredentialType.equals(TokenCredentialType.NOT_APPLICABLE); diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 88d43450ab82b..b8171284ec376 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -33,9 +33,9 @@ package org.opensearch.repositories.azure; import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.identity.CredentialUnavailableException; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.common.policy.RequestRetryPolicy; -import com.azure.identity.CredentialUnavailableException; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; @@ -143,13 +143,11 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr } public void testGetStorageBlobEndpoint() throws IOException { - final Settings settings = Settings.builder() - .setSecureSettings(buildSecureSettings()) - .build(); + final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; - final AzureStorageSettings storageSettings= azureStorageService.storageSettings.get("azure1"); + final AzureStorageSettings storageSettings = azureStorageService.storageSettings.get("azure1"); String primaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getPrimaryUri(); String secondaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getSecondaryUri(); assertThat(primaryUri, equalTo("https://myaccount1.blob.core.windows.net")); @@ -165,8 +163,11 @@ public void testGetStorageBlobEndpointWithInvalidValues() throws IOException { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; - final AzureStorageSettings storageSettings= azureStorageService.storageSettings.get("azure1"); - final RuntimeException e = expectThrows(RuntimeException.class, () -> azureStorageService.getStorageBlobEndpoint(storageSettings)); + final AzureStorageSettings storageSettings = azureStorageService.storageSettings.get("azure1"); + final RuntimeException e = expectThrows( + RuntimeException.class, + () -> azureStorageService.getStorageBlobEndpoint(storageSettings) + ); } } @@ -180,7 +181,8 @@ public void testClientUsingManagedIdentity() throws IOException { final AzureStorageService azureStorageService = plugin.azureStoreService; final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is not setup in the test + // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is not + // setup in the test final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo()); assertThat(e.getMessage(), is("Managed Identity authentication is not available.")); } @@ -520,8 +522,14 @@ public void testSettingUnsupportedTokenCredentialForAuthentication() { .put("azure.client.azure.token_credential_type", unsupported_token_credential_type) .build(); - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> storageServiceWithSettingsValidation(settings)); - assertEquals("No enum constant org.opensearch.repositories.azure.TokenCredentialType."+ unsupported_token_credential_type, e.getMessage()); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> storageServiceWithSettingsValidation(settings) + ); + assertEquals( + "No enum constant org.opensearch.repositories.azure.TokenCredentialType." + unsupported_token_credential_type, + e.getMessage() + ); } public void testBuildConnectStringWhenATokenCredentialIsEnabled() { @@ -572,7 +580,10 @@ public void testTokenCredentialWhenAccountIsNotProvided() { final Exception e = expectThrows(Exception.class, () -> storageServiceWithSettingsValidation(settings)); // Expect failure due to missing account name - assertEquals("missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]", e.getMessage()); + assertEquals( + "missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]", + e.getMessage() + ); } public void testAuthenticationMethodNotProvided() { @@ -581,9 +592,7 @@ public void testAuthenticationMethodNotProvided() { secureSettings.setString("azure.client.azure.account", "myaccount"); // Disabled Managed Identity in the settings by default - final Settings settings = Settings.builder() - .setSecureSettings(secureSettings) - .build(); + final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); // Expect fall back to authentication via sas token or account key when token credential is not specified. From 122af8fb4c0970d305e253d93d63332a458f1a45 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 4 Apr 2024 16:31:08 +1100 Subject: [PATCH 07/24] added transitive dependencies Signed-off-by: Chengwu Shi --- plugins/repository-azure/build.gradle | 90 +++++++- .../licenses/accessors-smart-2.5.0.jar.sha1 | 1 + .../licenses/accessors-smart-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/accessors-smart-NOTICE.txt | 0 .../licenses/asm-9.3.jar.sha1 | 1 + .../repository-azure/licenses/asm-LICENSE.txt | 27 +++ .../repository-azure/licenses/asm-NOTICE.txt | 0 .../licenses/azure-identity-1.11.2.jar.sha1 | 1 - .../licenses/azure-identity-1.11.4.jar.sha1 | 1 + .../licenses/content-type-2.3.jar.sha1 | 1 + .../licenses/content-type-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/content-type-NOTICE.txt | 0 .../licenses/jna-platform-5.13.0.jar.sha1 | 1 + .../licenses/jna-platform-LICENSE.txt | 26 +++ .../licenses/jna-platform-NOTICE.txt | 0 .../licenses/lang-tag-1.7.jar.sha1 | 1 + .../licenses/lang-tag-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/lang-tag-NOTICE.txt | 0 .../licenses/msal4j-1.14.2.jar.sha1 | 1 - .../licenses/msal4j-1.14.3.jar.sha1 | 1 + ...sal4j-persistence-extension-1.2.0.jar.sha1 | 1 + .../msal4j-persistence-extension-LICENSE.txt | 21 ++ .../msal4j-persistence-extension-NOTICE.txt | 0 .../licenses/nimbus-jose-jwt-9.37.3.jar.sha1 | 1 + .../licenses/nimbus-jose-jwt-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/nimbus-jose-jwt-NOTICE.txt | 0 .../licenses/oauth2-oidc-sdk-11.10.jar.sha1 | 1 - .../licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 | 1 + 28 files changed, 978 insertions(+), 7 deletions(-) create mode 100644 plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/accessors-smart-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/accessors-smart-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/asm-9.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/asm-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/asm-NOTICE.txt delete mode 100644 plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 create mode 100644 plugins/repository-azure/licenses/content-type-2.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/content-type-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/content-type-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jna-platform-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/jna-platform-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 create mode 100644 plugins/repository-azure/licenses/lang-tag-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/lang-tag-NOTICE.txt delete mode 100644 plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt delete mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index e521a1f853ba2..6e9fea6b24b48 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -56,10 +56,23 @@ dependencies { api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') api 'com.azure:azure-storage-blob:12.23.0' - api 'com.azure:azure-identity:1.11.2' - api 'com.microsoft.azure:msal4j:1.14.2' - api 'com.nimbusds:oauth2-oidc-sdk:11.10' + api 'com.azure:azure-identity:1.11.4' + // Start of transitive dependencies for azure-identity + api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0' + // Both azure-identity:1.11.4 and msal4j-persistence-extension:1.2.0 has compile dependency on different versions of jna-platform, + // selected the higher version which is 5.13.0 + api 'net.java.dev.jna:jna-platform:5.13.0' + api 'com.microsoft.azure:msal4j:1.14.3' + api 'com.nimbusds:oauth2-oidc-sdk:11.9.1' + api 'com.nimbusds:nimbus-jose-jwt:9.37.3' + api 'com.nimbusds:content-type:2.3' + api 'com.nimbusds:lang-tag:1.7' + // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart, + // selected the higher version which is 2.5.0 api 'net.minidev:json-smart:2.5.0' + api 'net.minidev:accessors-smart:2.5.0' + api 'org.ow2.asm:asm:9.3' + // End of transitive dependencies for azure-identity api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}" api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}" api "org.slf4j:slf4j-api:${versions.slf4j}" @@ -184,7 +197,76 @@ thirdPartyAudit { 'io.micrometer.observation.ObservationHandler', 'io.micrometer.observation.ObservationRegistry', 'io.micrometer.observation.ObservationRegistry$ObservationConfig', - 'io.micrometer.tracing.handler.DefaultTracingObservationHandler' + 'io.micrometer.tracing.handler.DefaultTracingObservationHandler', + // Start of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk". + 'com.google.crypto.tink.subtle.Ed25519Sign', + 'com.google.crypto.tink.subtle.Ed25519Sign$KeyPair', + 'com.google.crypto.tink.subtle.Ed25519Verify', + 'com.google.crypto.tink.subtle.X25519', + 'com.google.crypto.tink.subtle.XChaCha20Poly1305', + 'jakarta.servlet.ServletRequest', + 'jakarta.servlet.http.HttpServletRequest', + 'jakarta.servlet.http.HttpServletResponse', + 'javax.servlet.ServletRequest', + 'javax.servlet.http.HttpServletRequest', + 'javax.servlet.http.HttpServletResponse', + // net.shibboleth.utilities:java-support.* is declared as optional in the plugin `bnd-maven-plugin` used in "com.nimbusds:oauth2-oidc-sdk" + // Worth nothing that, the latest dependency "net.shibboleth.utilities:java-support:8.0.0" has many vulnerabilities. + // Hence ignored. + 'net.shibboleth.utilities.java.support.xml.SerializeSupport', + 'org.bouncycastle.asn1.pkcs.PrivateKeyInfo', + 'org.bouncycastle.asn1.x509.AlgorithmIdentifier', + 'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo', + 'org.bouncycastle.cert.X509CertificateHolder', + 'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder', + 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', + 'org.bouncycastle.crypto.InvalidCipherTextException', + 'org.bouncycastle.crypto.engines.AESEngine', + 'org.bouncycastle.crypto.modes.GCMBlockCipher', + 'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider', + 'org.bouncycastle.jce.provider.BouncyCastleProvider', + 'org.bouncycastle.openssl.PEMKeyPair', + 'org.bouncycastle.openssl.PEMParser', + 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter', + 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + 'org.cryptomator.siv.SivMode', + 'org.opensaml.core.config.InitializationException', + 'org.opensaml.core.config.InitializationService', + 'org.opensaml.core.xml.XMLObject', + 'org.opensaml.core.xml.XMLObjectBuilder', + 'org.opensaml.core.xml.XMLObjectBuilderFactory', + 'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport', + 'org.opensaml.core.xml.io.Marshaller', + 'org.opensaml.core.xml.io.MarshallerFactory', + 'org.opensaml.core.xml.io.MarshallingException', + 'org.opensaml.core.xml.io.Unmarshaller', + 'org.opensaml.core.xml.io.UnmarshallerFactory', + 'org.opensaml.core.xml.schema.XSString', + 'org.opensaml.core.xml.schema.impl.XSStringBuilder', + 'org.opensaml.saml.saml2.core.Assertion', + 'org.opensaml.saml.saml2.core.Attribute', + 'org.opensaml.saml.saml2.core.AttributeStatement', + 'org.opensaml.saml.saml2.core.AttributeValue', + 'org.opensaml.saml.saml2.core.Audience', + 'org.opensaml.saml.saml2.core.AudienceRestriction', + 'org.opensaml.saml.saml2.core.AuthnContext', + 'org.opensaml.saml.saml2.core.AuthnContextClassRef', + 'org.opensaml.saml.saml2.core.AuthnStatement', + 'org.opensaml.saml.saml2.core.Conditions', + 'org.opensaml.saml.saml2.core.Issuer', + 'org.opensaml.saml.saml2.core.NameID', + 'org.opensaml.saml.saml2.core.Subject', + 'org.opensaml.saml.saml2.core.SubjectConfirmation', + 'org.opensaml.saml.saml2.core.SubjectConfirmationData', + 'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator', + 'org.opensaml.security.credential.BasicCredential', + 'org.opensaml.security.credential.Credential', + 'org.opensaml.security.credential.UsageType', + 'org.opensaml.xmlsec.signature.Signature', + 'org.opensaml.xmlsec.signature.support.SignatureException', + 'org.opensaml.xmlsec.signature.support.SignatureValidator', + 'org.opensaml.xmlsec.signature.support.Signer', + // End of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk". ) ignoreViolations( diff --git a/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 new file mode 100644 index 0000000000000..1578c94fcdc7b --- /dev/null +++ b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 @@ -0,0 +1 @@ +aca011492dfe9c26f4e0659028a4fe0970829dd8 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt b/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/asm-9.3.jar.sha1 b/plugins/repository-azure/licenses/asm-9.3.jar.sha1 new file mode 100644 index 0000000000000..71d3966a6f6f9 --- /dev/null +++ b/plugins/repository-azure/licenses/asm-9.3.jar.sha1 @@ -0,0 +1 @@ +8e6300ef51c1d801a7ed62d07cd221aca3a90640 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/asm-LICENSE.txt b/plugins/repository-azure/licenses/asm-LICENSE.txt new file mode 100644 index 0000000000000..c71bb7bac5d4d --- /dev/null +++ b/plugins/repository-azure/licenses/asm-LICENSE.txt @@ -0,0 +1,27 @@ +ASM: a very small and fast Java bytecode manipulation framework +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/repository-azure/licenses/asm-NOTICE.txt b/plugins/repository-azure/licenses/asm-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 deleted file mode 100644 index fc459b573417c..0000000000000 --- a/plugins/repository-azure/licenses/azure-identity-1.11.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -31a85e3a591a2513736bf5cf787eeab9cd542590 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 new file mode 100644 index 0000000000000..c8d98ba9c8ad2 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 @@ -0,0 +1 @@ +59b5ce48888f638b80d85ef5aa0e22a265d3dc89 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/content-type-2.3.jar.sha1 b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1 new file mode 100644 index 0000000000000..e18bbaec9a89c --- /dev/null +++ b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1 @@ -0,0 +1 @@ +e3aa0be212d7a42839a8f3f506f5b990bcce0222 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/content-type-LICENSE.txt b/plugins/repository-azure/licenses/content-type-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/content-type-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/content-type-NOTICE.txt b/plugins/repository-azure/licenses/content-type-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 new file mode 100644 index 0000000000000..e2a8ba1c1bbd3 --- /dev/null +++ b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 @@ -0,0 +1 @@ +88e9a306715e9379f3122415ef4ae759a352640d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jna-platform-LICENSE.txt b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt new file mode 100644 index 0000000000000..c5a025f0c3e6d --- /dev/null +++ b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt @@ -0,0 +1,26 @@ +SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1 + +Java Native Access (JNA) is licensed under the LGPL, version 2.1 +or later, or (from version 4.0 onward) the Apache License, +version 2.0. + +You can freely decide which license you want to apply to the project. + +You may obtain a copy of the LGPL License at: + +http://www.gnu.org/licenses/licenses.html + +A copy is also included in the downloadable source code package +containing JNA, in file "LGPL2.1", under the same directory +as this file. + +You may obtain a copy of the Apache License at: + +http://www.apache.org/licenses/ + +A copy is also included in the downloadable source code package +containing JNA, in file "AL2.0", under the same directory +as this file. + +Commercial support may be available, please e-mail +twall[at]users[dot]sf[dot]net. diff --git a/plugins/repository-azure/licenses/jna-platform-NOTICE.txt b/plugins/repository-azure/licenses/jna-platform-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 new file mode 100644 index 0000000000000..9cd79d1dba715 --- /dev/null +++ b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 @@ -0,0 +1 @@ +97c73ecd70bc7e8eefb26c5eea84f251a63f1031 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/lang-tag-LICENSE.txt b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/lang-tag-NOTICE.txt b/plugins/repository-azure/licenses/lang-tag-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 deleted file mode 100644 index 39c7e6691ccca..0000000000000 --- a/plugins/repository-azure/licenses/msal4j-1.14.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bbf1c786b4ef555ad0111e59c35938da4271f592 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 new file mode 100644 index 0000000000000..2a6e42e3f2b48 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 @@ -0,0 +1 @@ +117b28c41bd760f979ed1b6467c5ec491f0d4d60 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 new file mode 100644 index 0000000000000..cfcf7548b7694 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 @@ -0,0 +1 @@ +1111a95878de8745ddc9de132df18ebd9ca7024d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt new file mode 100644 index 0000000000000..21071075c2459 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 new file mode 100644 index 0000000000000..7278cd8994f71 --- /dev/null +++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 @@ -0,0 +1 @@ +700f71ffefd60c16bd8ce711a956967ea9071cec \ No newline at end of file diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 deleted file mode 100644 index 102458c2aa7e6..0000000000000 --- a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -385b80a438f7678eb430681b362284a37db0c18d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 new file mode 100644 index 0000000000000..96d9a196a172a --- /dev/null +++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 @@ -0,0 +1 @@ +fa9a2e447e2cef4dfda40a854dd7ec35624a7799 \ No newline at end of file From abe98f66f4cae2ab0bc7c801b884c80663cd3ee3 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Fri, 12 Apr 2024 13:16:34 +1000 Subject: [PATCH 08/24] changed getStorageBlobEndpoint to private and using asm version from buildSrc/version.properties Signed-off-by: Chengwu Shi --- plugins/repository-azure/build.gradle | 2 +- .../azure/AzureStorageService.java | 2 +- .../azure/AzureStorageServiceTests.java | 43 +++++++++---------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 6e9fea6b24b48..f52d332a995b1 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -71,7 +71,7 @@ dependencies { // selected the higher version which is 2.5.0 api 'net.minidev:json-smart:2.5.0' api 'net.minidev:accessors-smart:2.5.0' - api 'org.ow2.asm:asm:9.3' + api "org.ow2.asm:asm:${versions.asm}" // End of transitive dependencies for azure-identity api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}" api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}" diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 8fdd6e4740286..e5ac08a4205cc 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -162,7 +162,7 @@ public Tuple> client(String clientName, BiC return new Tuple<>(state.getClient(), () -> buildOperationContext(azureStorageSettings)); } - public StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { + private StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { try { String endpointSuffix = settings.getEndpointSuffix(); if (!Strings.hasText(endpointSuffix)) { diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index b8171284ec376..f471972a3305b 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -120,14 +120,18 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr // Azure clients without account key and sas token. secureSettings.setString("azure.client.azure1.account", "myaccount1"); secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure3.account", "myaccount3"); final Settings settings = Settings.builder() .setSecureSettings(secureSettings) - // Enabled managed identity for both clients + // Enabled managed identity for all clients .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .put("azure.client.azure2.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .put("azure.client.azure3.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) // Defined an endpoint suffic for azure client 1 only. .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix") + // Defined an invalid endpoint suffix for azure client 3 only. + .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix") .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; @@ -139,35 +143,30 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr // Expect azure client 2 to use the default endpoint suffix final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); - } - } - - public void testGetStorageBlobEndpoint() throws IOException { - final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build(); - try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final AzureStorageSettings storageSettings = azureStorageService.storageSettings.get("azure1"); - String primaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getPrimaryUri(); - String secondaryUri = azureStorageService.getStorageBlobEndpoint(storageSettings).getSecondaryUri(); - assertThat(primaryUri, equalTo("https://myaccount1.blob.core.windows.net")); - assertThat(secondaryUri, equalTo("https://myaccount1-secondary.blob.core.windows.net")); + // Expect azure client 3 to fail due to invalid endpoint suffix + final RuntimeException e = expectThrows( + RuntimeException.class, + () -> azureStorageService.client("azure3").v1() + ); } } - public void testGetStorageBlobEndpointWithInvalidValues() throws IOException { + public void testGettingSecondaryStorageBlobEndpoint() throws IOException { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("azure.client.azure1.account", "myaccount1"); final Settings settings = Settings.builder() - .setSecureSettings(buildSecureSettings()) - .put("azure.client.azure1.endpoint_suffix", "my endpoint suffix") + .setSecureSettings(secureSettings) + // Enabled managed identity + .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .build(); - try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; - final AzureStorageSettings storageSettings = azureStorageService.storageSettings.get("azure1"); - final RuntimeException e = expectThrows( - RuntimeException.class, - () -> azureStorageService.getStorageBlobEndpoint(storageSettings) - ); + final Map prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap()); + final Map newSettings = AzureStorageSettings.overrideLocationMode(prevSettings, LocationMode.SECONDARY_ONLY); + azureStorageService.refreshAndClearCache(newSettings); + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net")); } } From ac1c51598aa903fb3bb329bccc449f0e2f840027 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Fri, 12 Apr 2024 15:48:47 +1000 Subject: [PATCH 09/24] run spotlessApply Signed-off-by: Chengwu Shi --- .../repositories/azure/AzureStorageServiceTests.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index f471972a3305b..8522ab8a8b8fb 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -145,10 +145,7 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); // Expect azure client 3 to fail due to invalid endpoint suffix - final RuntimeException e = expectThrows( - RuntimeException.class, - () -> azureStorageService.client("azure3").v1() - ); + final RuntimeException e = expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1()); } } @@ -163,7 +160,10 @@ public void testGettingSecondaryStorageBlobEndpoint() throws IOException { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; final Map prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap()); - final Map newSettings = AzureStorageSettings.overrideLocationMode(prevSettings, LocationMode.SECONDARY_ONLY); + final Map newSettings = AzureStorageSettings.overrideLocationMode( + prevSettings, + LocationMode.SECONDARY_ONLY + ); azureStorageService.refreshAndClearCache(newSettings); final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net")); From 7783b8c12c2ec08c4689e52ae6a81b939dc52b15 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Fri, 12 Apr 2024 18:24:02 +1000 Subject: [PATCH 10/24] update shas for asm 9.7 Signed-off-by: Chengwu Shi --- plugins/repository-azure/licenses/asm-9.3.jar.sha1 | 1 - plugins/repository-azure/licenses/asm-9.7.jar.sha1 | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 plugins/repository-azure/licenses/asm-9.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/asm-9.7.jar.sha1 diff --git a/plugins/repository-azure/licenses/asm-9.3.jar.sha1 b/plugins/repository-azure/licenses/asm-9.3.jar.sha1 deleted file mode 100644 index 71d3966a6f6f9..0000000000000 --- a/plugins/repository-azure/licenses/asm-9.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e6300ef51c1d801a7ed62d07cd221aca3a90640 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/asm-9.7.jar.sha1 b/plugins/repository-azure/licenses/asm-9.7.jar.sha1 new file mode 100644 index 0000000000000..84c9a9703af6d --- /dev/null +++ b/plugins/repository-azure/licenses/asm-9.7.jar.sha1 @@ -0,0 +1 @@ +073d7b3086e14beb604ced229c302feff6449723 \ No newline at end of file From e0a0bbf6368a90f121760761628d5b3fe8dbaa5e Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Mon, 15 Apr 2024 09:43:41 +1000 Subject: [PATCH 11/24] use version.jna for jna-platform Signed-off-by: Chengwu Shi --- plugins/repository-azure/build.gradle | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index f52d332a995b1..ff62c328c7e74 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -59,9 +59,7 @@ dependencies { api 'com.azure:azure-identity:1.11.4' // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0' - // Both azure-identity:1.11.4 and msal4j-persistence-extension:1.2.0 has compile dependency on different versions of jna-platform, - // selected the higher version which is 5.13.0 - api 'net.java.dev.jna:jna-platform:5.13.0' + api "net.java.dev.jna:jna-platform:${versions.jna}" api 'com.microsoft.azure:msal4j:1.14.3' api 'com.nimbusds:oauth2-oidc-sdk:11.9.1' api 'com.nimbusds:nimbus-jose-jwt:9.37.3' From 368d2adbfb866807f3c3079ebce8206d630b095b Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 18 Apr 2024 10:37:29 +1000 Subject: [PATCH 12/24] change string 'core.windows.net' to be a constant, use uri.create instead of new uri, and added a few comments for clarity Signed-off-by: Chengwu Shi --- .../repositories/azure/AzureStorageService.java | 17 +++++++---------- .../repositories/azure/TokenCredentialType.java | 3 +++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index e5ac08a4205cc..17af6196550bb 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -48,6 +48,7 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.ParallelTransferOptions; import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.connectionstring.StorageConnectionString; import com.azure.storage.common.implementation.connectionstring.StorageEndpoint; import com.azure.storage.common.policy.RequestRetryOptions; @@ -163,17 +164,13 @@ public Tuple> client(String clientName, BiC } private StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { - try { - String endpointSuffix = settings.getEndpointSuffix(); - if (!Strings.hasText(endpointSuffix)) { - endpointSuffix = "core.windows.net"; - } - final URI primaryBlobEndpoint = new URI("https://" + settings.getAccount() + ".blob." + endpointSuffix); - final URI secondaryBlobEndpoint = new URI("https://" + settings.getAccount() + "-secondary.blob." + endpointSuffix); - return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); - } catch (URISyntaxException var14) { - throw logger.logExceptionAsError(new RuntimeException(var14)); + String endpointSuffix = settings.getEndpointSuffix(); + if (!Strings.hasText(endpointSuffix)) { + endpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; } + final URI primaryBlobEndpoint = URI.create("https://" + settings.getAccount() + ".blob." + endpointSuffix); + final URI secondaryBlobEndpoint = URI.create("https://" + settings.getAccount() + "-secondary.blob." + endpointSuffix); + return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); } private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiConsumer statsCollector) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java index 75346617c91ac..4ed0e383cd439 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java @@ -10,6 +10,9 @@ // Type of token credentials that the plugin supports public enum TokenCredentialType { + // This represents the support for ManagedIdentityCredential MANAGED_IDENTITY, + // This is the default when token credential is not configure. + // SAS token or Account Key will be used for authentication instead. NOT_APPLICABLE } From ebcd568cf94cc4dd169064890cc8a6071564ae8a Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 18 Apr 2024 10:45:48 +1000 Subject: [PATCH 13/24] added one more comment line Signed-off-by: Chengwu Shi --- .../org/opensearch/repositories/azure/AzureStorageService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 17af6196550bb..c59a80174fe80 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -166,6 +166,7 @@ public Tuple> client(String clientName, BiC private StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { String endpointSuffix = settings.getEndpointSuffix(); if (!Strings.hasText(endpointSuffix)) { + // endpointSuffix is default to "core.windows.net". endpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; } final URI primaryBlobEndpoint = URI.create("https://" + settings.getAccount() + ".blob." + endpointSuffix); From efa20e513e4cbfd94b751900e2cec1ea5485e357 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Thu, 18 Apr 2024 17:12:21 +1000 Subject: [PATCH 14/24] refactor TokenCredentialType to not have NOT_APPLICABLE Signed-off-by: Chengwu Shi --- .../azure/AzureStorageSettings.java | 34 ++++++++++++++----- .../azure/TokenCredentialType.java | 6 +--- .../azure/AzureStorageServiceTests.java | 10 ++---- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 5e42c541ab06f..fda34e8395530 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -50,6 +50,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; final class AzureStorageSettings { @@ -78,10 +79,10 @@ final class AzureStorageSettings { ); /** Azure token credentials such as Managed Identity */ - public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( + public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( AZURE_CLIENT_PREFIX_KEY, "token_credential_type", - (key) -> new Setting<>(key, "not_applicable", s -> TokenCredentialType.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope), + key -> Setting.simpleString(key, Property.NodeScope), () -> ACCOUNT_SETTING ); @@ -204,7 +205,7 @@ final class AzureStorageSettings { private final String account; private final String connectString; private final String endpointSuffix; - private final TokenCredentialType tokenCredentialType; + private final String tokenCredentialType; private final TimeValue timeout; private final int maxRetries; private final LocationMode locationMode; @@ -218,7 +219,7 @@ final class AzureStorageSettings { private AzureStorageSettings( String account, String connectString, - TokenCredentialType tokenCredentialType, + String tokenCredentialType, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -247,7 +248,7 @@ private AzureStorageSettings( String account, String key, String sasToken, - TokenCredentialType tokenCredentialType, + String tokenCredentialType, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -259,7 +260,7 @@ private AzureStorageSettings( ) { this.account = account; this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, tokenCredentialType); - this.tokenCredentialType = tokenCredentialType; + this.tokenCredentialType = validateTokenCredentialType(tokenCredentialType); this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -272,7 +273,7 @@ private AzureStorageSettings( } public boolean usesManagedIdentityCredential() { - return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY); + return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY.name()); } public String getEndpointSuffix() { @@ -299,16 +300,31 @@ public String getAccount() { return account; } + private static String validateTokenCredentialType(String tokenCredentialType) { + tokenCredentialType = tokenCredentialType.toUpperCase(Locale.ROOT); + final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); + boolean isValidTokenCredentialType = false; + for (TokenCredentialType type : TokenCredentialType.values()) { + if (Objects.equals(type.toString(), tokenCredentialType)) { + isValidTokenCredentialType = true; + } + } + if (hasTokenCredentialType && !isValidTokenCredentialType) { + throw new SettingsException(String.format("'%s' is currently not supported.", tokenCredentialType)); + } + return tokenCredentialType; + } + private static String buildConnectString( String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix, - TokenCredentialType tokenCredentialType + String tokenCredentialType ) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); - final boolean hasTokenCredentialType = !tokenCredentialType.equals(TokenCredentialType.NOT_APPLICABLE); + final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); // When a valid token credential type is declared, we are no longer using connection string if (hasTokenCredentialType) { return ""; diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java index 4ed0e383cd439..9048873053e49 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java @@ -10,9 +10,5 @@ // Type of token credentials that the plugin supports public enum TokenCredentialType { - // This represents the support for ManagedIdentityCredential - MANAGED_IDENTITY, - // This is the default when token credential is not configure. - // SAS token or Account Key will be used for authentication instead. - NOT_APPLICABLE + MANAGED_IDENTITY } diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 8522ab8a8b8fb..299a694f9cee3 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -521,14 +521,8 @@ public void testSettingUnsupportedTokenCredentialForAuthentication() { .put("azure.client.azure.token_credential_type", unsupported_token_credential_type) .build(); - final IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> storageServiceWithSettingsValidation(settings) - ); - assertEquals( - "No enum constant org.opensearch.repositories.azure.TokenCredentialType." + unsupported_token_credential_type, - e.getMessage() - ); + final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); + assertEquals("'" + unsupported_token_credential_type + "' is currently not supported.", e.getMessage()); } public void testBuildConnectStringWhenATokenCredentialIsEnabled() { From 74d1d8c8e13d1b54e04a5d5682c8b15e7379aabb Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Mon, 22 Apr 2024 17:27:20 +1000 Subject: [PATCH 15/24] refactored code based on recommended changes from Andriy Redko Signed-off-by: Chengwu Shi --- .../azure/AzureStorageService.java | 35 +---- .../azure/AzureStorageSettings.java | 103 +++++++------ .../azure/TokenCredentialType.java | 28 +++- .../azure/AzureStorageServiceTests.java | 142 +++++++++++++++--- 4 files changed, 210 insertions(+), 98 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index c59a80174fe80..84e4f1ecc8c1d 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -43,13 +43,10 @@ import com.azure.core.util.Configuration; import com.azure.core.util.Context; import com.azure.core.util.logging.ClientLogger; -import com.azure.identity.ManagedIdentityCredentialBuilder; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.ParallelTransferOptions; import com.azure.storage.blob.specialized.BlockBlobAsyncClient; -import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.implementation.connectionstring.StorageConnectionString; import com.azure.storage.common.implementation.connectionstring.StorageEndpoint; import com.azure.storage.common.policy.RequestRetryOptions; import com.azure.storage.common.policy.RetryPolicyType; @@ -58,13 +55,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import java.net.Authenticator; import java.net.PasswordAuthentication; -import java.net.URI; import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.time.Duration; @@ -163,24 +158,9 @@ public Tuple> client(String clientName, BiC return new Tuple<>(state.getClient(), () -> buildOperationContext(azureStorageSettings)); } - private StorageEndpoint getStorageBlobEndpoint(final AzureStorageSettings settings) { - String endpointSuffix = settings.getEndpointSuffix(); - if (!Strings.hasText(endpointSuffix)) { - // endpointSuffix is default to "core.windows.net". - endpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; - } - final URI primaryBlobEndpoint = URI.create("https://" + settings.getAccount() + ".blob." + endpointSuffix); - final URI secondaryBlobEndpoint = URI.create("https://" + settings.getAccount() + "-secondary.blob." + endpointSuffix); - return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); - } - private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiConsumer statsCollector) throws InvalidKeyException, URISyntaxException { final BlobServiceClientBuilder builder = createClientBuilder(azureStorageSettings); - // When managed identity is enabled, no connection string will be generated, need to declare the primary uri - if (azureStorageSettings.usesManagedIdentityCredential()) { - builder.endpoint(getStorageBlobEndpoint(azureStorageSettings).getPrimaryUri()); - } final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(new NioThreadFactory()); final NettyAsyncHttpClientBuilder clientBuilder = new NettyAsyncHttpClientBuilder().eventLoopGroup(eventLoopGroup); @@ -234,13 +214,7 @@ protected PasswordAuthentication getPasswordAuthentication() { * migration guide for mode details: */ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) { - final StorageEndpoint endpoint; - if (settings.usesManagedIdentityCredential()) { - endpoint = getStorageBlobEndpoint(settings); - } else { - final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger); - endpoint = storageConnectionString.getBlobEndpoint(); - } + final StorageEndpoint endpoint = settings.getStorageEndpoint(); if (endpoint == null || endpoint.getPrimaryUri() == null) { throw new IllegalArgumentException("connectionString missing required settings to derive blob service primary endpoint."); @@ -272,12 +246,7 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException { - if (settings.usesManagedIdentityCredential()) { - return SocketAccess.doPrivilegedException( - () -> new BlobServiceClientBuilder().credential(new ManagedIdentityCredentialBuilder().build()) - ); - } - return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString())); + return SocketAccess.doPrivilegedException(() -> settings.configure(new BlobServiceClientBuilder())); } /** diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index fda34e8395530..0266f1aa676e1 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -32,6 +32,12 @@ package org.opensearch.repositories.azure; +import com.azure.core.util.logging.ClientLogger; +import com.azure.identity.ManagedIdentityCredentialBuilder; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.connectionstring.StorageConnectionString; +import com.azure.storage.common.implementation.connectionstring.StorageEndpoint; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.SecureSetting; @@ -45,14 +51,16 @@ import org.opensearch.core.common.settings.SecureString; import java.net.InetAddress; +import java.net.URI; import java.net.UnknownHostException; import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.Objects; +import java.util.function.Function; final class AzureStorageSettings { + private final ClientLogger logger = new ClientLogger(AzureStorageSettings.class); // prefix for azure client settings private static final String AZURE_CLIENT_PREFIX_KEY = "azure.client."; @@ -82,7 +90,11 @@ final class AzureStorageSettings { public static final AffixSetting TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting( AZURE_CLIENT_PREFIX_KEY, "token_credential_type", - key -> Setting.simpleString(key, Property.NodeScope), + key -> Setting.simpleString(key, value -> { + if (value != null && value != "") { + TokenCredentialType.valueOfType(value); + } + }, Property.NodeScope), () -> ACCOUNT_SETTING ); @@ -203,9 +215,10 @@ final class AzureStorageSettings { ); private final String account; - private final String connectString; - private final String endpointSuffix; private final String tokenCredentialType; + private final Function clientBuilder; + private final StorageEndpoint storageEndpoint; + private final String endpointSuffix; private final TimeValue timeout; private final int maxRetries; private final LocationMode locationMode; @@ -218,8 +231,9 @@ final class AzureStorageSettings { // copy-constructor private AzureStorageSettings( String account, - String connectString, String tokenCredentialType, + Function clientBuilder, + StorageEndpoint storageEndpoint, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -231,8 +245,9 @@ private AzureStorageSettings( ProxySettings proxySettings ) { this.account = account; - this.connectString = connectString; this.tokenCredentialType = tokenCredentialType; + this.clientBuilder = clientBuilder; + this.storageEndpoint = storageEndpoint; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -259,8 +274,17 @@ private AzureStorageSettings( ProxySettings proxySettings ) { this.account = account; - this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, tokenCredentialType); - this.tokenCredentialType = validateTokenCredentialType(tokenCredentialType); + this.tokenCredentialType = tokenCredentialType; + if (this.tokenCredentialType == null || this.tokenCredentialType.isEmpty()) { + String connectString = buildConnectString(account, key, sasToken, endpointSuffix); + this.clientBuilder = (builder) -> builder.connectionString(connectString); + this.storageEndpoint = getStorageEndpointFromConnectString(connectString); + } else { + StorageEndpoint storageEndpoint = buildStorageEndpoint(account, endpointSuffix); + this.clientBuilder = (builder) -> builder.credential(new ManagedIdentityCredentialBuilder().build()) + .endpoint(storageEndpoint.getPrimaryUri()); + this.storageEndpoint = storageEndpoint; + } this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -272,8 +296,12 @@ private AzureStorageSettings( this.proxySettings = proxySettings; } - public boolean usesManagedIdentityCredential() { - return tokenCredentialType.equals(TokenCredentialType.MANAGED_IDENTITY.name()); + public String getTokenCredentialType() { + return tokenCredentialType; + } + + public StorageEndpoint getStorageEndpoint() { + return storageEndpoint; } public String getEndpointSuffix() { @@ -292,43 +320,13 @@ public ProxySettings getProxySettings() { return proxySettings; } - public String getConnectString() { - return connectString; - } - public String getAccount() { return account; } - private static String validateTokenCredentialType(String tokenCredentialType) { - tokenCredentialType = tokenCredentialType.toUpperCase(Locale.ROOT); - final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); - boolean isValidTokenCredentialType = false; - for (TokenCredentialType type : TokenCredentialType.values()) { - if (Objects.equals(type.toString(), tokenCredentialType)) { - isValidTokenCredentialType = true; - } - } - if (hasTokenCredentialType && !isValidTokenCredentialType) { - throw new SettingsException(String.format("'%s' is currently not supported.", tokenCredentialType)); - } - return tokenCredentialType; - } - - private static String buildConnectString( - String account, - @Nullable String key, - @Nullable String sasToken, - String endpointSuffix, - String tokenCredentialType - ) { + private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); - final boolean hasTokenCredentialType = Strings.hasText(tokenCredentialType); - // When a valid token credential type is declared, we are no longer using connection string - if (hasTokenCredentialType) { - return ""; - } if (hasSasToken == false && hasKey == false) { throw new SettingsException("Neither a secret key nor a shared access token was set."); } @@ -480,8 +478,9 @@ static Map overrideLocationMode( entry.getKey(), new AzureStorageSettings( entry.getValue().account, - entry.getValue().connectString, entry.getValue().tokenCredentialType, + entry.getValue().clientBuilder, + entry.getValue().storageEndpoint, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, @@ -496,4 +495,24 @@ static Map overrideLocationMode( } return mapBuilder.immutableMap(); } + + public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder) { + return clientBuilder.apply(builder); + } + + private StorageEndpoint buildStorageEndpoint(String account, String endpointSuffix) { + String tokenCredentialEndpointSuffix = endpointSuffix; + if (!Strings.hasText(tokenCredentialEndpointSuffix)) { + // Default to "core.windows.net". + tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; + } + final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix); + final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix); + return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); + } + + private StorageEndpoint getStorageEndpointFromConnectString(String connectionString) { + final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectionString, logger); + return storageConnectionString.getBlobEndpoint(); + } } diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java index 9048873053e49..1f78f73934231 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java @@ -8,7 +8,33 @@ package org.opensearch.repositories.azure; +import java.util.Arrays; + // Type of token credentials that the plugin supports public enum TokenCredentialType { - MANAGED_IDENTITY + MANAGED_IDENTITY("managed"); + + private final String type; + + TokenCredentialType(String type) { + this.type = type; + } + + public static String[] getTokenCredentialTypes() { + return Arrays.stream(TokenCredentialType.values()).map(tokenCredentialType -> tokenCredentialType.type).toArray(String[]::new); + } + + static TokenCredentialType valueOfType(String type) { + for (TokenCredentialType value : values()) { + if (value.type.equalsIgnoreCase(type) || value.name().equalsIgnoreCase(type)) { + return value; + } + } + throw new IllegalArgumentException( + "The token credential type '" + + type + + "' is unsupported, please use one of the following values: " + + String.join(", ", getTokenCredentialTypes()) + ); + } } diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 299a694f9cee3..3ed11039bb278 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -120,35 +120,70 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr // Azure clients without account key and sas token. secureSettings.setString("azure.client.azure1.account", "myaccount1"); secureSettings.setString("azure.client.azure2.account", "myaccount2"); - secureSettings.setString("azure.client.azure3.account", "myaccount3"); final Settings settings = Settings.builder() .setSecureSettings(secureSettings) // Enabled managed identity for all clients .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .put("azure.client.azure2.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) - .put("azure.client.azure3.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) - // Defined an endpoint suffic for azure client 1 only. + // Defined an endpoint suffix for azure client 1 only. .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix") - // Defined an invalid endpoint suffix for azure client 3 only. - .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix") .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; - + System.out.println(azureStorageService.storageSettings.get("azure1").getStorageEndpoint().getPrimaryUri()); // Expect azure client 1 to use the custom endpoint suffix final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); - // Expect azure client 2 to use the default endpoint suffix final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); - - // Expect azure client 3 to fail due to invalid endpoint suffix - final RuntimeException e = expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1()); } } + public void testCreateClientWithInvalidEndpointSuffix() { + final MockSecureSettings secureSettings1 = new MockSecureSettings(); + secureSettings1.setString("azure.client.azure1.account", "myaccount1"); + + final MockSecureSettings secureSettings2 = new MockSecureSettings(); + secureSettings2.setString("azure.client.azure2.account", "myaccount2"); + secureSettings2.setString("azure.client.azure2.key", encodeKey("mykey12")); + + final MockSecureSettings secureSettings3 = new MockSecureSettings(); + secureSettings3.setString("azure.client.azure3.account", "myaccount1"); + secureSettings3.setString("azure.client.azure3.sas_token", encodeKey("mysastoken")); + + // Test invalid endpointsuffix when token credential is used + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings1) + .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) + .put("azure.client.azure1.endpoint_suffix", "invalid endpoint suffix") + .build(); + + final IllegalArgumentException e1 = expectThrows( + IllegalArgumentException.class, + () -> storageServiceWithSettingsValidation(settings) + ); + + // Test invalid endpointsuffix when account key is used + final Settings settings2 = Settings.builder() + .setSecureSettings(secureSettings2) + // Defined an invalid endpoint suffix + .put("azure.client.azure2.endpoint_suffix", "invalid endpoint suffix") + .build(); + + final RuntimeException e2 = expectThrows(RuntimeException.class, () -> storageServiceWithSettingsValidation(settings2)); + + // Test invalid endpointsuffix when sas token key is used + final Settings settings3 = Settings.builder() + .setSecureSettings(secureSettings3) + // Defined an invalid endpoint suffix + .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix") + .build(); + + final RuntimeException e3 = expectThrows(RuntimeException.class, () -> storageServiceWithSettingsValidation(settings3)); + } + public void testGettingSecondaryStorageBlobEndpoint() throws IOException { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.azure1.account", "myaccount1"); @@ -497,19 +532,37 @@ public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. secureSettings.setString("azure.client.azure.account", "myaccount"); + secureSettings.setString("azure.client.azure2.account", "myaccount"); - // Enabled Managed Identity in the settings using lower case + // Enabled Managed Identity in the settings using lower case and mixed case final Settings settings = Settings.builder() .setSecureSettings(secureSettings) .put("azure.client.azure.token_credential_type", "managed_identity") + .put("azure.client.azure2.token_credential_type", "managed_IDENTITY") .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(true, mock.storageSettings.get("azure").usesManagedIdentityCredential()); + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed_identity"); + assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "managed_IDENTITY"); + } + + public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure client without account key or sas token. + secureSettings.setString("azure.client.azure.account", "myaccount"); + + // Enabled Managed Identity in the settings using lower case + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure.token_credential_type", "managed") + .build(); + + final AzureStorageService mock = storageServiceWithSettingsValidation(settings); + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed"); } public void testSettingUnsupportedTokenCredentialForAuthentication() { - final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_THAT_DOES_NOT_EXIST"; + final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_TYPE_THAT_DOES_NOT_EXIST"; final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. @@ -521,11 +574,20 @@ public void testSettingUnsupportedTokenCredentialForAuthentication() { .put("azure.client.azure.token_credential_type", unsupported_token_credential_type) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); - assertEquals("'" + unsupported_token_credential_type + "' is currently not supported.", e.getMessage()); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> storageServiceWithSettingsValidation(settings) + ); + assertEquals( + "The token credential type '" + + unsupported_token_credential_type + + "' is unsupported, please use one of the following values: " + + String.join(", ", TokenCredentialType.getTokenCredentialTypes()), + e.getMessage() + ); } - public void testBuildConnectStringWhenATokenCredentialIsEnabled() { + public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications() { final String token_credential_type = TokenCredentialType.MANAGED_IDENTITY.name(); final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure1 with account key @@ -554,11 +616,11 @@ public void testBuildConnectStringWhenATokenCredentialIsEnabled() { .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - // Expect connection string to be empty when managed identity is selected for authentication - assertEquals("", mock.storageSettings.get("azure1").getConnectString()); - assertEquals("", mock.storageSettings.get("azure2").getConnectString()); - assertEquals("", mock.storageSettings.get("azure3").getConnectString()); - assertEquals("", mock.storageSettings.get("azure4").getConnectString()); + // Expect token credential authentication is selected over account key or sas token. + assertEquals(token_credential_type, mock.storageSettings.get("azure1").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure2").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure3").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure4").getTokenCredentialType()); } public void testTokenCredentialWhenAccountIsNotProvided() { @@ -592,6 +654,24 @@ public void testAuthenticationMethodNotProvided() { assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage()); } + public void testSettingTokenCredentialTypeToBeEmpty() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure clients without account key and sas token. + secureSettings.setString("azure.client.azure1.account", "myaccount"); + secureSettings.setString("azure.client.azure2.account", "myaccount"); + + // Disabled Managed Identity in the settings by default + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("azure.client.azure1.token_credential_type", "") + .put("azure.client.azure2.token_credential_type", " ") + .build(); + final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); + + // Expect fall back to authentication via sas token or account key when token credential is not specified. + assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage()); + } + public void testManagedIdentityIsEnabled() { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. @@ -604,7 +684,25 @@ public void testManagedIdentityIsEnabled() { .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(true, mock.storageSettings.get("azure").usesManagedIdentityCredential()); + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), TokenCredentialType.MANAGED_IDENTITY.name()); + } + + public void testNonTokenCredentialAuthenticationEnabled() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + // Azure client account key. + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure1.sas_token", encodeKey("mysastoken")); + + // Azure client with sas token + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.key", encodeKey("mykey")); + + final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); + final AzureStorageService mock = storageServiceWithSettingsValidation(settings); + + // Expect token credential is not enabled + assertEquals(mock.storageSettings.get("azure1").getTokenCredentialType(), ""); + assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), ""); } private static MockSecureSettings buildSecureSettings() { From 305cafa98d161c03cde5ec36fe9ba3592820392a Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Mon, 22 Apr 2024 17:33:55 +1000 Subject: [PATCH 16/24] added a jvm security policy for reactor-core jar Signed-off-by: Chengwu Shi --- .../main/resources/org/opensearch/bootstrap/security.policy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index e1226345ef961..ddb08053dde42 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -79,6 +79,10 @@ grant codeBase "${codebase.jna}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; +grant codeBase "${codebase.reactor-core}" { + // azure-identity need this permission to perform authentication for managed identity + permission java.net.SocketPermission "*", "connect,resolve"; +}; // ZSTD compression grant codeBase "${codebase.zstd-jni}" { From 223f737030680857fc598d7f138f21595636db37 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Tue, 23 Apr 2024 09:37:57 +1000 Subject: [PATCH 17/24] fixed failing forbidden api fix Signed-off-by: Chengwu Shi --- .../opensearch/repositories/azure/AzureStorageServiceTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 3ed11039bb278..0337d026b9623 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -131,7 +131,6 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService; - System.out.println(azureStorageService.storageSettings.get("azure1").getStorageEndpoint().getPrimaryUri()); // Expect azure client 1 to use the custom endpoint suffix final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); From ce947ff2dae913f9eb70ba0725d3849ed664b211 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Fri, 26 Apr 2024 17:13:56 +1000 Subject: [PATCH 18/24] removed the jvm security policy for reactor-core which was added Signed-off-by: Chengwu Shi --- .../main/resources/org/opensearch/bootstrap/security.policy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index ddb08053dde42..e1226345ef961 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -79,10 +79,6 @@ grant codeBase "${codebase.jna}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; -grant codeBase "${codebase.reactor-core}" { - // azure-identity need this permission to perform authentication for managed identity - permission java.net.SocketPermission "*", "connect,resolve"; -}; // ZSTD compression grant codeBase "${codebase.zstd-jni}" { From 4873f3f46beb7d0475466d1e9227e35f331c1ec6 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Sun, 28 Apr 2024 21:09:24 +1000 Subject: [PATCH 19/24] refactored code, such that storage endpoint is not evaluated at compiled time Signed-off-by: Chengwu Shi --- .../azure/AzureStorageSettings.java | 62 +++++++++---------- .../azure/AzureStorageServiceTests.java | 48 +++++--------- 2 files changed, 43 insertions(+), 67 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 0266f1aa676e1..26c59a599cd75 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -91,7 +91,7 @@ final class AzureStorageSettings { AZURE_CLIENT_PREFIX_KEY, "token_credential_type", key -> Setting.simpleString(key, value -> { - if (value != null && value != "") { + if (usesTokenCredential(value)) { TokenCredentialType.valueOfType(value); } }, Property.NodeScope), @@ -215,9 +215,9 @@ final class AzureStorageSettings { ); private final String account; + private final String connectString; private final String tokenCredentialType; private final Function clientBuilder; - private final StorageEndpoint storageEndpoint; private final String endpointSuffix; private final TimeValue timeout; private final int maxRetries; @@ -231,9 +231,9 @@ final class AzureStorageSettings { // copy-constructor private AzureStorageSettings( String account, + String connectString, String tokenCredentialType, Function clientBuilder, - StorageEndpoint storageEndpoint, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -245,9 +245,9 @@ private AzureStorageSettings( ProxySettings proxySettings ) { this.account = account; + this.connectString = connectString; this.tokenCredentialType = tokenCredentialType; this.clientBuilder = clientBuilder; - this.storageEndpoint = storageEndpoint; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -275,15 +275,13 @@ private AzureStorageSettings( ) { this.account = account; this.tokenCredentialType = tokenCredentialType; - if (this.tokenCredentialType == null || this.tokenCredentialType.isEmpty()) { - String connectString = buildConnectString(account, key, sasToken, endpointSuffix); - this.clientBuilder = (builder) -> builder.connectionString(connectString); - this.storageEndpoint = getStorageEndpointFromConnectString(connectString); - } else { - StorageEndpoint storageEndpoint = buildStorageEndpoint(account, endpointSuffix); + if (usesTokenCredential(tokenCredentialType)) { + this.connectString = ""; this.clientBuilder = (builder) -> builder.credential(new ManagedIdentityCredentialBuilder().build()) - .endpoint(storageEndpoint.getPrimaryUri()); - this.storageEndpoint = storageEndpoint; + .endpoint(getStorageEndpoint().getPrimaryUri()); + } else { + this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); + this.clientBuilder = (builder) -> builder.connectionString(connectString); } this.endpointSuffix = endpointSuffix; this.timeout = timeout; @@ -300,8 +298,24 @@ public String getTokenCredentialType() { return tokenCredentialType; } + private static Boolean usesTokenCredential(String tokenCredential) { + return tokenCredential != null && !tokenCredential.isEmpty() && !tokenCredential.isBlank(); + } + public StorageEndpoint getStorageEndpoint() { - return storageEndpoint; + if (usesTokenCredential(tokenCredentialType)) { + String tokenCredentialEndpointSuffix = endpointSuffix; + if (!Strings.hasText(tokenCredentialEndpointSuffix)) { + // Default to "core.windows.net". + tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; + } + final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix); + final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix); + return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); + } else { + final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectString, logger); + return storageConnectionString.getBlobEndpoint(); + } } public String getEndpointSuffix() { @@ -320,10 +334,6 @@ public ProxySettings getProxySettings() { return proxySettings; } - public String getAccount() { - return account; - } - private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) { final boolean hasSasToken = Strings.hasText(sasToken); final boolean hasKey = Strings.hasText(key); @@ -478,9 +488,9 @@ static Map overrideLocationMode( entry.getKey(), new AzureStorageSettings( entry.getValue().account, + entry.getValue().connectString, entry.getValue().tokenCredentialType, entry.getValue().clientBuilder, - entry.getValue().storageEndpoint, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, @@ -499,20 +509,4 @@ static Map overrideLocationMode( public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder) { return clientBuilder.apply(builder); } - - private StorageEndpoint buildStorageEndpoint(String account, String endpointSuffix) { - String tokenCredentialEndpointSuffix = endpointSuffix; - if (!Strings.hasText(tokenCredentialEndpointSuffix)) { - // Default to "core.windows.net". - tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; - } - final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix); - final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix); - return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); - } - - private StorageEndpoint getStorageEndpointFromConnectString(String connectionString) { - final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectionString, logger); - return storageConnectionString.getBlobEndpoint(); - } } diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 0337d026b9623..a965624813d17 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -140,47 +140,29 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr } } - public void testCreateClientWithInvalidEndpointSuffix() { - final MockSecureSettings secureSettings1 = new MockSecureSettings(); - secureSettings1.setString("azure.client.azure1.account", "myaccount1"); - - final MockSecureSettings secureSettings2 = new MockSecureSettings(); - secureSettings2.setString("azure.client.azure2.account", "myaccount2"); - secureSettings2.setString("azure.client.azure2.key", encodeKey("mykey12")); - - final MockSecureSettings secureSettings3 = new MockSecureSettings(); - secureSettings3.setString("azure.client.azure3.account", "myaccount1"); - secureSettings3.setString("azure.client.azure3.sas_token", encodeKey("mysastoken")); + public void testCreateClientWithInvalidEndpointSuffix() throws IOException { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.key", encodeKey("mykey12")); + secureSettings.setString("azure.client.azure3.account", "myaccount1"); + secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken")); - // Test invalid endpointsuffix when token credential is used final Settings settings = Settings.builder() - .setSecureSettings(secureSettings1) + .setSecureSettings(secureSettings) .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .put("azure.client.azure1.endpoint_suffix", "invalid endpoint suffix") - .build(); - - final IllegalArgumentException e1 = expectThrows( - IllegalArgumentException.class, - () -> storageServiceWithSettingsValidation(settings) - ); - - // Test invalid endpointsuffix when account key is used - final Settings settings2 = Settings.builder() - .setSecureSettings(secureSettings2) - // Defined an invalid endpoint suffix .put("azure.client.azure2.endpoint_suffix", "invalid endpoint suffix") - .build(); - - final RuntimeException e2 = expectThrows(RuntimeException.class, () -> storageServiceWithSettingsValidation(settings2)); - - // Test invalid endpointsuffix when sas token key is used - final Settings settings3 = Settings.builder() - .setSecureSettings(secureSettings3) - // Defined an invalid endpoint suffix .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix") .build(); - final RuntimeException e3 = expectThrows(RuntimeException.class, () -> storageServiceWithSettingsValidation(settings3)); + try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { + final AzureStorageService azureStorageService = plugin.azureStoreService; + // Expect all clients 1 to fail due to invalid endpoint suffix + expectThrows(SettingsException.class, () -> azureStorageService.client("azure1").v1()); + expectThrows(RuntimeException.class, () -> azureStorageService.client("azure2").v1()); + expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1()); + } } public void testGettingSecondaryStorageBlobEndpoint() throws IOException { From 76276eaffc2b4eb98514db639ce4f917da3ada33 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Sun, 28 Apr 2024 22:36:14 +1000 Subject: [PATCH 20/24] refactored token credential types checks Signed-off-by: Chengwu Shi --- .../azure/AzureStorageSettings.java | 2 +- .../azure/AzureStorageServiceTests.java | 27 ++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 26c59a599cd75..5ac4e705f2801 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -299,7 +299,7 @@ public String getTokenCredentialType() { } private static Boolean usesTokenCredential(String tokenCredential) { - return tokenCredential != null && !tokenCredential.isEmpty() && !tokenCredential.isBlank(); + return tokenCredential != null && !tokenCredential.isEmpty(); } public StorageEndpoint getStorageEndpoint() { diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index a965624813d17..7b00a9bc75c75 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -636,21 +636,34 @@ public void testAuthenticationMethodNotProvided() { } public void testSettingTokenCredentialTypeToBeEmpty() { - final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure clients without account key and sas token. + final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.azure1.account", "myaccount"); - secureSettings.setString("azure.client.azure2.account", "myaccount"); - - // Disabled Managed Identity in the settings by default final Settings settings = Settings.builder() .setSecureSettings(secureSettings) .put("azure.client.azure1.token_credential_type", "") - .put("azure.client.azure2.token_credential_type", " ") .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); - // Expect fall back to authentication via sas token or account key when token credential is not specified. + final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings)); assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage()); + + // Azure clients without account key and sas token. + final MockSecureSettings secureSettings2 = new MockSecureSettings(); + secureSettings2.setString("azure.client.azure2.account", "myaccount"); + final Settings settings2 = Settings.builder() + .setSecureSettings(secureSettings2) + .put("azure.client.azure2.token_credential_type", " ") + .build(); + // Expect failing token credential type checks + final IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> storageServiceWithSettingsValidation(settings2) + ); + assertEquals( + "The token credential type ' ' is unsupported, please use one of the following values: " + + String.join(", ", TokenCredentialType.getTokenCredentialTypes()), + e2.getMessage() + ); } public void testManagedIdentityIsEnabled() { From 5caae91b1606c9e7ad5657f2bea091248db617f0 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Tue, 30 Apr 2024 11:32:11 +1000 Subject: [PATCH 21/24] changed clientlogger in azurestoragesetting to 'AzureStorageService' Signed-off-by: Chengwu Shi --- .../org/opensearch/repositories/azure/AzureStorageSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index 5ac4e705f2801..d35fc943d4ff2 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -60,7 +60,7 @@ import java.util.function.Function; final class AzureStorageSettings { - private final ClientLogger logger = new ClientLogger(AzureStorageSettings.class); + private final ClientLogger logger = new ClientLogger(AzureStorageService.class); // prefix for azure client settings private static final String AZURE_CLIENT_PREFIX_KEY = "azure.client."; From 3d34c8e4d10bf12f285481af3b3418163b133b38 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Tue, 30 Apr 2024 14:09:51 +1000 Subject: [PATCH 22/24] added a nullable argument to getStorageEndpoint Signed-off-by: Chengwu Shi --- .../opensearch/repositories/azure/AzureStorageService.java | 2 +- .../opensearch/repositories/azure/AzureStorageSettings.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 84e4f1ecc8c1d..bfa7f092a0f9c 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -214,7 +214,7 @@ protected PasswordAuthentication getPasswordAuthentication() { * migration guide for mode details: */ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) { - final StorageEndpoint endpoint = settings.getStorageEndpoint(); + final StorageEndpoint endpoint = settings.getStorageEndpoint(logger); if (endpoint == null || endpoint.getPrimaryUri() == null) { throw new IllegalArgumentException("connectionString missing required settings to derive blob service primary endpoint."); diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index d35fc943d4ff2..ade75e94721dc 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -60,7 +60,6 @@ import java.util.function.Function; final class AzureStorageSettings { - private final ClientLogger logger = new ClientLogger(AzureStorageService.class); // prefix for azure client settings private static final String AZURE_CLIENT_PREFIX_KEY = "azure.client."; @@ -278,7 +277,7 @@ private AzureStorageSettings( if (usesTokenCredential(tokenCredentialType)) { this.connectString = ""; this.clientBuilder = (builder) -> builder.credential(new ManagedIdentityCredentialBuilder().build()) - .endpoint(getStorageEndpoint().getPrimaryUri()); + .endpoint(getStorageEndpoint(null).getPrimaryUri()); } else { this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); this.clientBuilder = (builder) -> builder.connectionString(connectString); @@ -302,7 +301,7 @@ private static Boolean usesTokenCredential(String tokenCredential) { return tokenCredential != null && !tokenCredential.isEmpty(); } - public StorageEndpoint getStorageEndpoint() { + public StorageEndpoint getStorageEndpoint(@Nullable ClientLogger logger) { if (usesTokenCredential(tokenCredentialType)) { String tokenCredentialEndpointSuffix = endpointSuffix; if (!Strings.hasText(tokenCredentialEndpointSuffix)) { From d15d2fbd78064af8b96f65d469201dd3d35ec821 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 1 May 2024 09:11:18 -0400 Subject: [PATCH 23/24] Fix IdentityClient security permissions, get rid of connection string (since it is not applicable to managed identity configuration) Signed-off-by: Andriy Redko --- .../repositories/azure/AzureBlobStore.java | 2 +- .../azure/AzureStorageService.java | 54 ++- .../azure/AzureStorageSettings.java | 76 ++-- .../plugin-metadata/plugin-security.policy | 3 + .../azure/AzureStorageServiceTests.java | 402 ++++++++++-------- .../org/opensearch/bootstrap/security.policy | 5 + 6 files changed, 315 insertions(+), 227 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java index e76a6bdd16764..acaaa043df3ac 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java @@ -175,7 +175,7 @@ public BlobContainer blobContainer(BlobPath path) { } @Override - public void close() { + public void close() throws IOException { service.close(); } diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index bfa7f092a0f9c..70e5f241fc2bb 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -58,14 +58,19 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; +import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; import java.net.URISyntaxException; +import java.security.AccessController; import java.security.InvalidKeyException; +import java.security.PrivilegedAction; import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -98,6 +103,37 @@ public class AzureStorageService implements AutoCloseable { // 'package' for testing volatile Map storageSettings = emptyMap(); private final Map clients = new ConcurrentHashMap<>(); + private final ExecutorService executor; + + private static final class IdentityClientThreadFactory implements ThreadFactory { + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + + @SuppressWarnings("removal") + IdentityClientThreadFactory(String namePrefix) { + this.namePrefix = namePrefix; + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, new Runnable() { + @SuppressWarnings("removal") + public void run() { + AccessController.doPrivileged(new PrivilegedAction<>() { + public Void run() { + r.run(); + return null; + } + }); + } + }, namePrefix + "[T#" + threadNumber.getAndIncrement() + "]", 0); + t.setDaemon(true); + return t; + } + } static { // See please: @@ -111,6 +147,9 @@ public AzureStorageService(Settings settings) { // eagerly load client settings so that secure settings are read final Map clientsSettings = AzureStorageSettings.load(settings); refreshAndClearCache(clientsSettings); + executor = SocketAccess.doPrivilegedException( + () -> Executors.newCachedThreadPool(new IdentityClientThreadFactory("azure-identity-client")) + ); } /** @@ -244,9 +283,8 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde return builder; } - private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, - URISyntaxException { - return SocketAccess.doPrivilegedException(() -> settings.configure(new BlobServiceClientBuilder())); + private BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException { + return SocketAccess.doPrivilegedException(() -> settings.configure(new BlobServiceClientBuilder(), executor, logger)); } /** @@ -292,9 +330,17 @@ public Map refreshAndClearCache(Map Setting.simpleString(key, value -> { - if (usesTokenCredential(value)) { + if (Strings.hasText(value) == true) { TokenCredentialType.valueOfType(value); } }, Property.NodeScope), @@ -214,9 +218,9 @@ final class AzureStorageSettings { ); private final String account; - private final String connectString; private final String tokenCredentialType; - private final Function clientBuilder; + private final TriFunction clientBuilder; + private final Function endpointBuilder; private final String endpointSuffix; private final TimeValue timeout; private final int maxRetries; @@ -230,9 +234,9 @@ final class AzureStorageSettings { // copy-constructor private AzureStorageSettings( String account, - String connectString, String tokenCredentialType, - Function clientBuilder, + TriFunction clientBuilder, + Function endpointBuilder, String endpointSuffix, TimeValue timeout, int maxRetries, @@ -244,9 +248,9 @@ private AzureStorageSettings( ProxySettings proxySettings ) { this.account = account; - this.connectString = connectString; this.tokenCredentialType = tokenCredentialType; this.clientBuilder = clientBuilder; + this.endpointBuilder = endpointBuilder; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -274,13 +278,35 @@ private AzureStorageSettings( ) { this.account = account; this.tokenCredentialType = tokenCredentialType; - if (usesTokenCredential(tokenCredentialType)) { - this.connectString = ""; - this.clientBuilder = (builder) -> builder.credential(new ManagedIdentityCredentialBuilder().build()) - .endpoint(getStorageEndpoint(null).getPrimaryUri()); + if (Strings.hasText(tokenCredentialType) == true) { + this.endpointBuilder = (logger) -> { + String tokenCredentialEndpointSuffix = endpointSuffix; + if (Strings.hasText(tokenCredentialEndpointSuffix) == false) { + // Default to "core.windows.net". + tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; + } + final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix); + final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix); + return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); + }; + + this.clientBuilder = (builder, executor, logger) -> builder.credential(new ManagedIdentityCredentialBuilder() { + @Override + public ManagedIdentityCredential build() { + // Use the privileged executor with IdentityClient instance + CredentialBuilderBaseHelper.getClientOptions(this).setExecutorService(executor); + return super.build(); + } + }.build()).endpoint(endpointBuilder.apply(logger).getPrimaryUri()); } else { - this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); - this.clientBuilder = (builder) -> builder.connectionString(connectString); + final String connectString = buildConnectString(account, key, sasToken, endpointSuffix); + + this.endpointBuilder = (logger) -> { + final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectString, logger); + return storageConnectionString.getBlobEndpoint(); + }; + + this.clientBuilder = (builder, executor, logger) -> builder.connectionString(connectString); } this.endpointSuffix = endpointSuffix; this.timeout = timeout; @@ -297,24 +323,8 @@ public String getTokenCredentialType() { return tokenCredentialType; } - private static Boolean usesTokenCredential(String tokenCredential) { - return tokenCredential != null && !tokenCredential.isEmpty(); - } - - public StorageEndpoint getStorageEndpoint(@Nullable ClientLogger logger) { - if (usesTokenCredential(tokenCredentialType)) { - String tokenCredentialEndpointSuffix = endpointSuffix; - if (!Strings.hasText(tokenCredentialEndpointSuffix)) { - // Default to "core.windows.net". - tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS; - } - final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix); - final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix); - return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint); - } else { - final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectString, logger); - return storageConnectionString.getBlobEndpoint(); - } + public StorageEndpoint getStorageEndpoint(ClientLogger logger) { + return endpointBuilder.apply(logger); } public String getEndpointSuffix() { @@ -487,9 +497,9 @@ static Map overrideLocationMode( entry.getKey(), new AzureStorageSettings( entry.getValue().account, - entry.getValue().connectString, entry.getValue().tokenCredentialType, entry.getValue().clientBuilder, + entry.getValue().endpointBuilder, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, @@ -505,7 +515,7 @@ static Map overrideLocationMode( return mapBuilder.immutableMap(); } - public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder) { - return clientBuilder.apply(builder); + public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder, ExecutorService executor, ClientLogger logger) { + return clientBuilder.apply(builder, executor, logger); } } diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy index f3bf52ea46505..e8fbe35ebab1d 100644 --- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -41,4 +41,7 @@ grant { // azure client set Authenticator for proxy username/password permission java.net.NetPermission "setDefaultAuthenticator"; + + // azure identity + permission java.util.PropertyPermission "os.name", "read"; }; diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java index 7b00a9bc75c75..ea74a49e593cf 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java @@ -51,7 +51,6 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; -import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Base64; @@ -107,11 +106,12 @@ public void testCreateClientWithEndpointSuffix() throws IOException { .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix") .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); - final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); - assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); + final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); + assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); + } } } @@ -130,13 +130,14 @@ public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() thr .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix") .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - // Expect azure client 1 to use the custom endpoint suffix - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); - // Expect azure client 2 to use the default endpoint suffix - final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); - assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + // Expect azure client 1 to use the custom endpoint suffix + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); + // Expect azure client 2 to use the default endpoint suffix + final BlobServiceClient client2 = azureStorageService.client("azure2").v1(); + assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net")); + } } } @@ -157,11 +158,12 @@ public void testCreateClientWithInvalidEndpointSuffix() throws IOException { .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - // Expect all clients 1 to fail due to invalid endpoint suffix - expectThrows(SettingsException.class, () -> azureStorageService.client("azure1").v1()); - expectThrows(RuntimeException.class, () -> azureStorageService.client("azure2").v1()); - expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1()); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + // Expect all clients 1 to fail due to invalid endpoint suffix + expectThrows(SettingsException.class, () -> azureStorageService.client("azure1").v1()); + expectThrows(RuntimeException.class, () -> azureStorageService.client("azure2").v1()); + expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1()); + } } } @@ -174,15 +176,16 @@ public void testGettingSecondaryStorageBlobEndpoint() throws IOException { .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final Map prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap()); - final Map newSettings = AzureStorageSettings.overrideLocationMode( - prevSettings, - LocationMode.SECONDARY_ONLY - ); - azureStorageService.refreshAndClearCache(newSettings); - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final Map prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap()); + final Map newSettings = AzureStorageSettings.overrideLocationMode( + prevSettings, + LocationMode.SECONDARY_ONLY + ); + azureStorageService.refreshAndClearCache(newSettings); + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net")); + } } } @@ -193,13 +196,15 @@ public void testClientUsingManagedIdentity() throws IOException { .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - - // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is not - // setup in the test - final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo()); - assertThat(e.getMessage(), is("Managed Identity authentication is not available.")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + + // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is + // not + // setup in the test + final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo()); + assertThat(e.getMessage(), is("Managed Identity authentication is not available.")); + } } } @@ -217,29 +222,30 @@ public void testReinitClientSettings() throws IOException { secureSettings2.setString("azure.client.azure3.key", encodeKey("mykey23")); final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); - assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net")); - final BlobServiceClient client12 = azureStorageService.client("azure2").v1(); - assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net")); - // client 3 is missing - final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3")); - assertThat(e1.getMessage(), is("Unable to find client with name [azure3]")); - // update client settings - plugin.reload(settings2); - // old client 1 not changed - assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net")); - // new client 1 is changed - final BlobServiceClient client21 = azureStorageService.client("azure1").v1(); - assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net")); - // old client 2 not changed - assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net")); - // new client2 is gone - final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2")); - assertThat(e2.getMessage(), is("Unable to find client with name [azure2]")); - // client 3 emerged - final BlobServiceClient client23 = azureStorageService.client("azure3").v1(); - assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); + assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net")); + final BlobServiceClient client12 = azureStorageService.client("azure2").v1(); + assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net")); + // client 3 is missing + final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3")); + assertThat(e1.getMessage(), is("Unable to find client with name [azure3]")); + // update client settings + plugin.reload(settings2); + // old client 1 not changed + assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net")); + // new client 1 is changed + final BlobServiceClient client21 = azureStorageService.client("azure1").v1(); + assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net")); + // old client 2 not changed + assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net")); + // new client2 is gone + final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2")); + assertThat(e2.getMessage(), is("Unable to find client with name [azure2]")); + // client 3 emerged + final BlobServiceClient client23 = azureStorageService.client("azure3").v1(); + assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net")); + } } } @@ -249,17 +255,18 @@ public void testReinitClientEmptySettings() throws IOException { secureSettings.setString("azure.client.azure1.key", encodeKey("mykey11")); final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); - assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); - // reinit with empty settings - final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY)); - assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration.")); - // existing client untouched - assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); - // new client also untouched - final BlobServiceClient client21 = azureStorageService.client("azure1").v1(); - assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); + assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + // reinit with empty settings + final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY)); + assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration.")); + // existing client untouched + assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + // new client also untouched + final BlobServiceClient client21 = azureStorageService.client("azure1").v1(); + assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + } } } @@ -278,138 +285,150 @@ public void testReinitClientWrongSettings() throws IOException { secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33")); final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) { - final AzureStorageService azureStorageService = plugin.azureStoreService; - final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); - assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); - final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2)); - assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set.")); - final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3)); - assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set.")); - // existing client untouched - assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + try (final AzureStorageService azureStorageService = plugin.azureStoreService) { + final BlobServiceClient client11 = azureStorageService.client("azure1").v1(); + assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2)); + assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set.")); + final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3)); + assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set.")); + // existing client untouched + assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net")); + } } } - public void testGetSelectedClientNonExisting() { - final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings()); - final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4")); - assertThat(e.getMessage(), is("Unable to find client with name [azure4]")); + public void testGetSelectedClientNonExisting() throws IOException { + try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) { + final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4")); + assertThat(e.getMessage(), is("Unable to find client with name [azure4]")); + } } - public void testGetSelectedClientDefaultTimeout() { + public void testGetSelectedClientDefaultTimeout() throws IOException { final Settings timeoutSettings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure3.timeout", "30s") .build(); - final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings); - assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue()); - assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30))); + try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) { + assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue()); + assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30))); + } } - public void testClientDefaultConnectTimeout() { + public void testClientDefaultConnectTimeout() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure3.connect.timeout", "25s") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout(); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout(); - assertThat(timeout, notNullValue()); - assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25))); - assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue()); - assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10))); + assertThat(timeout, notNullValue()); + assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25))); + assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue()); + assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10))); + } } - public void testClientDefaultWriteTimeout() { + public void testClientDefaultWriteTimeout() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure3.write.timeout", "85s") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout(); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout(); - assertThat(timeout, notNullValue()); - assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85))); - assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue()); - assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + assertThat(timeout, notNullValue()); + assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85))); + assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue()); + assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + } } - public void testClientDefaultReadTimeout() { + public void testClientDefaultReadTimeout() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure3.read.timeout", "120s") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout(); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout(); - assertThat(timeout, notNullValue()); - assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120))); - assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue()); - assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + assertThat(timeout, notNullValue()); + assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120))); + assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue()); + assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + } } - public void testClientDefaultResponseTimeout() { + public void testClientDefaultResponseTimeout() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure3.response.timeout", "1ms") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout(); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout(); - assertThat(timeout, notNullValue()); - assertThat(timeout, equalTo(TimeValue.timeValueMillis(1))); - assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue()); - assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + assertThat(timeout, notNullValue()); + assertThat(timeout, equalTo(TimeValue.timeValueMillis(1))); + assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue()); + assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60))); + } } - public void testGetSelectedClientNoTimeout() { - final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings()); - assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue()); + public void testGetSelectedClientNoTimeout() throws IOException { + try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) { + assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue()); + } } - public void testGetSelectedClientBackoffPolicy() { - final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings()); - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - assertThat(requestRetryOptions(client1), is(notNullValue())); + public void testGetSelectedClientBackoffPolicy() throws IOException { + try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) { + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(requestRetryOptions(client1), is(notNullValue())); + } } - public void testGetSelectedClientBackoffPolicyNbRetries() { + public void testGetSelectedClientBackoffPolicyNbRetries() throws IOException { final Settings timeoutSettings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure1.max_retries", 7) .build(); - final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings); - final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); - assertThat(requestRetryOptions(client1), is(notNullValue())); + try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) { + final BlobServiceClient client1 = azureStorageService.client("azure1").v1(); + assertThat(requestRetryOptions(client1), is(notNullValue())); + } } - public void testNoProxy() { + public void testNoProxy() throws IOException { final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); - assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); - assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); + assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); + assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); + } } - public void testProxyHttp() throws UnknownHostException { + public void testProxyHttp() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure1.proxy.host", "127.0.0.1") .put("azure.client.azure1.proxy.port", 8080) .put("azure.client.azure1.proxy.type", "http") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); - - assertThat(azure1Proxy, notNullValue()); - assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); - assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings()); - assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings()); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); + + assertThat(azure1Proxy, notNullValue()); + assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); + assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); + assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings()); + assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings()); + } } - public void testMultipleProxies() throws UnknownHostException { + public void testMultipleProxies() throws IOException { final Settings settings = Settings.builder() .setSecureSettings(buildSecureSettings()) .put("azure.client.azure1.proxy.host", "127.0.0.1") @@ -419,21 +438,22 @@ public void testMultipleProxies() throws UnknownHostException { .put("azure.client.azure2.proxy.port", 8081) .put("azure.client.azure2.proxy.type", "http") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); - assertThat(azure1Proxy, notNullValue()); - assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); - assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings(); - assertThat(azure2Proxy, notNullValue()); - assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); - assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081))); - assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername())); - assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword())); - assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); - } - - public void testProxySocks() throws UnknownHostException { + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); + assertThat(azure1Proxy, notNullValue()); + assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); + assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); + final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings(); + assertThat(azure2Proxy, notNullValue()); + assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP)); + assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081))); + assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername())); + assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword())); + assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS); + } + } + + public void testProxySocks() throws IOException { final MockSecureSettings secureSettings = buildSecureSettings(); secureSettings.setString("azure.client.azure1.proxy.username", "user"); secureSettings.setString("azure.client.azure1.proxy.password", "pwd"); @@ -443,15 +463,16 @@ public void testProxySocks() throws UnknownHostException { .put("azure.client.azure1.proxy.type", "socks5") .setSecureSettings(secureSettings) .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); - assertThat(azure1Proxy, notNullValue()); - assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5)); - assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - assertEquals("user", azure1Proxy.getUsername()); - assertEquals("pwd", azure1Proxy.getPassword()); - assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings()); - assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings()); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings(); + assertThat(azure1Proxy, notNullValue()); + assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5)); + assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); + assertEquals("user", azure1Proxy.getUsername()); + assertEquals("pwd", azure1Proxy.getPassword()); + assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings()); + assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings()); + } } public void testProxyNoHost() { @@ -509,7 +530,7 @@ public void testBlobNameFromUri() throws URISyntaxException { assertThat(name, is("path/to/myfile")); } - public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() { + public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() throws IOException { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. secureSettings.setString("azure.client.azure.account", "myaccount"); @@ -522,12 +543,13 @@ public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() { .put("azure.client.azure2.token_credential_type", "managed_IDENTITY") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed_identity"); - assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "managed_IDENTITY"); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed_identity"); + assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "managed_IDENTITY"); + } } - public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue() { + public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue() throws IOException { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. secureSettings.setString("azure.client.azure.account", "myaccount"); @@ -538,8 +560,9 @@ public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue( .put("azure.client.azure.token_credential_type", "managed") .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed"); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed"); + } } public void testSettingUnsupportedTokenCredentialForAuthentication() { @@ -568,7 +591,7 @@ public void testSettingUnsupportedTokenCredentialForAuthentication() { ); } - public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications() { + public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications() throws IOException { final String token_credential_type = TokenCredentialType.MANAGED_IDENTITY.name(); final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure1 with account key @@ -595,13 +618,13 @@ public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications .put("azure.client.azure3.token_credential_type", token_credential_type) .put("azure.client.azure4.token_credential_type", token_credential_type) .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - - // Expect token credential authentication is selected over account key or sas token. - assertEquals(token_credential_type, mock.storageSettings.get("azure1").getTokenCredentialType()); - assertEquals(token_credential_type, mock.storageSettings.get("azure2").getTokenCredentialType()); - assertEquals(token_credential_type, mock.storageSettings.get("azure3").getTokenCredentialType()); - assertEquals(token_credential_type, mock.storageSettings.get("azure4").getTokenCredentialType()); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + // Expect token credential authentication is selected over account key or sas token. + assertEquals(token_credential_type, mock.storageSettings.get("azure1").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure2").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure3").getTokenCredentialType()); + assertEquals(token_credential_type, mock.storageSettings.get("azure4").getTokenCredentialType()); + } } public void testTokenCredentialWhenAccountIsNotProvided() { @@ -652,7 +675,7 @@ public void testSettingTokenCredentialTypeToBeEmpty() { secureSettings2.setString("azure.client.azure2.account", "myaccount"); final Settings settings2 = Settings.builder() .setSecureSettings(secureSettings2) - .put("azure.client.azure2.token_credential_type", " ") + .put("azure.client.azure2.token_credential_type", "x") .build(); // Expect failing token credential type checks final IllegalArgumentException e2 = expectThrows( @@ -660,13 +683,13 @@ public void testSettingTokenCredentialTypeToBeEmpty() { () -> storageServiceWithSettingsValidation(settings2) ); assertEquals( - "The token credential type ' ' is unsupported, please use one of the following values: " + "The token credential type 'x' is unsupported, please use one of the following values: " + String.join(", ", TokenCredentialType.getTokenCredentialTypes()), e2.getMessage() ); } - public void testManagedIdentityIsEnabled() { + public void testManagedIdentityIsEnabled() throws IOException { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client without account key or sas token. secureSettings.setString("azure.client.azure.account", "myaccount"); @@ -677,11 +700,12 @@ public void testManagedIdentityIsEnabled() { .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name()) .build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), TokenCredentialType.MANAGED_IDENTITY.name()); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), TokenCredentialType.MANAGED_IDENTITY.name()); + } } - public void testNonTokenCredentialAuthenticationEnabled() { + public void testNonTokenCredentialAuthenticationEnabled() throws IOException { final MockSecureSettings secureSettings = new MockSecureSettings(); // Azure client account key. secureSettings.setString("azure.client.azure1.account", "myaccount1"); @@ -692,11 +716,11 @@ public void testNonTokenCredentialAuthenticationEnabled() { secureSettings.setString("azure.client.azure2.key", encodeKey("mykey")); final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - - // Expect token credential is not enabled - assertEquals(mock.storageSettings.get("azure1").getTokenCredentialType(), ""); - assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), ""); + try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) { + // Expect token credential is not enabled + assertEquals(mock.storageSettings.get("azure1").getTokenCredentialType(), ""); + assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), ""); + } } private static MockSecureSettings buildSecureSettings() { diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index e1226345ef961..55e8db0d9c6a3 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -85,6 +85,11 @@ grant codeBase "${codebase.zstd-jni}" { permission java.lang.RuntimePermission "loadLibrary.*"; }; +// repository-azure plugin and server side streaming +grant codeBase "${codebase.reactor-core}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + //// Everything else: grant { From 280dd0a14fdfda3c3caa0814063f2168e7e544f0 Mon Sep 17 00:00:00 2001 From: Chengwu Shi Date: Tue, 14 May 2024 14:24:20 +1000 Subject: [PATCH 24/24] Responded to feedback from Andrew Ross, fixed typo, spelling, added shudownNow() and Thread.currentThread().interrupt() Signed-off-by: Chengwu Shi --- CHANGELOG.md | 2 +- .../opensearch/repositories/azure/AzureStorageService.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d08100728ecd..5487d7555c970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Added support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423)) +- Add support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423)) - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478)) - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293)) diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index 70e5f241fc2bb..f39ed185d8b35 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -336,9 +336,11 @@ public void close() throws IOException { this.executor.shutdown(); try { if (this.executor.awaitTermination(30, TimeUnit.SECONDS) == false) { - logger.warning("The executor was not shutdown gracefuly with 30 seconds"); + this.executor.shutdownNow(); + logger.warning("The executor was not shutdown gracefully with 30 seconds"); } } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); throw new IOException(ex); } }