diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7d119eeb206..53ff23e0a1a78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) -- Added encryption-sdk lib to provide encryption and decryption capabilities ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466)) +- Added encryption-sdk lib to provide encryption and decryption capabilities ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466) [#9289](https://github.com/opensearch-project/OpenSearch/pull/9289)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 5103999428814..d78c16d17de72 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -48,6 +48,10 @@ dependencies { implementation 'org.apache.commons:commons-compress:1.23.0' } +configurations.implementation { + exclude group: 'org.bouncycastle', module: 'bcprov-jdk15to18' +} + tasks.named("dependencyLicenses").configure { mapping from: /bc.*/, to: 'bouncycastle' } diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java index bdcaa295f1cc4..f8fd983543117 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java @@ -8,6 +8,7 @@ package org.opensearch.common.crypto; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.io.InputStreamContainer; import java.io.IOException; @@ -20,6 +21,7 @@ * T - Encryption Metadata / CryptoContext * U - Parsed Encryption Metadata / CryptoContext */ +@ExperimentalApi public interface CryptoHandler { /** diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 83c4db80b7798..9baf2092c6f29 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -131,10 +131,6 @@ thirdPartyAudit { 'com.aayushatharva.brotli4j.encoder.Encoder$Parameters', // classes are missing - // from io.netty.logging.CommonsLoggerFactory (netty) - 'org.apache.commons.logging.Log', - 'org.apache.commons.logging.LogFactory', - // from Log4j (deliberate, Netty will fallback to Log4j 2) 'org.apache.log4j.Level', 'org.apache.log4j.Logger', @@ -177,13 +173,6 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional - 'org.slf4j.helpers.FormattingTuple', - 'org.slf4j.helpers.MessageFormatter', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.spi.LocationAwareLogger', - 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/plugins/discovery-azure-classic/licenses/commons-logging-1.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-logging-LICENSE.txt b/plugins/discovery-azure-classic/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/discovery-azure-classic/licenses/commons-logging-NOTICE.txt b/plugins/discovery-azure-classic/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index d3d6e140ce4f3..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/discovery-ec2/licenses/commons-logging-1.2.jar.sha1 b/plugins/discovery-ec2/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-logging-LICENSE.txt b/plugins/discovery-ec2/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index 57bc88a15a0ee..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/plugins/discovery-ec2/licenses/commons-logging-NOTICE.txt b/plugins/discovery-ec2/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 72eb32a902458..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt b/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 2be7689435062..0000000000000 --- a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2022 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt b/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/discovery-gce/licenses/commons-logging-1.2.jar.sha1 b/plugins/discovery-gce/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/discovery-gce/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-logging-LICENSE.txt b/plugins/discovery-gce/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/discovery-gce/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/discovery-gce/licenses/commons-logging-NOTICE.txt b/plugins/discovery-gce/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index d3d6e140ce4f3..0000000000000 --- a/plugins/discovery-gce/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index 9911bb75f9209..0000000000000 --- a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt b/plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt deleted file mode 100644 index 9f27bafe96885..0000000000000 --- a/plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. - (http://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt b/plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/identity-shiro/licenses/commons-logging-1.2.jar.sha1 b/plugins/identity-shiro/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/identity-shiro/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/commons-logging-LICENSE.txt b/plugins/identity-shiro/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/identity-shiro/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/identity-shiro/licenses/commons-logging-NOTICE.txt b/plugins/identity-shiro/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 1a45218353e87..0000000000000 --- a/plugins/identity-shiro/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Logging -Copyright 2003-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt b/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt b/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index 9911bb75f9209..0000000000000 --- a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt deleted file mode 100644 index e1fc4a1506db5..0000000000000 --- a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt +++ /dev/null @@ -1,23 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. - (http://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 deleted file mode 100644 index d0c2f2486ee1f..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-lang3-LICENSE.txt b/plugins/ingest-attachment/licenses/commons-lang3-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-lang3-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/ingest-attachment/licenses/commons-lang3-NOTICE.txt b/plugins/ingest-attachment/licenses/commons-lang3-NOTICE.txt deleted file mode 100644 index 13a3140897472..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-lang3-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Lang -Copyright 2001-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/ingest-attachment/licenses/commons-logging-1.2.jar.sha1 b/plugins/ingest-attachment/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-logging-LICENSE.txt b/plugins/ingest-attachment/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/ingest-attachment/licenses/commons-logging-NOTICE.txt b/plugins/ingest-attachment/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index d3d6e140ce4f3..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt b/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 52055e61de46f..0000000000000 --- a/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt b/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 deleted file mode 100644 index d0c2f2486ee1f..0000000000000 --- a/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/commons-lang3-LICENSE.txt b/plugins/repository-azure/licenses/commons-lang3-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/repository-azure/licenses/commons-lang3-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt b/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt deleted file mode 100644 index 078282451b679..0000000000000 --- a/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt +++ /dev/null @@ -1,8 +0,0 @@ -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) diff --git a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt b/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt b/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 b/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt b/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index 57bc88a15a0ee..0000000000000 --- a/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt b/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 72eb32a902458..0000000000000 --- a/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 deleted file mode 100644 index d0c2f2486ee1f..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt deleted file mode 100644 index 8dfa22157abc3..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt +++ /dev/null @@ -1,9 +0,0 @@ -Apache Commons Lang -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - diff --git a/plugins/repository-hdfs/licenses/commons-logging-1.2.jar.sha1 b/plugins/repository-hdfs/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-logging-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/repository-hdfs/licenses/commons-logging-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 556bd03951d4b..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt b/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt b/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-s3/licenses/commons-logging-1.2.jar.sha1 b/plugins/repository-s3/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/plugins/repository-s3/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-logging-LICENSE.txt b/plugins/repository-s3/licenses/commons-logging-LICENSE.txt deleted file mode 100644 index 57bc88a15a0ee..0000000000000 --- a/plugins/repository-s3/licenses/commons-logging-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/plugins/repository-s3/licenses/commons-logging-NOTICE.txt b/plugins/repository-s3/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 72eb32a902458..0000000000000 --- a/plugins/repository-s3/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt b/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt b/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index 4b3c1bfbd53ea..bb1643faecc95 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -63,12 +63,12 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.blobstore.DeleteResult; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; @@ -99,7 +99,7 @@ import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART; import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART; -class S3BlobContainer extends AbstractBlobContainer implements VerifyingMultiStreamBlobContainer { +class S3BlobContainer extends AbstractBlobContainer implements AsyncMultiStreamBlobContainer { private static final Logger logger = LogManager.getLogger(S3BlobContainer.class); @@ -217,6 +217,10 @@ public void readBlobAsync(String blobName, ActionListener listener) throw new UnsupportedOperationException(); } + public boolean remoteIntegrityCheckSupported() { + return true; + } + // package private for testing long getLargeBlobThresholdInBytes() { return blobStore.bufferSizeInBytes(); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index c9486c8bbfd78..ecad68474b601 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -41,9 +41,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.StreamContext; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.StreamContextSupplier; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; @@ -161,7 +161,7 @@ protected Class unresponsiveExceptionType() { } @Override - protected VerifyingMultiStreamBlobContainer createBlobContainer( + protected AsyncMultiStreamBlobContainer createBlobContainer( final @Nullable Integer maxRetries, final @Nullable TimeValue readTimeout, final @Nullable Boolean disableChunkedEncoding, @@ -320,7 +320,7 @@ public void testWriteBlobByStreamsWithRetries() throws Exception { } }); - final VerifyingMultiStreamBlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null); + final AsyncMultiStreamBlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null); List openInputStreams = new ArrayList<>(); CountDownLatch countDownLatch = new CountDownLatch(1); AtomicReference exceptionRef = new AtomicReference<>(); diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 8c0ee8ba718ac..128dc6078c608 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -65,10 +65,6 @@ thirdPartyAudit { 'com.aayushatharva.brotli4j.encoder.Encoder$Mode', 'com.aayushatharva.brotli4j.encoder.Encoder$Parameters', - // from io.netty.logging.CommonsLoggerFactory (netty) - 'org.apache.commons.logging.Log', - 'org.apache.commons.logging.LogFactory', - // from Log4j (deliberate, Netty will fallback to Log4j 2) 'org.apache.log4j.Level', 'org.apache.log4j.Logger', @@ -103,13 +99,6 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional - 'org.slf4j.helpers.FormattingTuple', - 'org.slf4j.helpers.MessageFormatter', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.spi.LocationAwareLogger', - 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/server/build.gradle b/server/build.gradle index f6db3d53a0dcc..af56032897ee1 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -106,6 +106,7 @@ dependencies { api project(':libs:opensearch-x-content') api project(":libs:opensearch-geo") api project(":libs:opensearch-telemetry") + api project(":libs:opensearch-encryption-sdk") compileOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java similarity index 94% rename from server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java rename to server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java index 887a4cc6ba9a8..079753de95680 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java @@ -10,8 +10,8 @@ import org.apache.lucene.index.CorruptIndexException; import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -31,13 +31,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -public class MockFsVerifyingBlobContainer extends FsBlobContainer implements VerifyingMultiStreamBlobContainer { +public class MockFsAsyncBlobContainer extends FsBlobContainer implements AsyncMultiStreamBlobContainer { private static final int TRANSFER_TIMEOUT_MILLIS = 30000; private final boolean triggerDataIntegrityFailure; - public MockFsVerifyingBlobContainer(FsBlobStore blobStore, BlobPath blobPath, Path path, boolean triggerDataIntegrityFailure) { + public MockFsAsyncBlobContainer(FsBlobStore blobStore, BlobPath blobPath, Path path, boolean triggerDataIntegrityFailure) { super(blobStore, blobPath, path); this.triggerDataIntegrityFailure = triggerDataIntegrityFailure; } @@ -138,6 +138,10 @@ public void readBlobAsync(String blobName, ActionListener listener) }).start(); } + public boolean remoteIntegrityCheckSupported() { + return true; + } + private boolean isSegmentFile(String filename) { return !filename.endsWith(".tlog") && !filename.endsWith(".ckp"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java index f1d9fbba84528..77b0cac922014 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java @@ -28,7 +28,7 @@ public MockFsBlobStore(int bufferSizeInBytes, Path path, boolean readonly, boole @Override public BlobContainer blobContainer(BlobPath path) { try { - return new MockFsVerifyingBlobContainer(this, path, buildAndCreate(path), triggerDataIntegrityFailure); + return new MockFsAsyncBlobContainer(this, path, buildAndCreate(path), triggerDataIntegrityFailure); } catch (IOException ex) { throw new OpenSearchException("failed to create blob container", ex); } diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 11d30178e0930..d8054e91c9b7a 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -9,6 +9,7 @@ package org.opensearch; import org.opensearch.core.index.snapshots.IndexShardSnapshotException; +import org.opensearch.crypto.CryptoRegistryException; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle; @@ -1172,6 +1173,7 @@ public static void registerExceptions() { V_2_7_0 ) ); + registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_3_0_0)); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java new file mode 100644 index 0000000000000..0d289e3552cdf --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.crypto; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +import static org.opensearch.action.ValidateActions.addValidationError; +import static org.opensearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import static org.opensearch.common.settings.Settings.readSettingsFromStream; +import static org.opensearch.common.settings.Settings.writeSettingsToStream; + +/** + * Crypto settings supplied during a put repository request + * + * @opensearch.internal + */ +public class CryptoSettings implements Writeable, ToXContentObject { + + private String keyProviderName; + private String keyProviderType; + private Settings settings = EMPTY_SETTINGS; + + public CryptoSettings(StreamInput in) throws IOException { + keyProviderName = in.readString(); + keyProviderType = in.readString(); + settings = readSettingsFromStream(in); + } + + public CryptoSettings(String keyProviderName) { + this.keyProviderName = keyProviderName; + } + + /** + * Validate settings supplied in put repository request. + * @return Exception in case validation fails. + */ + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (keyProviderName == null) { + validationException = addValidationError("key_provider_name is missing", validationException); + } + if (keyProviderType == null) { + validationException = addValidationError("key_provider_type is missing", validationException); + } + return validationException; + } + + /** + * Returns key provider name + * @return keyProviderName + */ + public String getKeyProviderName() { + return keyProviderName; + } + + /** + * Returns key provider type + * @return keyProviderType + */ + public String getKeyProviderType() { + return keyProviderType; + } + + /** + * Returns crypto settings + * @return settings + */ + public Settings getSettings() { + return settings; + } + + /** + * Constructs a new crypto settings with provided key provider name. + * @param keyProviderName Name of the key provider + */ + public CryptoSettings keyProviderName(String keyProviderName) { + this.keyProviderName = keyProviderName; + return this; + } + + /** + * Constructs a new crypto settings with provided key provider type. + * @param keyProviderType Type of key provider to be used in encryption. + */ + public CryptoSettings keyProviderType(String keyProviderType) { + this.keyProviderType = keyProviderType; + return this; + } + + /** + * Sets the encryption settings + * + * @param settings for encryption + * @return this request + */ + public CryptoSettings settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + /** + * Sets the encryption settings. + * + * @param source encryption settings in json or yaml format + * @param xContentType the content type of the source + * @return this request + */ + public CryptoSettings settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * Sets the encryption settings. + * + * @param source encryption settings + * @return this request + */ + public CryptoSettings settings(Map source) { + this.settings = Settings.builder().loadFromMap(source).build(); + return this; + } + + /** + * Parses crypto settings definition. + * + * @param cryptoDefinition crypto settings definition + */ + public CryptoSettings(Map cryptoDefinition) { + for (Map.Entry entry : cryptoDefinition.entrySet()) { + if (entry.getKey().equals("key_provider_name")) { + keyProviderName(entry.getValue().toString()); + } else if (entry.getKey().equals("key_provider_type")) { + keyProviderType(entry.getValue().toString()); + } else if (entry.getKey().equals("settings")) { + if (!(entry.getValue() instanceof Map)) { + throw new IllegalArgumentException("Malformed settings section in crypto settings, should include an inner object"); + } + @SuppressWarnings("unchecked") + Map sub = (Map) entry.getValue(); + settings(sub); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(keyProviderName); + out.writeString(keyProviderType); + writeSettingsToStream(settings, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("key_provider_name", keyProviderName); + builder.field("key_provider_type", keyProviderType); + + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + + builder.endObject(); + return builder; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/crypto/package-info.java b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/package-info.java new file mode 100644 index 0000000000000..bb9375c20c87e --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Crypto client request and settings handlers. + */ +package org.opensearch.action.admin.cluster.crypto; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index ad6be17821337..8fd74bc718bd4 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -32,7 +32,9 @@ package org.opensearch.action.admin.cluster.repositories.put; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.admin.cluster.crypto.CryptoSettings; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; @@ -67,12 +69,18 @@ public class PutRepositoryRequest extends AcknowledgedRequest repositoryDefinition) { @SuppressWarnings("unchecked") Map sub = (Map) entry.getValue(); settings(sub); + } else if (name.equals("crypto_settings")) { + if (!(entry.getValue() instanceof Map)) { + throw new IllegalArgumentException("Malformed encryption_settings section, should include an inner object"); + } + @SuppressWarnings("unchecked") + Map sub = (Map) entry.getValue(); + CryptoSettings cryptoSettings = new CryptoSettings(sub); + cryptoSettings(cryptoSettings); } } return this; @@ -236,6 +275,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(type); writeSettingsToStream(settings, out); out.writeBoolean(verify); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalWriteable(cryptoSettings); + } } @Override @@ -249,6 +291,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field("verify", verify); + + if (cryptoSettings != null) { + builder.startObject("crypto_settings"); + cryptoSettings.toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 6e1b2795b6375..cf649ee6b4cbf 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.repositories.put; +import org.opensearch.action.admin.cluster.crypto.CryptoSettings; import org.opensearch.action.support.master.AcknowledgedRequestBuilder; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; @@ -141,4 +142,15 @@ public PutRepositoryRequestBuilder setVerify(boolean verify) { request.verify(verify); return this; } + + /** + * Sets the repository encryption settings + * + * @param cryptoSettings repository crypto settings builder + * @return this builder + */ + public PutRepositoryRequestBuilder setEncryptionSettings(CryptoSettings cryptoSettings) { + request.cryptoSettings(cryptoSettings); + return this; + } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java new file mode 100644 index 0000000000000..bf55ef0837ea9 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.action.admin.cluster.crypto.CryptoSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +/** + * Metadata about encryption and decryption + * + * @opensearch.internal + */ +public class CryptoMetadata implements Writeable { + private final String keyProviderName; + private final String keyProviderType; + private final Settings settings; + + /** + * Constructs new crypto metadata + * + * @param keyProviderName key provider name + * @param keyProviderType key provider type + * @param settings crypto settings + */ + public CryptoMetadata(String keyProviderName, String keyProviderType, Settings settings) { + this.keyProviderName = keyProviderName; + this.keyProviderType = keyProviderType; + this.settings = settings; + } + + /** + * Returns key provider name + * + * @return Key provider name + */ + public String keyProviderName() { + return this.keyProviderName; + } + + /** + * Returns key provider type + * + * @return key provider type + */ + public String keyProviderType() { + return this.keyProviderType; + } + + /** + * Returns crypto settings + * + * @return crypto settings + */ + public Settings settings() { + return this.settings; + } + + public CryptoMetadata(StreamInput in) throws IOException { + keyProviderName = in.readString(); + keyProviderType = in.readString(); + settings = Settings.readSettingsFromStream(in); + } + + public static CryptoMetadata fromRequest(CryptoSettings cryptoSettings) { + if (cryptoSettings == null) { + return null; + } + return new CryptoMetadata(cryptoSettings.getKeyProviderName(), cryptoSettings.getKeyProviderType(), cryptoSettings.getSettings()); + } + + /** + * Writes crypto metadata to stream output + * + * @param out stream output + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(keyProviderName); + out.writeString(keyProviderType); + Settings.writeSettingsToStream(settings, out); + } + + public static CryptoMetadata fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token; + String keyProviderType = null; + Settings settings = null; + String keyProviderName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if ("key_provider_name".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { + throw new OpenSearchParseException("failed to parse crypto metadata [{}], unknown type"); + } + keyProviderName = parser.text(); + } else if ("key_provider_type".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { + throw new OpenSearchParseException("failed to parse crypto metadata [{}], unknown type"); + } + keyProviderType = parser.text(); + } else if ("settings".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.START_OBJECT) { + throw new OpenSearchParseException("failed to parse crypto metadata [{}], unknown type"); + } + settings = Settings.fromXContent(parser); + } else { + throw new OpenSearchParseException("failed to parse crypto metadata, unknown field [{}]", currentFieldName); + } + } else { + throw new OpenSearchParseException("failed to parse repositories"); + } + } + return new CryptoMetadata(keyProviderName, keyProviderType, settings); + } + + public void toXContent(CryptoMetadata cryptoMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject("crypto_metadata"); + builder.field("key_provider_name", cryptoMetadata.keyProviderName()); + builder.field("key_provider_type", cryptoMetadata.keyProviderType()); + builder.startObject("settings"); + cryptoMetadata.settings().toXContent(builder, params); + builder.endObject(); + builder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CryptoMetadata that = (CryptoMetadata) o; + + if (!keyProviderName.equals(that.keyProviderName)) return false; + if (!keyProviderType.equals(that.keyProviderType)) return false; + return settings.equals(that.settings); + } + + @Override + public int hashCode() { + return Objects.hash(keyProviderName, keyProviderType, settings); + } + + @Override + public String toString() { + return "CryptoMetadata{" + keyProviderName + "}{" + keyProviderType + "}{" + settings + "}"; + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 90dba581df33b..3cde53c3ae74b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -208,6 +208,7 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO Settings settings = Settings.EMPTY; long generation = RepositoryData.UNKNOWN_REPO_GEN; long pendingGeneration = RepositoryData.EMPTY_REPO_GEN; + CryptoMetadata cryptoMetadata = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); @@ -231,6 +232,11 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO throw new OpenSearchParseException("failed to parse repository [{}], unknown type", name); } pendingGeneration = parser.longValue(); + } else if ("crypto_metadata".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.START_OBJECT) { + throw new OpenSearchParseException("failed to parse repository [{}], unknown type", name); + } + cryptoMetadata = CryptoMetadata.fromXContent(parser); } else { throw new OpenSearchParseException( "failed to parse repository [{}], unknown field [{}]", @@ -245,7 +251,7 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO if (type == null) { throw new OpenSearchParseException("failed to parse repository [{}], missing repository type", name); } - repository.add(new RepositoryMetadata(name, type, settings, generation, pendingGeneration)); + repository.add(new RepositoryMetadata(name, type, settings, generation, pendingGeneration, cryptoMetadata)); } else { throw new OpenSearchParseException("failed to parse repositories"); } @@ -279,6 +285,9 @@ public EnumSet context() { public static void toXContent(RepositoryMetadata repository, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(repository.name()); builder.field("type", repository.type()); + if (repository.cryptoMetadata() != null) { + repository.cryptoMetadata().toXContent(repository.cryptoMetadata(), builder, params); + } builder.startObject("settings"); repository.settings().toXContent(builder, params); builder.endObject(); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java index e8a2cfe9d07d5..181b8743fbfe6 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java @@ -31,6 +31,7 @@ package org.opensearch.cluster.metadata; +import org.opensearch.Version; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -50,6 +51,7 @@ public class RepositoryMetadata implements Writeable { private final String name; private final String type; private final Settings settings; + private final CryptoMetadata cryptoMetadata; /** * Safe repository generation. @@ -69,14 +71,29 @@ public class RepositoryMetadata implements Writeable { * @param settings repository settings */ public RepositoryMetadata(String name, String type, Settings settings) { - this(name, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN); + this(name, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN, null); + } + + public RepositoryMetadata(String name, String type, Settings settings, CryptoMetadata cryptoMetadata) { + this(name, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN, cryptoMetadata); } public RepositoryMetadata(RepositoryMetadata metadata, long generation, long pendingGeneration) { - this(metadata.name, metadata.type, metadata.settings, generation, pendingGeneration); + this(metadata.name, metadata.type, metadata.settings, generation, pendingGeneration, metadata.cryptoMetadata); } public RepositoryMetadata(String name, String type, Settings settings, long generation, long pendingGeneration) { + this(name, type, settings, generation, pendingGeneration, null); + } + + public RepositoryMetadata( + String name, + String type, + Settings settings, + long generation, + long pendingGeneration, + CryptoMetadata cryptoMetadata + ) { this.name = name; this.type = type; this.settings = settings; @@ -87,6 +104,7 @@ public RepositoryMetadata(String name, String type, Settings settings, long gene + "] must be greater or equal to generation [" + generation + "]"; + this.cryptoMetadata = cryptoMetadata; } /** @@ -116,6 +134,15 @@ public Settings settings() { return this.settings; } + /** + * Returns crypto metadata of repository + * + * @return crypto metadata of repository + */ + public CryptoMetadata cryptoMetadata() { + return this.cryptoMetadata; + } + /** * Returns the safe repository generation. {@link RepositoryData} for this generation is assumed to exist in the repository. * All operations on the repository must be based on the {@link RepositoryData} at this generation. @@ -146,6 +173,11 @@ public RepositoryMetadata(StreamInput in) throws IOException { settings = Settings.readSettingsFromStream(in); generation = in.readLong(); pendingGeneration = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + cryptoMetadata = in.readOptionalWriteable(CryptoMetadata::new); + } else { + cryptoMetadata = null; + } } /** @@ -160,6 +192,9 @@ public void writeTo(StreamOutput out) throws IOException { Settings.writeSettingsToStream(settings, out); out.writeLong(generation); out.writeLong(pendingGeneration); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalWriteable(cryptoMetadata); + } } /** @@ -169,7 +204,10 @@ public void writeTo(StreamOutput out) throws IOException { * @return {@code true} if both instances equal in all fields but the generation fields */ public boolean equalsIgnoreGenerations(RepositoryMetadata other) { - return name.equals(other.name) && type.equals(other.type()) && settings.equals(other.settings()); + return name.equals(other.name) + && type.equals(other.type()) + && settings.equals(other.settings()) + && Objects.equals(cryptoMetadata, other.cryptoMetadata()); } @Override @@ -183,16 +221,21 @@ public boolean equals(Object o) { if (!type.equals(that.type)) return false; if (generation != that.generation) return false; if (pendingGeneration != that.pendingGeneration) return false; - return settings.equals(that.settings); + if (!settings.equals(that.settings)) return false; + return Objects.equals(cryptoMetadata, that.cryptoMetadata); } @Override public int hashCode() { - return Objects.hash(name, type, settings, generation, pendingGeneration); + return Objects.hash(name, type, settings, generation, pendingGeneration, cryptoMetadata); } @Override public String toString() { - return "RepositoryMetadata{" + name + "}{" + type + "}{" + settings + "}{" + generation + "}{" + pendingGeneration + "}"; + String toStr = "RepositoryMetadata{" + name + "}{" + type + "}{" + settings + "}{" + generation + "}{" + pendingGeneration + "}"; + if (cryptoMetadata != null) { + return toStr + "{" + cryptoMetadata + "}"; + } + return toStr; } } diff --git a/server/src/main/java/org/opensearch/common/StreamContext.java b/server/src/main/java/org/opensearch/common/StreamContext.java index 32f095f8488b7..b163ba65dc7db 100644 --- a/server/src/main/java/org/opensearch/common/StreamContext.java +++ b/server/src/main/java/org/opensearch/common/StreamContext.java @@ -44,6 +44,16 @@ public StreamContext( this.numberOfParts = numberOfParts; } + /** + * Copy constructor for overriding class + */ + protected StreamContext(StreamContext streamContext) { + this.streamSupplier = streamContext.streamSupplier; + this.partSize = streamContext.partSize; + this.numberOfParts = streamContext.numberOfParts; + this.lastPartSize = streamContext.lastPartSize; + } + /** * Vendor plugins can use this method to create new streams only when they are required for processing * New streams won't be created till this method is called with the specific partNumber diff --git a/server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java similarity index 83% rename from server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java rename to server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java index 1764c9e634781..e73a9f5cd0bc9 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java @@ -19,12 +19,12 @@ import java.nio.file.Path; /** - * An extension of {@link BlobContainer} that adds {@link VerifyingMultiStreamBlobContainer#asyncBlobUpload} to allow + * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow * multipart uploads and performs integrity checks on transferred files * * @opensearch.internal */ -public interface VerifyingMultiStreamBlobContainer extends BlobContainer { +public interface AsyncMultiStreamBlobContainer extends BlobContainer { /** * Reads blob content from multiple streams, each from a specific part of the file, which is provided by the @@ -57,4 +57,11 @@ default void asyncBlobDownload(String blobName, Path fileLocation, ThreadPool th ReadContextListener readContextListener = new ReadContextListener(blobName, fileLocation, threadPool, completionListener); readBlobAsync(blobName, readContextListener); } + + /* + * Wether underlying blobContainer can verify integrity of data after transfer. If true and if expected + * checksum is provided in WriteContext, then the checksum of transferred data is compared with expected checksum + * by underlying blobContainer. In this case, caller doesn't need to ensure integrity of data. + */ + boolean remoteIntegrityCheckSupported(); } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java new file mode 100644 index 0000000000000..07a0b49df47ff --- /dev/null +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.stream.read.ReadContext; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.core.action.ActionListener; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * EncryptedBlobContainer is an encrypted BlobContainer that is backed by a + * {@link AsyncMultiStreamBlobContainer} + * + * @opensearch.internal + */ +public class AsyncMultiStreamEncryptedBlobContainer extends EncryptedBlobContainer implements AsyncMultiStreamBlobContainer { + + private final AsyncMultiStreamBlobContainer blobContainer; + private final CryptoHandler cryptoHandler; + + public AsyncMultiStreamEncryptedBlobContainer(AsyncMultiStreamBlobContainer blobContainer, CryptoHandler cryptoHandler) { + super(blobContainer, cryptoHandler); + this.blobContainer = blobContainer; + this.cryptoHandler = cryptoHandler; + } + + @Override + public void asyncBlobUpload(WriteContext writeContext, ActionListener completionListener) throws IOException { + EncryptedWriteContext encryptedWriteContext = new EncryptedWriteContext<>(writeContext, cryptoHandler); + blobContainer.asyncBlobUpload(encryptedWriteContext, completionListener); + } + + @Override + public void readBlobAsync(String blobName, ActionListener listener) { + throw new UnsupportedOperationException(); + } + + @Override + public void asyncBlobDownload(String blobName, Path fileLocation, ThreadPool threadPool, ActionListener completionListener) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remoteIntegrityCheckSupported() { + return false; + } + + static class EncryptedWriteContext extends WriteContext { + + private final T encryptionMetadata; + private final CryptoHandler cryptoHandler; + private final long fileSize; + + /** + * Construct a new encrypted WriteContext object + */ + public EncryptedWriteContext(WriteContext writeContext, CryptoHandler cryptoHandler) { + super(writeContext); + this.cryptoHandler = cryptoHandler; + this.encryptionMetadata = this.cryptoHandler.initEncryptionMetadata(); + this.fileSize = this.cryptoHandler.estimateEncryptedLengthOfEntireContent(encryptionMetadata, writeContext.getFileSize()); + } + + public StreamContext getStreamProvider(long partSize) { + long adjustedPartSize = cryptoHandler.adjustContentSizeForPartialEncryption(encryptionMetadata, partSize); + StreamContext streamContext = super.getStreamProvider(adjustedPartSize); + return new EncryptedStreamContext<>(streamContext, cryptoHandler, encryptionMetadata); + } + + /** + * @return The total size of the encrypted file + */ + public long getFileSize() { + return fileSize; + } + } + + static class EncryptedStreamContext extends StreamContext { + + private final CryptoHandler cryptoHandler; + private final T encryptionMetadata; + + /** + * Construct a new encrypted StreamContext object + */ + public EncryptedStreamContext(StreamContext streamContext, CryptoHandler cryptoHandler, T encryptionMetadata) { + super(streamContext); + this.cryptoHandler = cryptoHandler; + this.encryptionMetadata = encryptionMetadata; + } + + @Override + public InputStreamContainer provideStream(int partNumber) throws IOException { + InputStreamContainer inputStreamContainer = super.provideStream(partNumber); + return cryptoHandler.createEncryptingStreamOfPart(encryptionMetadata, inputStreamContainer, getNumberOfParts(), partNumber); + } + + } +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java new file mode 100644 index 0000000000000..475d891ea9336 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.common.CheckedBiConsumer; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; +import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.core.action.ActionListener; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * EncryptedBlobContainer is a wrapper around BlobContainer that encrypts the data on the fly. + */ +public class EncryptedBlobContainer implements BlobContainer { + + private final BlobContainer blobContainer; + private final CryptoHandler cryptoHandler; + + public EncryptedBlobContainer(BlobContainer blobContainer, CryptoHandler cryptoHandler) { + this.blobContainer = blobContainer; + this.cryptoHandler = cryptoHandler; + } + + @Override + public BlobPath path() { + return blobContainer.path(); + } + + @Override + public boolean blobExists(String blobName) throws IOException { + return blobContainer.blobExists(blobName); + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + InputStream inputStream = blobContainer.readBlob(blobName); + return cryptoHandler.createDecryptingStream(inputStream); + } + + private EncryptedHeaderContentSupplier getEncryptedHeaderContentSupplier(String blobName) { + return (start, end) -> { + byte[] buffer; + int length = (int) (end - start + 1); + try (InputStream inputStream = blobContainer.readBlob(blobName, start, length)) { + buffer = new byte[length]; + inputStream.readNBytes(buffer, (int) start, buffer.length); + } + return buffer; + }; + } + + @Override + public InputStream readBlob(String blobName, long position, long length) throws IOException { + U encryptionMetadata = cryptoHandler.loadEncryptionMetadata(getEncryptedHeaderContentSupplier(blobName)); + DecryptedRangedStreamProvider decryptedStreamProvider = cryptoHandler.createDecryptingStreamOfRange( + encryptionMetadata, + position, + position + length - 1 + ); + long adjustedPos = decryptedStreamProvider.getAdjustedRange()[0]; + long adjustedLength = decryptedStreamProvider.getAdjustedRange()[1] - adjustedPos + 1; + InputStream encryptedStream = blobContainer.readBlob(blobName, adjustedPos, adjustedLength); + return decryptedStreamProvider.getDecryptedStreamProvider().apply(encryptedStream); + } + + @Override + public long readBlobPreferredLength() { + return blobContainer.readBlobPreferredLength(); + } + + private void executeWrite(InputStream inputStream, long blobSize, CheckedBiConsumer writeConsumer) + throws IOException { + T cryptoContext = cryptoHandler.initEncryptionMetadata(); + InputStreamContainer streamContainer = new InputStreamContainer(inputStream, blobSize, 0); + InputStreamContainer encryptedStream = cryptoHandler.createEncryptingStream(cryptoContext, streamContainer); + long cryptoLength = cryptoHandler.estimateEncryptedLengthOfEntireContent(cryptoContext, blobSize); + writeConsumer.accept(encryptedStream.getInputStream(), cryptoLength); + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + executeWrite( + inputStream, + blobSize, + (encryptedStream, encryptedLength) -> blobContainer.writeBlob(blobName, encryptedStream, encryptedLength, failIfAlreadyExists) + ); + } + + @Override + public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + executeWrite( + inputStream, + blobSize, + (encryptedStream, encryptedLength) -> blobContainer.writeBlobAtomic( + blobName, + encryptedStream, + encryptedLength, + failIfAlreadyExists + ) + ); + } + + @Override + public DeleteResult delete() throws IOException { + return blobContainer.delete(); + } + + @Override + public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOException { + blobContainer.deleteBlobsIgnoringIfNotExists(blobNames); + } + + @Override + public Map listBlobs() throws IOException { + Map blobMetadataMap = blobContainer.listBlobs(); + return convertToEncryptedMetadataMap(blobMetadataMap); + } + + @Override + public Map children() throws IOException { + Map children = blobContainer.children(); + if (children != null) { + return children.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> new EncryptedBlobContainer<>(entry.getValue(), cryptoHandler))); + } else { + return null; + } + } + + @Override + public Map listBlobsByPrefix(String blobNamePrefix) throws IOException { + Map blobMetadataMap = blobContainer.listBlobsByPrefix(blobNamePrefix); + return convertToEncryptedMetadataMap(blobMetadataMap); + } + + private Map convertToEncryptedMetadataMap(Map blobMetadataMap) { + if (blobMetadataMap == null) { + return null; + } + + return blobMetadataMap.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> new EncryptedBlobMetadata<>(entry.getValue(), cryptoHandler, getEncryptedHeaderContentSupplier(entry.getKey())) + ) + ); + + } + + @Override + public void listBlobsByPrefixInSortedOrder( + String blobNamePrefix, + int limit, + BlobNameSortOrder blobNameSortOrder, + ActionListener> listener + ) { + ActionListener> encryptedMetadataListener = ActionListener.delegateFailure( + listener, + (delegatedListener, metadataList) -> { + if (metadataList != null) { + List encryptedMetadata = metadataList.stream() + .map( + blobMetadata -> new EncryptedBlobMetadata<>( + blobMetadata, + cryptoHandler, + getEncryptedHeaderContentSupplier(blobMetadata.name()) + ) + ) + .collect(Collectors.toList()); + delegatedListener.onResponse(encryptedMetadata); + } else { + delegatedListener.onResponse(null); + } + } + ); + blobContainer.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder, encryptedMetadataListener); + } +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobMetadata.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobMetadata.java new file mode 100644 index 0000000000000..8917bba806d08 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobMetadata.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; + +import java.io.IOException; + +/** + * Adjusts length of encrypted blob to raw length + */ +public class EncryptedBlobMetadata implements BlobMetadata { + private final EncryptedHeaderContentSupplier encryptedHeaderContentSupplier; + private final BlobMetadata delegate; + private final CryptoHandler cryptoHandler; + + public EncryptedBlobMetadata( + BlobMetadata delegate, + CryptoHandler cryptoHandler, + EncryptedHeaderContentSupplier encryptedHeaderContentSupplier + ) { + this.encryptedHeaderContentSupplier = encryptedHeaderContentSupplier; + this.delegate = delegate; + this.cryptoHandler = cryptoHandler; + } + + @Override + public String name() { + return delegate.name(); + } + + @Override + public long length() { + U cryptoContext; + try { + cryptoContext = cryptoHandler.loadEncryptionMetadata(encryptedHeaderContentSupplier); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + return cryptoHandler.estimateDecryptedLength(cryptoContext, delegate.length()); + } +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java new file mode 100644 index 0000000000000..5365b5a9d8bdf --- /dev/null +++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.crypto.CryptoManagerRegistry; +import org.opensearch.crypto.CryptoRegistryException; +import org.opensearch.encryption.CryptoManager; + +import java.io.IOException; +import java.util.Map; + +/** + * The EncryptedBlobStore is a decorator class that wraps an existing BlobStore and provides encryption and decryption + * capabilities for the stored data. It uses a CryptoManager to handle encryption and decryption operations based on + * the provided CryptoMetadata. The EncryptedBlobStore ensures that all data written to and read from the underlying + * BlobStore is encrypted and decrypted transparently. + */ +public class EncryptedBlobStore implements BlobStore { + + private final BlobStore blobStore; + private final CryptoManager cryptoManager; + + /** + * Constructs an EncryptedBlobStore that wraps the provided BlobStore with encryption capabilities based on the + * given CryptoMetadata. + * + * @param blobStore The underlying BlobStore to be wrapped and used for storing encrypted data. + * @param cryptoMetadata The CryptoMetadata containing information about the key provider and settings for encryption. + * @throws CryptoRegistryException If the CryptoManager is not found during encrypted BlobStore creation. + */ + public EncryptedBlobStore(BlobStore blobStore, CryptoMetadata cryptoMetadata) { + CryptoManagerRegistry cryptoManagerRegistry = CryptoManagerRegistry.getInstance(); + assert cryptoManagerRegistry != null : "CryptoManagerRegistry is not initialized"; + this.cryptoManager = cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata); + if (cryptoManager == null) { + throw new CryptoRegistryException( + cryptoMetadata.keyProviderName(), + cryptoMetadata.keyProviderType(), + "Crypto manager not found during encrypted blob store creation." + ); + } + this.cryptoManager.incRef(); + this.blobStore = blobStore; + } + + /** + * Retrieves a BlobContainer from the underlying BlobStore based on the provided BlobPath. The returned BlobContainer + * is wrapped in an EncryptedBlobContainer to enable transparent encryption and decryption of data. + * + * @param path The BlobPath specifying the location of the BlobContainer. + * @return An EncryptedBlobContainer wrapping the BlobContainer retrieved from the underlying BlobStore. + */ + @Override + public BlobContainer blobContainer(BlobPath path) { + BlobContainer blobContainer = blobStore.blobContainer(path); + if (blobContainer instanceof AsyncMultiStreamBlobContainer) { + return new AsyncMultiStreamEncryptedBlobContainer<>( + (AsyncMultiStreamBlobContainer) blobContainer, + cryptoManager.getCryptoProvider() + ); + } + return new EncryptedBlobContainer<>(blobContainer, cryptoManager.getCryptoProvider()); + } + + /** + * Retrieves statistics about the BlobStore. Delegates the call to the underlying BlobStore's stats() method. + * + * @return A map containing statistics about the BlobStore. + */ + @Override + public Map stats() { + return blobStore.stats(); + } + + /** + * Closes the EncryptedBlobStore by decrementing the reference count of the CryptoManager and closing the + * underlying BlobStore. This ensures proper cleanup of resources. + * + * @throws IOException If an I/O error occurs while closing the BlobStore. + */ + @Override + public void close() throws IOException { + cryptoManager.decRef(); + blobStore.close(); + } + +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java index ef5e3d1e8c26c..e74462f82400d 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java @@ -61,6 +61,20 @@ public WriteContext( this.expectedChecksum = expectedChecksum; } + /** + * Copy constructor used by overriding class + */ + protected WriteContext(WriteContext writeContext) { + this.fileName = writeContext.fileName; + this.streamContextSupplier = writeContext.streamContextSupplier; + this.fileSize = writeContext.fileSize; + this.failIfAlreadyExists = writeContext.failIfAlreadyExists; + this.writePriority = writeContext.writePriority; + this.uploadFinalizer = writeContext.uploadFinalizer; + this.doRemoteDataIntegrityCheck = writeContext.doRemoteDataIntegrityCheck; + this.expectedChecksum = writeContext.expectedChecksum; + } + /** * @return The file name */ diff --git a/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java b/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java new file mode 100644 index 0000000000000..c6a8b56a8d8c8 --- /dev/null +++ b/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.crypto; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.SetOnce; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.encryption.CryptoManager; +import org.opensearch.encryption.CryptoManagerFactory; +import org.opensearch.plugins.CryptoKeyProviderPlugin; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * During node bootstrap, installed key provider extensions responsible for generating data keys are loaded. + * Crypto factories against the respective extensions are cached. A crypto factory is used to register crypto + * manager against an {@link org.opensearch.common.blobstore.EncryptedBlobStore} + */ +public class CryptoManagerRegistry { + private static final Logger logger = LogManager.getLogger(CryptoManagerRegistry.class); + // Package private for tests + SetOnce> registry = new SetOnce<>(); + + // Package private for tests + SetOnce cryptoManagerFactory = new SetOnce(); + private final Map registeredCryptoManagers = new HashMap<>(); + + private static volatile CryptoManagerRegistry instance; + private static final Object lock = new Object(); + + /** + * Initializes the registry with crypto factories for the installed crypto key providers. + * + * @param cryptoPlugins The list of installed crypto key provider plugins. + * @param settings Crypto settings. + */ + protected CryptoManagerRegistry(List cryptoPlugins, Settings settings) { + cryptoManagerFactory.set(new CryptoManagerFactory("ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY", TimeValue.timeValueDays(2), 500)); + registry.set(loadCryptoFactories(cryptoPlugins)); + } + + public static CryptoManagerRegistry getInstance() { + return instance; + } + + public static CryptoManagerRegistry initRegistry(List cryptoPlugins, Settings settings) { + CryptoManagerRegistry curInstance = instance; + if (curInstance == null) { + synchronized (lock) { + curInstance = instance; + if (curInstance == null) { + instance = curInstance = new CryptoManagerRegistry(cryptoPlugins, settings); + } + } + } + return curInstance; + } + + // For tests + protected Map loadCryptoFactories(List cryptoPlugins) { + Map cryptoFactories = new HashMap<>(); + for (CryptoKeyProviderPlugin cryptoPlugin : cryptoPlugins) { + if (cryptoFactories.containsKey(cryptoPlugin.type())) { + throw new IllegalArgumentException("Crypto plugin key provider type [" + cryptoPlugin.type() + "] is already registered"); + } + cryptoFactories.put(cryptoPlugin.type(), cryptoPlugin); + } + + return Map.copyOf(cryptoFactories); + } + + /** + * Retrieves the crypto factory associated with the given key provider type (extension id). + * + * @param keyProviderType The unique extension type for which the factory is to be fetched. + * @return The crypto factory used to create {@link CryptoManager} + * instances in a {@link org.opensearch.common.blobstore.EncryptedBlobStore}. + * @throws IllegalStateException If the crypto registry is not yet loaded. + */ + public CryptoKeyProviderPlugin getCryptoKeyProviderPlugin(String keyProviderType) { + if (registry.get() == null) { + throw new IllegalStateException("Crypto registry is not yet loaded"); + } + return Objects.requireNonNull(registry.get()).get(keyProviderType); + } + + /** + * Fetches the cached crypto manager for the provided crypto metadata or creates a new one if not found. + * If the key provider is not installed, it throws a {@link CryptoRegistryException}. + * + * @param cryptoMetadata The crypto metadata for which the key provider is to be created. + * @return The crypto manager for performing encrypt/decrypt operations. + * @throws CryptoRegistryException If the key provider is not installed or there is an error during crypto manager creation. + */ + public CryptoManager fetchCryptoManager(CryptoMetadata cryptoMetadata) { + CryptoManager cryptoManager = registeredCryptoManagers.get(cryptoMetadata); + if (cryptoManager == null) { + synchronized (registeredCryptoManagers) { + cryptoManager = registeredCryptoManagers.get(cryptoMetadata); + if (cryptoManager == null) { + Runnable onClose = () -> { + synchronized (registeredCryptoManagers) { + registeredCryptoManagers.remove(cryptoMetadata); + } + }; + cryptoManager = createCryptoManager(cryptoMetadata, onClose); + registeredCryptoManagers.put(cryptoMetadata, cryptoManager); + } + } + } + return cryptoManager; + } + + private CryptoManager createCryptoManager(CryptoMetadata cryptoMetadata, Runnable onClose) { + logger.debug("creating crypto client [{}][{}]", cryptoMetadata.keyProviderType(), cryptoMetadata.keyProviderName()); + CryptoKeyProviderPlugin keyProviderPlugin = getCryptoKeyProviderPlugin(cryptoMetadata.keyProviderType()); + if (keyProviderPlugin == null) { + throw new CryptoRegistryException(cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType()); + } + + try { + MasterKeyProvider masterKeyProvider = keyProviderPlugin.createKeyProvider(cryptoMetadata); + return Objects.requireNonNull(cryptoManagerFactory.get()) + .getOrCreateCryptoManager(masterKeyProvider, cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType(), onClose); + + } catch (Exception e) { + logger.warn( + new ParameterizedMessage( + "failed to create crypto manager of name [{}] and type [{}]", + cryptoMetadata.keyProviderName(), + cryptoMetadata.keyProviderType() + ), + e + ); + throw new CryptoRegistryException(cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType(), e); + } + } + +} diff --git a/server/src/main/java/org/opensearch/crypto/CryptoRegistryException.java b/server/src/main/java/org/opensearch/crypto/CryptoRegistryException.java new file mode 100644 index 0000000000000..a1b065649079d --- /dev/null +++ b/server/src/main/java/org/opensearch/crypto/CryptoRegistryException.java @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.crypto; + +import org.opensearch.OpenSearchException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; + +import java.io.IOException; + +/** + * Thrown when crypto manager creation or retrieval fails. + * + * @opensearch.internal + */ +public class CryptoRegistryException extends OpenSearchException { + private final String name; + private final String type; + private final RestStatus restStatus; + + /** + * Constructs a new CryptoRegistryException with the given client name and client type. + * + * @param clientName The name of the client for which the crypto registry is missing. + * @param clientType The type of the client for which the crypto registry is missing. + */ + public CryptoRegistryException(String clientName, String clientType) { + super("[Missing crypto registry for client name : " + clientName + " of type " + clientType + " ]"); + this.name = clientName; + this.type = clientType; + this.restStatus = RestStatus.NOT_FOUND; + } + + /** + * Constructs a new CryptoRegistryException with the given client name, client type, and a cause. + * + * @param clientName The name of the client that caused the exception. + * @param clientType The type of the client that caused the exception. + * @param cause The cause of the exception, which could be another throwable. + */ + public CryptoRegistryException(String clientName, String clientType, Throwable cause) { + super("[Client name : " + clientName + " Type " + clientType + " ]", cause); + this.name = clientName; + this.type = clientType; + if (cause instanceof IllegalArgumentException) { + this.restStatus = RestStatus.BAD_REQUEST; + } else { + this.restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + } + + /** + * Constructs a new CryptoRegistryException with the given client name, client type, and a custom message. + * + * @param clientName The name of the client that caused the exception. + * @param clientType The type of the client that caused the exception. + * @param msg A custom message to be included in the exception. + */ + public CryptoRegistryException(String clientName, String clientType, String msg) { + super("[ " + msg + " Client name : " + clientName + " type " + clientType + " ] "); + this.name = clientName; + this.type = clientType; + this.restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + + /** + * Get the HTTP status associated with this exception. + * + * @return The HTTP status code representing the nature of the exception. + */ + @Override + public RestStatus status() { + return restStatus; + } + + /** + * Get the name of the client associated with this exception. + * + * @return The name of the client for which the exception was raised. + */ + public String getName() { + return name; + } + + /** + * Get the type of the client associated with this exception. + * + * @return The type of the client for which the exception was raised. + */ + public String getType() { + return type; + } + + /** + * Constructs a new CryptoRegistryException by deserializing it from the provided input stream. + * + * @param in The input stream containing the serialized exception data. + * @throws IOException If an I/O error occurs while reading from the input stream. + */ + public CryptoRegistryException(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.type = in.readString(); + this.restStatus = RestStatus.fromCode(in.readInt()); + } + + /** + * Write the exception data to the provided output stream for serialization. + * + * @param out The output stream to which the exception data should be written. + * @throws IOException If an I/O error occurs while writing to the output stream. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeString(type); + out.writeInt(restStatus.getStatus()); + } +} diff --git a/server/src/main/java/org/opensearch/crypto/package-info.java b/server/src/main/java/org/opensearch/crypto/package-info.java new file mode 100644 index 0000000000000..742960ac1cf97 --- /dev/null +++ b/server/src/main/java/org/opensearch/crypto/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Package for crypto client abstractions and exceptions. + */ +package org.opensearch.crypto; diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 04b5d7eb7c6bd..5250ce6230ffa 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -20,9 +20,9 @@ import org.apache.lucene.store.Lock; import org.opensearch.ExceptionsHelper; import org.opensearch.action.LatchedActionListener; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.exception.CorruptFileException; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; @@ -63,6 +63,7 @@ public class RemoteDirectory extends Directory { protected final BlobContainer blobContainer; + private static final Logger logger = LogManager.getLogger(RemoteDirectory.class); protected final UnaryOperator uploadRateLimiter; @@ -73,8 +74,6 @@ public class RemoteDirectory extends Directory { */ private static final int SEGMENT_CHECKSUM_BYTES = 8; - private static final Logger logger = LogManager.getLogger(RemoteDirectory.class); - public BlobContainer getBlobContainer() { return blobContainer; } @@ -146,6 +145,17 @@ public void onFailure(Exception e) { } } + /** + * Returns stream emitted from by blob object. Should be used with a closeable block. + * + * @param fileName Name of file + * @return Stream from the blob object + * @throws IOException if fetch of stream fails with IO error + */ + public InputStream getBlobStream(String fileName) throws IOException { + return blobContainer.readBlob(fileName); + } + /** * Removes an existing file in the directory. * @@ -192,7 +202,14 @@ public IndexInput openInput(String name, IOContext context) throws IOException { return new RemoteIndexInput(name, downloadRateLimiter.apply(inputStream), fileLength(name)); } catch (Exception e) { // Incase the RemoteIndexInput creation fails, close the input stream to avoid file handler leak. - if (inputStream != null) inputStream.close(); + if (inputStream != null) { + try { + inputStream.close(); + } catch (Exception closeEx) { + e.addSuppressed(closeEx); + } + } + logger.error("Exception while reading blob for file: " + name + " for path " + blobContainer.path()); throw e; } } @@ -308,7 +325,7 @@ public boolean copyFrom( Runnable postUploadRunner, ActionListener listener ) { - if (blobContainer instanceof VerifyingMultiStreamBlobContainer) { + if (blobContainer instanceof AsyncMultiStreamBlobContainer) { try { uploadBlob(from, src, remoteFileName, context, postUploadRunner, listener); } catch (Exception e) { @@ -332,6 +349,10 @@ private void uploadBlob( try (IndexInput indexInput = from.openInput(src, ioContext)) { contentLength = indexInput.length(); } + boolean remoteIntegrityEnabled = false; + if (getBlobContainer() instanceof AsyncMultiStreamBlobContainer) { + remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) getBlobContainer()).remoteIntegrityCheckSupported(); + } RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( src, remoteFileName, @@ -340,7 +361,7 @@ private void uploadBlob( WritePriority.NORMAL, (size, position) -> uploadRateLimiter.apply(new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)), expectedChecksum, - this.getBlobContainer() instanceof VerifyingMultiStreamBlobContainer + remoteIntegrityEnabled ); ActionListener completionListener = ActionListener.wrap(resp -> { try { @@ -375,7 +396,7 @@ private void uploadBlob( }); WriteContext writeContext = remoteTransferContainer.createWriteContext(); - ((VerifyingMultiStreamBlobContainer) blobContainer).asyncBlobUpload(writeContext, completionListener); + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(writeContext, completionListener); } private long calculateChecksumOfChecksum(Directory directory, String file) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 0f6ca2a61b67d..266552cea93d9 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -38,6 +38,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.nio.file.NoSuchFileException; import java.util.Collection; import java.util.Collections; @@ -190,9 +191,8 @@ public RemoteSegmentMetadata readLatestMetadataFile() throws IOException { } private RemoteSegmentMetadata readMetadataFile(String metadataFilename) throws IOException { - try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { - byte[] metadataBytes = new byte[(int) indexInput.length()]; - indexInput.readBytes(metadataBytes, 0, (int) indexInput.length()); + try (InputStream inputStream = remoteMetadataDirectory.getBlobStream(metadataFilename)) { + byte[] metadataBytes = inputStream.readAllBytes(); return metadataStreamWrapper.readStream(new ByteArrayIndexInput(metadataFilename, metadataBytes)); } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index f9b126e2de4a4..82dd6301ef79f 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -12,10 +12,11 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.ActionRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; @@ -92,7 +93,7 @@ public void uploadBlobs( ) { fileSnapshots.forEach(fileSnapshot -> { BlobPath blobPath = blobPaths.get(fileSnapshot.getPrimaryTerm()); - if (!(blobStore.blobContainer(blobPath) instanceof VerifyingMultiStreamBlobContainer)) { + if (!(blobStore.blobContainer(blobPath) instanceof AsyncMultiStreamBlobContainer)) { uploadBlob(ThreadPool.Names.TRANSLOG_TRANSFER, fileSnapshot, blobPath, listener, writePriority); } else { uploadBlob(fileSnapshot, listener, blobPath, writePriority); @@ -114,6 +115,11 @@ private void uploadBlob( try (FileChannel channel = channelFactory.open(fileSnapshot.getPath(), StandardOpenOption.READ)) { contentLength = channel.size(); } + boolean remoteIntegrityEnabled = false; + BlobContainer blobContainer = blobStore.blobContainer(blobPath); + if (blobContainer instanceof AsyncMultiStreamBlobContainer) { + remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported(); + } RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( fileSnapshot.getName(), fileSnapshot.getName(), @@ -122,7 +128,7 @@ private void uploadBlob( writePriority, (size, position) -> new OffsetRangeFileInputStream(fileSnapshot.getPath(), size, position), Objects.requireNonNull(fileSnapshot.getChecksum()), - blobStore.blobContainer(blobPath) instanceof VerifyingMultiStreamBlobContainer + remoteIntegrityEnabled ); ActionListener completionListener = ActionListener.wrap(resp -> listener.onResponse(fileSnapshot), ex -> { logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), ex); @@ -138,7 +144,7 @@ private void uploadBlob( }); WriteContext writeContext = remoteTransferContainer.createWriteContext(); - ((VerifyingMultiStreamBlobContainer) blobStore.blobContainer(blobPath)).asyncBlobUpload(writeContext, completionListener); + ((AsyncMultiStreamBlobContainer) blobStore.blobContainer(blobPath)).asyncBlobUpload(writeContext, completionListener); } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), e); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index e7c6eaebbf8ed..46f023910427c 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -115,6 +115,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.crypto.CryptoManagerRegistry; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; @@ -171,6 +172,7 @@ import org.opensearch.plugins.AnalysisPlugin; import org.opensearch.plugins.CircuitBreakerPlugin; import org.opensearch.plugins.ClusterPlugin; +import org.opensearch.plugins.CryptoKeyProviderPlugin; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.ExtensionAwarePlugin; @@ -929,6 +931,7 @@ protected Node( xContentRegistry, recoverySettings ); + CryptoManagerRegistry.initRegistry(pluginsService.filterPlugins(CryptoKeyProviderPlugin.class), settings); RepositoriesService repositoryService = repositoriesModule.getRepositoryService(); repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService( diff --git a/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java b/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java new file mode 100644 index 0000000000000..47dc75c7c908a --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.crypto.MasterKeyProvider; + +/** + * Crypto plugin to provide support for custom key providers. + * @opensearch.api + */ +@ExperimentalApi +public interface CryptoKeyProviderPlugin { + + /** + * Every call to this method should return a new key provider. + * @param cryptoMetadata These are crypto settings needed for creation of a new key provider. + * @return master key provider. + */ + MasterKeyProvider createKeyProvider(CryptoMetadata cryptoMetadata); + + /** + * One crypto plugin extension implementation refers to a unique key provider type. + * @return key provider type + */ + String type(); +} diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index f00bf3942c9a9..5c2a19965c532 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.admin.cluster.crypto.CryptoSettings; import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.cluster.AckedClusterStateUpdateTask; @@ -48,6 +49,7 @@ import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; +import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -164,9 +166,17 @@ public RepositoriesService( public void registerRepository(final PutRepositoryRequest request, final ActionListener listener) { assert lifecycle.started() : "Trying to register new repository but service is in state [" + lifecycle.state() + "]"; - final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata(request.name(), request.type(), request.settings()); + final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata( + request.name(), + request.type(), + request.settings(), + CryptoMetadata.fromRequest(request.cryptoSettings()) + ); validate(request.name()); validateRepositoryMetadataSettings(clusterService, request.name(), request.settings()); + if (newRepositoryMetadata.cryptoMetadata() != null) { + validate(newRepositoryMetadata.cryptoMetadata().keyProviderName()); + } final ActionListener registrationListener; if (request.verify()) { @@ -213,7 +223,14 @@ public ClusterState execute(ClusterState currentState) { if (repositories == null) { logger.info("put repository [{}]", request.name()); repositories = new RepositoriesMetadata( - Collections.singletonList(new RepositoryMetadata(request.name(), request.type(), request.settings())) + Collections.singletonList( + new RepositoryMetadata( + request.name(), + request.type(), + request.settings(), + CryptoMetadata.fromRequest(request.cryptoSettings()) + ) + ) ); } else { boolean found = false; @@ -225,6 +242,7 @@ public ClusterState execute(ClusterState currentState) { // Previous version is the same as this one no update is needed. return currentState; } + ensureCryptoSettingsAreSame(repositoryMetadata, request); found = true; repositoriesMetadata.add(newRepositoryMetadata); } else { @@ -233,7 +251,14 @@ public ClusterState execute(ClusterState currentState) { } if (!found) { logger.info("put repository [{}]", request.name()); - repositoriesMetadata.add(new RepositoryMetadata(request.name(), request.type(), request.settings())); + repositoriesMetadata.add( + new RepositoryMetadata( + request.name(), + request.type(), + request.settings(), + CryptoMetadata.fromRequest(request.cryptoSettings()) + ) + ); } else { logger.info("update repository [{}]", request.name()); } @@ -600,15 +625,15 @@ private Repository createRepository(RepositoryMetadata repositoryMetadata, Map, Integer> reverse = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java b/server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java new file mode 100644 index 0000000000000..f6c8f71bd653c --- /dev/null +++ b/server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java @@ -0,0 +1,158 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.crypto; + +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.common.settings.Settings; +import org.opensearch.encryption.CryptoManager; +import org.opensearch.plugins.CryptoKeyProviderPlugin; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +public class CryptoManagerRegistryTests extends OpenSearchTestCase { + + private TestCryptoManagerRegistry cryptoManagerRegistry; + private String pluginTypeWithCreationFailure; + private CryptoKeyProviderPlugin cryptoPlugin1; + private CryptoKeyProviderPlugin cryptoPlugin2; + + @Before + public void setup() { + List cryptoPlugins = new ArrayList<>(); + CryptoKeyProviderPlugin cryptoPlugin1 = Mockito.mock(CryptoKeyProviderPlugin.class); + String pluginType1 = UUID.randomUUID().toString(); + Mockito.when(cryptoPlugin1.type()).thenReturn(pluginType1); + MasterKeyProvider masterKeyProvider1 = Mockito.mock(MasterKeyProvider.class); + Mockito.when(cryptoPlugin1.createKeyProvider(ArgumentMatchers.any())).thenReturn(masterKeyProvider1); + this.cryptoPlugin1 = cryptoPlugin1; + cryptoPlugins.add(cryptoPlugin1); + + CryptoKeyProviderPlugin cryptoPlugin2 = Mockito.mock(CryptoKeyProviderPlugin.class); + String pluginType2 = UUID.randomUUID().toString(); + Mockito.when(cryptoPlugin2.type()).thenReturn(pluginType2); + MasterKeyProvider masterKeyProvider2 = Mockito.mock(MasterKeyProvider.class); + Mockito.when(cryptoPlugin2.createKeyProvider(ArgumentMatchers.any())).thenReturn(masterKeyProvider2); + cryptoPlugins.add(cryptoPlugin2); + this.cryptoPlugin2 = cryptoPlugin2; + + CryptoKeyProviderPlugin cryptoPluginCreationFailure = Mockito.mock(CryptoKeyProviderPlugin.class); + pluginTypeWithCreationFailure = UUID.randomUUID().toString(); + Mockito.when(cryptoPluginCreationFailure.type()).thenReturn(pluginTypeWithCreationFailure); + Mockito.when(cryptoPluginCreationFailure.createKeyProvider(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("Injected failure")); + cryptoPlugins.add(cryptoPluginCreationFailure); + + cryptoManagerRegistry = new TestCryptoManagerRegistry(cryptoPlugins, Settings.EMPTY); + } + + static class TestCryptoManagerRegistry extends CryptoManagerRegistry { + + protected TestCryptoManagerRegistry(List cryptoPlugins, Settings settings) { + super(cryptoPlugins, settings); + } + + @Override + public Map loadCryptoFactories(List cryptoPlugins) { + return super.loadCryptoFactories(cryptoPlugins); + } + } + + public void testInitRegistryWithDuplicateKPType() { + List cryptoPlugins = new ArrayList<>(); + CryptoKeyProviderPlugin cryptoPlugin1 = Mockito.mock(CryptoKeyProviderPlugin.class); + String pluginType = UUID.randomUUID().toString(); + Mockito.when(cryptoPlugin1.type()).thenReturn(pluginType); + cryptoPlugins.add(cryptoPlugin1); + CryptoKeyProviderPlugin cryptoPlugin2 = Mockito.mock(CryptoKeyProviderPlugin.class); + Mockito.when(cryptoPlugin2.type()).thenReturn(pluginType); + cryptoPlugins.add(cryptoPlugin2); + expectThrows(IllegalArgumentException.class, () -> cryptoManagerRegistry.loadCryptoFactories(cryptoPlugins)); + } + + public void testRegistry() { + List cryptoPlugins = new ArrayList<>(); + CryptoKeyProviderPlugin cryptoPlugin1 = Mockito.mock(CryptoKeyProviderPlugin.class); + String pluginType1 = UUID.randomUUID().toString(); + Mockito.when(cryptoPlugin1.type()).thenReturn(pluginType1); + MasterKeyProvider masterKeyProvider1 = Mockito.mock(MasterKeyProvider.class); + Mockito.when(cryptoPlugin1.createKeyProvider(Mockito.any())).thenReturn(masterKeyProvider1); + cryptoPlugins.add(cryptoPlugin1); + + CryptoKeyProviderPlugin cryptoPlugin2 = Mockito.mock(CryptoKeyProviderPlugin.class); + String pluginType2 = UUID.randomUUID().toString(); + Mockito.when(cryptoPlugin2.type()).thenReturn(pluginType2); + MasterKeyProvider masterKeyProvider2 = Mockito.mock(MasterKeyProvider.class); + Mockito.when(cryptoPlugin2.createKeyProvider(Mockito.any())).thenReturn(masterKeyProvider2); + cryptoPlugins.add(cryptoPlugin2); + + Map loadedPlugins = cryptoManagerRegistry.loadCryptoFactories(cryptoPlugins); + + CryptoKeyProviderPlugin keyProviderPlugin = loadedPlugins.get(pluginType1); + assertNotNull(keyProviderPlugin); + assertEquals(cryptoPlugin1, keyProviderPlugin); + + keyProviderPlugin = loadedPlugins.get(pluginType2); + assertNotNull(keyProviderPlugin); + assertEquals(cryptoPlugin2, keyProviderPlugin); + } + + public void testCryptoManagerMissing() { + String pluginName = UUID.randomUUID().toString(); + String pluginType = UUID.randomUUID().toString(); + CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName, pluginType, Settings.EMPTY); + expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata)); + } + + public void testCryptoManagerCreationFailure() { + String pluginName = UUID.randomUUID().toString(); + CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName, pluginTypeWithCreationFailure, Settings.EMPTY); + expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata)); + } + + public void testCryptoManagerCreationSuccess() { + + String pluginName1 = UUID.randomUUID().toString(); + CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName1, cryptoPlugin1.type(), Settings.EMPTY); + CryptoManager createdCryptoManager1 = cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata); + assertNotNull(createdCryptoManager1); + assertEquals(cryptoPlugin1.type(), createdCryptoManager1.type()); + assertEquals(cryptoMetadata.keyProviderName(), createdCryptoManager1.name()); + assertEquals(cryptoMetadata.keyProviderType(), createdCryptoManager1.type()); + + String pluginName2 = UUID.randomUUID().toString(); + CryptoManager createdCryptoManager2 = cryptoManagerRegistry.fetchCryptoManager( + new CryptoMetadata(pluginName2, cryptoPlugin2.type(), Settings.EMPTY) + ); + assertNotNull(createdCryptoManager2); + assertEquals(pluginName2, createdCryptoManager2.name()); + assertEquals(cryptoPlugin2.type(), createdCryptoManager2.type()); + CryptoManager createdCryptoManager3 = cryptoManagerRegistry.fetchCryptoManager( + new CryptoMetadata(pluginName1, cryptoPlugin1.type(), Settings.EMPTY) + ); + assertNotNull(createdCryptoManager3); + assertEquals(createdCryptoManager1, createdCryptoManager3); + + CryptoManager createdCryptoMgrNewType = cryptoManagerRegistry.fetchCryptoManager( + new CryptoMetadata(pluginName1, cryptoPlugin2.type(), Settings.EMPTY) + ); + assertNotNull(createdCryptoMgrNewType); + assertNotEquals(createdCryptoManager1, createdCryptoMgrNewType); + assertNotEquals(createdCryptoManager2, createdCryptoMgrNewType); + assertNotEquals(createdCryptoManager3, createdCryptoMgrNewType); + } +} diff --git a/server/src/test/java/org/opensearch/crypto/CryptoRegistryExceptionTests.java b/server/src/test/java/org/opensearch/crypto/CryptoRegistryExceptionTests.java new file mode 100644 index 0000000000000..91f86a2b4104a --- /dev/null +++ b/server/src/test/java/org/opensearch/crypto/CryptoRegistryExceptionTests.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.crypto; + +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class CryptoRegistryExceptionTests extends OpenSearchTestCase { + + public void testConstructorWithClientNameAndType() { + String clientName = "test-client"; + String clientType = "test-type"; + CryptoRegistryException exception = new CryptoRegistryException(clientName, clientType); + + assertEquals(RestStatus.NOT_FOUND, exception.status()); + assertEquals(clientName, exception.getName()); + assertEquals(clientType, exception.getType()); + } + + public void testConstructorWithClientNameTypeAndCause() { + String clientName = "test-client"; + String clientType = "test-type"; + String causeMessage = "Something went wrong."; + Throwable cause = new Throwable(causeMessage); + CryptoRegistryException exception = new CryptoRegistryException(clientName, clientType, cause); + + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, exception.status()); + assertEquals(clientName, exception.getName()); + assertEquals(clientType, exception.getType()); + assertEquals(cause, exception.getCause()); + } + + public void testConstructorWithClientNameTypeAndIllegalArgsCause() { + String clientName = "test-client"; + String clientType = "test-type"; + String causeMessage = "Bad arguments."; + IllegalArgumentException cause = new IllegalArgumentException(causeMessage); + ; + CryptoRegistryException exception = new CryptoRegistryException(clientName, clientType, cause); + + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + assertEquals(clientName, exception.getName()); + assertEquals(clientType, exception.getType()); + assertEquals(cause, exception.getCause()); + } + + public void testConstructorWithClientNameTypeAndCustomMessage() { + String clientName = "TestClient"; + String clientType = "TestType"; + String customMessage = "Invalid client data."; + CryptoRegistryException exception = new CryptoRegistryException(clientName, clientType, customMessage); + + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, exception.status()); + assertEquals(clientName, exception.getName()); + assertEquals(clientType, exception.getType()); + } + + public void testSerializationAndDeserialization() throws IOException { + String clientName = "TestClient"; + String clientType = "TestType"; + CryptoRegistryException originalException = new CryptoRegistryException(clientName, clientType); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + StreamOutput streamOutput = new OutputStreamStreamOutput(outputStream); + originalException.writeTo(streamOutput); + + byte[] byteArray = outputStream.toByteArray(); + ByteArrayInputStream inputStream = new ByteArrayInputStream(byteArray); + StreamInput streamInput = new InputStreamStreamInput(inputStream); + CryptoRegistryException deserializedException = new CryptoRegistryException(streamInput); + + assertEquals(originalException.getMessage(), deserializedException.getMessage()); + assertEquals(originalException.status(), deserializedException.status()); + assertEquals(originalException.getName(), deserializedException.getName()); + assertEquals(originalException.getType(), deserializedException.getType()); + } +} diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 896bbffb10d09..188e371ae223e 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -145,7 +145,7 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { segmentInfos = indexShardStore.readLastCommittedSegmentsInfo(); } - when(remoteMetadataDirectory.openInput(any(), any())).thenAnswer( + when(remoteMetadataDirectory.getBlobStream(any())).thenAnswer( I -> createMetadataFileBytes(getDummyMetadata("_0", 1), indexShard.getLatestReplicationCheckpoint(), segmentInfos) ); RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( @@ -162,9 +162,9 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { // Since the thrown IOException is caught in the constructor, ctor should be invoked successfully. new RemoteStoreRefreshListener(shard, SegmentReplicationCheckpointPublisher.EMPTY, mock(RemoteSegmentTransferTracker.class)); - // Validate that the openInput method of remoteMetadataDirectory has been opened only once and the + // Validate that the stream of metadata file of remoteMetadataDirectory has been opened only once and the // listFilesByPrefixInLexicographicOrder has been called twice. - verify(remoteMetadataDirectory, times(1)).openInput(any(), any()); + verify(remoteMetadataDirectory, times(1)).getBlobStream(any()); verify(remoteMetadataDirectory, times(2)).listFilesByPrefixInLexicographicOrder(MetadataFilenameUtils.METADATA_PREFIX, 1); } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index 7655690685889..3740abc57b02d 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -15,9 +15,9 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.action.LatchedActionListener; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.core.action.ActionListener; @@ -71,7 +71,7 @@ public void testListAllEmpty() throws IOException { public void testCopyFrom() throws IOException, InterruptedException { AtomicReference postUploadInvoked = new AtomicReference<>(false); String filename = "_100.si"; - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); Mockito.doAnswer(invocation -> { ActionListener completionListener = invocation.getArgument(1); completionListener.onResponse(null); @@ -113,7 +113,7 @@ public void onFailure(Exception e) { public void testCopyFromWithException() throws IOException, InterruptedException { AtomicReference postUploadInvoked = new AtomicReference<>(false); String filename = "_100.si"; - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); Mockito.doAnswer(invocation -> { ActionListener completionListener = invocation.getArgument(1); completionListener.onResponse(null); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 44dfb44eb9a15..2948528ad82e1 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -22,7 +22,7 @@ import org.apache.lucene.util.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -43,6 +43,7 @@ import org.junit.After; import org.junit.Before; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.NoSuchFileException; import java.util.ArrayList; @@ -223,21 +224,21 @@ private Map> populateMetadata() throws IOException { getDummyMetadata("_0", 1) ); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenAnswer( + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenAnswer( I -> createMetadataFileBytes( metadataFilenameContentMapping.get(metadataFilename), indexShard.getLatestReplicationCheckpoint(), segmentInfos ) ); - when(remoteMetadataDirectory.openInput(metadataFilename2, IOContext.DEFAULT)).thenAnswer( + when(remoteMetadataDirectory.getBlobStream(metadataFilename2)).thenAnswer( I -> createMetadataFileBytes( metadataFilenameContentMapping.get(metadataFilename2), indexShard.getLatestReplicationCheckpoint(), segmentInfos ) ); - when(remoteMetadataDirectory.openInput(metadataFilename3, IOContext.DEFAULT)).thenAnswer( + when(remoteMetadataDirectory.getBlobStream(metadataFilename3)).thenAnswer( I -> createMetadataFileBytes( metadataFilenameContentMapping.get(metadataFilename3), indexShard.getLatestReplicationCheckpoint(), @@ -491,7 +492,7 @@ public void testCopyFilesFromMultipart() throws Exception { assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); Mockito.doAnswer(invocation -> { ActionListener completionListener = invocation.getArgument(1); @@ -517,7 +518,7 @@ public void onFailure(Exception e) {} public void testCopyFilesFromMultipartIOException() throws Exception { String filename = "_100.si"; - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); remoteDataDirectory = new RemoteDirectory(blobContainer); remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( remoteDataDirectory, @@ -584,7 +585,7 @@ public void testContainsFile() throws IOException { metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major); metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn( + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn( createMetadataFileBytes(metadata, indexShard.getLatestReplicationCheckpoint(), segmentInfos) ); @@ -647,7 +648,7 @@ public void testUploadMetadataNonEmpty() throws IOException { latestMetadataFileName, getDummyMetadata("_0", (int) generation) ); - when(remoteMetadataDirectory.openInput(latestMetadataFileName, IOContext.DEFAULT)).thenReturn( + when(remoteMetadataDirectory.getBlobStream(latestMetadataFileName)).thenReturn( createMetadataFileBytes( metadataFilenameContentMapping.get(latestMetadataFileName), indexShard.getLatestReplicationCheckpoint(), @@ -750,8 +751,8 @@ public void testNoMetadataHeaderCorruptIndexException() throws IOException { OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); indexOutput.writeMapOfStrings(metadata); indexOutput.close(); - ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + ByteArrayInputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn(inputStream); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } @@ -775,8 +776,8 @@ public void testInvalidCodecHeaderCorruptIndexException() throws IOException { indexOutput.writeMapOfStrings(metadata); CodecUtil.writeFooter(indexOutput); indexOutput.close(); - ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + ByteArrayInputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn(inputStream); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } @@ -800,8 +801,8 @@ public void testHeaderMinVersionCorruptIndexException() throws IOException { indexOutput.writeMapOfStrings(metadata); CodecUtil.writeFooter(indexOutput); indexOutput.close(); - ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + ByteArrayInputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn(inputStream); assertThrows(IndexFormatTooOldException.class, () -> remoteSegmentStoreDirectory.init()); } @@ -825,8 +826,8 @@ public void testHeaderMaxVersionCorruptIndexException() throws IOException { indexOutput.writeMapOfStrings(metadata); CodecUtil.writeFooter(indexOutput); indexOutput.close(); - ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + ByteArrayInputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn(inputStream); assertThrows(IndexFormatTooNewException.class, () -> remoteSegmentStoreDirectory.init()); } @@ -854,8 +855,8 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException { CodecUtil.writeFooter(indexOutputSpy); indexOutputSpy.close(); - ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + ByteArrayInputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.getBlobStream(metadataFilename)).thenReturn(inputStream); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java index e629929a9196a..a806eea381297 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java @@ -9,9 +9,9 @@ package org.opensearch.index.translog.transfer; import org.opensearch.action.LatchedActionListener; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.core.action.ActionListener; @@ -60,7 +60,7 @@ public void testUploadBlobs() throws Exception { 0L ); - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); Mockito.doAnswer(invocation -> { ActionListener completionListener = invocation.getArgument(1); completionListener.onResponse(null); @@ -107,7 +107,7 @@ public void testUploadBlobsIOException() throws Exception { 0L ); - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); doThrow(new IOException()).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer); @@ -146,7 +146,7 @@ public void testUploadBlobsUploadFutureCompletedExceptionally() throws Exception 0L ); - VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); Mockito.doAnswer(invocation -> { ActionListener completionListener = invocation.getArgument(1); completionListener.onFailure(new Exception("Test exception")); diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 62bc4016d892d..1ab48b30af2f9 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -34,21 +34,30 @@ import org.apache.lucene.index.IndexCommit; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.crypto.CryptoSettings; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.ack.ClusterStateUpdateResponse; +import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; +import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.common.settings.Settings; @@ -56,6 +65,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.encryption.CryptoManager; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; @@ -71,14 +81,21 @@ import org.opensearch.transport.Transport; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -89,6 +106,14 @@ public class RepositoriesServiceTests extends OpenSearchTestCase { @Override public void setUp() throws Exception { super.setUp(); + ThreadPool threadPool = mock(ThreadPool.class); + final ClusterApplierService clusterApplierService = mock(ClusterApplierService.class); + when(clusterApplierService.threadPool()).thenReturn(threadPool); + final ClusterService clusterService = mock(ClusterService.class); + repositoriesService = createRepositoriesServiceWithMockedClusterService(clusterService); + } + + private RepositoriesService createRepositoriesServiceWithMockedClusterService(ClusterService clusterService) { ThreadPool threadPool = mock(ThreadPool.class); final TransportService transportService = new TransportService( Settings.EMPTY, @@ -101,7 +126,6 @@ public void setUp() throws Exception { ); final ClusterApplierService clusterApplierService = mock(ClusterApplierService.class); when(clusterApplierService.threadPool()).thenReturn(threadPool); - final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); Map typesRegistry = Map.of( TestRepository.TYPE, @@ -111,15 +135,24 @@ public void setUp() throws Exception { MeteredRepositoryTypeB.TYPE, metadata -> new MeteredRepositoryTypeB(metadata, clusterService) ); - repositoriesService = new RepositoriesService( + + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + when(nodes.getMinNodeVersion()).thenReturn(Version.V_2_9_0); + ClusterState currentClusterState = mock(ClusterState.class); + when(currentClusterState.getNodes()).thenReturn(nodes); + when(clusterService.state()).thenReturn(currentClusterState); + + RepositoriesService repositoriesService = new RepositoriesService( Settings.EMPTY, - mock(ClusterService.class), + clusterService, transportService, typesRegistry, typesRegistry, threadPool ); + repositoriesService.start(); + return repositoriesService; } public void testRegisterInternalRepository() { @@ -190,6 +223,254 @@ public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { assertThat(repositoryStatsTypeB.getRepositoryStats(), equalTo(MeteredRepositoryTypeB.STATS)); } + public void testWithSameKeyProviderNames() { + String keyProviderName = "kp-name"; + ClusterState clusterStateWithRepoTypeA = createClusterStateWithKeyProvider( + "repoName", + MeteredRepositoryTypeA.TYPE, + keyProviderName, + TestCryptoManagerTypeA.TYPE + ); + + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); + assertNotNull(repository); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + + ClusterState clusterStateWithRepoTypeB = createClusterStateWithKeyProvider( + "repoName", + MeteredRepositoryTypeB.TYPE, + keyProviderName, + TestCryptoManagerTypeB.TYPE + ); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeB, emptyState())); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(2)); + MeteredRepositoryTypeB repositoryB = (MeteredRepositoryTypeB) repositoriesService.repository("repoName"); + assertNotNull(repositoryB); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + } + + public void testCryptoManagersUnchangedWithSameCryptoMetadata() { + String keyProviderName = "kp-name"; + ClusterState clusterStateWithRepoTypeA = createClusterStateWithKeyProvider( + "repoName", + MeteredRepositoryTypeA.TYPE, + keyProviderName, + TestCryptoManagerTypeA.TYPE + ); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); + assertNotNull(repository); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); + assertNotNull(repository); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + } + + public void testRepositoryUpdateWithDifferentCryptoMetadata() { + String keyProviderName = "kp-name"; + + ClusterState clusterStateWithRepoTypeA = createClusterStateWithKeyProvider( + "repoName", + MeteredRepositoryTypeA.TYPE, + keyProviderName, + TestCryptoManagerTypeA.TYPE + ); + ClusterService clusterService = mock(ClusterService.class); + + PutRepositoryRequest request = new PutRepositoryRequest("repoName"); + request.type(MeteredRepositoryTypeA.TYPE); + request.settings(Settings.EMPTY); + + doAnswer((invocation) -> { + AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask< + ClusterStateUpdateResponse>) invocation.getArguments()[1]; + task.execute(clusterStateWithRepoTypeA); + return null; + }).when(clusterService).submitStateUpdateTask(any(), any()); + + RepositoriesService repositoriesService = createRepositoriesServiceWithMockedClusterService(clusterService); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); + assertNotNull(repository); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + + CryptoSettings cryptoSettings = new CryptoSettings(keyProviderName); + cryptoSettings.keyProviderType(TestCryptoManagerTypeA.TYPE); + cryptoSettings.settings(Settings.builder().put("key-1", "val-1")); + request.cryptoSettings(cryptoSettings); + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + + cryptoSettings.settings(Settings.builder()); + cryptoSettings.keyProviderName("random"); + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + + cryptoSettings.keyProviderName(keyProviderName); + + cryptoSettings.keyProviderType(TestCryptoManagerTypeA.TYPE); + repositoriesService.registerRepository(request, null); + } + + public void testCryptoManagerClusterStateChanges() { + + ClusterService clusterService = mock(ClusterService.class); + AtomicBoolean verified = new AtomicBoolean(); + List repositoryMetadata = new ArrayList<>(); + + String keyProviderName = "kp-name-1"; + String repoName = "repoName"; + String keyProviderType = TestCryptoManagerTypeA.TYPE; + Settings.Builder settings = Settings.builder(); + PutRepositoryRequest request = createPutRepositoryEncryptedRequest( + repoName, + MeteredRepositoryTypeA.TYPE, + keyProviderName, + settings, + keyProviderType + ); + verified.set(false); + RepositoriesService repositoriesService = createRepositoriesServiceAndMockCryptoClusterState( + clusterService, + repoName, + keyProviderName, + keyProviderType, + settings.build(), + verified, + repositoryMetadata + ); + repositoriesService.registerRepository(request, null); + MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertTrue(verified.get()); + + // No change + keyProviderType = TestCryptoManagerTypeA.TYPE; + settings = Settings.builder(); + request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); + verified.set(false); + repositoriesService = createRepositoriesServiceAndMockCryptoClusterState( + clusterService, + repoName, + keyProviderName, + keyProviderType, + settings.build(), + verified, + repositoryMetadata + ); + repositoriesService.registerRepository(request, null); + + repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertTrue(verified.get()); + + // Same crypto client in new repo + repoName = "repoName-2"; + keyProviderType = TestCryptoManagerTypeA.TYPE; + settings = Settings.builder(); + request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); + verified.set(false); + repositoriesService = createRepositoriesServiceAndMockCryptoClusterState( + clusterService, + repoName, + keyProviderName, + keyProviderType, + settings.build(), + verified, + repositoryMetadata + ); + repositoriesService.registerRepository(request, null); + repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertTrue(verified.get()); + + // Different crypto client in new repo + repoName = "repoName-3"; + keyProviderType = TestCryptoManagerTypeB.TYPE; + settings = Settings.builder(); + request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); + verified.set(false); + repositoriesService = createRepositoriesServiceAndMockCryptoClusterState( + clusterService, + repoName, + keyProviderName, + keyProviderType, + settings.build(), + verified, + repositoryMetadata + ); + repositoriesService.registerRepository(request, null); + repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); + assertNotNull(repository.cryptoManager); + assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeB); + assertTrue(verified.get()); + + } + + private RepositoriesService createRepositoriesServiceAndMockCryptoClusterState( + ClusterService clusterService, + String repoName, + String keyProviderName, + String keyProviderType, + Settings settings, + AtomicBoolean verified, + List repositoryMetadataList + ) { + + ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); + CryptoMetadata newCryptoMetadata = new CryptoMetadata(keyProviderName, keyProviderType, Settings.EMPTY); + Metadata.Builder mdBuilder = Metadata.builder(); + + RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata( + repoName, + MeteredRepositoryTypeA.TYPE, + Settings.EMPTY, + newCryptoMetadata + ); + if (!repositoryMetadataList.contains(newRepositoryMetadata)) { + repositoryMetadataList.add(newRepositoryMetadata); + } + RepositoriesMetadata newRepositoriesMetadata = new RepositoriesMetadata(repositoryMetadataList); + mdBuilder.putCustom(RepositoriesMetadata.TYPE, newRepositoriesMetadata); + state.metadata(mdBuilder); + ClusterState clusterStateWithRepoTypeA = state.build(); + + RepositoriesService repositoriesService = createRepositoriesServiceWithMockedClusterService(clusterService); + + doAnswer((invocation) -> { + AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask< + ClusterStateUpdateResponse>) invocation.getArguments()[1]; + ClusterState clusterState = task.execute(clusterStateWithRepoTypeA); + RepositoriesMetadata repositories = clusterState.metadata().custom(RepositoriesMetadata.TYPE); + RepositoryMetadata repositoryMetadata = repositories.repositories().get(repositoryMetadataList.size() - 1); + CryptoMetadata cryptoMetadata = repositoryMetadata.cryptoMetadata(); + assertNotNull(cryptoMetadata); + assertEquals(keyProviderName, cryptoMetadata.keyProviderName()); + assertEquals(keyProviderType, cryptoMetadata.keyProviderType()); + assertEquals(cryptoMetadata.settings(), settings); + verified.set(true); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); + return null; + }).when(clusterService).submitStateUpdateTask(any(), any()); + + return repositoriesService; + } + private ClusterState createClusterStateWithRepo(String repoName, String repoType) { ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); Metadata.Builder mdBuilder = Metadata.builder(); @@ -202,6 +483,43 @@ private ClusterState createClusterStateWithRepo(String repoName, String repoType return state.build(); } + private ClusterState createClusterStateWithKeyProvider( + String repoName, + String repoType, + String keyProviderName, + String keyProviderType + ) { + ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); + Metadata.Builder mdBuilder = Metadata.builder(); + CryptoMetadata cryptoMetadata = new CryptoMetadata(keyProviderName, keyProviderType, Settings.EMPTY); + mdBuilder.putCustom( + RepositoriesMetadata.TYPE, + new RepositoriesMetadata(Collections.singletonList(new RepositoryMetadata(repoName, repoType, Settings.EMPTY, cryptoMetadata))) + ); + state.metadata(mdBuilder); + + return state.build(); + } + + private PutRepositoryRequest createPutRepositoryEncryptedRequest( + String repoName, + String repoType, + String keyProviderName, + Settings.Builder settings, + String keyProviderType + ) { + PutRepositoryRequest repositoryRequest = new PutRepositoryRequest(repoName); + repositoryRequest.type(repoType); + repositoryRequest.settings(Settings.EMPTY); + CryptoSettings cryptoSettings = new CryptoSettings(keyProviderName); + cryptoSettings.keyProviderName(keyProviderName); + cryptoSettings.keyProviderType(keyProviderType); + cryptoSettings.settings(settings); + repositoryRequest.cryptoSettings(cryptoSettings); + + return repositoryRequest; + } + private ClusterState emptyState() { return ClusterState.builder(new ClusterName("test")).build(); } @@ -211,6 +529,133 @@ private void assertThrowsOnRegister(String repoName) { expectThrows(RepositoryException.class, () -> repositoriesService.registerRepository(request, null)); } + private static class TestCryptoProvider implements CryptoHandler { + + @Override + public Object initEncryptionMetadata() { + return new Object(); + } + + @Override + public long adjustContentSizeForPartialEncryption(Object cryptoContextObj, long contentSize) { + return 0; + } + + @Override + public long estimateEncryptedLengthOfEntireContent(Object cryptoContextObj, long contentLength) { + return 0; + } + + @Override + public InputStreamContainer createEncryptingStream(Object encryptionMetadata, InputStreamContainer streamContainer) { + return null; + } + + @Override + public InputStreamContainer createEncryptingStreamOfPart( + Object cryptoContextObj, + InputStreamContainer stream, + int totalStreams, + int streamIdx + ) { + return null; + } + + @Override + public InputStream createDecryptingStream(InputStream encryptingStream) { + return null; + } + + @Override + public Object loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException { + return null; + } + + @Override + public DecryptedRangedStreamProvider createDecryptingStreamOfRange( + Object cryptoContext, + long startPosOfRawContent, + long endPosOfRawContent + ) { + return null; + } + + @Override + public long estimateDecryptedLength(Object cryptoContext, long contentLength) { + return 0; + } + } + + private static abstract class TestCryptoManager implements CryptoManager { + private final String name; + private final AtomicInteger ref; + + private final CryptoHandler cryptoProvider; + + public TestCryptoManager(Settings settings, String keyProviderName) { + this.name = keyProviderName; + this.ref = new AtomicInteger(1); + this.cryptoProvider = new TestCryptoProvider(); + } + + @Override + public void incRef() { + ref.incrementAndGet(); + } + + @Override + public boolean tryIncRef() { + ref.incrementAndGet(); + return true; + } + + @Override + public boolean decRef() { + ref.decrementAndGet(); + return true; + } + + public int getReferenceCount() { + return ref.get(); + } + + @Override + public String name() { + return name; + } + + public CryptoHandler getCryptoProvider() { + return cryptoProvider; + } + } + + private static class TestCryptoManagerTypeA extends TestCryptoManager { + public static final String TYPE = "type-A"; + + public TestCryptoManagerTypeA(Settings settings, String keyProviderName) { + super(settings, keyProviderName); + } + + @Override + public String type() { + return TYPE; + } + + } + + private static class TestCryptoManagerTypeB extends TestCryptoManager { + public static final String TYPE = "type-B"; + + public TestCryptoManagerTypeB(Settings settings, String keyProviderName) { + super(settings, keyProviderName); + } + + @Override + public String type() { + return TYPE; + } + } + private static class TestRepository implements Repository { private static final String TYPE = "internal"; @@ -435,6 +880,7 @@ public void close() { private static class MeteredRepositoryTypeA extends MeteredBlobStoreRepository { private static final String TYPE = "type-a"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("GET", 10L)); + private final CryptoManager cryptoManager; private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clusterService) { super( @@ -445,6 +891,21 @@ private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clust mock(RecoverySettings.class), Map.of("bucket", "bucket-a") ); + + if (metadata.cryptoMetadata() != null) { + switch (metadata.cryptoMetadata().keyProviderType()) { + case TestCryptoManagerTypeA.TYPE: + cryptoManager = new TestCryptoManagerTypeA(null, metadata.cryptoMetadata().keyProviderName()); + break; + case TestCryptoManagerTypeB.TYPE: + cryptoManager = new TestCryptoManagerTypeB(null, metadata.cryptoMetadata().keyProviderName()); + break; + default: + cryptoManager = null; + } + } else { + cryptoManager = null; + } } @Override @@ -466,6 +927,7 @@ public BlobPath basePath() { private static class MeteredRepositoryTypeB extends MeteredBlobStoreRepository { private static final String TYPE = "type-b"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("LIST", 20L)); + private final CryptoManager cryptoManager; private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clusterService) { super( @@ -476,6 +938,21 @@ private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clust mock(RecoverySettings.class), Map.of("bucket", "bucket-b") ); + + if (metadata.cryptoMetadata() != null) { + switch (metadata.cryptoMetadata().keyProviderType()) { + case TestCryptoManagerTypeA.TYPE: + cryptoManager = new TestCryptoManagerTypeA(null, metadata.cryptoMetadata().keyProviderName()); + break; + case TestCryptoManagerTypeB.TYPE: + cryptoManager = new TestCryptoManagerTypeB(null, metadata.cryptoMetadata().keyProviderName()); + break; + default: + cryptoManager = null; + } + } else { + cryptoManager = null; + } } @Override diff --git a/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java index e47f6d87a95e5..8550316a666e8 100644 --- a/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java @@ -34,6 +34,7 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -57,13 +58,18 @@ protected Custom createTestInstance() { for (int i = 0; i < numberOfRepositories; i++) { // divide by 2 to not overflow when adding to this number for the pending generation below final long generation = randomNonNegativeLong() / 2L; + CryptoMetadata cryptoMetadata = null; + if (randomBoolean()) { + cryptoMetadata = new CryptoMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings()); + } entries.add( new RepositoryMetadata( randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings(), generation, - generation + randomLongBetween(0, generation) + generation + randomLongBetween(0, generation), + cryptoMetadata ) ); } @@ -81,7 +87,11 @@ protected Custom mutateInstance(Custom instance) { List entries = new ArrayList<>(((RepositoriesMetadata) instance).repositories()); boolean addEntry = entries.isEmpty() ? true : randomBoolean(); if (addEntry) { - entries.add(new RepositoryMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings())); + CryptoMetadata cryptoMetadata = null; + if (randomBoolean()) { + cryptoMetadata = new CryptoMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings()); + } + entries.add(new RepositoryMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings(), cryptoMetadata)); } else { entries.remove(randomIntBetween(0, entries.size() - 1)); } @@ -114,7 +124,11 @@ protected Custom makeTestChanges(Custom testInstance) { // add some elements int addElements = randomInt(10); for (int i = 0; i < addElements; i++) { - repos.add(new RepositoryMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings())); + CryptoMetadata cryptoMetadata = null; + if (randomBoolean()) { + cryptoMetadata = new CryptoMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings()); + } + repos.add(new RepositoryMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings(), cryptoMetadata)); } } return new RepositoriesMetadata(repos); diff --git a/test/framework/src/main/java/org/opensearch/test/RemoteStoreTestUtils.java b/test/framework/src/main/java/org/opensearch/test/RemoteStoreTestUtils.java index 0744d5fca853b..2e0b846d801e2 100644 --- a/test/framework/src/main/java/org/opensearch/test/RemoteStoreTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/RemoteStoreTestUtils.java @@ -16,12 +16,13 @@ import org.apache.lucene.util.Version; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Map; @@ -43,7 +44,7 @@ private RemoteStoreTestUtils() { * @return ByteArrayIndexInput: metadata file bytes with header and footer * @throws IOException IOException */ - public static ByteArrayIndexInput createMetadataFileBytes( + public static InputStream createMetadataFileBytes( Map segmentFilesMap, ReplicationCheckpoint replicationCheckpoint, SegmentInfos segmentInfos @@ -61,7 +62,7 @@ public static ByteArrayIndexInput createMetadataFileBytes( indexOutput.writeBytes(byteArray, byteArray.length); CodecUtil.writeFooter(indexOutput); indexOutput.close(); - return new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + return new ByteArrayInputStream(BytesReference.toBytes(output.bytes())); } public static Map getDummyMetadata(String prefix, int commitGeneration) {