From 72cf26ef80522f436f9cd3871719a75a91148bc5 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 12 May 2020 13:16:30 -0700 Subject: [PATCH 1/2] Enforce strict license distribution requirements This commit tightens certain dependency license checks in our build. Firstly, the build will not fail if it cannot accurately identify the type of license in one of our LICENSE.txt files. Secondly, dependencies for licenses identified as requiring source redistribution will fail if a corresponding SOURCES.txt file does not exist. This file should include a hyperlink to a source artifact for the given dependency to be used for redistribution during the release process. --- .../gradle/DependenciesInfoTask.groovy | 177 +--- .../precommit/DependencyLicensesTask.java | 54 +- .../gradle/precommit/LicenseAnalyzer.java | 205 +++++ .../DependencyLicensesTaskTests.java | 37 +- .../precommit/LicenseAnalyzerTests.java | 79 ++ .../licenses/hamcrest-core-LICENSE.txt | 22 + .../licenses/junit-LICENSE.txt | 22 + distribution/build.gradle | 2 +- .../licenses/jaxb-LICENSE.txt | 795 ++++-------------- .../repository-s3/licenses/jaxb-LICENSE.txt | 793 ++++------------- .../licenses/unboundid-ldapsdk-SOURCES.txt | 1 + .../licenses/jakarta.activation-LICENSE.txt | 40 +- 12 files changed, 811 insertions(+), 1416 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/LicenseAnalyzer.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/LicenseAnalyzerTests.java create mode 100644 x-pack/plugin/core/licenses/unboundid-ldapsdk-SOURCES.txt diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy index 49dfdbfb0ce3a..518609fc26dd0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy @@ -20,6 +20,7 @@ package org.elasticsearch.gradle import org.elasticsearch.gradle.precommit.DependencyLicensesTask +import org.elasticsearch.gradle.precommit.LicenseAnalyzer import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency import org.gradle.api.artifacts.DependencySet @@ -109,7 +110,8 @@ class DependenciesInfoTask extends ConventionTask { void setMappings(LinkedHashMap mappings) { this.mappings = mappings } -/** + + /** * Create an URL on Maven Central * based on dependency coordinates. */ @@ -134,159 +136,44 @@ class DependenciesInfoTask extends ConventionTask { * @return SPDX identifier, UNKNOWN or a Custom license */ protected String getLicenseType(final String group, final String name) { - File license + File license = getDependencyInfoFile(group, name, 'LICENSE') + String licenseType + + final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license) + if (licenseInfo.spdxLicense == false) { + // License has not be identified as SPDX. + // As we have the license file, we create a Custom entry with the URL to this license file. + final gitBranch = System.getProperty('build.branch', 'master') + final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/" + licenseType = "${licenseInfo.identifier};${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}" + } else { + licenseType = licenseInfo.identifier + } + + if (licenseInfo.sourceRedistributionRequired) { + File sources = getDependencyInfoFile(group, name, 'SOURCES') + licenseType += ",${sources.text.trim()}" + } + + return licenseType + } + + protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) { + File license = null if (licensesDir != null) { - licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file -> - String prefix = file.name.split('-LICENSE.*')[0] + licensesDir.eachFileMatch({ it ==~ /.*-${infoFileSuffix}.*/ }) { File file -> + String prefix = file.name.split("-${infoFileSuffix}.*")[0] if (group.contains(prefix) || name.contains(prefix)) { license = file.getAbsoluteFile() } } } - if (license) { - // replace * because they are sometimes used at the beginning lines as if the license was a multi-line comment - final String content = new String(license.readBytes(), "UTF-8").replaceAll("\\s+", " ").replaceAll("\\*", " ") - final String spdx = checkSPDXLicense(content) - if (spdx == null) { - // License has not be identified as SPDX. - // As we have the license file, we create a Custom entry with the URL to this license file. - final gitBranch = System.getProperty('build.branch', 'master') - final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/" - return "Custom;${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}" - } - return spdx - } else { - return "UNKNOWN" + if (license == null) { + throw new IllegalStateException("Unable to find ${infoFileSuffix} file for dependency ${group}:${name} in ${licensesDir}") } - } - /** - * Check the license content to identify an SPDX license type. - * - * @param licenseText LICENSE file content. - * @return SPDX identifier or null. - */ - private String checkSPDXLicense(final String licenseText) { - String spdx = null - - final String APACHE_2_0 = "Apache.*License.*(v|V)ersion.*2\\.0" - - final String BSD_2 = """ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1\\. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer\\. - 2\\. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution\\. - -THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\. -IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String BSD_3 = """ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - (1\\.)? Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer\\. - (2\\.)? Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution\\. - ((3\\.)? The name of .+ may not be used to endorse or promote products - derived from this software without specific prior written permission\\.| - (3\\.)? Neither the name of .+ nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission\\.) - -THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\. -IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0" - final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1" - final String ICU = "ICU License - ICU 1.8.1 and later" - final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3" - - final String MIT = """ -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files \\(the "Software"\\), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software\\. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\\. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1" - - final String MOZILLA_2_0 = "Mozilla\\s*Public\\s*License\\s*Version\\s*2\\.0" - - switch (licenseText) { - case ~/.*${APACHE_2_0}.*/: - spdx = 'Apache-2.0' - break - case ~/.*${MIT}.*/: - spdx = 'MIT' - break - case ~/.*${BSD_2}.*/: - spdx = 'BSD-2-Clause' - break - case ~/.*${BSD_3}.*/: - spdx = 'BSD-3-Clause' - break - case ~/.*${LGPL_3}.*/: - spdx = 'LGPL-3.0' - break - case ~/.*${CDDL_1_0}.*/: - spdx = 'CDDL-1.0' - break - case ~/.*${CDDL_1_1}.*/: - spdx = 'CDDL-1.1' - break - case ~/.*${ICU}.*/: - spdx = 'ICU' - break - case ~/.*${MOZILLA_1_1}.*/: - spdx = 'MPL-1.1' - break - case ~/.*${MOZILLA_2_0}.*/: - spdx = 'MPL-2.0' - break - default: - break - } - return spdx + return license } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java index c5a6fe10febd2..73817199245b2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.codec.binary.Hex; +import org.elasticsearch.gradle.precommit.LicenseAnalyzer.LicenseInfo; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.InvalidUserDataException; @@ -53,13 +54,13 @@ /** * A task to check licenses for dependencies. - * + *

* There are two parts to the check: *

- * + *

* The directory to find the license and sha files in defaults to the dir @{code licenses} * in the project directory for this task. You can override this directory: * 

@@ -67,7 +68,7 @@
  *     licensesDir = getProject().file("mybetterlicensedir")
  *   }
  *
- * + *

* The jar files to check default to the dependencies from the default configuration. You * can override this, for example, to only check compile dependencies: * 

@@ -75,12 +76,12 @@
  *     dependencies = getProject().configurations.compile
  *   }
  *
- * + *

* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed * automatically using the {@code updateShas} helper task that is created along * with this task. It will add {@code .sha1} files for new jars that are in dependencies * and remove old {@code .sha1} files that are no longer needed. - * + *

* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share * LICENSE and NOTICE files by mapping a pattern to the same name. * 

@@ -88,6 +89,10 @@
  *     mapping from: /lucene-.*/, to: "lucene"
  *   }
  *
+ * Dependencies using licenses with stricter distribution requirements (such as LGPL) + * require a SOURCES file as well. The file should include a URL to a source distribution + * for the dependency. This artifact will be redistributed by us with the release to + * comply with the license terms. */ public class DependencyLicensesTask extends DefaultTask { @@ -99,16 +104,24 @@ public class DependencyLicensesTask extends DefaultTask { // TODO: we should be able to default this to eg compile deps, but we need to move the licenses // check from distribution to core (ie this should only be run on java projects) - /** A collection of jar files that should be checked. */ + /** + * A collection of jar files that should be checked. + */ private FileCollection dependencies; - /** The directory to find the license and sha files in. */ + /** + * The directory to find the license and sha files in. + */ private File licensesDir = new File(getProject().getProjectDir(), "licenses"); - /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */ + /** + * A map of patterns to prefix, used to find the LICENSE and NOTICE file. + */ private Map mappings = new LinkedHashMap<>(); - /** Names of dependencies whose shas should not exist. */ + /** + * Names of dependencies whose shas should not exist. + */ private Set ignoreShas = new HashSet<>(); /** @@ -178,6 +191,7 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException { Map licenses = new HashMap<>(); Map notices = new HashMap<>(); + Map sources = new HashMap<>(); Set shaFiles = new HashSet<>(); for (File file : licensesDir.listFiles()) { @@ -189,15 +203,19 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException { licenses.put(name, false); } else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) { notices.put(name, false); + } else if (name.contains("-SOURCES") || name.contains("-SOURCES.txt")) { + sources.put(name, false); } } - checkDependencies(licenses, notices, shaFiles); + checkDependencies(licenses, notices, sources, shaFiles); licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license")); notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice")); + sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources")); + if (shaFiles.isEmpty() == false) { throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles)); } @@ -209,8 +227,12 @@ private void failIfAnyMissing(String item, Boolean exists, String type) { } } - private void checkDependencies(Map licenses, Map notices, Set shaFiles) - throws NoSuchAlgorithmException, IOException { + private void checkDependencies( + Map licenses, + Map notices, + Map sources, + Set shaFiles + ) throws NoSuchAlgorithmException, IOException { for (File dependency : dependencies) { String jarName = dependency.getName(); String depName = regex.matcher(jarName).replaceFirst(""); @@ -221,6 +243,12 @@ private void checkDependencies(Map licenses, Map Date: Tue, 12 May 2020 13:20:22 -0700 Subject: [PATCH 2/2] Remove temporary test --- .../precommit/LicenseAnalyzerTests.java | 79 ------------------- 1 file changed, 79 deletions(-) delete mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/LicenseAnalyzerTests.java diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/LicenseAnalyzerTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/LicenseAnalyzerTests.java deleted file mode 100644 index eca16631ddd00..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/LicenseAnalyzerTests.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle.precommit; - -import org.elasticsearch.gradle.test.GradleUnitTestCase; - -import java.io.File; - -public class LicenseAnalyzerTests extends GradleUnitTestCase { - - public void testCanDetectApacheLicense() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/server/licenses/lucene-LICENSE.txt"); - assertEquals("Apache-2.0", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectBSD2License() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/server/licenses/HdrHistogram-LICENSE.txt"); - assertEquals("BSD-2-Clause", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectBSD3License() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/x-pack/plugin/watcher/licenses/jakarta.activation-LICENSE.txt"); - assertEquals("BSD-3-Clause", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectCDDL1_0License() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/plugins/repository-hdfs/licenses/servlet-api-LICENSE.txt"); - assertEquals("CDDL-1.0", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectCDDL1_1License() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/plugins/discovery-azure-classic/licenses/jaxb-LICENSE.txt"); - assertEquals("CDDL-1.1", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectICULicense() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/x-pack/plugin/ml/licenses/icu4j-LICENSE.txt"); - assertEquals("ICU", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectLGPL2_1License() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/x-pack/plugin/core/licenses/unboundid-ldapsdk-LICENSE.txt"); - assertEquals("LGPL-2.1", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectMITLicense() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/distribution/tools/plugin-cli/licenses/bouncycastle-LICENSE.txt"); - assertEquals("MIT", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectMPL1_1License() { - File licenseFile = new File( - "/Users/mark/workspaces/elasticsearch/plugins/ingest-attachment/licenses/juniversalchardet-LICENSE.txt" - ); - assertEquals("MPL-1.1", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } - - public void testCanDetectXZLicense() { - File licenseFile = new File("/Users/mark/workspaces/elasticsearch/plugins/ingest-attachment/licenses/xz-LICENSE.txt"); - assertEquals("XZ", LicenseAnalyzer.licenseType(licenseFile).getIdentifier()); - } -}