From bbdf250c8a044305da9df0ecec3827ca0736bea8 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 14 May 2020 13:57:42 -0700 Subject: [PATCH] Enforce strict license distribution requirements (#56642) --- .../gradle/DependenciesInfoTask.groovy | 177 +--- .../precommit/DependencyLicensesTask.java | 54 +- .../gradle/precommit/LicenseAnalyzer.java | 207 +++++ .../DependencyLicensesTaskTests.java | 37 +- .../licenses/hamcrest-core-LICENSE.txt | 22 + .../licenses/junit-LICENSE.txt | 22 + distribution/build.gradle | 2 +- .../licenses/jaxb-LICENSE.txt | 795 ++++-------------- .../repository-s3/licenses/jaxb-LICENSE.txt | 793 ++++------------- .../licenses/unboundid-ldapsdk-SOURCES.txt | 1 + .../licenses/jakarta.activation-LICENSE.txt | 40 +- 11 files changed, 734 insertions(+), 1416 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/LicenseAnalyzer.java create mode 100644 x-pack/plugin/core/licenses/unboundid-ldapsdk-SOURCES.txt diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy index 49dfdbfb0ce3a..518609fc26dd0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy @@ -20,6 +20,7 @@ package org.elasticsearch.gradle import org.elasticsearch.gradle.precommit.DependencyLicensesTask +import org.elasticsearch.gradle.precommit.LicenseAnalyzer import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency import org.gradle.api.artifacts.DependencySet @@ -109,7 +110,8 @@ class DependenciesInfoTask extends ConventionTask { void setMappings(LinkedHashMap mappings) { this.mappings = mappings } -/** + + /** * Create an URL on Maven Central * based on dependency coordinates. */ @@ -134,159 +136,44 @@ class DependenciesInfoTask extends ConventionTask { * @return SPDX identifier, UNKNOWN or a Custom license */ protected String getLicenseType(final String group, final String name) { - File license + File license = getDependencyInfoFile(group, name, 'LICENSE') + String licenseType + + final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license) + if (licenseInfo.spdxLicense == false) { + // License has not be identified as SPDX. + // As we have the license file, we create a Custom entry with the URL to this license file. + final gitBranch = System.getProperty('build.branch', 'master') + final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/" + licenseType = "${licenseInfo.identifier};${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}" + } else { + licenseType = licenseInfo.identifier + } + + if (licenseInfo.sourceRedistributionRequired) { + File sources = getDependencyInfoFile(group, name, 'SOURCES') + licenseType += ",${sources.text.trim()}" + } + + return licenseType + } + + protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) { + File license = null if (licensesDir != null) { - licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file -> - String prefix = file.name.split('-LICENSE.*')[0] + licensesDir.eachFileMatch({ it ==~ /.*-${infoFileSuffix}.*/ }) { File file -> + String prefix = file.name.split("-${infoFileSuffix}.*")[0] if (group.contains(prefix) || name.contains(prefix)) { license = file.getAbsoluteFile() } } } - if (license) { - // replace * because they are sometimes used at the beginning lines as if the license was a multi-line comment - final String content = new String(license.readBytes(), "UTF-8").replaceAll("\\s+", " ").replaceAll("\\*", " ") - final String spdx = checkSPDXLicense(content) - if (spdx == null) { - // License has not be identified as SPDX. - // As we have the license file, we create a Custom entry with the URL to this license file. - final gitBranch = System.getProperty('build.branch', 'master') - final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/" - return "Custom;${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}" - } - return spdx - } else { - return "UNKNOWN" + if (license == null) { + throw new IllegalStateException("Unable to find ${infoFileSuffix} file for dependency ${group}:${name} in ${licensesDir}") } - } - /** - * Check the license content to identify an SPDX license type. - * - * @param licenseText LICENSE file content. - * @return SPDX identifier or null. - */ - private String checkSPDXLicense(final String licenseText) { - String spdx = null - - final String APACHE_2_0 = "Apache.*License.*(v|V)ersion.*2\\.0" - - final String BSD_2 = """ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1\\. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer\\. - 2\\. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution\\. - -THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\. -IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String BSD_3 = """ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - (1\\.)? Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer\\. - (2\\.)? Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution\\. - ((3\\.)? The name of .+ may not be used to endorse or promote products - derived from this software without specific prior written permission\\.| - (3\\.)? Neither the name of .+ nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission\\.) - -THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\. -IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0" - final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1" - final String ICU = "ICU License - ICU 1.8.1 and later" - final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3" - - final String MIT = """ -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files \\(the "Software"\\), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software\\. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\\. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE\\. -""".replaceAll("\\s+", "\\\\s*") - - final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1" - - final String MOZILLA_2_0 = "Mozilla\\s*Public\\s*License\\s*Version\\s*2\\.0" - - switch (licenseText) { - case ~/.*${APACHE_2_0}.*/: - spdx = 'Apache-2.0' - break - case ~/.*${MIT}.*/: - spdx = 'MIT' - break - case ~/.*${BSD_2}.*/: - spdx = 'BSD-2-Clause' - break - case ~/.*${BSD_3}.*/: - spdx = 'BSD-3-Clause' - break - case ~/.*${LGPL_3}.*/: - spdx = 'LGPL-3.0' - break - case ~/.*${CDDL_1_0}.*/: - spdx = 'CDDL-1.0' - break - case ~/.*${CDDL_1_1}.*/: - spdx = 'CDDL-1.1' - break - case ~/.*${ICU}.*/: - spdx = 'ICU' - break - case ~/.*${MOZILLA_1_1}.*/: - spdx = 'MPL-1.1' - break - case ~/.*${MOZILLA_2_0}.*/: - spdx = 'MPL-2.0' - break - default: - break - } - return spdx + return license } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java index c5a6fe10febd2..73817199245b2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.codec.binary.Hex; +import org.elasticsearch.gradle.precommit.LicenseAnalyzer.LicenseInfo; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.InvalidUserDataException; @@ -53,13 +54,13 @@ /** * A task to check licenses for dependencies. - * + *

* There are two parts to the check: *

- * + *

* The directory to find the license and sha files in defaults to the dir @{code licenses} * in the project directory for this task. You can override this directory: * 

@@ -67,7 +68,7 @@
  *     licensesDir = getProject().file("mybetterlicensedir")
  *   }
  *
- * + *

* The jar files to check default to the dependencies from the default configuration. You * can override this, for example, to only check compile dependencies: * 

@@ -75,12 +76,12 @@
  *     dependencies = getProject().configurations.compile
  *   }
  *
- * + *

* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed * automatically using the {@code updateShas} helper task that is created along * with this task. It will add {@code .sha1} files for new jars that are in dependencies * and remove old {@code .sha1} files that are no longer needed. - * + *

* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share * LICENSE and NOTICE files by mapping a pattern to the same name. * 

@@ -88,6 +89,10 @@
  *     mapping from: /lucene-.*/, to: "lucene"
  *   }
  *
+ * Dependencies using licenses with stricter distribution requirements (such as LGPL) + * require a SOURCES file as well. The file should include a URL to a source distribution + * for the dependency. This artifact will be redistributed by us with the release to + * comply with the license terms. */ public class DependencyLicensesTask extends DefaultTask { @@ -99,16 +104,24 @@ public class DependencyLicensesTask extends DefaultTask { // TODO: we should be able to default this to eg compile deps, but we need to move the licenses // check from distribution to core (ie this should only be run on java projects) - /** A collection of jar files that should be checked. */ + /** + * A collection of jar files that should be checked. + */ private FileCollection dependencies; - /** The directory to find the license and sha files in. */ + /** + * The directory to find the license and sha files in. + */ private File licensesDir = new File(getProject().getProjectDir(), "licenses"); - /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */ + /** + * A map of patterns to prefix, used to find the LICENSE and NOTICE file. + */ private Map mappings = new LinkedHashMap<>(); - /** Names of dependencies whose shas should not exist. */ + /** + * Names of dependencies whose shas should not exist. + */ private Set ignoreShas = new HashSet<>(); /** @@ -178,6 +191,7 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException { Map licenses = new HashMap<>(); Map notices = new HashMap<>(); + Map sources = new HashMap<>(); Set shaFiles = new HashSet<>(); for (File file : licensesDir.listFiles()) { @@ -189,15 +203,19 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException { licenses.put(name, false); } else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) { notices.put(name, false); + } else if (name.contains("-SOURCES") || name.contains("-SOURCES.txt")) { + sources.put(name, false); } } - checkDependencies(licenses, notices, shaFiles); + checkDependencies(licenses, notices, sources, shaFiles); licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license")); notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice")); + sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources")); + if (shaFiles.isEmpty() == false) { throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles)); } @@ -209,8 +227,12 @@ private void failIfAnyMissing(String item, Boolean exists, String type) { } } - private void checkDependencies(Map licenses, Map notices, Set shaFiles) - throws NoSuchAlgorithmException, IOException { + private void checkDependencies( + Map licenses, + Map notices, + Map sources, + Set shaFiles + ) throws NoSuchAlgorithmException, IOException { for (File dependency : dependencies) { String jarName = dependency.getName(); String depName = regex.matcher(jarName).replaceFirst(""); @@ -221,6 +243,12 @@ private void checkDependencies(Map licenses, Map