Include vendored code notices in distribution notice files (#57017) (#…

…57569)

(cherry picked from commit 627ef27)

buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -20,27 +20,32 @@
 package org.elasticsearch.gradle
 
 import org.gradle.api.DefaultTask
-import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
+import org.gradle.api.file.FileCollection
+import org.gradle.api.file.FileTree
+import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
 /**
  * A task to create a notice file which includes dependencies' notices.
  */
-public class NoticeTask extends DefaultTask {
+class NoticeTask extends DefaultTask {
 
     @InputFile
     File inputFile = project.rootProject.file('NOTICE.txt')
 
     @OutputFile
     File outputFile = new File(project.buildDir, "notices/${name}/NOTICE.txt")
 
+    private FileTree sources
+
     /** Directories to include notices from */
     private List<File> licensesDirs = new ArrayList<>()
 
-    public NoticeTask() {
+    NoticeTask() {
         description = 'Create a notice file from dependencies'
         // Default licenses directory is ${projectDir}/licenses (if it exists)
         File licensesDir = new File(project.projectDir, 'licenses')
@@ -50,46 +55,122 @@ public class NoticeTask extends DefaultTask {
     }
 
     /** Add notices from the specified directory. */
-    public void licensesDir(File licensesDir) {
+    void licensesDir(File licensesDir) {
         licensesDirs.add(licensesDir)
     }
 
+    void source(Object source) {
+        if (sources == null) {
+            sources = project.fileTree(source)
+        } else {
+            sources += project.fileTree(source)
+        }
+    }
+
+    void source(SourceDirectorySet source) {
+        if (sources == null) {
+            sources = source
+        } else {
+            sources += source
+        }
+    }
+
     @TaskAction
-    public void generateNotice() {
+    void generateNotice() {
         StringBuilder output = new StringBuilder()
         output.append(inputFile.getText('UTF-8'))
         output.append('\n\n')
         // This is a map rather than a set so that the sort order is the 3rd
         // party component names, unaffected by the full path to the various files
         Map<String, File> seen = new TreeMap<>()
-        for (File licensesDir : licensesDirs) {
-            licensesDir.eachFileMatch({ it ==~ /.*-NOTICE\.txt/ }) { File file ->
-                String name = file.name.substring(0, file.name.length() - '-NOTICE.txt'.length())
-                if (seen.containsKey(name)) {
-                    File prevFile = seen.get(name)
-                    if (prevFile.text != file.text) {
-                        throw new RuntimeException("Two different notices exist for dependency '" +
-                                name + "': " + prevFile + " and " + file)
-                    }
-                } else {
-                    seen.put(name, file)
+        noticeFiles.each { File file ->
+            String name = file.name.replaceFirst(/-NOTICE\.txt$/, "")
+            if (seen.containsKey(name)) {
+                File prevFile = seen.get(name)
+                if (prevFile.text != file.text) {
+                    throw new RuntimeException("Two different notices exist for dependency '" +
+                            name + "': " + prevFile + " and " + file)
                 }
+            } else {
+                seen.put(name, file)
             }
         }
+
+        // Add all LICENSE and NOTICE files in licenses directory
         for (Map.Entry<String, File> entry : seen.entrySet()) {
             String name = entry.getKey()
             File file = entry.getValue()
             appendFile(file, name, 'NOTICE', output)
             appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output)
         }
+
+        // Find any source files with "@notice" annotated license header
+        for (File sourceFile : sources.files) {
+            boolean isPackageInfo = sourceFile.name == 'package-info.java'
+            boolean foundNotice = false
+            boolean inNotice = false
+            StringBuilder header = new StringBuilder()
+            String packageDeclaration
+
+            for (String line : sourceFile.readLines()) {
+                if (isPackageInfo && packageDeclaration == null && line.startsWith('package')) {
+                    packageDeclaration = line
+                }
+
+                if (foundNotice == false) {
+                    foundNotice = line.contains('@notice')
+                    inNotice = true
+                } else {
+                    if (line.contains('*/')) {
+                        inNotice = false
+
+                        if (!isPackageInfo) {
+                            break
+                        }
+                    } else if (inNotice) {
+                        header.append(line.stripMargin('*'))
+                        header.append('\n')
+                    }
+                }
+            }
+
+            if (foundNotice) {
+                appendText(header.toString(), isPackageInfo ? packageDeclaration : sourceFile.name, '', output)
+            }
+        }
         outputFile.setText(output.toString(), 'UTF-8')
     }
 
+    @InputFiles
+    @Optional
+    FileCollection getNoticeFiles() {
+        FileTree tree
+        licensesDirs.each { dir ->
+            if (tree == null) {
+                tree = project.fileTree(dir)
+            } else {
+                tree += project.fileTree(dir)
+            }
+        }
+
+        return tree?.matching { include '**/*-NOTICE.txt' }
+    }
+
+    @InputFiles
+    @Optional
+    FileCollection getSources() {
+        return sources
+    }
+
     static void appendFile(File file, String name, String type, StringBuilder output) {
         String text = file.getText('UTF-8')
         if (text.trim().isEmpty()) {
             return
         }
+        appendText(text, name, type, output)
+    }
+
+    static void appendText(String text, String name, String type, StringBuilder output) {
         output.append('================================================================================\n')
         output.append("${name} ${type}\n")
         output.append('================================================================================\n')

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -28,6 +28,7 @@ import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.util.Util
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
@@ -256,6 +257,7 @@ class PluginBuildPlugin implements Plugin<Project> {
         if (noticeFile != null) {
             TaskProvider<NoticeTask> generateNotice = project.tasks.register('generateNotice', NoticeTask) {
                 inputFile = noticeFile
+                source(Util.getJavaMainSourceSet(project).get().allJava)
             }
             project.tasks.named('bundlePlugin').configure {
                 from(generateNotice)

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -27,7 +27,6 @@ import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.SkipWhenEmpty
-import org.gradle.api.tasks.SourceSet
 
 import java.nio.file.Files
 
@@ -43,7 +42,7 @@ public class LicenseHeadersTask extends AntTask {
 
     /** Allowed license families for this project. */
     @Input
-    List<String> approvedLicenses = ['Apache', 'Generated']
+    List<String> approvedLicenses = ['Apache', 'Generated', 'Vendored']
 
     /**
      * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
@@ -69,7 +68,7 @@ public class LicenseHeadersTask extends AntTask {
      */
     @InputFiles
     @SkipWhenEmpty
-    public List<FileCollection> getJavaFiles() {
+    List<FileCollection> getJavaFiles() {
         return project.sourceSets.collect({it.allJava})
     }
 
@@ -82,7 +81,7 @@ public class LicenseHeadersTask extends AntTask {
      * @param familyName An expanded string name for the license
      * @param pattern A pattern to search for, which if found, indicates a file contains the license
      */
-    public void additionalLicense(String categoryName, String familyName, String pattern) {
+    void additionalLicense(String categoryName, String familyName, String pattern) {
         if (categoryName.length() != 5) {
             throw new IllegalArgumentException("License category name must be exactly 5 characters, got ${categoryName}");
         }
@@ -120,10 +119,6 @@ public class LicenseHeadersTask extends AntTask {
                              licenseFamilyName:     "Apache") {
                // Apache license (ES)
                pattern(substring: "Licensed to Elasticsearch under one or more contributor")
-               // Apache license (ASF)
-               pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
-               // this is the old-school one under some files
-               pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")")
             }
 
             // Generated resources
@@ -133,6 +128,12 @@ public class LicenseHeadersTask extends AntTask {
                pattern(substring: "ANTLR GENERATED CODE")
             }
 
+            // Vendored Code
+            substringMatcher(licenseFamilyCategory: "VEN  ",
+                licenseFamilyName:     "Vendored") {
+                pattern(substring: "@notice")
+            }
+
             // license types added by the project
             for (Map.Entry<String, String[]> additional : additionalLicenses.entrySet()) {
                 String category = additional.getKey().substring(0, 5)

distribution/build.gradle

@@ -48,24 +48,29 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
  *****************************************************************************/
 
 // integ test zip only uses server, so a different notice file is needed there
-task buildServerNotice(type: NoticeTask) {
-  licensesDir new File(project(':server').projectDir, 'licenses')
-}
+task buildServerNotice(type: NoticeTask)
 
 // other distributions include notices from modules as well, which are added below later
 task buildDefaultNotice(type: NoticeTask) {
-  licensesDir new File(project(':server').projectDir, 'licenses')
   licensesDir new File(project(':distribution').projectDir, 'licenses')
 }
+
 task buildOssNotice(type: NoticeTask) {
-  licensesDir new File(project(':server').projectDir, 'licenses')
   licensesDir new File(project(':distribution').projectDir, 'licenses')
 }
-task buildDefaultNoJdkNotice(type: NoticeTask) {
-  licensesDir new File(project(':server').projectDir, 'licenses')
-}
-task buildOssNoJdkNotice(type: NoticeTask) {
-  licensesDir new File(project(':server').projectDir, 'licenses')
+
+task buildDefaultNoJdkNotice(type: NoticeTask)
+
+task buildOssNoJdkNotice(type: NoticeTask)
+
+// The :server and :libs projects belong to all distributions
+tasks.withType(NoticeTask) {
+  licensesDir project(':server').file('licenses')
+  source project(':server').file('src/main/java')
+  project(':libs').subprojects.each { Project lib ->
+    licensesDir lib.file('licenses')
+    source lib.file('src/main/java')
+  }
 }
 
 /*****************************************************************************
@@ -211,7 +216,9 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
   File licenses = new File(module.projectDir, 'licenses')
   if (licenses.exists()) {
     buildDefaultNotice.licensesDir licenses
+    buildDefaultNotice.source module.file('src/main/java')
     buildOssNotice.licensesDir licenses
+    buildOssNotice.source module.file('src/main/java')
   }
 
   copyModule(processOssOutputs, module)
@@ -235,6 +242,7 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
   File licenses = new File(xpackModule.projectDir, 'licenses')
   if (licenses.exists()) {
     buildDefaultNotice.licensesDir licenses
+    buildDefaultNotice.source xpackModule.file('src/main/java')
   }
   copyModule(processDefaultOutputs, xpackModule)
   copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)

libs/core/src/main/java/org/elasticsearch/common/util/FastMath.java

@@ -1,4 +1,4 @@
-/*
+/* @notice
  * Copyright 2012 Jeff Hain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +12,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- */
-
-/*
+ *
  * =============================================================================
  * Notice of fdlibm package this program is partially derived from:
  *

libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java

@@ -1,4 +1,4 @@
-/*
+/* @notice
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -13,6 +13,8 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Modifications copyright (C) 2020 Elasticsearch B.V.
  */
 
 package org.elasticsearch.core.internal.io;

libs/core/src/test/java/org/elasticsearch/core/internal/io/IOUtilsTests.java

@@ -1,4 +1,4 @@
-/*
+/* @notice
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DerParser.java

@@ -1,4 +1,4 @@
-/*
+/* @notice
   Copyright (c) 1998-2010 AOL Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");

plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java

@@ -1,6 +1,6 @@
 package org.elasticsearch.index.analysis;
 
-/*
+/* @notice
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.

plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IndexableBinaryStringTools.java

@@ -1,6 +1,6 @@
 package org.elasticsearch.index.analysis;
 
-/*
+/* @notice
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.

plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IndexableBinaryStringToolsTests.java

@@ -1,6 +1,6 @@
 package org.elasticsearch.index.analysis;
 
-/*
+/* @notice
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.

server/build.gradle

@@ -336,6 +336,11 @@ dependencyLicenses {
   }
 }
 
+licenseHeaders {
+  // Ignore our vendored version of Google Guice
+  excludes << 'org/elasticsearch/common/inject/**/*'
+}
+
 tasks.named('internalClusterTest').configure {
   if (org.elasticsearch.gradle.info.BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.datastreams_feature_enabled', 'true'