Backport 3334 & 3379 to 5.2: Notice file

CHANGELOG.asciidoc

@@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Added new flags to import_dashboards (-cacert, -cert, -key, -insecure). {pull}3139[3139] {pull}3163[3163]
 - The limit for the number of fields is increased via the mapping template. {pull}3275[3275]
 - Updated to Go 1.7.4. {pull}3277[3277]
+- Added a NOTICE file containing the notices and licenses of the dependencies. {pull}3334[3334].
 
 *Metricbeat*
 - Kafka module broker matching enhancements. {pull}3129[3129]

Makefile

@@ -113,3 +113,9 @@ upload-package:
 .PHONY: release-upload
 upload-release:
 	aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/
+
+.PHONY: notice
+notice: 
+	python dev-tools/generate_notice.py .
+
+

dev-tools/generate_notice.py

@@ -0,0 +1,98 @@
+import glob
+import sys
+import os
+import datetime
+import argparse
+
+
+def read_file(filename):
+
+    if not os.path.isfile(filename):
+        print("File not found {}".format(filename))
+        return ""
+
+    with open(filename, 'r') as f:
+        file_content = f.read()
+        return file_content
+
+
+def get_library_name(license):
+
+    lib = ""
+    path = os.path.dirname(license)
+    # get the last three directories
+    for i in range(0, 3):
+        path, x = os.path.split(path)
+        if len(lib) == 0:
+            lib = x
+        elif len(x) > 0:
+            lib = x + "/" + lib
+
+    return lib
+
+
+def add_licenses(f, licenses):
+
+    for license in licenses:
+        for license_file in glob.glob(license):
+            f.write("\n--------------------------------------------------------------------\n")
+            f.write("{}\n".format(get_library_name(license_file)))
+            f.write("--------------------------------------------------------------------\n")
+            copyright = read_file(license_file)
+            if "Apache License" not in copyright:
+                f.write(copyright)
+            else:
+                # it's an Apache License, so include only the NOTICE file
+                f.write("Apache License\n\n")
+                for notice_file in glob.glob(os.path.join(os.path.dirname(license_file), "NOTICE*")):
+                    f.write("-------{}-----\n".format(os.path.basename(notice_file)))
+                    f.write(read_file(notice_file))
+
+
+
+def create_notice(filename, beat, copyright, licenses):
+
+    now = datetime.datetime.now()
+
+    with open(filename, "w+") as f:
+
+        # Add header
+        f.write("{}\n".format(beat))
+        f.write("Copyright 2014-{0} {1}\n".format(now.year, copyright))
+        f.write("\n")
+        f.write("This product includes software developed by The Apache Software \nFoundation (http://www.apache.org/).\n\n")
+
+        # Add licenses for 3rd party libraries
+        f.write("==========================================================================\n")
+        f.write("Third party libraries used by the Beats project:\n")
+        f.write("==========================================================================\n\n")
+        add_licenses(f, licenses)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="Generate the NOTICE file from all vendor directories available in a given directory")
+    parser.add_argument("vendor",
+                        help="directory where to search for vendor directories")
+    parser.add_argument("-b", "--beat", default="Elastic Beats",
+                        help="Beat name")
+    parser.add_argument("-c", "--copyright", default="Elasticsearch BV",
+                        help="copyright owner")
+
+    args = parser.parse_args()
+
+    cwd = os.getcwd()
+    notice = os.path.join(cwd, "NOTICE")
+    licenses = []
+
+    for root, dirs, files in os.walk(args.vendor):
+        if 'vendor' in dirs:
+            license = os.path.join(os.path.join(root, 'vendor'),
+                                   '**/**/**/LICENSE*')
+            licenses.append(license)
+
+    print("Get the licenses available from {}".format(licenses))
+    create_notice(notice, args.beat, args.copyright, licenses)
+
+    print("Available at {}\n".format(notice))

generate/beat/{beat}/NOTICE

@@ -0,0 +1,5 @@
+{beat}
+Copyright 2017 {full_name}
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

generate/beat/{{cookiecutter.beat}}/Makefile

@@ -5,6 +5,7 @@ TEST_ENVIRONMENT=false
 ES_BEATS?=./vendor/github.com/elastic/beats
 GOPACKAGES=$(shell glide novendor)
 PREFIX?=.
+NOTICE_FILE=NOTICE
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/libbeat/scripts/Makefile

generate/metricbeat/metricset/{{cookiecutter.beat}}/Makefile

@@ -5,6 +5,7 @@ TEST_ENVIRONMENT=false
 ES_BEATS?=./vendor/github.com/elastic/beats
 GOPACKAGES=$(shell glide novendor)
 PREFIX?=.
+NOTICE_FILE=NOTICE
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile

generate/metricbeat/metricset/{{cookiecutter.beat}}/NOTICE

@@ -0,0 +1,5 @@
+{{cookiecutter.beat}}
+Copyright 2017 {{cookiecutter.full_name}}
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

libbeat/scripts/Makefile

@@ -9,6 +9,7 @@ BEAT_DIR?=github.com/elastic/beats/${BEATNAME}
 ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats
 GOPACKAGES?=${BEAT_DIR}/...## @community_beat Must be set to $(shell glide novendor)
 PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package"
+NOTICE_FILE?=../NOTICE
 
 # Makefile for a custom beat that includes this libbeat/scripts/Makefile:
 # if glide is used to manage vendor dependencies,
@@ -329,6 +330,9 @@ HOME_PREFIX?=/tmp/${BEATNAME}
 install-home:
 	install -d -m 755 ${HOME_PREFIX}/scripts/
 	install -m 755 ${ES_BEATS}/libbeat/scripts/migrate_beat_config_1_x_to_5_0.py ${HOME_PREFIX}/scripts/
+	if [ -a ${NOTICE_FILE} ]; then \
+		install -m 644 ${NOTICE_FILE} ${HOME_PREFIX}/; \
+	fi
 
 # Prepares for packaging. Builds binaries and creates homedir data
 .PHONY: prepare-package