From 1a9f6e99c8302aba1479be402441d0463276c17b Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Fri, 21 Jun 2019 11:43:43 -0400
Subject: [PATCH 1/6] Add a metadata_merge that merges YAML from security check
 & download_pkgs

Will use this rule to generate a combined metadata for FUS to upload for
the bazel & clang debs in layer-definitions. The combined metadata will
include vulnerability + package information.
---
 WORKSPACE                                     |  16 +--
 repositories/go_repositories.bzl              |  46 ++++++++
 rules/container/metadata_merge.bzl            |  85 ++++++++++++++
 src/go/cmd/metadata_merge/BUILD.bazel         |  18 +++
 src/go/cmd/metadata_merge/metadata_merge.go   | 110 ++++++++++++++++++
 src/go/cmd/packages_metadata/BUILD            |   1 +
 .../packages_metadata/packages_metadata.go    |  26 +----
 src/go/pkg/metadata/BUILD.bazel               |   8 ++
 src/go/pkg/metadata/packages.go               |  17 +++
 tests/rules/BUILD                             |  25 ++++
 tests/rules/metadata_merge_file1.yaml         |   3 +
 tests/rules/metadata_merge_file2.yaml         |   3 +
 tests/rules/metadata_merge_file3.yaml         |   2 +
 tests/rules/metadata_merge_file4.yaml         |   3 +
 14 files changed, 329 insertions(+), 34 deletions(-)
 create mode 100644 repositories/go_repositories.bzl
 create mode 100644 rules/container/metadata_merge.bzl
 create mode 100644 src/go/cmd/metadata_merge/BUILD.bazel
 create mode 100644 src/go/cmd/metadata_merge/metadata_merge.go
 create mode 100644 src/go/pkg/metadata/BUILD.bazel
 create mode 100644 src/go/pkg/metadata/packages.go
 create mode 100644 tests/rules/metadata_merge_file1.yaml
 create mode 100644 tests/rules/metadata_merge_file2.yaml
 create mode 100644 tests/rules/metadata_merge_file3.yaml
 create mode 100644 tests/rules/metadata_merge_file4.yaml

diff --git a/WORKSPACE b/WORKSPACE
index 48391b127..1bfc87303 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -59,9 +59,9 @@ go_register_toolchains()
 # is being imported from a *.bzl file.
 # gazelle:repo bazel_gazelle
 
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
+load("//repositories:go_repositories.bzl", bazel_toolchains_go_deps="go_deps")
 
-gazelle_dependencies()
+bazel_toolchains_go_deps()
 
 container_pull(
     name = "official_jessie",
@@ -591,15 +591,3 @@ pip_import(
 load("@pip_deps//:requirements.bzl", "pip_install")
 
 pip_install()
-
-go_repository(
-    name = "in_gopkg_yaml_v2",
-    commit = "51d6538a90f86fe93ac480b35f37b2be17fef232",  # v2.2.2
-    importpath = "gopkg.in/yaml.v2",
-)
-
-go_repository(
-    name = "com_github_pkg_errors",
-    commit = "05ac58a23b8798a296fa64f7d9c1559904db4b98",  # v0.8.1
-    importpath = "github.com/pkg/errors",
-)
diff --git a/repositories/go_repositories.bzl b/repositories/go_repositories.bzl
new file mode 100644
index 000000000..afef8163d
--- /dev/null
+++ b/repositories/go_repositories.bzl
@@ -0,0 +1,46 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Once recursive workspace is implemented in Bazel, this file should cease
+# to exist.
+"""
+Provides functions to pull all Go external package dependencies of this
+repository.
+"""
+
+load("@bazel_gazelle//:deps.bzl", "go_repository")
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+def go_deps():
+    """
+    Pull in all dependencies needed to build the Go binaries in this
+    repository. This function assumes the repositories imported by the macro
+    'repositories' in //repositories:repositories.bzl have been imported
+    already.
+    """
+    gazelle_dependencies()
+    excludes = native.existing_rules().keys()
+    if "in_gopkg_yaml_v2" not in excludes:
+        go_repository(
+            name = "in_gopkg_yaml_v2",
+            commit = "51d6538a90f86fe93ac480b35f37b2be17fef232",  # v2.2.2
+            importpath = "gopkg.in/yaml.v2",
+        )
+
+    if "com_github_pkg_errors" not in excludes:
+        go_repository(
+            name = "com_github_pkg_errors",
+            commit = "05ac58a23b8798a296fa64f7d9c1559904db4b98",  # v0.8.1
+            importpath = "github.com/pkg/errors",
+        )
diff --git a/rules/container/metadata_merge.bzl b/rules/container/metadata_merge.bzl
new file mode 100644
index 000000000..dc773b413
--- /dev/null
+++ b/rules/container/metadata_merge.bzl
@@ -0,0 +1,85 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+metadata_merge accepts a list of YAML metadata generated by the download_pkgs
+rule & packages_metadata merges their contents into a single YAML file.
+The tags are added in order & de-duplicated. The de-duplication removes later
+occurences of the same tag. The packages are merged in order including any
+duplicates.
+
+Example for the following input YAML files:-
+File 1 (generated by security_check rule):
+tags:
+- foo
+
+File 2 (generated by download_pkgs rule):
+packages:
+- name: foo
+version: 1
+
+File 3 (generated by security_check rule):
+tags:
+- bar
+
+File 4 (generated by download_pkgs rule)
+packages:
+- name: bar
+    version: 2
+
+The merged YAML will be as follows:
+tags:
+- foo
+- bar
+packages:
+- name: foo
+    version: 1
+- name: bar
+    version: 2
+"""
+
+def _impl(ctx):
+    yaml_files = ctx.files.yamls
+    if len(yaml_files) == 0:
+        fail("Attribute yamls to {} did not specify any YAML files.".format(ctx.label))
+    args = []
+    for yaml_file in yaml_files:
+        args.append("-yamlFile")
+        args.append(yaml_file.path)
+    args.append("-outFile")
+    args.append(ctx.outputs.yaml.path)
+    ctx.actions.run(
+        inputs = yaml_files,
+        outputs = [ctx.outputs.yaml],
+        executable = ctx.executable._merger,
+        arguments = args,
+        mnemonic = "MetadataYAMLMerge",
+    )
+
+metadata_merge = rule(
+    attrs = {
+        "yamls": attr.label_list(
+            mandatory = True,
+            allow_files = True,
+        ),
+        "_merger": attr.label(
+            default = "@bazel_toolchains//src/go/cmd/metadata_merge",
+            cfg = "host",
+            executable = True,
+        ),
+    },
+    outputs = {
+        "yaml": "%{name}.yaml",
+    },
+    implementation = _impl,
+)
diff --git a/src/go/cmd/metadata_merge/BUILD.bazel b/src/go/cmd/metadata_merge/BUILD.bazel
new file mode 100644
index 000000000..021a9bb7b
--- /dev/null
+++ b/src/go/cmd/metadata_merge/BUILD.bazel
@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metadata_merge.go"],
+    importpath = "github.com/bazelbuild/bazel-toolchains/src/go/cmd/metadata_merge",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//src/go/pkg/metadata:go_default_library",
+        "@in_gopkg_yaml_v2//:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "metadata_merge",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
diff --git a/src/go/cmd/metadata_merge/metadata_merge.go b/src/go/cmd/metadata_merge/metadata_merge.go
new file mode 100644
index 000000000..e3aea2fb1
--- /dev/null
+++ b/src/go/cmd/metadata_merge/metadata_merge.go
@@ -0,0 +1,110 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/bazelbuild/bazel-toolchains/src/go/pkg/metadata"
+)
+
+var (
+	outFile = flag.String("outFile", "", "Output merged YAML file to generate.")
+)
+
+// strArgList implements a command line flag that can be specified multiple
+// times to define a list of values.
+type strArgList struct {
+	// Args is the list of command line flags.
+	Args []string
+}
+
+func (l *strArgList) String() string {
+	return fmt.Sprintf("%v", l.Args)
+}
+
+// Set appends the given value for a particular occurance of the flag to the
+// list of flag values.
+func (l *strArgList) Set(value string) error {
+	l.Args = append(l.Args, value)
+	return nil
+}
+
+// metadataYAML stores the metadata generated by security_check & download_pkgs
+// rules as a single YAML object.
+type metadataYAML struct {
+	// Tags is the list of vulnerability tags read from YAML files generated
+	// by the security_check rule.
+	Tags []string `yaml:"tags"`
+	// Packages is the list of software package entries read from YAML files
+	// generated by the download_pkgs rule.
+	Packages []metadata.PackageMetadata `yaml:"packages"`
+
+	// tagsLookup maintains a map of tags in the "Tags" field.
+	tagsLookup map[string]bool
+}
+
+// merge merges the contents of the metadataYaml 'from' into the metadataYAML
+// 'm'. This does the following:
+// 1. Add every tag that appears in 'from' into 'm' if it doesn't already exist
+//    in 'm'.
+// 2. Add every package that apppears in 'from' into 'm'.
+func (m *metadataYAML) merge(from *metadataYAML) error {
+	for _, t := range from.Tags {
+		if _, ok := m.tagsLookup[t]; ok {
+			// This tag has been added already.
+			continue
+		}
+		m.tagsLookup[t] = true
+		m.Tags = append(m.Tags, t)
+	}
+	for _, p := range from.Packages {
+		m.Packages = append(m.Packages, p)
+	}
+	return nil
+}
+
+func main() {
+	var yamlFiles strArgList
+	flag.Var(&yamlFiles, "yamlFile", "Path to an input YAML file to process. Can be specified multiple times to process more than one file.")
+	flag.Parse()
+	log.Println("Running the YAML Metadata merger.")
+	for _, f := range yamlFiles.Args {
+		log.Println("-yamlFile", f)
+	}
+	log.Println("-outFile", *outFile)
+	if len(yamlFiles.Args) == 0 {
+		log.Fatalf("No input YAML files provided. Use the -yamlFile flag to provide at least 1 YAML file.")
+	}
+	if *outFile == "" {
+		log.Fatalf("-outFile was not specified.")
+	}
+
+	result := metadataYAML{tagsLookup: make(map[string]bool)}
+	for _, yamlFile := range yamlFiles.Args {
+		log.Println("Loading metadata from", yamlFile)
+		blob, err := ioutil.ReadFile(yamlFile)
+		if err != nil {
+			log.Fatalf("Unable to read data from %s: %v", yamlFile, err)
+		}
+		m := new(metadataYAML)
+		if err := yaml.UnmarshalStrict(blob, m); err != nil {
+			log.Fatalf("Unable to parse data read from %s as metadata YAML: %v", yamlFile, err)
+		}
+		if err := result.merge(m); err != nil {
+			log.Fatalf("Unable to merge metadata read from %s into a single merged YAML: %v", yamlFile, err)
+		}
+	}
+	log.Printf("Merged YAML has %d tags and %d packages.", len(result.Tags), len(result.Packages))
+	blob, err := yaml.Marshal(&result)
+	if err != nil {
+		log.Fatalf("Unable to generate a merged YAML blob for the output merged YAML file: %v", err)
+	}
+	if err := ioutil.WriteFile(*outFile, blob, os.FileMode(0644)); err != nil {
+		log.Fatalf("Unable to write %d bytes of content to output YAML file %s: %v", len(blob), *outFile, err)
+	}
+	log.Printf("Successfully generated output %s that merged %d YAML files.", *outFile, len(yamlFiles.Args))
+}
diff --git a/src/go/cmd/packages_metadata/BUILD b/src/go/cmd/packages_metadata/BUILD
index 66cd0f86b..efac539db 100644
--- a/src/go/cmd/packages_metadata/BUILD
+++ b/src/go/cmd/packages_metadata/BUILD
@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/bazelbuild/bazel-toolchains/src/go/cmd/packages_metadata",
     visibility = ["//visibility:private"],
     deps = [
+        "//src/go/pkg/metadata:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@in_gopkg_yaml_v2//:go_default_library",
     ],
diff --git a/src/go/cmd/packages_metadata/packages_metadata.go b/src/go/cmd/packages_metadata/packages_metadata.go
index 8f254516e..26f34a895 100644
--- a/src/go/cmd/packages_metadata/packages_metadata.go
+++ b/src/go/cmd/packages_metadata/packages_metadata.go
@@ -25,6 +25,8 @@ import (
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/bazelbuild/bazel-toolchains/src/go/pkg/metadata"
 )
 
 var (
@@ -32,28 +34,12 @@ var (
 	outputYAML = flag.String("outputYAML", "", "Path to the output YAML file to generate.")
 )
 
-// packageMetada is the YAML entry for a single software package.
-type packageMetadata struct {
-	// Name is the name of the software package.
-	Name string `yaml:"name"`
-	// Version is the version string of the software package.
-	Version string `yaml:"version"`
-}
-
-// packageMetadata is the collection of software package metadata read from
-// the input CSV file to be serialized into a YAML file.
-type packagesMetadata struct {
-	// Packages is the list of software package entries read from the input
-	// CSV file.
-	Packages []packageMetadata `yaml:"packages"`
-}
-
 // newPackageMetadataFromCSV reads the package metadata from the given
 // CSV file with columns "Name" & "Version" and returns the corresponding
-// packagesMetadata object. The package entries in the returned packagesMetadata
+// PackagesMetadata object. The package entries in the returned PackagesMetadata
 // are sorted by their package names.
-func newPackagesMetadataFromCSV(csvFile string) (*packagesMetadata, error) {
-	result := new(packagesMetadata)
+func newPackagesMetadataFromCSV(csvFile string) (*metadata.PackagesMetadata, error) {
+	result := new(metadata.PackagesMetadata)
 	f, err := os.Open(csvFile)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to open %s for reading", csvFile)
@@ -74,7 +60,7 @@ func newPackagesMetadataFromCSV(csvFile string) (*packagesMetadata, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "error parsing record in CSV file %s", csvFile)
 		}
-		result.Packages = append(result.Packages, packageMetadata{
+		result.Packages = append(result.Packages, metadata.PackageMetadata{
 			Name:    record[0],
 			Version: record[1],
 		})
diff --git a/src/go/pkg/metadata/BUILD.bazel b/src/go/pkg/metadata/BUILD.bazel
new file mode 100644
index 000000000..73ddd7b8e
--- /dev/null
+++ b/src/go/pkg/metadata/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["packages.go"],
+    importpath = "github.com/bazelbuild/bazel-toolchains/src/go/pkg/metadata",
+    visibility = ["//visibility:public"],
+)
diff --git a/src/go/pkg/metadata/packages.go b/src/go/pkg/metadata/packages.go
new file mode 100644
index 000000000..031ef5118
--- /dev/null
+++ b/src/go/pkg/metadata/packages.go
@@ -0,0 +1,17 @@
+package metadata
+
+// PackageMetadata is the YAML entry for a single software package.
+type PackageMetadata struct {
+	// Name is the name of the software package.
+	Name string `yaml:"name"`
+	// Version is the version string of the software package.
+	Version string `yaml:"version"`
+}
+
+// PackagesMetadata is the collection of software package metadata read from
+// the input CSV file to be serialized into a YAML file.
+type PackagesMetadata struct {
+	// Packages is the list of software package entries read from the input
+	// CSV file.
+	Packages []PackageMetadata `yaml:"packages"`
+}
diff --git a/tests/rules/BUILD b/tests/rules/BUILD
index 810d78908..1b03b47ee 100644
--- a/tests/rules/BUILD
+++ b/tests/rules/BUILD
@@ -17,6 +17,7 @@ load(
     "file_test",
 )
 load("//rules/container:packages_metadata.bzl", "packages_metadata")
+load("//rules/container:metadata_merge.bzl", "metadata_merge")
 
 # Simply ensures the file supposed to be downloaded by the gce_file
 # test rule was actually downloaded and the contents of the file was
@@ -44,3 +45,27 @@ file_test(
 """,
     file = ":packages_metadata.yaml",
 )
+
+metadata_merge(
+    name = "metadata_merge",
+    yamls = [
+        "metadata_merge_file1.yaml",
+        "metadata_merge_file2.yaml",
+        "metadata_merge_file3.yaml",
+        "metadata_merge_file4.yaml",
+    ],
+)
+
+file_test(
+    name = "test_metadata_merge",
+    content = """tags:
+- foo
+- bar
+packages:
+- name: foo
+  version: "1"
+- name: bar
+  version: "2"
+""",
+    file = ":metadata_merge.yaml",
+)
diff --git a/tests/rules/metadata_merge_file1.yaml b/tests/rules/metadata_merge_file1.yaml
new file mode 100644
index 000000000..afdeec5d5
--- /dev/null
+++ b/tests/rules/metadata_merge_file1.yaml
@@ -0,0 +1,3 @@
+tags:
+- foo
+
diff --git a/tests/rules/metadata_merge_file2.yaml b/tests/rules/metadata_merge_file2.yaml
new file mode 100644
index 000000000..8516abb1e
--- /dev/null
+++ b/tests/rules/metadata_merge_file2.yaml
@@ -0,0 +1,3 @@
+packages:
+- name: foo
+  version: 1
diff --git a/tests/rules/metadata_merge_file3.yaml b/tests/rules/metadata_merge_file3.yaml
new file mode 100644
index 000000000..e7cc47ab7
--- /dev/null
+++ b/tests/rules/metadata_merge_file3.yaml
@@ -0,0 +1,2 @@
+tags:
+- bar
diff --git a/tests/rules/metadata_merge_file4.yaml b/tests/rules/metadata_merge_file4.yaml
new file mode 100644
index 000000000..16ce8fce9
--- /dev/null
+++ b/tests/rules/metadata_merge_file4.yaml
@@ -0,0 +1,3 @@
+packages:
+- name: bar
+  version: 2

From e1fe0eac878fd066522ccfc925a98cb0d36960fe Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Fri, 21 Jun 2019 11:47:13 -0400
Subject: [PATCH 2/6] buildifier & rename BUILD files.

---
 WORKSPACE                                        | 2 +-
 src/go/cmd/metadata_merge/{BUILD.bazel => BUILD} | 0
 src/go/pkg/metadata/{BUILD.bazel => BUILD}       | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename src/go/cmd/metadata_merge/{BUILD.bazel => BUILD} (100%)
 rename src/go/pkg/metadata/{BUILD.bazel => BUILD} (100%)

diff --git a/WORKSPACE b/WORKSPACE
index 1bfc87303..74bdf5db1 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -59,7 +59,7 @@ go_register_toolchains()
 # is being imported from a *.bzl file.
 # gazelle:repo bazel_gazelle
 
-load("//repositories:go_repositories.bzl", bazel_toolchains_go_deps="go_deps")
+load("//repositories:go_repositories.bzl", bazel_toolchains_go_deps = "go_deps")
 
 bazel_toolchains_go_deps()
 
diff --git a/src/go/cmd/metadata_merge/BUILD.bazel b/src/go/cmd/metadata_merge/BUILD
similarity index 100%
rename from src/go/cmd/metadata_merge/BUILD.bazel
rename to src/go/cmd/metadata_merge/BUILD
diff --git a/src/go/pkg/metadata/BUILD.bazel b/src/go/pkg/metadata/BUILD
similarity index 100%
rename from src/go/pkg/metadata/BUILD.bazel
rename to src/go/pkg/metadata/BUILD

From 13c4aabce5b515fb590fea59f4927579f25b2465 Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Fri, 21 Jun 2019 11:52:08 -0400
Subject: [PATCH 3/6] buildifier lint fixes

---
 repositories/go_repositories.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/repositories/go_repositories.bzl b/repositories/go_repositories.bzl
index afef8163d..8c41c3371 100644
--- a/repositories/go_repositories.bzl
+++ b/repositories/go_repositories.bzl
@@ -19,11 +19,11 @@ Provides functions to pull all Go external package dependencies of this
 repository.
 """
 
-load("@bazel_gazelle//:deps.bzl", "go_repository")
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
 
 def go_deps():
-    """
+    """Pull in external Go packages needed by Go binaries in this repo.
+
     Pull in all dependencies needed to build the Go binaries in this
     repository. This function assumes the repositories imported by the macro
     'repositories' in //repositories:repositories.bzl have been imported

From 4c4c43a68b856009a8a692a8958512f2c48f97e9 Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Fri, 21 Jun 2019 14:17:01 -0400
Subject: [PATCH 4/6] Add package comment.

---
 src/go/pkg/metadata/packages.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/go/pkg/metadata/packages.go b/src/go/pkg/metadata/packages.go
index 031ef5118..c1ba3f18b 100644
--- a/src/go/pkg/metadata/packages.go
+++ b/src/go/pkg/metadata/packages.go
@@ -1,3 +1,5 @@
+// Package metadata provides functionality to store metadata about debian
+// packages installed in a docker image layer.
 package metadata
 
 // PackageMetadata is the YAML entry for a single software package.

From 6ad809e6d17278c708e309369bc06cc8d5b5b53e Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Tue, 25 Jun 2019 12:48:56 -0400
Subject: [PATCH 5/6] Address review comments

---
 rules/container/metadata_merge.bzl          | 31 +++++++++++----------
 src/go/cmd/metadata_merge/metadata_merge.go | 13 +++++----
 tests/rules/BUILD                           |  4 +--
 tests/rules/metadata_merge_file4.yaml       |  2 +-
 4 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/rules/container/metadata_merge.bzl b/rules/container/metadata_merge.bzl
index dc773b413..ae6311b0b 100644
--- a/rules/container/metadata_merge.bzl
+++ b/rules/container/metadata_merge.bzl
@@ -13,12 +13,12 @@
 # limitations under the License.
 """
 metadata_merge accepts a list of YAML metadata generated by the download_pkgs
-rule & packages_metadata merges their contents into a single YAML file.
-The tags are added in order & de-duplicated. The de-duplication removes later
-occurences of the same tag. The packages are merged in order including any
-duplicates.
+& packages_metadata rules. metadata_metaga merges the input list into a single
+YAML file. The tags are added in order & de-duplicated. The de-duplication
+removes later occurences of the same tag. The packages are merged in order
+including any duplicates.
 
-Example for the following input YAML files:-
+Example for the following input YAML files:
 File 1 (generated by security_check rule):
 tags:
 - foo
@@ -26,7 +26,7 @@ tags:
 File 2 (generated by download_pkgs rule):
 packages:
 - name: foo
-version: 1
+  version: 1
 
 File 3 (generated by security_check rule):
 tags:
@@ -34,8 +34,8 @@ tags:
 
 File 4 (generated by download_pkgs rule)
 packages:
-- name: bar
-    version: 2
+- name: baz
+  version: 2
 
 The merged YAML will be as follows:
 tags:
@@ -43,13 +43,13 @@ tags:
 - bar
 packages:
 - name: foo
-    version: 1
-- name: bar
-    version: 2
+  version: 1
+- name: baz
+  version: 2
 """
 
 def _impl(ctx):
-    yaml_files = ctx.files.yamls
+    yaml_files = ctx.files.srcs
     if len(yaml_files) == 0:
         fail("Attribute yamls to {} did not specify any YAML files.".format(ctx.label))
     args = []
@@ -68,14 +68,17 @@ def _impl(ctx):
 
 metadata_merge = rule(
     attrs = {
-        "yamls": attr.label_list(
+        "srcs": attr.label_list(
             mandatory = True,
-            allow_files = True,
+            allow_files = [".yaml"],
+            doc = "YAML file targets to merge.",
         ),
         "_merger": attr.label(
             default = "@bazel_toolchains//src/go/cmd/metadata_merge",
             cfg = "host",
             executable = True,
+            doc = "The go binary that merges a given list of YAML files to " +
+                  "produce a single output YAML.",
         ),
     },
     outputs = {
diff --git a/src/go/cmd/metadata_merge/metadata_merge.go b/src/go/cmd/metadata_merge/metadata_merge.go
index e3aea2fb1..6b6039a09 100644
--- a/src/go/cmd/metadata_merge/metadata_merge.go
+++ b/src/go/cmd/metadata_merge/metadata_merge.go
@@ -33,14 +33,17 @@ func (l *strArgList) Set(value string) error {
 	return nil
 }
 
-// metadataYAML stores the metadata generated by security_check & download_pkgs
-// rules as a single YAML object.
+// metadataYAML stores the contents of one or more YAML file with the following
+// top level keys:
+// 1. "tags" (list of strings).
+// 2. "packages" (list of YAML objects with keys "name" & "version" which are
+//    strings).
 type metadataYAML struct {
-	// Tags is the list of vulnerability tags read from YAML files generated
-	// by the security_check rule.
+	// Tags is the list of tags read from YAML files with a top level "tags"
+	// key.
 	Tags []string `yaml:"tags"`
 	// Packages is the list of software package entries read from YAML files
-	// generated by the download_pkgs rule.
+	// with a top level "packages" key.
 	Packages []metadata.PackageMetadata `yaml:"packages"`
 
 	// tagsLookup maintains a map of tags in the "Tags" field.
diff --git a/tests/rules/BUILD b/tests/rules/BUILD
index 1b03b47ee..b56727a71 100644
--- a/tests/rules/BUILD
+++ b/tests/rules/BUILD
@@ -48,7 +48,7 @@ file_test(
 
 metadata_merge(
     name = "metadata_merge",
-    yamls = [
+    srcs = [
         "metadata_merge_file1.yaml",
         "metadata_merge_file2.yaml",
         "metadata_merge_file3.yaml",
@@ -64,7 +64,7 @@ file_test(
 packages:
 - name: foo
   version: "1"
-- name: bar
+- name: baz
   version: "2"
 """,
     file = ":metadata_merge.yaml",
diff --git a/tests/rules/metadata_merge_file4.yaml b/tests/rules/metadata_merge_file4.yaml
index 16ce8fce9..cef0e90cb 100644
--- a/tests/rules/metadata_merge_file4.yaml
+++ b/tests/rules/metadata_merge_file4.yaml
@@ -1,3 +1,3 @@
 packages:
-- name: bar
+- name: baz
   version: 2

From 15c84facc5b7f594ddf163ed6413d45d02c00915 Mon Sep 17 00:00:00 2001
From: Suvanjan Mukherjee <suvanjan@google.com>
Date: Tue, 25 Jun 2019 16:50:52 -0400
Subject: [PATCH 6/6] Add doc

---
 src/go/cmd/metadata_merge/metadata_merge.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/go/cmd/metadata_merge/metadata_merge.go b/src/go/cmd/metadata_merge/metadata_merge.go
index 6b6039a09..938567446 100644
--- a/src/go/cmd/metadata_merge/metadata_merge.go
+++ b/src/go/cmd/metadata_merge/metadata_merge.go
@@ -54,7 +54,9 @@ type metadataYAML struct {
 // 'm'. This does the following:
 // 1. Add every tag that appears in 'from' into 'm' if it doesn't already exist
 //    in 'm'.
-// 2. Add every package that apppears in 'from' into 'm'.
+// 2. Add every package that apppears in 'from' into 'm'. If the list of
+//    packages in 'from' have duplicates with the list of packages in 'm', the
+//    list of packages in 'm' will contain these duplicates after the merge.
 func (m *metadataYAML) merge(from *metadataYAML) error {
 	for _, t := range from.Tags {
 		if _, ok := m.tagsLookup[t]; ok {