testing changes to platforms to enable #1203 to work (#1205)

* testing changes to platforms

* fixes

* fix py2->py3 incompat

* fix more py

.bazelci/presubmit.yml

@@ -71,7 +71,7 @@ platforms:
     # For tests/container:set_env_make_vars_test
     - "--define=ENV_KEY=my_key"
     - "--define=ENV_VALUE=my_value"
-    # Needed because register_toolchains is apparently not respecting order of
+    # Needed because register_platforms is apparently not respecting order of
     # platforms passed:
     - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform"
     test_targets:
@@ -114,7 +114,7 @@ platforms:
     - "--define=ENV_KEY=my_key"
     - "--define=ENV_VALUE=my_value"
     - "--test_output=errors"
-    # Needed because register_toolchains is apparently not respecting order of
+    # Needed because register_platforms is apparently not respecting order of
     # platforms passed:
     - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform"
   rbe_ubuntu1604:

.bazelrc

@@ -15,7 +15,6 @@
 # The following flags are set to test use of new features for python toolchains
 # These flags will only work with Bazel 0.25.0 or above and are only needed for
 #  //tests/docker/security/... & the docker/package_managers rules.
-build --host_force_python=PY2
 build --incompatible_use_python_toolchains
 test --incompatible_use_python_toolchains
-test --host_force_python=PY2
+

docker/security/BUILD

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("@pip_deps//:requirements.bzl", "requirement")
 load("@subpar//:subpar.bzl", "par_binary")
 
 package(default_visibility = ["//visibility:public"])
@@ -26,11 +25,8 @@ par_binary(
     name = "security_check",
     srcs = ["security_check.py"],
     main = "security_check.py",
-    python_version = "PY2",
+    python_version = "PY3",
     visibility = ["//visibility:public"],
-    deps = [
-        requirement("PyYaml"),
-    ],
 )
 
 bzl_library(

testing/examples/.bazelrc → docker/security/cmd/json_to_yaml/BUILD

@@ -11,8 +11,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 
-build --host_force_python=PY2
-test --host_force_python=PY2
-run --host_force_python=PY2
+go_library(
+    name = "go_default_library",
+    srcs = ["json_to_yaml.go"],
+    importpath = "github.com/bazelbuild/rules_docker/docker/security/cmd/json_to_yaml",
+    visibility = ["//visibility:private"],
+    deps = ["@com_github_ghodss_yaml//:go_default_library"],
+)
 
+go_binary(
+    name = "json_to_yaml",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

docker/security/cmd/json_to_yaml/json_to_yaml.go

@@ -0,0 +1,50 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/ghodss/yaml"
+)
+
+var (
+	inJSON  = flag.String("in-json", "", "Path to input JSON file that will be converted to YAML.")
+	outYAML = flag.String("out-yaml", "", "Path to output YAML file.")
+)
+
+func main() {
+	flag.Parse()
+	if *inJSON == "" {
+		log.Fatalf("--in-json is required.")
+	}
+	if *outYAML == "" {
+		log.Fatalf("--out-yaml is required.")
+	}
+
+	j, err := ioutil.ReadFile(*inJSON)
+	if err != nil {
+		log.Fatalf("Unable to read input JSON file %q: %v", *inJSON, err)
+	}
+	y, err := yaml.JSONToYAML(j)
+	if err != nil {
+		log.Fatalf("Unable to convert JSON data loaded from %q to YAML: %v", *inJSON, err)
+	}
+	if err := ioutil.WriteFile(*outYAML, y, os.ModePerm); err != nil {
+		log.Fatalf("Unable to write output YAML to %q: %v", *outYAML, err)
+	}
+}

docker/security/security_check.bzl

@@ -19,7 +19,7 @@ def _impl(ctx):
     output_yaml = ctx.outputs.yaml
     args = ctx.actions.args()
     args.add(ctx.attr.image)
-    args.add("--output-yaml", ctx.outputs.yaml)
+    args.add("--output-json", ctx.outputs.json)
     args.add("--severity", ctx.attr.severity)
     if ctx.attr.whitelist != None:
         files = ctx.attr.whitelist.files.to_list()
@@ -35,7 +35,7 @@ def _impl(ctx):
     ctx.actions.run(
         executable = ctx.executable._security_check,
         arguments = [args],
-        outputs = [ctx.outputs.yaml],
+        outputs = [ctx.outputs.json],
         mnemonic = "ImageSecurityCheck",
         use_default_shell_env = True,
         execution_requirements = {
@@ -46,6 +46,16 @@ def _impl(ctx):
             "no-sandbox": "True",
         },
     )
+    args = ctx.actions.args()
+    args.add("--in-json", ctx.outputs.json)
+    args.add("--out-yaml", ctx.outputs.yaml)
+    ctx.actions.run(
+        executable = ctx.executable._json_to_yaml,
+        arguments = [args],
+        inputs = [ctx.outputs.json],
+        outputs = [ctx.outputs.yaml],
+        mnemonic = "JSONToYAML",
+    )
 
 # Run the security_check.py script on the given docker image to generate a
 # YAML output file with information about the types of vulnerabilities
@@ -68,6 +78,13 @@ security_check = rule(
             default = Label("@io_bazel_rules_docker//docker/security:security_check_whitelist.json"),
             allow_single_file = True,
         ),
+        # JSON to YAML converter.
+        "_json_to_yaml": attr.label(
+            default = Label("@io_bazel_rules_docker//docker/security/cmd/json_to_yaml"),
+            cfg = "host",
+            executable = True,
+            allow_files = True,
+        ),
         # The security checker python executable.
         "_security_check": attr.label(
             default = Label("@io_bazel_rules_docker//docker/security:security_check"),
@@ -77,6 +94,7 @@ security_check = rule(
         ),
     },
     outputs = {
+        "json": "%{name}.json",
         "yaml": "%{name}.yaml",
     },
 )

docker/security/security_check.py

@@ -19,7 +19,6 @@
 import subprocess
 import sys
 import logging
-import yaml
 
 import distutils.version as ver
 
@@ -72,7 +71,7 @@ def _run_gcloud(cmd):
   full_cmd = [gcloud_path(), 'alpha', 'container', 'images',
               '--format=json'] + cmd
   output = subprocess.check_output(full_cmd)
-  return json.loads(output)
+  return json.loads(output.decode('utf-8'))
 
 
 def _find_base_image(image):
@@ -213,7 +212,7 @@ def _check_image(image, severity, whitelist):
 
 
 def _get_relevant_severities(severity):
-  return [k for k, v in _SEV_MAP.iteritems()
+  return [k for k, v in _SEV_MAP.items()
           if v >= _SEV_MAP.get(severity, 1)]
 
 
@@ -265,17 +264,17 @@ def _get_version_number(version_obj):
 
   return ''.join([str(epoch), delimiter1, name, delimiter2, str(revision)])
 
-def _generate_yaml_output(output_yaml, vulnerabilities):
-  """Generate a YAML file mapping the key "tags" to the list of types of
+def _generate_json_output(output_json, vulnerabilities):
+  """Generate a JSON file mapping the key "tags" to the list of types of
   vulnerabilities found.
 
   Args:
-    output_yaml: Path to the output YAML file to generate.
+    output_json: Path to the output JSON file to generate.
     vulnerabilities: A dictionary mapping the name of the CVE entry to details
                      about the vulnerability.
   """
   tags = set()
-  for v in vulnerabilities.itervalues():
+  for v in vulnerabilities.values():
     details = v["vulnerabilityDetails"]
     # The service that consumes the metadata expects the tags as follows:
     # LOW -> cveLow
@@ -284,19 +283,19 @@ def _generate_yaml_output(output_yaml, vulnerabilities):
     sev = str(details['severity'])
     tags.add("cve{}".format(sev.lower().capitalize()))
   result = {"tags": list(tags)}
-  logging.info("Creating YAML output {}".format(output_yaml))
-  with open(output_yaml, "w") as ofp:
-    ofp.write(yaml.dump(result))
+  logging.info("Creating JSON output {}".format(output_json))
+  with open(output_json, "w") as ofp:
+    json.dump(result, ofp)
 
 def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json',
-                   output_yaml=None):
+                   output_json=None):
   """Main security check function.
 
   Args:
     image: full name of the docker image
     severity: the severity of vulnerability to trigger failure
     whitelist_file: file with list of whitelisted CVE
-    output_yaml: Output file which will be populated with a list of types of
+    output_json: Output file which will be populated with a list of types of
                  vulnerability that exist for the given image.
 
   Returns:
@@ -312,9 +311,9 @@ def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json',
 
   result = _check_for_vulnz(_sub_image(image), severity, whitelist)
 
-  if output_yaml:
-    logging.info("Creating YAML output {}".format(output_yaml))
-    _generate_yaml_output(output_yaml, result)
+  if output_json:
+    logging.info("Creating JSON output {}".format(output_json))
+    _generate_json_output(output_json, result)
   return result
 
 
@@ -330,13 +329,13 @@ def _main():
   parser.add_argument('--whitelist-file', dest='whitelist',
                       help='The path to the whitelist json file',
                       default='whitelist.json')
-  parser.add_argument('--output-yaml', dest='output_yaml',
-                      help='The path to the output YAML file to'+\
+  parser.add_argument('--output-json', dest='output_json',
+                      help='The path to the output JSON file to'+\
                       ' generate with a list of tags indicating the types of'+\
                       ' vulnerability fixes available for the given image.')
   args = parser.parse_args()
   security_check(args.image, args.severity, args.whitelist,
-                            args.output_yaml)
+                            args.output_json)
 
 
 if __name__ == '__main__':

docker/util/BUILD

@@ -21,11 +21,13 @@ licenses(["notice"])  # Apache 2.0
 py_binary(
     name = "config_stripper",
     srcs = ["config_stripper.py"],
+    python_version = "PY3",
 )
 
 py_binary(
     name = "to_json",
     srcs = ["to_json.py"],
+    python_version = "PY3",
 )
 
 exports_files([

docker/util/config_stripper.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import argparse
-import cStringIO
+import io
 import hashlib
 import json
 import os
@@ -39,6 +39,8 @@ def main():
                         required=True)
     args = parser.parse_args()
 
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+
     return strip_tar(args.in_tar_path, args.out_tar_path)
 
 
@@ -101,14 +103,14 @@ def strip_layer(path):
     # working directory is one level up from where layer.tar is.
     original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..'))
 
-    buf = cStringIO.StringIO()
+    buf = io.BytesIO()
 
     # Go through each file/dir in the layer
     # Set its mtime to 0
     # If it's a file, add its content to the running buffer
     # Add it to the new gzip'd tar.
     with tarfile.open(name=path, mode='r') as it:
-      with tarfile.open(fileobj=buf, mode='w') as ot:
+      with tarfile.open(fileobj=buf, encoding='utf-8', mode='w') as ot:
         for tarinfo in it:
           # Use a deterministic mtime that doesn't confuse other programs,
           # e.g. Python.
@@ -137,7 +139,7 @@ def strip_layer(path):
     # Calculate sha of layer
     sha = hashlib.sha256(gz).hexdigest()
     new_name = 'sha256:%s' % sha
-    with open(os.path.join(original_dir, new_name), 'w') as out:
+    with open(os.path.join(original_dir, new_name), 'wb') as out:
       out.write(gz)
 
     shutil.rmtree(os.path.dirname(path))
@@ -169,7 +171,7 @@ def strip_config(path, new_diff_ids):
         f.write(config_str)
 
     # Calculate the new file path
-    sha = hashlib.sha256(config_str).hexdigest()
+    sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest()
     new_path = 'sha256:%s' % sha
     os.rename(path, os.path.join(os.path.dirname(path), new_path))
     return new_path

repositories/go_repositories.bzl

@@ -59,3 +59,10 @@ def go_deps():
             commit = "9ff306d4fbead574800b66369df5b6144732d58e",  # v1.1.0
             importpath = "github.com/kylelemons/godebug",
         )
+    if "com_github_ghodss_yaml" not in excludes:
+        go_repository(
+            name = "com_github_ghodss_yaml",
+            importpath = "github.com/ghodss/yaml",
+            sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=",
+            version = "v1.0.0",
+        )