diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b1866dced..6da4db560 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -71,7 +71,7 @@ platforms: # For tests/container:set_env_make_vars_test - "--define=ENV_KEY=my_key" - "--define=ENV_VALUE=my_value" - # Needed because register_toolchains is apparently not respecting order of + # Needed because register_platforms is apparently not respecting order of # platforms passed: - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform" test_targets: @@ -114,7 +114,7 @@ platforms: - "--define=ENV_KEY=my_key" - "--define=ENV_VALUE=my_value" - "--test_output=errors" - # Needed because register_toolchains is apparently not respecting order of + # Needed because register_platforms is apparently not respecting order of # platforms passed: - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform" rbe_ubuntu1604: diff --git a/.bazelrc b/.bazelrc index 056d9a7b2..a607b74f9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -15,7 +15,6 @@ # The following flags are set to test use of new features for python toolchains # These flags will only work with Bazel 0.25.0 or above and are only needed for # //tests/docker/security/... & the docker/package_managers rules. -build --host_force_python=PY2 build --incompatible_use_python_toolchains test --incompatible_use_python_toolchains -test --host_force_python=PY2 + diff --git a/docker/security/BUILD b/docker/security/BUILD index 0937912e5..df6b76c44 100644 --- a/docker/security/BUILD +++ b/docker/security/BUILD @@ -13,7 +13,6 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@pip_deps//:requirements.bzl", "requirement") load("@subpar//:subpar.bzl", "par_binary") package(default_visibility = ["//visibility:public"]) @@ -26,11 +25,8 @@ par_binary( name = "security_check", srcs = ["security_check.py"], main = "security_check.py", - python_version = "PY2", + python_version = "PY3", visibility = ["//visibility:public"], - deps = [ - requirement("PyYaml"), - ], ) bzl_library( diff --git a/testing/examples/.bazelrc b/docker/security/cmd/json_to_yaml/BUILD similarity index 56% rename from testing/examples/.bazelrc rename to docker/security/cmd/json_to_yaml/BUILD index e9bb64632..023770c7b 100644 --- a/testing/examples/.bazelrc +++ b/docker/security/cmd/json_to_yaml/BUILD @@ -11,8 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") -build --host_force_python=PY2 -test --host_force_python=PY2 -run --host_force_python=PY2 +go_library( + name = "go_default_library", + srcs = ["json_to_yaml.go"], + importpath = "github.com/bazelbuild/rules_docker/docker/security/cmd/json_to_yaml", + visibility = ["//visibility:private"], + deps = ["@com_github_ghodss_yaml//:go_default_library"], +) +go_binary( + name = "json_to_yaml", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/docker/security/cmd/json_to_yaml/json_to_yaml.go b/docker/security/cmd/json_to_yaml/json_to_yaml.go new file mode 100644 index 000000000..3bfbc7415 --- /dev/null +++ b/docker/security/cmd/json_to_yaml/json_to_yaml.go @@ -0,0 +1,50 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "flag" + "io/ioutil" + "log" + "os" + + "github.com/ghodss/yaml" +) + +var ( + inJSON = flag.String("in-json", "", "Path to input JSON file that will be converted to YAML.") + outYAML = flag.String("out-yaml", "", "Path to output YAML file.") +) + +func main() { + flag.Parse() + if *inJSON == "" { + log.Fatalf("--in-json is required.") + } + if *outYAML == "" { + log.Fatalf("--out-yaml is required.") + } + + j, err := ioutil.ReadFile(*inJSON) + if err != nil { + log.Fatalf("Unable to read input JSON file %q: %v", *inJSON, err) + } + y, err := yaml.JSONToYAML(j) + if err != nil { + log.Fatalf("Unable to convert JSON data loaded from %q to YAML: %v", *inJSON, err) + } + if err := ioutil.WriteFile(*outYAML, y, os.ModePerm); err != nil { + log.Fatalf("Unable to write output YAML to %q: %v", *outYAML, err) + } +} diff --git a/docker/security/security_check.bzl b/docker/security/security_check.bzl index 68f7a7107..905f7dfe9 100644 --- a/docker/security/security_check.bzl +++ b/docker/security/security_check.bzl @@ -19,7 +19,7 @@ def _impl(ctx): output_yaml = ctx.outputs.yaml args = ctx.actions.args() args.add(ctx.attr.image) - args.add("--output-yaml", ctx.outputs.yaml) + args.add("--output-json", ctx.outputs.json) args.add("--severity", ctx.attr.severity) if ctx.attr.whitelist != None: files = ctx.attr.whitelist.files.to_list() @@ -35,7 +35,7 @@ def _impl(ctx): ctx.actions.run( executable = ctx.executable._security_check, arguments = [args], - outputs = [ctx.outputs.yaml], + outputs = [ctx.outputs.json], mnemonic = "ImageSecurityCheck", use_default_shell_env = True, execution_requirements = { @@ -46,6 +46,16 @@ def _impl(ctx): "no-sandbox": "True", }, ) + args = ctx.actions.args() + args.add("--in-json", ctx.outputs.json) + args.add("--out-yaml", ctx.outputs.yaml) + ctx.actions.run( + executable = ctx.executable._json_to_yaml, + arguments = [args], + inputs = [ctx.outputs.json], + outputs = [ctx.outputs.yaml], + mnemonic = "JSONToYAML", + ) # Run the security_check.py script on the given docker image to generate a # YAML output file with information about the types of vulnerabilities @@ -68,6 +78,13 @@ security_check = rule( default = Label("@io_bazel_rules_docker//docker/security:security_check_whitelist.json"), allow_single_file = True, ), + # JSON to YAML converter. + "_json_to_yaml": attr.label( + default = Label("@io_bazel_rules_docker//docker/security/cmd/json_to_yaml"), + cfg = "host", + executable = True, + allow_files = True, + ), # The security checker python executable. "_security_check": attr.label( default = Label("@io_bazel_rules_docker//docker/security:security_check"), @@ -77,6 +94,7 @@ security_check = rule( ), }, outputs = { + "json": "%{name}.json", "yaml": "%{name}.yaml", }, ) diff --git a/docker/security/security_check.py b/docker/security/security_check.py index 15a3dc471..38612dc7f 100644 --- a/docker/security/security_check.py +++ b/docker/security/security_check.py @@ -19,7 +19,6 @@ import subprocess import sys import logging -import yaml import distutils.version as ver @@ -72,7 +71,7 @@ def _run_gcloud(cmd): full_cmd = [gcloud_path(), 'alpha', 'container', 'images', '--format=json'] + cmd output = subprocess.check_output(full_cmd) - return json.loads(output) + return json.loads(output.decode('utf-8')) def _find_base_image(image): @@ -213,7 +212,7 @@ def _check_image(image, severity, whitelist): def _get_relevant_severities(severity): - return [k for k, v in _SEV_MAP.iteritems() + return [k for k, v in _SEV_MAP.items() if v >= _SEV_MAP.get(severity, 1)] @@ -265,17 +264,17 @@ def _get_version_number(version_obj): return ''.join([str(epoch), delimiter1, name, delimiter2, str(revision)]) -def _generate_yaml_output(output_yaml, vulnerabilities): - """Generate a YAML file mapping the key "tags" to the list of types of +def _generate_json_output(output_json, vulnerabilities): + """Generate a JSON file mapping the key "tags" to the list of types of vulnerabilities found. Args: - output_yaml: Path to the output YAML file to generate. + output_json: Path to the output JSON file to generate. vulnerabilities: A dictionary mapping the name of the CVE entry to details about the vulnerability. """ tags = set() - for v in vulnerabilities.itervalues(): + for v in vulnerabilities.values(): details = v["vulnerabilityDetails"] # The service that consumes the metadata expects the tags as follows: # LOW -> cveLow @@ -284,19 +283,19 @@ def _generate_yaml_output(output_yaml, vulnerabilities): sev = str(details['severity']) tags.add("cve{}".format(sev.lower().capitalize())) result = {"tags": list(tags)} - logging.info("Creating YAML output {}".format(output_yaml)) - with open(output_yaml, "w") as ofp: - ofp.write(yaml.dump(result)) + logging.info("Creating JSON output {}".format(output_json)) + with open(output_json, "w") as ofp: + json.dump(result, ofp) def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json', - output_yaml=None): + output_json=None): """Main security check function. Args: image: full name of the docker image severity: the severity of vulnerability to trigger failure whitelist_file: file with list of whitelisted CVE - output_yaml: Output file which will be populated with a list of types of + output_json: Output file which will be populated with a list of types of vulnerability that exist for the given image. Returns: @@ -312,9 +311,9 @@ def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json', result = _check_for_vulnz(_sub_image(image), severity, whitelist) - if output_yaml: - logging.info("Creating YAML output {}".format(output_yaml)) - _generate_yaml_output(output_yaml, result) + if output_json: + logging.info("Creating JSON output {}".format(output_json)) + _generate_json_output(output_json, result) return result @@ -330,13 +329,13 @@ def _main(): parser.add_argument('--whitelist-file', dest='whitelist', help='The path to the whitelist json file', default='whitelist.json') - parser.add_argument('--output-yaml', dest='output_yaml', - help='The path to the output YAML file to'+\ + parser.add_argument('--output-json', dest='output_json', + help='The path to the output JSON file to'+\ ' generate with a list of tags indicating the types of'+\ ' vulnerability fixes available for the given image.') args = parser.parse_args() security_check(args.image, args.severity, args.whitelist, - args.output_yaml) + args.output_json) if __name__ == '__main__': diff --git a/docker/util/BUILD b/docker/util/BUILD index a7f038d36..6a5454582 100644 --- a/docker/util/BUILD +++ b/docker/util/BUILD @@ -21,11 +21,13 @@ licenses(["notice"]) # Apache 2.0 py_binary( name = "config_stripper", srcs = ["config_stripper.py"], + python_version = "PY3", ) py_binary( name = "to_json", srcs = ["to_json.py"], + python_version = "PY3", ) exports_files([ diff --git a/docker/util/config_stripper.py b/docker/util/config_stripper.py index e88259806..739cdc770 100644 --- a/docker/util/config_stripper.py +++ b/docker/util/config_stripper.py @@ -15,7 +15,7 @@ # limitations under the License. import argparse -import cStringIO +import io import hashlib import json import os @@ -39,6 +39,8 @@ def main(): required=True) args = parser.parse_args() + os.environ["PYTHONIOENCODING"] = "utf-8" + return strip_tar(args.in_tar_path, args.out_tar_path) @@ -101,14 +103,14 @@ def strip_layer(path): # working directory is one level up from where layer.tar is. original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..')) - buf = cStringIO.StringIO() + buf = io.BytesIO() # Go through each file/dir in the layer # Set its mtime to 0 # If it's a file, add its content to the running buffer # Add it to the new gzip'd tar. with tarfile.open(name=path, mode='r') as it: - with tarfile.open(fileobj=buf, mode='w') as ot: + with tarfile.open(fileobj=buf, encoding='utf-8', mode='w') as ot: for tarinfo in it: # Use a deterministic mtime that doesn't confuse other programs, # e.g. Python. @@ -137,7 +139,7 @@ def strip_layer(path): # Calculate sha of layer sha = hashlib.sha256(gz).hexdigest() new_name = 'sha256:%s' % sha - with open(os.path.join(original_dir, new_name), 'w') as out: + with open(os.path.join(original_dir, new_name), 'wb') as out: out.write(gz) shutil.rmtree(os.path.dirname(path)) @@ -169,7 +171,7 @@ def strip_config(path, new_diff_ids): f.write(config_str) # Calculate the new file path - sha = hashlib.sha256(config_str).hexdigest() + sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest() new_path = 'sha256:%s' % sha os.rename(path, os.path.join(os.path.dirname(path), new_path)) return new_path diff --git a/repositories/go_repositories.bzl b/repositories/go_repositories.bzl index ca3d5d7b2..c3244ad03 100644 --- a/repositories/go_repositories.bzl +++ b/repositories/go_repositories.bzl @@ -59,3 +59,10 @@ def go_deps(): commit = "9ff306d4fbead574800b66369df5b6144732d58e", # v1.1.0 importpath = "github.com/kylelemons/godebug", ) + if "com_github_ghodss_yaml" not in excludes: + go_repository( + name = "com_github_ghodss_yaml", + importpath = "github.com/ghodss/yaml", + sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=", + version = "v1.0.0", + )