From 596fdd1835328a84647ccfe3b9fb11cd2816b1bb Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 27 Sep 2022 11:38:08 +0200 Subject: [PATCH 1/5] add Image Referencer for Dockerfile --- checkov/common/images/image_referencer.py | 4 +- checkov/dockerfile/base_registry.py | 27 ++++++++- .../dockerfile/image_referencer/__init__.py | 0 .../dockerfile/image_referencer/manager.py | 22 +++++++ .../dockerfile/image_referencer/provider.py | 59 +++++++++++++++++++ checkov/dockerfile/parser.py | 12 ++-- checkov/dockerfile/runner.py | 53 +++++++++++++---- checkov/dockerfile/utils.py | 9 +++ 8 files changed, 165 insertions(+), 21 deletions(-) create mode 100644 checkov/dockerfile/image_referencer/__init__.py create mode 100644 checkov/dockerfile/image_referencer/manager.py create mode 100644 checkov/dockerfile/image_referencer/provider.py create mode 100644 checkov/dockerfile/utils.py diff --git a/checkov/common/images/image_referencer.py b/checkov/common/images/image_referencer.py index 7414ca388da..930838e3b14 100644 --- a/checkov/common/images/image_referencer.py +++ b/checkov/common/images/image_referencer.py @@ -117,10 +117,10 @@ class ImageReferencerMixin: def check_container_image_references( self, - graph_connector: DiGraph | None, root_path: str | Path | None, runner_filter: RunnerFilter, - definitions: dict[str, dict[str, Any] | list[dict[str, Any]]] | None = None + definitions: dict[str, dict[str, Any] | list[dict[str, Any]]] | None = None, + graph_connector: DiGraph | None = None, ) -> Report | None: """Tries to find image references in graph based IaC templates""" from checkov.common.bridgecrew.platform_integration import bc_integration diff --git a/checkov/dockerfile/base_registry.py b/checkov/dockerfile/base_registry.py index 6de8095ff5f..85b45fe0017 100644 --- a/checkov/dockerfile/base_registry.py +++ b/checkov/dockerfile/base_registry.py @@ -1,15 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from checkov.common.bridgecrew.check_type import CheckType from checkov.common.checks.base_check_registry import BaseCheckRegistry from checkov.common.models.enums import CheckResult +if TYPE_CHECKING: + from dockerfile_parse.parser import _Instruction # only in extra_stubs + from checkov.common.checks.base_check import BaseCheck + from checkov.common.typing import _SkippedCheck, _CheckResult + from checkov.runner_filter import RunnerFilter + class Registry(BaseCheckRegistry): - def __init__(self): + def __init__(self) -> None: super().__init__(CheckType.DOCKERFILE) - def scan(self, scanned_file, entity, skipped_checks, runner_filter): + def extract_entity_details(self, entity: dict[str, Any]) -> tuple[str, str, dict[str, Any]]: + # not needed + pass + + def scan( + self, + scanned_file: str, + entity: dict[str, list[_Instruction]], + skipped_checks: list[_SkippedCheck], + runner_filter: RunnerFilter, + report_type: str | None = None, + ) -> dict[BaseCheck, _CheckResult]: - results = {} + results: "dict[BaseCheck, _CheckResult]" = {} if not entity: return results for instruction, checks in self.checks.items(): diff --git a/checkov/dockerfile/image_referencer/__init__.py b/checkov/dockerfile/image_referencer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/checkov/dockerfile/image_referencer/manager.py b/checkov/dockerfile/image_referencer/manager.py new file mode 100644 index 00000000000..ccba4627f82 --- /dev/null +++ b/checkov/dockerfile/image_referencer/manager.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from checkov.dockerfile.image_referencer.provider import DockerfileProvider + +if TYPE_CHECKING: + from checkov.common.images.image_referencer import Image + + +class DockerfileImageReferencerManager: + __slots__ = ("definitions",) + + def __init__(self, definitions: dict[str, Any]) -> None: + self.definitions = definitions + + def extract_images_from_resources(self) -> list[Image]: + provider = DockerfileProvider(definitions=self.definitions) + + images = provider.extract_images_from_resources() + + return images diff --git a/checkov/dockerfile/image_referencer/provider.py b/checkov/dockerfile/image_referencer/provider.py new file mode 100644 index 00000000000..3d253e16ca1 --- /dev/null +++ b/checkov/dockerfile/image_referencer/provider.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Callable, Any + +from checkov.common.images.image_referencer import Image +from checkov.common.util.str_utils import removeprefix +from checkov.dockerfile.utils import DOCKERFILE_STARTLINE, DOCKERFILE_ENDLINE + +if TYPE_CHECKING: + from typing_extensions import TypeAlias + +_ExtractImagesCallableAlias: TypeAlias = Callable[["dict[str, Any]"], "list[str]"] + + +class DockerfileProvider: + __slots__ = ("definitions", "supported_resource_types") + + def __init__(self, definitions: dict[str, Any]) -> None: + self.definitions = definitions + + def extract_images_from_resources(self) -> list[Image]: + images = [] + + for file_path, config in self.definitions.items(): + instructions = config.get("FROM") + if not isinstance(instructions, list): + continue + + for instruction in instructions: + name = instruction["value"] + + if " AS " in name: + # indicates a multi-stage build, therefore remove everything starting from AS + # ex. FROM maven:3.8-openjdk-17-slim AS build + name = name.split(" AS ")[0] + + if name.startswith("--platform"): + # indicates a multi-platform build, therefore remove the platform flag and its value + # ex. FROM --platform=$BUILDPLATFORM golang:alpine AS build + multi_platform = name.split(" ", maxsplit=1) + if len(multi_platform) != 2: + logging.info(f"Invalid FROM statement {instruction['value']}") + continue + + name = multi_platform[1] + + images.append( + Image( + file_path=file_path, + name=name, + start_line=instruction[DOCKERFILE_STARTLINE] + 1, # starts with 0 + end_line=instruction[DOCKERFILE_ENDLINE] + 1, + related_resource_id=f'{removeprefix(file_path, os.getenv("BC_ROOT_DIR", ""))}.FROM' + ) + ) + + return images diff --git a/checkov/dockerfile/parser.py b/checkov/dockerfile/parser.py index 6887fe2e4e7..1af3ce6c242 100644 --- a/checkov/dockerfile/parser.py +++ b/checkov/dockerfile/parser.py @@ -2,6 +2,7 @@ from collections import OrderedDict from pathlib import Path +from typing import TYPE_CHECKING from dockerfile_parse import DockerfileParser from dockerfile_parse.constants import COMMENT_INSTRUCTION @@ -9,15 +10,18 @@ from checkov.common.typing import _SkippedCheck from checkov.common.util.suppression import collect_suppressions_for_context +if TYPE_CHECKING: + from dockerfile_parse.parser import _Instruction # only in extra_stubs -def parse(filename: str | Path) -> tuple[dict[str, list[dict[str, int | str]]], list[str]]: + +def parse(filename: str | Path) -> tuple[dict[str, list[_Instruction]], list[str]]: with open(filename) as dockerfile: dfp = DockerfileParser(fileobj=dockerfile) return dfp_group_by_instructions(dfp) -def dfp_group_by_instructions(dfp: DockerfileParser) -> tuple[dict[str, list[dict[str, int | str]]], list[str]]: - result: dict[str, list[dict[str, int | str]]] = OrderedDict() +def dfp_group_by_instructions(dfp: DockerfileParser) -> tuple[dict[str, list[_Instruction]], list[str]]: + result: dict[str, list[_Instruction]] = OrderedDict() for instruction in dfp.structure: instruction_literal = instruction["instruction"] if instruction_literal not in result: @@ -26,7 +30,7 @@ def dfp_group_by_instructions(dfp: DockerfileParser) -> tuple[dict[str, list[dic return result, dfp.lines -def collect_skipped_checks(parse_result: dict[str, list[dict[str, int | str]]]) -> list[_SkippedCheck]: +def collect_skipped_checks(parse_result: dict[str, list[_Instruction]]) -> list[_SkippedCheck]: skipped_checks = [] if COMMENT_INSTRUCTION in parse_result: diff --git a/checkov/dockerfile/runner.py b/checkov/dockerfile/runner.py index a8381136cb4..b49e2453ac2 100644 --- a/checkov/dockerfile/runner.py +++ b/checkov/dockerfile/runner.py @@ -2,8 +2,9 @@ import logging import os -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Callable, Any +from checkov.common.images.image_referencer import ImageReferencerMixin from checkov.common.output.record import Record from checkov.common.output.report import Report from checkov.common.bridgecrew.check_type import CheckType @@ -11,16 +12,20 @@ from checkov.common.runners.base_runner import BaseRunner, filter_ignored_paths from checkov.common.util.dockerfile import is_docker_file from checkov.common.typing import _CheckResult +from checkov.dockerfile.image_referencer.manager import DockerfileImageReferencerManager from checkov.dockerfile.parser import parse, collect_skipped_checks from checkov.dockerfile.registry import registry +from checkov.dockerfile.utils import DOCKERFILE_STARTLINE, DOCKERFILE_ENDLINE from checkov.runner_filter import RunnerFilter if TYPE_CHECKING: - from checkov.common.parsers.node import DictNode - from checkov.dockerfile.base_dockerfile_check import BaseDockerfileCheck + from dockerfile_parse.parser import _Instruction # only in extra_stubs + from networkx import DiGraph + from checkov.common.checks.base_check import BaseCheck + from checkov.common.images.image_referencer import Image -class Runner(BaseRunner): +class Runner(ImageReferencerMixin, BaseRunner[None]): check_type = CheckType.DOCKERFILE # noqa: CCE003 # a static attribute def should_scan_file(self, filename: str) -> bool: @@ -33,7 +38,7 @@ def run( files: list[str] | None = None, runner_filter: RunnerFilter | None = None, collect_skip_comments: bool = True, - ) -> Report: + ) -> Report | list[Report]: runner_filter = runner_filter or RunnerFilter() if not runner_filter.show_progress_bar: self.pbar.turn_off_progress_bar() @@ -86,8 +91,8 @@ def run( if result_configuration: if isinstance(result_configuration, list): for res in result_configuration: - startline = res['startline'] - endline = res['endline'] + startline = res[DOCKERFILE_STARTLINE] + endline = res[DOCKERFILE_ENDLINE] result_instruction = res["instruction"] self.build_record(report, definitions_raw, @@ -99,8 +104,8 @@ def run( endline, result_instruction) else: - startline = result_configuration['startline'] - endline = result_configuration['endline'] + startline = result_configuration[DOCKERFILE_STARTLINE] + endline = result_configuration[DOCKERFILE_ENDLINE] result_instruction = result_configuration["instruction"] self.build_record(report, definitions_raw, @@ -123,6 +128,18 @@ def run( result_instruction) self.pbar.update() self.pbar.close() + + if runner_filter.run_image_referencer: + image_report = self.check_container_image_references( + root_path=root_folder, + runner_filter=runner_filter, + definitions=definitions, + ) + + if image_report: + # due too many tests failing only return a list, if there is an image report + return [report, image_report] + return report def calc_record_codeblock( @@ -142,7 +159,7 @@ def build_record( definitions_raw: dict[str, list[str]], docker_file_path: str, file_abs_path: str, - check: BaseDockerfileCheck, + check: BaseCheck, check_result: _CheckResult, startline: int, endline: int, @@ -168,11 +185,23 @@ def build_record( record.set_guideline(check.guideline) report.add_record(record=record) + def extract_images( + self, graph_connector: DiGraph | None = None, definitions: dict[str, Any] | None = None + ) -> list[Image]: + if not definitions: + # should not happen + return [] + + manager = DockerfileImageReferencerManager(definitions=definitions) + images = manager.extract_images_from_resources() + + return images + def get_files_definitions( files: list[str], filepath_fn: Callable[[str], str] | None = None -) -> tuple[dict[str, DictNode], dict[str, list[str]]]: - def _parse_file(file: str) -> tuple[str, tuple[dict[str, list[dict[str, int | str]]], list[str]] | None]: +) -> tuple[dict[str, dict[str, list[_Instruction]]], dict[str, list[str]]]: + def _parse_file(file: str) -> tuple[str, tuple[dict[str, list[_Instruction]], list[str]] | None]: try: return file, parse(file) except TypeError: diff --git a/checkov/dockerfile/utils.py b/checkov/dockerfile/utils.py new file mode 100644 index 00000000000..0eef8492db1 --- /dev/null +++ b/checkov/dockerfile/utils.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing_extensions import Literal + +DOCKERFILE_STARTLINE: Literal["startline"] = "startline" +DOCKERFILE_ENDLINE: Literal["endline"] = "endline" From f0c383855c6525079ff61fa8b983af3259da8726 Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 27 Sep 2022 11:38:30 +0200 Subject: [PATCH 2/5] add stubs for dockerfile_parse --- extra_stubs/dockerfile_parse/__init__.pyi | 5 ++++ extra_stubs/dockerfile_parse/constants.pyi | 3 +++ extra_stubs/dockerfile_parse/parser.pyi | 29 ++++++++++++++++++++++ mypy.ini | 2 ++ 4 files changed, 39 insertions(+) create mode 100644 extra_stubs/dockerfile_parse/__init__.pyi create mode 100644 extra_stubs/dockerfile_parse/constants.pyi create mode 100644 extra_stubs/dockerfile_parse/parser.pyi diff --git a/extra_stubs/dockerfile_parse/__init__.pyi b/extra_stubs/dockerfile_parse/__init__.pyi new file mode 100644 index 00000000000..7eac7739997 --- /dev/null +++ b/extra_stubs/dockerfile_parse/__init__.pyi @@ -0,0 +1,5 @@ +from .parser import DockerfileParser + +__all__ = [ + "DockerfileParser", +] diff --git a/extra_stubs/dockerfile_parse/constants.pyi b/extra_stubs/dockerfile_parse/constants.pyi new file mode 100644 index 00000000000..bc2f0450744 --- /dev/null +++ b/extra_stubs/dockerfile_parse/constants.pyi @@ -0,0 +1,3 @@ +from typing_extensions import Literal + +COMMENT_INSTRUCTION: Literal["COMMENT"] diff --git a/extra_stubs/dockerfile_parse/parser.pyi b/extra_stubs/dockerfile_parse/parser.pyi new file mode 100644 index 00000000000..e434247f236 --- /dev/null +++ b/extra_stubs/dockerfile_parse/parser.pyi @@ -0,0 +1,29 @@ +from typing import TextIO + +from typing_extensions import TypedDict + + +class _Instruction(TypedDict): + instruction: str + startline: int + endline: int + content: str + value: str + + +class DockerfileParser: + def __init__( + self, + path: str | None = ..., + cache_content: bool = ..., + env_replace: bool = ..., + parent_env: dict[str, str] | None = None, + fileobj: TextIO | None = None, + build_args: dict[str, str] | None = None, + ) -> None: ... + + @property + def lines(self) -> list[str]: ... + + @property + def structure(self) -> list[_Instruction]: ... diff --git a/mypy.ini b/mypy.ini index 0eb59bd551d..68fe7903d1b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,6 @@ [mypy] +mypy_path = extra_stubs + files = checkov exclude = checkov/(arm|cloudformation|dockerfile|gitlab_ci|github_actions|bitbucket|helm|kubernetes|kustomize|sca_package|serverless|terraform) strict = True From 83b413ab3723ac3aa3aef2c67d44aa0cb61f2758 Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 27 Sep 2022 19:10:43 +0200 Subject: [PATCH 3/5] add tests --- checkov/dockerfile/runner.py | 4 + tests/dockerfile/image_referencer/__init__.py | 0 tests/dockerfile/image_referencer/conftest.py | 63 ++++++++ .../resources/Dockerfile.multi_platform | 10 ++ .../resources/Dockerfile.multi_stage | 16 ++ .../resources/Dockerfile.simple | 3 + .../image_referencer/test_manager.py | 42 +++++ .../image_referencer/test_provider.py | 81 ++++++++++ .../test_runner_dockerfile_resources.py | 152 ++++++++++++++++++ 9 files changed, 371 insertions(+) create mode 100644 tests/dockerfile/image_referencer/__init__.py create mode 100644 tests/dockerfile/image_referencer/conftest.py create mode 100644 tests/dockerfile/image_referencer/resources/Dockerfile.multi_platform create mode 100644 tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage create mode 100644 tests/dockerfile/image_referencer/resources/Dockerfile.simple create mode 100644 tests/dockerfile/image_referencer/test_manager.py create mode 100644 tests/dockerfile/image_referencer/test_provider.py create mode 100644 tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py diff --git a/checkov/dockerfile/runner.py b/checkov/dockerfile/runner.py index b49e2453ac2..5e6686a8d64 100644 --- a/checkov/dockerfile/runner.py +++ b/checkov/dockerfile/runner.py @@ -130,6 +130,10 @@ def run( self.pbar.close() if runner_filter.run_image_referencer: + if files: + # 'root_folder' shouldn't be empty to remove the whole path later and only leave the shortened form + root_folder = os.path.split(os.path.commonprefix(files))[0] + image_report = self.check_container_image_references( root_path=root_folder, runner_filter=runner_filter, diff --git a/tests/dockerfile/image_referencer/__init__.py b/tests/dockerfile/image_referencer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/dockerfile/image_referencer/conftest.py b/tests/dockerfile/image_referencer/conftest.py new file mode 100644 index 00000000000..c2f623daf28 --- /dev/null +++ b/tests/dockerfile/image_referencer/conftest.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import Any + +import pytest + + +@pytest.fixture() +def image_cached_result() -> dict[str, Any]: + return { + "results": [ + { + "id": "sha256:f9b91f78b0344fa0efc5583d79e78a90556ab0bb3f93fcbc8728b0b70d29a5db", + "name": "python:3.9-alpine", + "distro": "Alpine Linux v3.16", + "distroRelease": "3.16.1", + "digest": "sha256:83a343afa488ff14d0c807b62770140d2ec30ef2e83a3a45c4ce62c29623e240", + "collections": ["All"], + "packages": [{"type": "os", "name": "zlib", "version": "1.2.12-r1", "licenses": ["Zlib"]}], + "compliances": [], + "complianceDistribution": {"critical": 0, "high": 0, "medium": 0, "low": 0, "total": 0}, + "complianceScanPassed": True, + "vulnerabilities": [ + { + "id": "CVE-2022-37434", + "status": "fixed in 1.2.12-r2", + "description": "zlib through 1.2.12 has a heap-based buffer over-read ...", + "severity": "low", + "packageName": "zlib", + "packageVersion": "1.2.12-r1", + "link": "https://nvd.nist.gov/vuln/detail/CVE-2022-37434", + "riskFactors": ["Has fix", "Recent vulnerability"], + "impactedVersions": ["<1.2.12-r2"], + "publishedDate": "2022-08-05T07:15:00Z", + "discoveredDate": "2022-08-08T13:45:43Z", + "fixDate": "2022-08-05T07:15:00Z", + } + ], + "vulnerabilityDistribution": {"critical": 0, "high": 0, "medium": 0, "low": 1, "total": 1}, + "vulnerabilityScanPassed": True, + } + ] + } + + +@pytest.fixture() +def license_statuses_result() -> list[dict[str, str]]: + return [ + { + "package_name": "openssl", + "package_version": "1.1.1q-r0", + "policy": "BC_LIC_1", + "license": "OpenSSL", + "status": "OPEN", + }, + { + "package_name": "musl", + "package_version": "1.2.3-r0", + "policy": "BC_LIC_1", + "license": "MIT", + "status": "COMPLIANT", + }, + ] diff --git a/tests/dockerfile/image_referencer/resources/Dockerfile.multi_platform b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_platform new file mode 100644 index 00000000000..f65d849f0c4 --- /dev/null +++ b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_platform @@ -0,0 +1,10 @@ +FROM --platform=$BUILDPLATFORM golang:alpine AS build + +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +RUN echo "I am running on $BUILDPLATFORM, building for $TARGETPLATFORM" > /log + +FROM alpine + +COPY --from=build /log /log diff --git a/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage new file mode 100644 index 00000000000..f74e543994b --- /dev/null +++ b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage @@ -0,0 +1,16 @@ +# Build stage +FROM maven:3.8-openjdk-17-slim AS build + +COPY pom.xml /app/pom.xml +COPY src /app/src + +RUN mvn -f /app/pom.xml clean package + +# Run stage +FROM amazonlinux:2 + +RUN yum install -y java-17-amazon-corretto-headless + +COPY --from=build /app/target/main-1.0-SNAPSHOT-jar-with-dependencies.jar /app/app.jar + +CMD ["java","-jar","/app/app.jar"] diff --git a/tests/dockerfile/image_referencer/resources/Dockerfile.simple b/tests/dockerfile/image_referencer/resources/Dockerfile.simple new file mode 100644 index 00000000000..a101066917d --- /dev/null +++ b/tests/dockerfile/image_referencer/resources/Dockerfile.simple @@ -0,0 +1,3 @@ +FROM php:7.1-apache + +RUN apk --no-cache add nginx diff --git a/tests/dockerfile/image_referencer/test_manager.py b/tests/dockerfile/image_referencer/test_manager.py new file mode 100644 index 00000000000..16745d603d8 --- /dev/null +++ b/tests/dockerfile/image_referencer/test_manager.py @@ -0,0 +1,42 @@ +from checkov.common.images.image_referencer import Image +from checkov.dockerfile.image_referencer.manager import DockerfileImageReferencerManager + + +def test_extract_images_from_resources(): + # given + definitions = { + "/Dockerfile": { + "FROM": [ + { + "instruction": "FROM", + "startline": 0, + "endline": 0, + "content": "FROM php:7.1-apache\n", + "value": "php:7.1-apache", + } + ], + "RUN": [ + { + "instruction": "RUN", + "startline": 2, + "endline": 2, + "content": "RUN apk --no-cache add nginx\n", + "value": "apk --no-cache add nginx", + }, + ], + }, + } + + # when + images = DockerfileImageReferencerManager(definitions=definitions).extract_images_from_resources() + + # then + assert images == [ + Image( + file_path="/Dockerfile", + name="php:7.1-apache", + start_line=1, + end_line=1, + related_resource_id="/Dockerfile.FROM", + ), + ] diff --git a/tests/dockerfile/image_referencer/test_provider.py b/tests/dockerfile/image_referencer/test_provider.py new file mode 100644 index 00000000000..f18976b33b4 --- /dev/null +++ b/tests/dockerfile/image_referencer/test_provider.py @@ -0,0 +1,81 @@ +from checkov.common.images.image_referencer import Image +from checkov.dockerfile.image_referencer.provider import DockerfileProvider + + +def test_extract_images_from_resources(): + # given + definitions = { + "/Dockerfile": { + "FROM": [ + { + "instruction": "FROM", + "startline": 0, + "endline": 0, + "content": "FROM maven:3.8-openjdk-17-slim AS build\n", + "value": "maven:3.8-openjdk-17-slim AS build", + }, + { + "instruction": "FROM", + "startline": 4, + "endline": 4, + "content": "FROM amazonlinux:2\n", + "value": "amazonlinux:2", + } + ], + "RUN": [ + { + "instruction": "RUN", + "startline": 2, + "endline": 2, + "content": "RUN apt-get install -y curl\n", + "value": "apt-get install -y curl", + }, + ], + }, + } + + # when + provider = DockerfileProvider(definitions=definitions) + images = provider.extract_images_from_resources() + + # then + assert images == [ + Image( + file_path="/Dockerfile", + name="maven:3.8-openjdk-17-slim", + start_line=1, + end_line=1, + related_resource_id="/Dockerfile.FROM", + ), + Image( + file_path="/Dockerfile", + name="amazonlinux:2", + start_line=5, + end_line=5, + related_resource_id="/Dockerfile.FROM", + ), + ] + + +def test_extract_images_from_resources_with_no_image(): + # given + definitions = { + "/Dockerfile": { + "RUN": [ + { + "instruction": "RUN", + "startline": 2, + "endline": 2, + "content": "RUN apt-get install -y curl\n", + "value": "apt-get install -y curl", + }, + ], + }, + } + + # when + provider = DockerfileProvider(definitions=definitions) + images = provider.extract_images_from_resources() + + # then + assert not images diff --git a/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py b/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py new file mode 100644 index 00000000000..ed35a769816 --- /dev/null +++ b/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py @@ -0,0 +1,152 @@ +from pathlib import Path + +from pytest_mock import MockerFixture + +from checkov.common.bridgecrew.bc_source import get_source_type +from checkov.common.bridgecrew.check_type import CheckType +from checkov.dockerfile.runner import Runner +from checkov.runner_filter import RunnerFilter + +RESOURCES_PATH = Path(__file__).parent / "resources" + + +def test_simple_dockerfile(mocker: MockerFixture, image_cached_result, license_statuses_result): + from checkov.common.bridgecrew.platform_integration import bc_integration + + # given + file_name = "Dockerfile.simple" + image_name = "php:7.1-apache" + code_lines = "1-1" + test_file = RESOURCES_PATH / file_name + runner_filter = RunnerFilter(run_image_referencer=True) + bc_integration.bc_source = get_source_type("disabled") + + mocker.patch( + "checkov.common.images.image_referencer.image_scanner.get_scan_results_from_cache", + return_value=image_cached_result, + ) + mocker.patch( + "checkov.common.images.image_referencer.get_license_statuses", + return_value=license_statuses_result, + ) + + # when + reports = Runner().run(root_folder="", files=[str(test_file)], runner_filter=runner_filter) + + # then + assert len(reports) == 2 + + tf_report = next(report for report in reports if report.check_type == CheckType.DOCKERFILE) + sca_image_report = next(report for report in reports if report.check_type == CheckType.SCA_IMAGE) + + assert len(tf_report.resources) == 1 + assert len(tf_report.passed_checks) == 4 + assert len(tf_report.failed_checks) == 2 + assert len(tf_report.skipped_checks) == 0 + assert len(tf_report.parsing_errors) == 0 + + assert len(sca_image_report.resources) == 3 + assert sca_image_report.resources == { + f"{file_name} ({image_name} lines:{code_lines} (sha256:f9b91f78b0)).musl", + f"{file_name} ({image_name} lines:{code_lines} (sha256:f9b91f78b0)).openssl", + f"{file_name} ({image_name} lines:{code_lines} (sha256:f9b91f78b0)).zlib", + } + assert sca_image_report.image_cached_results[0]["dockerImageName"] == "php:7.1-apache" + assert "Dockerfile.simple.FROM" in sca_image_report.image_cached_results[0]["relatedResourceId"] + assert sca_image_report.image_cached_results[0]["packages"] == [ + {"type": "os", "name": "zlib", "version": "1.2.12-r1", "licenses": ["Zlib"]} + ] + + assert len(sca_image_report.passed_checks) == 1 + assert len(sca_image_report.failed_checks) == 2 + assert len(sca_image_report.image_cached_results) == 1 + assert len(sca_image_report.skipped_checks) == 0 + assert len(sca_image_report.parsing_errors) == 0 + + +def test_multi_stage_dockerfile(mocker: MockerFixture, image_cached_result): + from checkov.common.bridgecrew.platform_integration import bc_integration + + # given + file_name = "Dockerfile.multi_stage" + image_name_1 = "maven:3.8-openjdk-17-slim" + image_name_2 = "amazonlinux:2" + code_lines_1 = "2-2" + code_lines_2 = "10-10" + test_file = RESOURCES_PATH / file_name + runner_filter = RunnerFilter(run_image_referencer=True) + bc_integration.bc_source = get_source_type("disabled") + + mocker.patch( + "checkov.common.images.image_referencer.image_scanner.get_scan_results_from_cache", + return_value=image_cached_result, + ) + + # when + reports = Runner().run(root_folder="", files=[str(test_file)], runner_filter=runner_filter) + + # then + assert len(reports) == 2 + + tf_report = next(report for report in reports if report.check_type == CheckType.DOCKERFILE) + sca_image_report = next(report for report in reports if report.check_type == CheckType.SCA_IMAGE) + + assert len(tf_report.resources) == 1 + assert len(tf_report.passed_checks) == 4 + assert len(tf_report.failed_checks) == 2 + assert len(tf_report.skipped_checks) == 0 + assert len(tf_report.parsing_errors) == 0 + + assert len(sca_image_report.resources) == 2 + assert sca_image_report.resources == { + f"{file_name} ({image_name_1} lines:{code_lines_1} (sha256:f9b91f78b0)).zlib", + f"{file_name} ({image_name_2} lines:{code_lines_2} (sha256:f9b91f78b0)).zlib", + } + assert len(sca_image_report.passed_checks) == 0 + assert len(sca_image_report.failed_checks) == 2 + assert len(sca_image_report.skipped_checks) == 0 + assert len(sca_image_report.parsing_errors) == 0 + + +def test_multi_platform_dockerfile(mocker: MockerFixture, image_cached_result): + from checkov.common.bridgecrew.platform_integration import bc_integration + + # given + file_name = "Dockerfile.multi_platform" + image_name_1 = "golang:alpine" + image_name_2 = "alpine" + code_lines_1 = "1-1" + code_lines_2 = "8-8" + test_file = RESOURCES_PATH / file_name + runner_filter = RunnerFilter(run_image_referencer=True) + bc_integration.bc_source = get_source_type("disabled") + + mocker.patch( + "checkov.common.images.image_referencer.image_scanner.get_scan_results_from_cache", + return_value=image_cached_result, + ) + + # when + reports = Runner().run(root_folder="", files=[str(test_file)], runner_filter=runner_filter) + + # then + assert len(reports) == 2 + + tf_report = next(report for report in reports if report.check_type == CheckType.DOCKERFILE) + sca_image_report = next(report for report in reports if report.check_type == CheckType.SCA_IMAGE) + + assert len(tf_report.resources) == 1 + assert len(tf_report.passed_checks) == 3 + assert len(tf_report.failed_checks) == 3 + assert len(tf_report.skipped_checks) == 0 + assert len(tf_report.parsing_errors) == 0 + + assert len(sca_image_report.resources) == 2 + assert sca_image_report.resources == { + f"{file_name} ({image_name_1} lines:{code_lines_1} (sha256:f9b91f78b0)).zlib", + f"{file_name} ({image_name_2} lines:{code_lines_2} (sha256:f9b91f78b0)).zlib", + } + assert len(sca_image_report.passed_checks) == 0 + assert len(sca_image_report.failed_checks) == 2 + assert len(sca_image_report.skipped_checks) == 0 + assert len(sca_image_report.parsing_errors) == 0 From 790eea9a7253403c50c613f8c2ad88f85ff77ee3 Mon Sep 17 00:00:00 2001 From: gruebel Date: Mon, 3 Oct 2022 10:23:24 +0200 Subject: [PATCH 4/5] only scan last image --- .../dockerfile/image_referencer/provider.py | 50 +++++++++---------- .../resources/Dockerfile.multi_stage | 2 +- .../image_referencer/test_provider.py | 11 +--- .../test_runner_dockerfile_resources.py | 10 ++-- 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/checkov/dockerfile/image_referencer/provider.py b/checkov/dockerfile/image_referencer/provider.py index 3d253e16ca1..a6d3b8aa10f 100644 --- a/checkov/dockerfile/image_referencer/provider.py +++ b/checkov/dockerfile/image_referencer/provider.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import os from typing import TYPE_CHECKING, Callable, Any @@ -28,32 +27,29 @@ def extract_images_from_resources(self) -> list[Image]: if not isinstance(instructions, list): continue - for instruction in instructions: - name = instruction["value"] - - if " AS " in name: - # indicates a multi-stage build, therefore remove everything starting from AS - # ex. FROM maven:3.8-openjdk-17-slim AS build - name = name.split(" AS ")[0] - - if name.startswith("--platform"): - # indicates a multi-platform build, therefore remove the platform flag and its value - # ex. FROM --platform=$BUILDPLATFORM golang:alpine AS build - multi_platform = name.split(" ", maxsplit=1) - if len(multi_platform) != 2: - logging.info(f"Invalid FROM statement {instruction['value']}") - continue - - name = multi_platform[1] - - images.append( - Image( - file_path=file_path, - name=name, - start_line=instruction[DOCKERFILE_STARTLINE] + 1, # starts with 0 - end_line=instruction[DOCKERFILE_ENDLINE] + 1, - related_resource_id=f'{removeprefix(file_path, os.getenv("BC_ROOT_DIR", ""))}.FROM' - ) + # just scan the last one + instruction = instructions[-1] + + name = instruction["value"] + + if name.startswith("--platform"): + # indicates a multi-platform build, therefore skip it + # ex. FROM --platform=$BUILDPLATFORM golang:alpine AS build + continue + + if " AS " in name: + # indicates a multi-stage build, therefore remove everything starting from AS + # ex. FROM amazonlinux:2 as run + name = name.split(" AS ")[0] + + images.append( + Image( + file_path=file_path, + name=name, + start_line=instruction[DOCKERFILE_STARTLINE] + 1, # starts with 0 + end_line=instruction[DOCKERFILE_ENDLINE] + 1, + related_resource_id=f'{removeprefix(file_path, os.getenv("BC_ROOT_DIR", ""))}.FROM', ) + ) return images diff --git a/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage index f74e543994b..7ebfe220dd2 100644 --- a/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage +++ b/tests/dockerfile/image_referencer/resources/Dockerfile.multi_stage @@ -7,7 +7,7 @@ COPY src /app/src RUN mvn -f /app/pom.xml clean package # Run stage -FROM amazonlinux:2 +FROM amazonlinux:2 AS run RUN yum install -y java-17-amazon-corretto-headless diff --git a/tests/dockerfile/image_referencer/test_provider.py b/tests/dockerfile/image_referencer/test_provider.py index f18976b33b4..f173ba44e8c 100644 --- a/tests/dockerfile/image_referencer/test_provider.py +++ b/tests/dockerfile/image_referencer/test_provider.py @@ -18,9 +18,9 @@ def test_extract_images_from_resources(): "instruction": "FROM", "startline": 4, "endline": 4, - "content": "FROM amazonlinux:2\n", + "content": "FROM amazonlinux:2 AS run\n", "value": "amazonlinux:2", - } + }, ], "RUN": [ { @@ -40,13 +40,6 @@ def test_extract_images_from_resources(): # then assert images == [ - Image( - file_path="/Dockerfile", - name="maven:3.8-openjdk-17-slim", - start_line=1, - end_line=1, - related_resource_id="/Dockerfile.FROM", - ), Image( file_path="/Dockerfile", name="amazonlinux:2", diff --git a/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py b/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py index ed35a769816..f7a9a5c94e5 100644 --- a/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py +++ b/tests/dockerfile/image_referencer/test_runner_dockerfile_resources.py @@ -97,13 +97,12 @@ def test_multi_stage_dockerfile(mocker: MockerFixture, image_cached_result): assert len(tf_report.skipped_checks) == 0 assert len(tf_report.parsing_errors) == 0 - assert len(sca_image_report.resources) == 2 + assert len(sca_image_report.resources) == 1 assert sca_image_report.resources == { - f"{file_name} ({image_name_1} lines:{code_lines_1} (sha256:f9b91f78b0)).zlib", f"{file_name} ({image_name_2} lines:{code_lines_2} (sha256:f9b91f78b0)).zlib", } assert len(sca_image_report.passed_checks) == 0 - assert len(sca_image_report.failed_checks) == 2 + assert len(sca_image_report.failed_checks) == 1 assert len(sca_image_report.skipped_checks) == 0 assert len(sca_image_report.parsing_errors) == 0 @@ -141,12 +140,11 @@ def test_multi_platform_dockerfile(mocker: MockerFixture, image_cached_result): assert len(tf_report.skipped_checks) == 0 assert len(tf_report.parsing_errors) == 0 - assert len(sca_image_report.resources) == 2 + assert len(sca_image_report.resources) == 1 assert sca_image_report.resources == { - f"{file_name} ({image_name_1} lines:{code_lines_1} (sha256:f9b91f78b0)).zlib", f"{file_name} ({image_name_2} lines:{code_lines_2} (sha256:f9b91f78b0)).zlib", } assert len(sca_image_report.passed_checks) == 0 - assert len(sca_image_report.failed_checks) == 2 + assert len(sca_image_report.failed_checks) == 1 assert len(sca_image_report.skipped_checks) == 0 assert len(sca_image_report.parsing_errors) == 0 From a51b2cd0c29f58dfdf2d0b74815f8f0708a3c56a Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 4 Oct 2022 18:00:51 +0200 Subject: [PATCH 5/5] fix merge conflicts --- checkov/dockerfile/image_referencer/provider.py | 2 +- checkov/dockerfile/runner.py | 5 ++++- mypy.ini | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/checkov/dockerfile/image_referencer/provider.py b/checkov/dockerfile/image_referencer/provider.py index a6d3b8aa10f..cca5e783f2a 100644 --- a/checkov/dockerfile/image_referencer/provider.py +++ b/checkov/dockerfile/image_referencer/provider.py @@ -14,7 +14,7 @@ class DockerfileProvider: - __slots__ = ("definitions", "supported_resource_types") + __slots__ = ("definitions",) def __init__(self, definitions: dict[str, Any]) -> None: self.definitions = definitions diff --git a/checkov/dockerfile/runner.py b/checkov/dockerfile/runner.py index 67553626d12..43ced65b4d6 100644 --- a/checkov/dockerfile/runner.py +++ b/checkov/dockerfile/runner.py @@ -201,7 +201,10 @@ def build_record( report.add_record(record=record) def extract_images( - self, graph_connector: DiGraph | None = None, definitions: dict[str, Any] | None = None + self, + graph_connector: DiGraph | None = None, + definitions: dict[str, dict[str, Any] | list[dict[str, Any]]] | None = None, + definitions_raw: dict[str, list[tuple[int, str]]] | None = None ) -> list[Image]: if not definitions: # should not happen diff --git a/mypy.ini b/mypy.ini index f506713e9e5..59e283de4af 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,7 @@ mypy_path = extra_stubs files = checkov exclude = checkov/(arm|cloudformation|dockerfile|bitbucket_pipelines|helm|kubernetes|kustomize|sca_package|serverless|terraform) strict = True +pretty = True disallow_subclassing_any = False implicit_reexport = True show_error_codes = True