From 1660ed3fb59c2c7d811a94bb9354396ad4f1e3e6 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Sun, 11 Aug 2019 11:57:30 -0700 Subject: [PATCH 1/4] Simplified the build_docker_image function --- sdk/python/kfp/compiler/_component_builder.py | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index 8e37278225b..b7d8ec12cd6 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -282,23 +282,6 @@ def __init__(self, gcs_staging, target_image, namespace): self._arc_requirement_filename = 'requirements.txt' self._container_builder = ContainerBuilder(gcs_staging, gcr_image_tag=target_image, namespace=namespace) - def _prepare_files(self, local_dir, docker_filename, python_filename=None, requirement_filename=None): - """ _prepare_buildfiles generates the tarball with all the build files - Args: - local_dir (dir): a directory that stores all the build files - docker_filename (str): docker filename - python_filename (str): python filename - requirement_filename (str): requirement filename - """ - dst_docker_filepath = os.path.join(local_dir, self._arc_docker_filename) - shutil.copyfile(docker_filename, dst_docker_filepath) - if python_filename is not None: - dst_python_filepath = os.path.join(local_dir, self._arc_python_filename) - shutil.copyfile(python_filename, dst_python_filepath) - if requirement_filename is not None: - dst_requirement_filepath = os.path.join(local_dir, self._arc_requirement_filename) - shutil.copyfile(requirement_filename, dst_requirement_filepath) - def build_image_from_func(self, component_func, base_image, timeout, dependency, python_version='python3'): """ build_image builds an image for the given python function args: @@ -325,12 +308,6 @@ def build_image_from_func(self, component_func, base_image, timeout, dependency, logging.info('Generate build files.') return self._container_builder.build(local_build_dir, self._arc_docker_filename, timeout=timeout) - def build_image_from_dockerfile(self, docker_filename, timeout): - """ build_image_from_dockerfile builds an image based on the dockerfile """ - with tempfile.TemporaryDirectory() as local_build_dir: - self._prepare_files(local_build_dir, docker_filename) - return self._container_builder.build(local_build_dir, self._arc_docker_filename, timeout=timeout) - def _configure_logger(logger): """ _configure_logger configures the logger such that the info level logs go to the stdout and the error(or above) level logs go to the stderr. @@ -455,7 +432,13 @@ def build_docker_image(staging_gcs_path, target_image, dockerfile_path, timeout= namespace (str): the namespace within which to run the kubernetes kaniko job, default is "kubeflow" """ _configure_logger(logging.getLogger()) - builder = ComponentBuilder(gcs_staging=staging_gcs_path, target_image=target_image, namespace=namespace) - image_name_with_digest = builder.build_image_from_dockerfile(docker_filename=dockerfile_path, timeout=timeout) + + with tempfile.TemporaryDirectory() as local_build_dir: + dst_dockerfile_path = os.path.join(local_build_dir, 'Dockerfile') + shutil.copyfile(dockerfile_path, dst_dockerfile_path) + + container_builder = ContainerBuilder(staging_gcs_path, namespace) + image_name_with_digest = container_builder.build(local_build_dir, 'Dockerfile', target_image, timeout) + logging.info('Build image complete.') return image_name_with_digest From e7c126cd8d7ad71f76880f236aead1b21502e5b6 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 19 Aug 2019 17:43:39 -0700 Subject: [PATCH 2/4] Extracted 'Dockerfile' into a variable Addressed PR feedback --- sdk/python/kfp/compiler/_component_builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index b7d8ec12cd6..cbe1722eae4 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -434,11 +434,12 @@ def build_docker_image(staging_gcs_path, target_image, dockerfile_path, timeout= _configure_logger(logging.getLogger()) with tempfile.TemporaryDirectory() as local_build_dir: - dst_dockerfile_path = os.path.join(local_build_dir, 'Dockerfile') + dockerfile_rel_path = 'Dockerfile' + dst_dockerfile_path = os.path.join(local_build_dir, dockerfile_rel_path) shutil.copyfile(dockerfile_path, dst_dockerfile_path) container_builder = ContainerBuilder(staging_gcs_path, namespace) - image_name_with_digest = container_builder.build(local_build_dir, 'Dockerfile', target_image, timeout) + image_name_with_digest = container_builder.build(local_build_dir, dockerfile_rel_path, target_image, timeout) logging.info('Build image complete.') return image_name_with_digest From f8b17bfbe8910af105b9ae502ec073b9d507f41b Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 20 Aug 2019 14:48:58 -0700 Subject: [PATCH 3/4] Fixed conflict with updated master. --- sdk/python/kfp/compiler/_component_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index cbe1722eae4..0aba942a2e8 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -438,7 +438,7 @@ def build_docker_image(staging_gcs_path, target_image, dockerfile_path, timeout= dst_dockerfile_path = os.path.join(local_build_dir, dockerfile_rel_path) shutil.copyfile(dockerfile_path, dst_dockerfile_path) - container_builder = ContainerBuilder(staging_gcs_path, namespace) + container_builder = ContainerBuilder(staging_gcs_path, target_image, namespace=namespace) image_name_with_digest = container_builder.build(local_build_dir, dockerfile_rel_path, target_image, timeout) logging.info('Build image complete.') From dd33171e0fd7752f4fc13ab0b58e71ce47939512 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Wed, 21 Aug 2019 11:27:52 -0700 Subject: [PATCH 4/4] Addressed the feedback --- sdk/python/kfp/compiler/_component_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index 0aba942a2e8..ece5dd6362a 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -439,7 +439,7 @@ def build_docker_image(staging_gcs_path, target_image, dockerfile_path, timeout= shutil.copyfile(dockerfile_path, dst_dockerfile_path) container_builder = ContainerBuilder(staging_gcs_path, target_image, namespace=namespace) - image_name_with_digest = container_builder.build(local_build_dir, dockerfile_rel_path, target_image, timeout) + image_name_with_digest = container_builder.build(local_build_dir, dockerfile_rel_path, timeout) logging.info('Build image complete.') return image_name_with_digest