From d2896249c40105f33f0480cd3058ec4b67258d45 Mon Sep 17 00:00:00 2001 From: Allan Benson <95691842+AllanBenson001@users.noreply.github.com> Date: Wed, 19 Oct 2022 20:50:58 +0100 Subject: [PATCH 1/4] Add docker_additional_options and pre_package_commands --- README.md | 12 +++++ examples/build-package/main.tf | 70 +++++++++++++++++++++++++-- package.py | 86 +++++++++++++++++++++++++++------- package.tf | 11 +++-- variables.tf | 6 +++ 5 files changed, 158 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d90105d1..c7617011 100644 --- a/README.md +++ b/README.md @@ -455,6 +455,7 @@ source_path = [ - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. +- `pre_package_commands` - List of commands to run before doing a `pip install` or `npm install`. Works in conjunction with `pip_requirements` and `npm_requirements`. ### Building in Docker @@ -470,6 +471,17 @@ Using this module you can install dependencies from private hosts. To do this, y docker_with_ssh_agent = true +#### Passing additional Docker options + +To add flexibility when building in docker, you can pass any number of additional options that you require (see [Docker run reference](https://docs.docker.com/engine/reference/run/) for available options): + +```hcl + docker_additional_options = [ + "-e", "MY_ENV_VAR='My environment variable value'", + "-v", "/local:/docker-vol", + ] +``` + ## Deployment package - Create or use existing By default, this module creates deployment package and uses it to create or update Lambda Function or Lambda Layer. diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index 61dd0102..fc0321cd 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -79,8 +79,12 @@ module "package_file_with_pip_requirements" { source_path = [ "${path.module}/../fixtures/python3.8-app1/index.py", { - pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" - prefix_in_zip = "vendor" + pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" + prefix_in_zip = "vendor" + pre_package_commands = [ + "echo I will run before the pip install command", + "echo so will I", + ] } ] } @@ -100,11 +104,20 @@ module "package_with_pip_requirements_in_docker" { "${path.module}/../fixtures/python3.8-app1/index.py", "${path.module}/../fixtures/python3.8-app1/dir1/dir2", { - pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" + pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" + pre_package_commands = [ + "echo I will run before the pip install command in the docker container", + "echo I can read $MY_ENV_VAR and my volume:", + "ls -la /my-vol", + ] } ] - build_in_docker = true + build_in_docker = true + docker_additional_options = [ + "-e", "MY_ENV_VAR='My environment variable value'", + "-v", "${abspath(path.module)}:/my-vol:ro", + ] } # Create zip-archive which contains content of directory with commands and patterns applied. @@ -233,6 +246,26 @@ module "package_dir_with_npm_install" { source_path = "${path.module}/../fixtures/nodejs14.x-app1" } +# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime) +# Also run some additional commands before the npm install +module "package_dir_with_npm_install_and_additional_commands" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = [ + { + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = true + pre_package_commands = [ + "echo I will run before the npm install command", + "echo so will I", + ] + } + ] +} + # Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime) module "package_dir_without_npm_install" { source = "../../" @@ -261,6 +294,35 @@ module "package_with_npm_requirements_in_docker" { hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" } +# Create zip-archive of a single directory where "npm install" will also be executed using docker +# Also set additional docker run options and also run some commands before the npm install +module "package_with_npm_requirements_in_docker_and_additional_commands" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = [ + { + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = true + pre_package_commands = [ + "echo I will run before the npm install command in the docker container", + "echo I can read $MY_ENV_VAR and my volume:", + "ls -l /my-vol", + ] + } + ] + + build_in_docker = true + docker_additional_options = [ + "-e", "MY_ENV_VAR='My environment variable value'", + "-v", "${abspath(path.module)}:/my-vol:ro", + ] + + hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" +} + ################################ # Build package in Docker and # use it to deploy Lambda Layer diff --git a/package.py b/package.py index 1b966085..fe9b9e9f 100644 --- a/package.py +++ b/package.py @@ -648,7 +648,7 @@ def plan(self, source_path, query): step = lambda *x: build_plan.append(x) hash = source_paths.append - def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): + def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_package_commands=None): requirements = path if os.path.isdir(path): requirements = os.path.join(path, 'requirements.txt') @@ -657,10 +657,13 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): raise RuntimeError( 'File not found: {}'.format(requirements)) else: - step('pip', runtime, requirements, prefix, tmp_dir) + if pre_package_commands and not query.docker: + additional_commands_step(path, pre_package_commands) + pre_package_commands = None + step('pip', runtime, requirements, prefix, tmp_dir, pre_package_commands) hash(requirements) - def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): + def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_package_commands=None): requirements = path if os.path.isdir(path): requirements = os.path.join(path, 'package.json') @@ -669,9 +672,29 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): raise RuntimeError( 'File not found: {}'.format(requirements)) else: - step('npm', runtime, requirements, prefix, tmp_dir) + if pre_package_commands and not query.docker: + additional_commands_step(path, pre_package_commands) + pre_package_commands = None + step('npm', runtime, requirements, prefix, tmp_dir, pre_package_commands) hash(requirements) + def additional_commands_step(path, commands): + if not commands: + return + + if isinstance(commands, str): + commands = map(str.strip, commands.splitlines()) + + if path: + path = os.path.normpath(path) + if os.path.isfile(path): + path = os.path.dirname(path) + else: + path = query.paths.cwd + + hash(path) + step('sh', path, '\n'.join(commands)) + def commands_step(path, commands): if not commands: return @@ -747,21 +770,30 @@ def commands_step(path, commands): prefix = claim.get('prefix_in_zip') pip_requirements = claim.get('pip_requirements') npm_requirements = claim.get('npm_package_json') + if not npm_requirements: + # npm_package_json is an undocumented setting!!!! + # Set correct value here rather than create breaking change + npm_requirements = claim.get('npm_requirements') + pre_package_commands = claim.get('pre_package_commands') runtime = claim.get('runtime', query.runtime) if pip_requirements and runtime.startswith('python'): if isinstance(pip_requirements, bool) and path: - pip_requirements_step(path, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir')) + pip_requirements_step(path, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir'), + pre_package_commands=pre_package_commands) else: pip_requirements_step(pip_requirements, prefix, - required=True, tmp_dir=claim.get('pip_tmp_dir')) + required=True, tmp_dir=claim.get('pip_tmp_dir'), + pre_package_commands=pre_package_commands) if npm_requirements and runtime.startswith('nodejs'): if isinstance(npm_requirements, bool) and path: - npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir')) + npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir'), + pre_package_commands=pre_package_commands) else: npm_requirements_step(npm_requirements, prefix, - required=True, tmp_dir=claim.get('npm_tmp_dir')) + required=True, tmp_dir=claim.get('npm_tmp_dir'), + pre_package_commands=pre_package_commands) if path: step('zip', path, prefix) @@ -807,8 +839,8 @@ def execute(self, build_plan, zip_stream, query): else: zs.write_file(source_path, prefix=prefix, timestamp=ts) elif cmd == 'pip': - runtime, pip_requirements, prefix, tmp_dir = action[1:] - with install_pip_requirements(query, pip_requirements, tmp_dir) as rd: + runtime, pip_requirements, prefix, tmp_dir, pre_package_commands = action[1:] + with install_pip_requirements(query, pip_requirements, tmp_dir, pre_package_commands) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, @@ -817,8 +849,8 @@ def execute(self, build_plan, zip_stream, query): # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == 'npm': - runtime, npm_requirements, prefix, tmp_dir = action[1:] - with install_npm_requirements(query, npm_requirements, tmp_dir) as rd: + runtime, npm_requirements, prefix, tmp_dir, pre_package_commands = action[1:] + with install_npm_requirements(query, npm_requirements, tmp_dir, pre_package_commands) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, @@ -858,7 +890,7 @@ def _zip_write_with_filter(zip_stream, path_filter, source_path, prefix, @contextmanager -def install_pip_requirements(query, requirements_file, tmp_dir): +def install_pip_requirements(query, requirements_file, tmp_dir, pre_package_commands): # TODO: # 1. Emit files instead of temp_dir @@ -941,7 +973,13 @@ def install_pip_requirements(query, requirements_file, tmp_dir): working_dir, artifacts_dir, 'cache/pip')) chown_mask = '{}:{}'.format(os.getuid(), os.getgid()) - shell_command = [shlex_join(pip_command), '&&', + if pre_package_commands: + additional_commands = '{} &&'.format(' && '.join(pre_package_commands)) + else: + additional_commands = "" + + shell_command = [additional_commands, + shlex_join(pip_command), '&&', shlex_join(['chown', '-R', chown_mask, '.'])] shell_command = [' '.join(shell_command)] @@ -950,6 +988,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): image=docker_image_tag_id, shell=True, ssh_agent=with_ssh_agent, pip_cache_dir=pip_cache_dir, + docker_additional_options=docker.docker_additional_options, )) else: cmd_log.info(shlex_join(pip_command)) @@ -968,7 +1007,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_npm_requirements(query, requirements_file, tmp_dir): +def install_npm_requirements(query, requirements_file, tmp_dir, pre_package_commands): # TODO: # 1. Emit files instead of temp_dir @@ -1025,14 +1064,21 @@ def install_npm_requirements(query, requirements_file, tmp_dir): if docker: with_ssh_agent = docker.with_ssh_agent chown_mask = '{}:{}'.format(os.getuid(), os.getgid()) - shell_command = [shlex_join(npm_command), '&&', + if pre_package_commands: + additional_commands = '{} &&'.format(' && '.join(pre_package_commands)) + else: + additional_commands = "" + + shell_command = [additional_commands, + shlex_join(npm_command), '&&', shlex_join(['chown', '-R', chown_mask, '.'])] shell_command = [' '.join(shell_command)] check_call(docker_run_command( '.', shell_command, runtime, image=docker_image_tag_id, - shell=True, ssh_agent=with_ssh_agent + shell=True, ssh_agent=with_ssh_agent, + docker_additional_options=docker.docker_additional_options, )) else: cmd_log.info(shlex_join(npm_command)) @@ -1082,7 +1128,8 @@ def docker_build_command(tag=None, docker_file=None, build_root=False): def docker_run_command(build_root, command, runtime, image=None, shell=None, ssh_agent=False, - interactive=False, pip_cache_dir=None): + interactive=False, pip_cache_dir=None, + docker_additional_options=None): """""" if platform.system() not in ('Linux', 'Darwin'): raise RuntimeError("Unsupported platform for docker building") @@ -1103,6 +1150,9 @@ def docker_run_command(build_root, command, runtime, '-v', '{}/.ssh/known_hosts:/root/.ssh/known_hosts:z'.format(home), ]) + if docker_additional_options: + docker_cmd.extend(docker_additional_options) + if ssh_agent: if platform.system() == 'Darwin': # https://docs.docker.com/docker-for-mac/osxfs/#ssh-agent-forwarding diff --git a/package.tf b/package.tf index ca473a48..917a55c2 100644 --- a/package.tf +++ b/package.tf @@ -17,11 +17,12 @@ data "external" "archive_prepare" { }) docker = var.build_in_docker ? jsonencode({ - docker_pip_cache = var.docker_pip_cache - docker_build_root = var.docker_build_root - docker_file = var.docker_file - docker_image = var.docker_image - with_ssh_agent = var.docker_with_ssh_agent + docker_pip_cache = var.docker_pip_cache + docker_build_root = var.docker_build_root + docker_file = var.docker_file + docker_image = var.docker_image + with_ssh_agent = var.docker_with_ssh_agent + docker_additional_options = var.docker_additional_options }) : null artifacts_dir = var.artifacts_dir diff --git a/variables.tf b/variables.tf index 55f9f61d..c45803d7 100644 --- a/variables.tf +++ b/variables.tf @@ -671,6 +671,12 @@ variable "docker_pip_cache" { default = null } +variable "docker_additional_options" { + description = "Additional options to pass to the docker run command (e.g. to set environment variables, volumes, etc.)" + type = list(string) + default = [] +} + variable "recreate_missing_package" { description = "Whether to recreate missing Lambda package if it is missing locally or not" type = bool From ccf12bb27f281ba9cef102bf8244c1dfb2a07d06 Mon Sep 17 00:00:00 2001 From: Allan Benson <95691842+AllanBenson001@users.noreply.github.com> Date: Wed, 19 Oct 2022 20:53:26 +0100 Subject: [PATCH 2/4] TF Format --- examples/build-package/main.tf | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index fc0321cd..acc7097f 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -79,8 +79,8 @@ module "package_file_with_pip_requirements" { source_path = [ "${path.module}/../fixtures/python3.8-app1/index.py", { - pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" - prefix_in_zip = "vendor" + pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" + prefix_in_zip = "vendor" pre_package_commands = [ "echo I will run before the pip install command", "echo so will I", @@ -104,7 +104,7 @@ module "package_with_pip_requirements_in_docker" { "${path.module}/../fixtures/python3.8-app1/index.py", "${path.module}/../fixtures/python3.8-app1/dir1/dir2", { - pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" + pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt" pre_package_commands = [ "echo I will run before the pip install command in the docker container", "echo I can read $MY_ENV_VAR and my volume:", @@ -113,11 +113,11 @@ module "package_with_pip_requirements_in_docker" { } ] - build_in_docker = true + build_in_docker = true docker_additional_options = [ - "-e", "MY_ENV_VAR='My environment variable value'", - "-v", "${abspath(path.module)}:/my-vol:ro", - ] + "-e", "MY_ENV_VAR='My environment variable value'", + "-v", "${abspath(path.module)}:/my-vol:ro", + ] } # Create zip-archive which contains content of directory with commands and patterns applied. @@ -256,8 +256,8 @@ module "package_dir_with_npm_install_and_additional_commands" { runtime = "nodejs14.x" source_path = [ { - path = "${path.module}/../fixtures/nodejs14.x-app1" - npm_requirements = true + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = true pre_package_commands = [ "echo I will run before the npm install command", "echo so will I", @@ -301,11 +301,11 @@ module "package_with_npm_requirements_in_docker_and_additional_commands" { create_function = false - runtime = "nodejs14.x" + runtime = "nodejs14.x" source_path = [ { - path = "${path.module}/../fixtures/nodejs14.x-app1" - npm_requirements = true + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = true pre_package_commands = [ "echo I will run before the npm install command in the docker container", "echo I can read $MY_ENV_VAR and my volume:", @@ -314,13 +314,13 @@ module "package_with_npm_requirements_in_docker_and_additional_commands" { } ] - build_in_docker = true + build_in_docker = true docker_additional_options = [ - "-e", "MY_ENV_VAR='My environment variable value'", - "-v", "${abspath(path.module)}:/my-vol:ro", - ] + "-e", "MY_ENV_VAR='My environment variable value'", + "-v", "${abspath(path.module)}:/my-vol:ro", + ] - hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" + hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" } ################################ From d559578160ba843528b174b24c12e62b24e9718f Mon Sep 17 00:00:00 2001 From: Allan Benson <95691842+AllanBenson001@users.noreply.github.com> Date: Wed, 19 Oct 2022 21:16:26 +0100 Subject: [PATCH 3/4] Update docs --- README.md | 1 + examples/build-package/README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index c7617011..22676360 100644 --- a/README.md +++ b/README.md @@ -722,6 +722,7 @@ No modules. | [description](#input\_description) | Description of your Lambda Function (or Layer) | `string` | `""` | no | | [destination\_on\_failure](#input\_destination\_on\_failure) | Amazon Resource Name (ARN) of the destination resource for failed asynchronous invocations | `string` | `null` | no | | [destination\_on\_success](#input\_destination\_on\_success) | Amazon Resource Name (ARN) of the destination resource for successful asynchronous invocations | `string` | `null` | no | +| [docker\_additional\_options](#input\_docker\_additional\_options) | Additional options to pass to the docker run command (e.g. to set environment variables, volumes, etc.) | `list(string)` | `[]` | no | | [docker\_build\_root](#input\_docker\_build\_root) | Root dir where to build in Docker | `string` | `""` | no | | [docker\_file](#input\_docker\_file) | Path to a Dockerfile when building in Docker | `string` | `""` | no | | [docker\_image](#input\_docker\_image) | Docker image to use for the build | `string` | `""` | no | diff --git a/examples/build-package/README.md b/examples/build-package/README.md index 739e8b0a..0b4af884 100644 --- a/examples/build-package/README.md +++ b/examples/build-package/README.md @@ -39,6 +39,7 @@ Note that this example may create resources which cost money. Run `terraform des | [package\_dir](#module\_package\_dir) | ../../ | n/a | | [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a | | [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a | +| [package\_dir\_with\_npm\_install\_and\_additional\_commands](#module\_package\_dir\_with\_npm\_install\_and\_additional\_commands) | ../../ | n/a | | [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a | | [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a | | [package\_file](#module\_package\_file) | ../../ | n/a | @@ -46,6 +47,7 @@ Note that this example may create resources which cost money. Run `terraform des | [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a | | [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a | | [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a | +| [package\_with\_npm\_requirements\_in\_docker\_and\_additional\_commands](#module\_package\_with\_npm\_requirements\_in\_docker\_and\_additional\_commands) | ../../ | n/a | | [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a | | [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a | From 0984afa5afca3167dbdf8949da7f4613b9fe691a Mon Sep 17 00:00:00 2001 From: Allan Benson <95691842+AllanBenson001@users.noreply.github.com> Date: Thu, 20 Oct 2022 17:26:19 +0100 Subject: [PATCH 4/4] Fix issues introduced by #358/#359 --- package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.py b/package.py index 0d551820..556d779f 100644 --- a/package.py +++ b/package.py @@ -657,7 +657,7 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_p raise RuntimeError( 'File not found: {}'.format(requirements)) else: - if not shutil.which(runtime): + if not query.docker and not shutil.which(runtime): raise RuntimeError( "Python interpreter version equal " "to defined lambda runtime ({}) should be " @@ -679,7 +679,7 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_p raise RuntimeError( 'File not found: {}'.format(requirements)) else: - if not shutil.which(runtime): + if not query.docker and not shutil.which(runtime): raise RuntimeError( "Nodejs interpreter version equal " "to defined lambda runtime ({}) should be "