feat: Add support for additional Docker options and pre pip/npm commands

README.md

@@ -455,6 +455,7 @@ source_path = [
 - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead.
 - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds.
 - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive.
+- `pre_package_commands` - List of commands to run before doing a `pip install` or `npm install`. Works in conjunction with `pip_requirements` and `npm_requirements`.
 
 ### Building in Docker
 
@@ -470,6 +471,17 @@ Using this module you can install dependencies from private hosts. To do this, y
 
     docker_with_ssh_agent = true
 
+#### Passing additional Docker options
+
+To add flexibility when building in docker, you can pass any number of additional options that you require (see [Docker run reference](https://docs.docker.com/engine/reference/run/) for available options):
+
+```hcl
+  docker_additional_options = [
+        "-e", "MY_ENV_VAR='My environment variable value'",
+        "-v", "/local:/docker-vol",
+  ] 
+```
+
 ## <a name="package"></a> Deployment package - Create or use existing
 
 By default, this module creates deployment package and uses it to create or update Lambda Function or Lambda Layer.
@@ -710,6 +722,7 @@ No modules.
 | <a name="input_description"></a> [description](#input\_description) | Description of your Lambda Function (or Layer) | `string` | `""` | no |
 | <a name="input_destination_on_failure"></a> [destination\_on\_failure](#input\_destination\_on\_failure) | Amazon Resource Name (ARN) of the destination resource for failed asynchronous invocations | `string` | `null` | no |
 | <a name="input_destination_on_success"></a> [destination\_on\_success](#input\_destination\_on\_success) | Amazon Resource Name (ARN) of the destination resource for successful asynchronous invocations | `string` | `null` | no |
+| <a name="input_docker_additional_options"></a> [docker\_additional\_options](#input\_docker\_additional\_options) | Additional options to pass to the docker run command (e.g. to set environment variables, volumes, etc.) | `list(string)` | `[]` | no |
 | <a name="input_docker_build_root"></a> [docker\_build\_root](#input\_docker\_build\_root) | Root dir where to build in Docker | `string` | `""` | no |
 | <a name="input_docker_file"></a> [docker\_file](#input\_docker\_file) | Path to a Dockerfile when building in Docker | `string` | `""` | no |
 | <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Docker image to use for the build | `string` | `""` | no |

examples/build-package/README.md

@@ -39,13 +39,15 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_package_dir"></a> [package\_dir](#module\_package\_dir) | ../../ | n/a |
 | <a name="module_package_dir_pip_dir"></a> [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a |
 | <a name="module_package_dir_with_npm_install"></a> [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a |
+| <a name="module_package_dir_with_npm_install_and_additional_commands"></a> [package\_dir\_with\_npm\_install\_and\_additional\_commands](#module\_package\_dir\_with\_npm\_install\_and\_additional\_commands) | ../../ | n/a |
 | <a name="module_package_dir_without_npm_install"></a> [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a |
 | <a name="module_package_dir_without_pip_install"></a> [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a |
 | <a name="module_package_file"></a> [package\_file](#module\_package\_file) | ../../ | n/a |
 | <a name="module_package_file_with_pip_requirements"></a> [package\_file\_with\_pip\_requirements](#module\_package\_file\_with\_pip\_requirements) | ../../ | n/a |
 | <a name="module_package_with_commands_and_patterns"></a> [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a |
 | <a name="module_package_with_docker"></a> [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a |
 | <a name="module_package_with_npm_requirements_in_docker"></a> [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a |
+| <a name="module_package_with_npm_requirements_in_docker_and_additional_commands"></a> [package\_with\_npm\_requirements\_in\_docker\_and\_additional\_commands](#module\_package\_with\_npm\_requirements\_in\_docker\_and\_additional\_commands) | ../../ | n/a |
 | <a name="module_package_with_patterns"></a> [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a |
 | <a name="module_package_with_pip_requirements_in_docker"></a> [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a |
 

examples/build-package/main.tf

@@ -81,6 +81,10 @@ module "package_file_with_pip_requirements" {
     {
       pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt"
       prefix_in_zip    = "vendor"
+      pre_package_commands = [
+        "echo I will run before the pip install command",
+        "echo so will I",
+      ]
     }
   ]
 }
@@ -101,10 +105,19 @@ module "package_with_pip_requirements_in_docker" {
     "${path.module}/../fixtures/python3.8-app1/dir1/dir2",
     {
       pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt"
+      pre_package_commands = [
+        "echo I will run before the pip install command in the docker container",
+        "echo I can read $MY_ENV_VAR and my volume:",
+        "ls -la /my-vol",
+      ]
     }
   ]
 
   build_in_docker = true
+  docker_additional_options = [
+    "-e", "MY_ENV_VAR='My environment variable value'",
+    "-v", "${abspath(path.module)}:/my-vol:ro",
+  ]
 }
 
 # Create zip-archive which contains content of directory with commands and patterns applied.
@@ -233,6 +246,26 @@ module "package_dir_with_npm_install" {
   source_path = "${path.module}/../fixtures/nodejs14.x-app1"
 }
 
+# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime)
+# Also run some additional commands before the npm install
+module "package_dir_with_npm_install_and_additional_commands" {
+  source = "../../"
+
+  create_function = false
+
+  runtime = "nodejs14.x"
+  source_path = [
+    {
+      path             = "${path.module}/../fixtures/nodejs14.x-app1"
+      npm_requirements = true
+      pre_package_commands = [
+        "echo I will run before the npm install command",
+        "echo so will I",
+      ]
+    }
+  ]
+}
+
 # Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime)
 module "package_dir_without_npm_install" {
   source = "../../"
@@ -261,6 +294,35 @@ module "package_with_npm_requirements_in_docker" {
   hash_extra      = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
 }
 
+# Create zip-archive of a single directory where "npm install" will also be executed using docker
+# Also set additional docker run options and also run some commands before the npm install
+module "package_with_npm_requirements_in_docker_and_additional_commands" {
+  source = "../../"
+
+  create_function = false
+
+  runtime = "nodejs14.x"
+  source_path = [
+    {
+      path             = "${path.module}/../fixtures/nodejs14.x-app1"
+      npm_requirements = true
+      pre_package_commands = [
+        "echo I will run before the npm install command in the docker container",
+        "echo I can read $MY_ENV_VAR and my volume:",
+        "ls -l /my-vol",
+      ]
+    }
+  ]
+
+  build_in_docker = true
+  docker_additional_options = [
+    "-e", "MY_ENV_VAR='My environment variable value'",
+    "-v", "${abspath(path.module)}:/my-vol:ro",
+  ]
+
+  hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
+}
+
 ################################
 # Build package in Docker and
 # use it to deploy Lambda Layer

package.py

@@ -648,7 +648,7 @@ def plan(self, source_path, query):
         step = lambda *x: build_plan.append(x)
         hash = source_paths.append
 
-        def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
+        def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_package_commands=None):
             requirements = path
             if os.path.isdir(path):
                 requirements = os.path.join(path, 'requirements.txt')
@@ -657,16 +657,20 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
                     raise RuntimeError(
                         'File not found: {}'.format(requirements))
             else:
-                if not shutil.which(runtime):
+                if not query.docker and not shutil.which(runtime):
                     raise RuntimeError(
                         "Python interpreter version equal "
                         "to defined lambda runtime ({}) should be "
                         "available in system PATH".format(runtime))
 
-                step('pip', runtime, requirements, prefix, tmp_dir)
+                if pre_package_commands and not query.docker:
+                    additional_commands_step(path, pre_package_commands)
+                    pre_package_commands = None
+
+                step('pip', runtime, requirements, prefix, tmp_dir, pre_package_commands)
                 hash(requirements)
 
-        def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
+        def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None, pre_package_commands=None):
             requirements = path
             if os.path.isdir(path):
                 requirements = os.path.join(path, 'package.json')
@@ -675,15 +679,36 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
                     raise RuntimeError(
                         'File not found: {}'.format(requirements))
             else:
-                if not shutil.which(runtime):
+                if not query.docker and not shutil.which(runtime):
                     raise RuntimeError(
                         "Nodejs interpreter version equal "
                         "to defined lambda runtime ({}) should be "
                         "available in system PATH".format(runtime))
 
-                step('npm', runtime, requirements, prefix, tmp_dir)
+                if pre_package_commands and not query.docker:
+                    additional_commands_step(path, pre_package_commands)
+                    pre_package_commands = None
+
+                step('npm', runtime, requirements, prefix, tmp_dir, pre_package_commands)
                 hash(requirements)
 
+        def additional_commands_step(path, commands):
+            if not commands:
+                return
+
+            if isinstance(commands, str):
+                commands = map(str.strip, commands.splitlines())
+
+            if path:
+                path = os.path.normpath(path)
+                if os.path.isfile(path):
+                    path = os.path.dirname(path)
+            else:
+                path = query.paths.cwd
+
+            hash(path)
+            step('sh', path, '\n'.join(commands))
+
         def commands_step(path, commands):
             if not commands:
                 return
@@ -759,21 +784,30 @@ def commands_step(path, commands):
                     prefix = claim.get('prefix_in_zip')
                     pip_requirements = claim.get('pip_requirements')
                     npm_requirements = claim.get('npm_package_json')
+                    if not npm_requirements:
+                        # npm_package_json is an undocumented setting!!!!
+                        # Set correct value here rather than create breaking change
+                        npm_requirements = claim.get('npm_requirements')
+                    pre_package_commands = claim.get('pre_package_commands')
                     runtime = claim.get('runtime', query.runtime)
 
                     if pip_requirements and runtime.startswith('python'):
                         if isinstance(pip_requirements, bool) and path:
-                            pip_requirements_step(path, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir'))
+                            pip_requirements_step(path, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir'),
+                                                  pre_package_commands=pre_package_commands)
                         else:
                             pip_requirements_step(pip_requirements, prefix,
-                                                  required=True, tmp_dir=claim.get('pip_tmp_dir'))
+                                                  required=True, tmp_dir=claim.get('pip_tmp_dir'),
+                                                  pre_package_commands=pre_package_commands)
 
                     if npm_requirements and runtime.startswith('nodejs'):
                         if isinstance(npm_requirements, bool) and path:
-                            npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir'))
+                            npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir'),
+                                                  pre_package_commands=pre_package_commands)
                         else:
                             npm_requirements_step(npm_requirements, prefix,
-                                                  required=True, tmp_dir=claim.get('npm_tmp_dir'))
+                                                  required=True, tmp_dir=claim.get('npm_tmp_dir'),
+                                                  pre_package_commands=pre_package_commands)
 
                     if path:
                         step('zip', path, prefix)
@@ -819,8 +853,8 @@ def execute(self, build_plan, zip_stream, query):
                 else:
                     zs.write_file(source_path, prefix=prefix, timestamp=ts)
             elif cmd == 'pip':
-                runtime, pip_requirements, prefix, tmp_dir = action[1:]
-                with install_pip_requirements(query, pip_requirements, tmp_dir) as rd:
+                runtime, pip_requirements, prefix, tmp_dir, pre_package_commands = action[1:]
+                with install_pip_requirements(query, pip_requirements, tmp_dir, pre_package_commands) as rd:
                     if rd:
                         if pf:
                             self._zip_write_with_filter(zs, pf, rd, prefix,
@@ -829,8 +863,8 @@ def execute(self, build_plan, zip_stream, query):
                             # XXX: timestamp=0 - what actually do with it?
                             zs.write_dirs(rd, prefix=prefix, timestamp=0)
             elif cmd == 'npm':
-                runtime, npm_requirements, prefix, tmp_dir = action[1:]
-                with install_npm_requirements(query, npm_requirements, tmp_dir) as rd:
+                runtime, npm_requirements, prefix, tmp_dir, pre_package_commands = action[1:]
+                with install_npm_requirements(query, npm_requirements, tmp_dir, pre_package_commands) as rd:
                     if rd:
                         if pf:
                             self._zip_write_with_filter(zs, pf, rd, prefix,
@@ -870,7 +904,7 @@ def _zip_write_with_filter(zip_stream, path_filter, source_path, prefix,
 
 
 @contextmanager
-def install_pip_requirements(query, requirements_file, tmp_dir):
+def install_pip_requirements(query, requirements_file, tmp_dir, pre_package_commands):
     # TODO:
     #  1. Emit files instead of temp_dir
 
@@ -953,7 +987,13 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
                             working_dir, artifacts_dir, 'cache/pip'))
 
                 chown_mask = '{}:{}'.format(os.getuid(), os.getgid())
-                shell_command = [shlex_join(pip_command), '&&',
+                if pre_package_commands:
+                    additional_commands = '{} &&'.format(' && '.join(pre_package_commands))
+                else:
+                    additional_commands = ""
+
+                shell_command = [additional_commands,
+                                 shlex_join(pip_command), '&&',
                                  shlex_join(['chown', '-R',
                                              chown_mask, '.'])]
                 shell_command = [' '.join(shell_command)]
@@ -962,6 +1002,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
                     image=docker_image_tag_id,
                     shell=True, ssh_agent=with_ssh_agent,
                     pip_cache_dir=pip_cache_dir,
+                    docker_additional_options=docker.docker_additional_options,
                 ))
             else:
                 cmd_log.info(shlex_join(pip_command))
@@ -980,7 +1021,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
 
 
 @contextmanager
-def install_npm_requirements(query, requirements_file, tmp_dir):
+def install_npm_requirements(query, requirements_file, tmp_dir, pre_package_commands):
     # TODO:
     #  1. Emit files instead of temp_dir
 
@@ -1037,14 +1078,21 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
             if docker:
                 with_ssh_agent = docker.with_ssh_agent
                 chown_mask = '{}:{}'.format(os.getuid(), os.getgid())
-                shell_command = [shlex_join(npm_command), '&&',
+                if pre_package_commands:
+                    additional_commands = '{} &&'.format(' && '.join(pre_package_commands))
+                else:
+                    additional_commands = ""
+
+                shell_command = [additional_commands,
+                                    shlex_join(npm_command), '&&',
                                     shlex_join(['chown', '-R',
                                                 chown_mask, '.'])]
                 shell_command = [' '.join(shell_command)]
                 check_call(docker_run_command(
                     '.', shell_command, runtime,
                     image=docker_image_tag_id,
-                    shell=True, ssh_agent=with_ssh_agent
+                    shell=True, ssh_agent=with_ssh_agent,
+                    docker_additional_options=docker.docker_additional_options,
                 ))
             else:
                 cmd_log.info(shlex_join(npm_command))
@@ -1094,7 +1142,8 @@ def docker_build_command(tag=None, docker_file=None, build_root=False):
 
 def docker_run_command(build_root, command, runtime,
                        image=None, shell=None, ssh_agent=False,
-                       interactive=False, pip_cache_dir=None):
+                       interactive=False, pip_cache_dir=None,
+                       docker_additional_options=None):
     """"""
     if platform.system() not in ('Linux', 'Darwin'):
         raise RuntimeError("Unsupported platform for docker building")
@@ -1115,6 +1164,9 @@ def docker_run_command(build_root, command, runtime,
         '-v', '{}/.ssh/known_hosts:/root/.ssh/known_hosts:z'.format(home),
     ])
 
+    if docker_additional_options:
+        docker_cmd.extend(docker_additional_options)
+
     if ssh_agent:
         if platform.system() == 'Darwin':
             # https://docs.docker.com/docker-for-mac/osxfs/#ssh-agent-forwarding

package.tf

@@ -17,11 +17,12 @@ data "external" "archive_prepare" {
     })
 
     docker = var.build_in_docker ? jsonencode({
-      docker_pip_cache  = var.docker_pip_cache
-      docker_build_root = var.docker_build_root
-      docker_file       = var.docker_file
-      docker_image      = var.docker_image
-      with_ssh_agent    = var.docker_with_ssh_agent
+      docker_pip_cache          = var.docker_pip_cache
+      docker_build_root         = var.docker_build_root
+      docker_file               = var.docker_file
+      docker_image              = var.docker_image
+      with_ssh_agent            = var.docker_with_ssh_agent
+      docker_additional_options = var.docker_additional_options
     }) : null
 
     artifacts_dir = var.artifacts_dir