From b01f2ee5f4f526ba00af1b1e81658eed7ff9bca3 Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Wed, 1 Sep 2021 12:35:41 -0700 Subject: [PATCH 1/7] wip --- Dockerfile | 10 +++++----- requirements.txt | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7067f11..4cfa6ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ -FROM amazonlinux:2018.03 +FROM amazonlinux:2 # Need to set "ulimit -n" to a small value to stop yum from hanging: # https://bugzilla.redhat.com/show_bug.cgi?id=1715254#c1 RUN ulimit -n 1024 && yum -y update && yum -y install \ git \ gcc \ - python36 \ - python36-pip \ - python36-devel \ + python38 \ + python38-pip \ + python38-devel \ jq \ nano \ unzip \ @@ -16,7 +16,7 @@ RUN ulimit -n 1024 && yum -y update && yum -y install \ COPY requirements.txt quilt/requirements.txt -RUN python3 -m pip install pip==18.1 +RUN python3 -m pip install pip==21.1.1 # Requirements copied from lambda Python 3.6, but not in base image # (Plus Cython which is a build-time requirement for numpy) diff --git a/requirements.txt b/requirements.txt index a709c7a..8d28e83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ -# from `pip freeze` under Python 3.6 lambda image 16-Apr-2020 -boto3==1.12.22 -botocore==1.15.22 -Cython==0.29.21 -docutils==0.15.2 -jmespath==0.9.5 -python-dateutil==2.8.1 -s3transfer==0.3.3 -six==1.14.0 -urllib3==1.25.8 +# from `pip freeze` under Python 3.8 lambda image 01-Sep-2021 +boto3==1.17.100 +botocore==1.20.100 +Cython==0.29.24 +jmespath==0.10.0 +python-dateutil==2.8.2 +rapid-client==0.0.0 +s3transfer==0.4.2 +six==1.16.0 +urllib3==1.26.6 + From a739c4d95704ace8ce06459852a966aa0c2e86a0 Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Wed, 1 Sep 2021 14:39:53 -0700 Subject: [PATCH 2/7] restore 38-devel, pip --- Dockerfile | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4cfa6ba..ee7d1e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,22 @@ FROM amazonlinux:2 # Need to set "ulimit -n" to a small value to stop yum from hanging: # https://bugzilla.redhat.com/show_bug.cgi?id=1715254#c1 -RUN ulimit -n 1024 && yum -y update && yum -y install \ - git \ - gcc \ - python38 \ - python38-pip \ - python38-devel \ - jq \ - nano \ - unzip \ - zip \ - && yum clean all +# See the following for installing 3.8 via yum on amazon-linux +# https://techviewleo.com/how-to-install-python-on-amazon-linux/ +RUN ulimit -n 1024 \ + && amazon-linux-extras enable python3.8 && yum clean metadata \ + && yum -y update && yum -y install \ + git \ + gcc \ + python38 \ + python38-devel \ + jq \ + nano \ + unzip \ + zip \ + && yum clean all + +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 COPY requirements.txt quilt/requirements.txt From 6f415f1f8b44b88a8dfadbc7e6a3f527505305ee Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Wed, 1 Sep 2021 14:53:22 -0700 Subject: [PATCH 3/7] Update docs --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af6f9bb..60d2ece 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,13 @@ is clumsy linked from [this AWS user guide](https://docs.aws.amazon.com/AmazonECR/latest/userguide/amazon_linux_container_image.html)) ## [Tags](https://hub.docker.com/r/quiltdata/lambda/tags) -* `build` - introduced to build wheels per [#2](https://github.com/quiltdata/lambda/pull/2). In theory this should not affect how the environment functions, but in practice use image id `823205fae3ed` if you want to be as close to the official lambda environment as possible. +* `build` - introduced to build wheels per [#2](https://github.com/quiltdata/lambda/pull/2). +In theory this should not affect how the environment functions, +but in practice use image id `823205fae3ed` +if you want to be as close to the official lambda environment as possible. +On 9/1/21 the `build` tag has been updated to support the Python3.8 runtime (only). -## Example: Python 3.6 deployment package +## Example: Python 3.8 deployment package ```sh docker pull quiltdata/lambda From 1f09fffb5ed15c5a68899710ae40fd8e02b0bfd4 Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Thu, 2 Sep 2021 14:09:03 -0700 Subject: [PATCH 4/7] Use aws lambda image --- Dockerfile | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee7d1e0..8f6d9c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,19 @@ -FROM amazonlinux:2 - +FROM amazon/aws-lambda-python:3.8 # Need to set "ulimit -n" to a small value to stop yum from hanging: # https://bugzilla.redhat.com/show_bug.cgi?id=1715254#c1 # See the following for installing 3.8 via yum on amazon-linux # https://techviewleo.com/how-to-install-python-on-amazon-linux/ -RUN ulimit -n 1024 \ - && amazon-linux-extras enable python3.8 && yum clean metadata \ - && yum -y update && yum -y install \ +RUN ulimit -n 1024 && yum -y update && yum -y install \ git \ gcc \ - python38 \ - python38-devel \ jq \ nano \ unzip \ zip \ && yum clean all -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 - COPY requirements.txt quilt/requirements.txt -RUN python3 -m pip install pip==21.1.1 - # Requirements copied from lambda Python 3.6, but not in base image # (Plus Cython which is a build-time requirement for numpy) RUN python3 -m pip install -r quilt/requirements.txt From a1d27121d37d7ce60d60135ac3e8623821a77d1d Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Thu, 2 Sep 2021 14:10:32 -0700 Subject: [PATCH 5/7] Revise README to point to amazon/ images --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 60d2ece..a42d3d8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Build AWS Lambda deployment packages with Docker +## Update +Amazon now provides images for Lambda runtimes on Docker Hub. +See [amazon/aws-lambda-python](https://hub.docker.com/r/amazon/aws-lambda-python). + ## Blog post (includes tips on reducing package size) [An easier way to build AWS Lambda deployment packages — with Docker instead of EC2](https://blog.quiltdata.com/an-easier-way-to-build-lambda-deployment-packages-with-docker-instead-of-ec2-9050cd486ba8) From 6c532c8faaf72d9450be80cfae4f4a09d4a0697f Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Fri, 3 Sep 2021 11:02:03 -0700 Subject: [PATCH 6/7] Clean whitespace, comments --- Dockerfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8f6d9c8..aeb1ef2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,15 @@ FROM amazon/aws-lambda-python:3.8 + # Need to set "ulimit -n" to a small value to stop yum from hanging: # https://bugzilla.redhat.com/show_bug.cgi?id=1715254#c1 -# See the following for installing 3.8 via yum on amazon-linux -# https://techviewleo.com/how-to-install-python-on-amazon-linux/ RUN ulimit -n 1024 && yum -y update && yum -y install \ - git \ - gcc \ - jq \ - nano \ - unzip \ - zip \ - && yum clean all + git \ + gcc \ + jq \ + nano \ + unzip \ + zip \ + && yum clean all COPY requirements.txt quilt/requirements.txt From 002ff79b4594c31f59c48038685781476fe1d234 Mon Sep 17 00:00:00 2001 From: Aneesh Karve Date: Fri, 3 Sep 2021 17:56:40 -0700 Subject: [PATCH 7/7] Remove sus package (even though AWS Lambda installs it) --- Dockerfile | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f6d9c8..6de596a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,3 +21,4 @@ RUN python3 -m pip install -r quilt/requirements.txt # Make it possible to build numpy: # https://github.com/numpy/numpy/issues/14147 ENV CFLAGS=-std=c99 + diff --git a/requirements.txt b/requirements.txt index 8d28e83..6529e49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ botocore==1.20.100 Cython==0.29.24 jmespath==0.10.0 python-dateutil==2.8.2 -rapid-client==0.0.0 s3transfer==0.4.2 six==1.16.0 urllib3==1.26.6