feat: Provide pinned requirements to allow for reproducible builds

Makefile

@@ -32,10 +32,15 @@ lint:
 dev: lint test
 
 black:
-	black samcli/* tests/*
+	black samcli/* tests/* scripts/*
 
 black-check:
-	black --check samcli/* tests/*
+	black --check samcli/* tests/* scripts/*
 
 # Verifications to run before sending a pull request
-pr: init dev black-check
+pr: init dev black-check
+
+update-isolated-req:
+	pipenv --three
+	pipenv run pip install -r requirements/base.txt
+	pipenv run pip freeze > requirements/isolated.txt

appveyor.yml

@@ -81,7 +81,8 @@ test_script:
 
     # Runs only in Linux
     - sh: "pytest -vv tests/integration"
-    - sh: "/tmp/black --check setup.py tests samcli"
+    - sh: "/tmp/black --check setup.py tests samcli scripts"
+    - sh: "python scripts/check-isolated-needs-update.py"
 
     # Smoke tests run in parallel - it runs on both Linux & Windows
     # Presence of the RUN_SMOKE envvar will run the smoke tests

requirements/isolated.txt

@@ -0,0 +1,37 @@
+arrow==0.14.6
+aws-lambda-builders==0.4.0
+aws-sam-translator==1.11.0
+binaryornot==0.4.4
+boto3==1.9.220
+botocore==1.12.220
+certifi==2019.6.16
+chardet==3.0.4
+chevron==0.13.1
+Click==7.0
+cookiecutter==1.6.0
+dateparser==0.7.1
+docker==4.0.2
+docutils==0.15.2
+Flask==1.0.4
+future==0.17.1
+idna==2.8
+itsdangerous==1.1.0
+Jinja2==2.10.1
+jinja2-time==0.2.0
+jmespath==0.9.4
+jsonschema==2.6.0
+MarkupSafe==1.1.1
+poyo==0.5.0
+python-dateutil==2.8.0
+pytz==2019.2
+PyYAML==5.1.2
+regex==2019.8.19
+requests==2.22.0
+s3transfer==0.2.1
+serverlessrepo==0.1.9
+six==1.11.0
+tzlocal==2.0.0
+urllib3==1.25.3
+websocket-client==0.56.0
+Werkzeug==0.15.5
+whichcraft==0.6.0

scripts/check-isolated-needs-update.py

@@ -0,0 +1,58 @@
+import io
+import sys
+import os
+from subprocess import Popen, PIPE
+
+
+def read(*filenames, **kwargs):
+    encoding = kwargs.get("encoding", "utf-8")
+    sep = kwargs.get("sep", os.linesep)
+    buf = []
+    for filename in filenames:
+        with io.open(filename, encoding=encoding) as f:
+            buf.append(f.read())
+    return sep.join(buf)
+
+
+def get_requirements_list(content):
+    pkgs_versions = []
+    for line in content.split(os.linesep):
+        if line:
+            # remove markers from the line, which are seperated by ';'
+            pkgs_versions.append(line.split(";")[0])
+
+    return pkgs_versions
+
+
+# Don't try and compare the isolated list with the Python2 version. SAM CLI installers
+# all use Python3.6+ and Python2.7 is going EOL
+if sys.version_info[0] < 3:
+    sys.exit(0)
+
+isolated_req_content = read(os.path.join("requirements", "isolated.txt"))
+base_req_content = read(os.path.join("requirements", "base.txt"))
+
+isolated_req_list = get_requirements_list(isolated_req_content)
+base_req_list = get_requirements_list(base_req_content)
+
+process = Popen(["pip", "freeze"], stdout=PIPE)
+
+all_installed_pkgs_list = []
+for package in process.stdout.readlines():
+    package = package.decode("utf-8").strip(os.linesep)
+    all_installed_pkgs_list.append(package)
+
+for installed_pkg_version in all_installed_pkgs_list:
+    for base_req in base_req_list:
+        # a base requirement can be defined with different specifiers (>, <, ==, etc.). Instead of doing tons of string parsing,
+        # brute force the check by assuming the installed_pkgs will have == as a specifier. This is true due to how pip freeze
+        # works. So check to make sure the installed pakcage we are looking at is in the base.txt file, if so make sure the
+        # full requirement==version is within the isolated list.
+        installed_pkg = installed_pkg_version.split("==")[0]
+        # There is a py library we use but due to how we are comparing requirements, we need to handle this as a special case. :(
+        if installed_pkg not in ("py", "boto3") and base_req.startswith(installed_pkg):
+            assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format(
+                installed_pkg_version
+            )
+            print ("{} is in the isolated.txt file".format(installed_pkg_version))
+            break

scripts/check-requirements.py

@@ -0,0 +1,43 @@
+import io
+import os
+from subprocess import Popen, PIPE
+
+
+def read(*filenames, **kwargs):
+    encoding = kwargs.get("encoding", "utf-8")
+    sep = kwargs.get("sep", os.linesep)
+    buf = []
+    for filename in filenames:
+        with io.open(filename, encoding=encoding) as f:
+            buf.append(f.read())
+    return sep.join(buf)
+
+
+exclude_packages = ("setuptools", "wheel", "pip", "aws-sam-cli")
+
+all_pkgs_list = []
+process = Popen(["pip", "freeze"], stdout=PIPE)
+
+for package in process.stdout.readlines():
+    package = package.decode("utf-8").strip(os.linesep)
+    if package.split("==")[0] not in exclude_packages:
+        all_pkgs_list.append(package)
+all_pkgs_list = sorted(all_pkgs_list)
+print ("installed package/versions" + os.linesep)
+print (",".join(all_pkgs_list))
+print (os.linesep)
+
+content = read(os.path.join("requirements", "isolated.txt"))
+
+locked_pkgs = []
+for line in content.split(os.linesep):
+    if line:
+        locked_pkgs.append(line)
+
+locked_pkgs = sorted(locked_pkgs)
+print ("locked package/versions" + os.linesep)
+print (",".join(locked_pkgs))
+print (os.linesep)
+
+assert len(locked_pkgs) == len(all_pkgs_list), "Number of expected dependencies do not match the number installed"
+assert locked_pkgs == all_pkgs_list, "The list of expected dependencies do not match what is installed"