Add backend build system to support source builds

Changes:
 * Add python interface for building from source under
   backends/build_system
 * Add autotools interface for calling the python build backend
 * Add requirements/lock files under requirements/ directory needed
   for building form source.
 * Add script for regenerating the requirements lock files from
   scratch.
 * add .gitignore entries for autotools

.changes/next-release/feature-SourceDistribution-54150.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "Source Distribution",
+  "description": "Add supported autotools interface for building from source."
+}

.github/workflows/source-dist-tests.yml

@@ -0,0 +1,33 @@
+
+name: Run source distribution tests
+
+on:
+  push:
+  pull_request:
+    branches-ignore: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python scripts/ci/install
+        python scripts/ci/install-build-system
+        python -m pip freeze --all
+    - name: Run build-system tests
+      run: |
+        pip uninstall -y awscli
+        python scripts/ci/run-build-system-tests

.gitignore

@@ -54,3 +54,27 @@ dist/
 
 # autocomplete index
 awscli/data/ac.index
+
+# Build system files
+Makefile
+config.log
+config.status
+
+# autoconf
+autom4te.cache
+/autoscan.log
+/autoscan-*.log
+/aclocal.m4
+/compile
+/config.cache
+/config.guess
+/config.h.in
+/config.log
+/config.status
+/config.sub
+/configure
+/configure.scan
+/depcomp
+/install-sh
+/missing
+/stamp-h1

Makefile.in

@@ -0,0 +1,41 @@
+prefix = @prefix@
+exec_prefix = @exec_prefix@
+bindir = @bindir@
+libdir = @libdir@
+aws_cli_builddir = $(builddir)/build
+build_backend = $(srcdir)/backends/build_system
+
+builddir = @builddir@
+srcdir = @srcdir@
+VPATH = @srcdir@
+
+PYTHON = @PYTHON@
+
+INSTALL_TYPE = @INSTALL_TYPE@
+DOWNLOAD_DEPS_FLAG = @DOWNLOAD_DEPS_FLAG@
+
+all: build
+
+build:
+	"$(PYTHON)" "$(build_backend)" \
+		build \
+		--artifact "$(INSTALL_TYPE)" \
+		--build-dir "$(aws_cli_builddir)" $(DOWNLOAD_DEPS_FLAG)
+
+clean:
+	rm -rf "$(aws_cli_builddir)"
+
+install:
+	"$(PYTHON)" "$(build_backend)" \
+		install \
+		--build-dir "$(aws_cli_builddir)" \
+		--lib-dir "$(DESTDIR)$(libdir)" \
+		--bin-dir "$(DESTDIR)$(bindir)"
+
+uninstall:
+	"$(PYTHON)" "$(build_backend)" \
+		uninstall \
+		--lib-dir "$(DESTDIR)$(libdir)" \
+		--bin-dir "$(DESTDIR)$(bindir)"
+
+.PHONY: all build install uninstall

backends/build_system/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.

backends/build_system/__main__.py

@@ -0,0 +1,125 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import argparse
+import os
+import shutil
+from awscli_venv import AwsCliVenv
+from constants import (
+    ArtifactType,
+    BUILD_DIR,
+    INSTALL_DIRNAME,
+)
+from exe import ExeBuilder
+from install import (
+    Installer,
+    Uninstaller,
+)
+from validate_env import validate_env
+
+
+def create_exe(aws_venv, build_dir):
+    exe_workspace = os.path.join(build_dir, "exe")
+    if os.path.exists(exe_workspace):
+        shutil.rmtree(exe_workspace)
+    builder = ExeBuilder(exe_workspace, aws_venv)
+    builder.build()
+
+
+def build(parsed_args):
+    aws_venv = _bootstap_venv(
+        parsed_args.build_dir,
+        parsed_args.artifact,
+        parsed_args.download_deps,
+    )
+    if parsed_args.artifact == ArtifactType.PORTABLE_EXE.value:
+        create_exe(aws_venv, parsed_args.build_dir)
+
+
+def validate(parsed_args):
+    validate_env(parsed_args.artifact)
+
+
+def install(parsed_args):
+    build_dir = parsed_args.build_dir
+    install_dir = os.path.join(parsed_args.lib_dir, INSTALL_DIRNAME)
+    bin_dir = parsed_args.bin_dir
+    installer = Installer(build_dir)
+    installer.install(install_dir, bin_dir)
+
+
+def uninstall(parsed_args):
+    install_dir = os.path.join(parsed_args.lib_dir, INSTALL_DIRNAME)
+    bin_dir = parsed_args.bin_dir
+    uninstaller = Uninstaller()
+    uninstaller.uninstall(install_dir, bin_dir)
+
+
+def _bootstap_venv(build_dir: str, artifact_type: str, download_deps: bool):
+    venv_path = os.path.join(build_dir, "venv")
+    if os.path.exists(venv_path):
+        shutil.rmtree(venv_path)
+    os.makedirs(venv_path)
+    aws_venv = AwsCliVenv(venv_path)
+    aws_venv.create()
+    aws_venv.bootstrap(artifact_type, download_deps)
+    return aws_venv
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    subparser = parser.add_subparsers()
+
+    validate_env_parser = subparser.add_parser("validate-env")
+    validate_env_parser.add_argument(
+        "--artifact", choices=[e.value for e in ArtifactType], required=True
+    )
+    validate_env_parser.set_defaults(func=validate)
+
+    build_parser = subparser.add_parser("build")
+    build_parser.add_argument(
+        "--artifact", choices=[e.value for e in ArtifactType], required=True
+    )
+    build_parser.add_argument(
+        "--build-dir", default=BUILD_DIR, type=os.path.abspath
+    )
+    build_parser.add_argument("--download-deps", action="store_true")
+    build_parser.set_defaults(func=build)
+
+    install_parser = subparser.add_parser("install")
+    install_parser.add_argument(
+        "--build-dir", default=BUILD_DIR, type=os.path.abspath
+    )
+    install_parser.add_argument(
+        "--lib-dir", required=True, type=os.path.abspath
+    )
+    install_parser.add_argument(
+        "--bin-dir", required=True, type=os.path.abspath
+    )
+    install_parser.set_defaults(func=install)
+
+    uninstall_parser = subparser.add_parser("uninstall")
+    uninstall_parser.add_argument(
+        "--lib-dir", required=True, type=os.path.abspath
+    )
+    uninstall_parser.add_argument(
+        "--bin-dir", required=True, type=os.path.abspath
+    )
+    uninstall_parser.set_defaults(func=uninstall)
+
+    parsed_args = parser.parse_args()
+    parsed_args.func(parsed_args)
+
+
+if __name__ == "__main__":
+    main()

backends/build_system/awscli_venv.py

@@ -0,0 +1,139 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+import subprocess
+import site
+import sys
+import pathlib
+
+from constants import (
+    ArtifactType,
+    DOWNLOAD_DEPS_BOOTSTRAP_LOCK,
+    PORTABLE_EXE_REQUIREMENTS_LOCK,
+    SYSTEM_SANDBOX_REQUIREMENTS_LOCK,
+    ROOT_DIR,
+    IS_WINDOWS,
+    BIN_DIRNAME,
+    PYTHON_EXE_NAME,
+    CLI_SCRIPTS,
+)
+from utils import Utils
+
+
+class AwsCliVenv:
+    _PARENT_SCRIPTS_TO_COPY = [
+        "pyinstaller",
+    ]
+
+    def __init__(self, venv_dir: str, utils: Utils = None):
+        self._venv_dir = venv_dir
+        if utils is None:
+            utils = Utils()
+        self._utils = utils
+
+    def create(self):
+        self._utils.create_venv(self._venv_dir, with_pip=True)
+
+    def bootstrap(
+        self, artifact_type: ArtifactType, download_deps: bool = False
+    ):
+        if download_deps:
+            self._install_requirements(DOWNLOAD_DEPS_BOOTSTRAP_LOCK)
+            if artifact_type == ArtifactType.PORTABLE_EXE.value:
+                self._install_requirements(PORTABLE_EXE_REQUIREMENTS_LOCK)
+            else:
+                self._install_requirements(SYSTEM_SANDBOX_REQUIREMENTS_LOCK)
+        else:
+            self._copy_parent_packages()
+        self._install_awscli()
+        self._update_windows_script_header()
+
+    def _copy_parent_packages(self):
+        for site_package in site.getsitepackages():
+            self._utils.copy_directory_contents_into(
+                site_package, self._site_packages()
+            )
+        parent_scripts = pathlib.Path(sys.executable).parents[0]
+        for script in self._PARENT_SCRIPTS_TO_COPY:
+            source = os.path.join(parent_scripts, script)
+            if self._utils.path_exists(source):
+                self._utils.copy_file(
+                    source, os.path.join(self.bin_dir, script)
+                )
+
+    def _install_requirements(self, requirements_file, cwd=None):
+        self._pip_install(
+            ["--no-build-isolation", "-r", requirements_file],
+            cwd=cwd,
+        )
+
+    def _install_awscli(self):
+        self._pip_install(
+            [
+                ROOT_DIR,
+                "--no-build-isolation",
+                "--no-cache-dir",
+                "--no-index",
+            ]
+        )
+
+    def _update_windows_script_header(self):
+        # When installing to a venv pip will rewrite shebang lines
+        # to reference the relevant virutalenv directly. This is not
+        # the case for aws.cmd which is our own windows cmd file
+        # and does not have a shebang that is re-writable.
+        # We need to manually overwrite the header line in this script
+        # to reference the current virtualenv.
+        # If we are not on Windows then this is not relevant.
+        if not IS_WINDOWS:
+            return
+        python_exe_path = os.path.join(self.bin_dir, "python.exe")
+        exe_path = os.path.join(self.bin_dir, "aws.cmd")
+        lines = self._utils.read_file_lines(exe_path)
+        lines[0] = self._utils.get_script_header(python_exe_path)
+        self._utils.write_file(exe_path, "".join(lines))
+
+    @property
+    def bin_dir(self):
+        return os.path.join(self._venv_dir, BIN_DIRNAME)
+
+    @property
+    def python_exe(self):
+        return os.path.join(self.bin_dir, PYTHON_EXE_NAME)
+
+    def _pip_install(self, args, cwd=None):
+        args = [self.python_exe, "-m", "pip", "install"] + args
+        run_kwargs = {"check": True}
+        if IS_WINDOWS:
+            args = " ".join([str(a) for a in args])
+            # The tests on windows will fail when executed with
+            # the wrapper test runner script in scripts/ci if this
+            # is not executed from shell.
+            run_kwargs["shell"] = True
+        if cwd is not None:
+            run_kwargs["cwd"] = cwd
+        self._utils.run(args, **run_kwargs)
+
+    def _site_packages(self) -> str:
+        site_path = (
+            subprocess.check_output(
+                [
+                    self.python_exe,
+                    "-c",
+                    "import site; print(site.getsitepackages()[0])",
+                ]
+            )
+            .decode()
+            .strip()
+        )
+        return site_path