build-sync-wheels

#!/usr/bin/env python3

import os
import sys
import glob
import subprocess
import tempfile
import shutil
import argparse
from pathlib import Path

import utils


if os.geteuid() == 0:
    # tar has issues when resetting permissions and ends up
    # causing very subtle reproducibility issues.
    print("This script cannot be run as root.", file=sys.stderr)
    # sys.exit(1)
if "VIRTUAL_ENV" not in os.environ:
    print(
        "This script should be run in a virtualenv: `source .venv/bin/activate`",
        file=sys.stderr,
    )
    sys.exit(1)

# Set SOURCE_DATE_EPOCH to a predictable value. Using the first
# commit to the SecureDrop project:
#
#   git show -s 62bbe590afd77a6af2dcaed46c93da6e0cf40951 --date=unix
#
# which yields 1309379017
os.environ["SOURCE_DATE_EPOCH"] = "1309379017"

# Force sane umask for reproducibility's sake.
os.umask(0o022)

# When building wheels, pip defaults to a safe dynamic tmpdir path.
# Some shared objects include the path from build, so we must make
# the path predictable across builds.
WHEEL_BUILD_DIR = "/tmp/pip-wheel-build"


def main():
    if "PKG_DIR" in os.environ and not any(
        arg.startswith("--pkg-dir") for arg in sys.argv
    ):
        sys.argv.extend(["--pkg-dir", os.environ["PKG_DIR"]])

    parser = argparse.ArgumentParser(description="Builds and stores sources and wheels")
    parser.add_argument("--pkg-dir", help="Package directory", required=True)
    parser.add_argument("--project", help="Project name to update")
    parser.add_argument(
        "--clobber",
        action="store_true",
        default=False,
        help="Whether to overwrite wheels and source tarballs",
    )
    parser.add_argument(
        "--requirements",
        default="requirements",
        help="Directory that contains requirements.txt inside the package directory",
    )
    args = parser.parse_args()

    if args.pkg_dir.startswith("https://"):
        git_clone_directory = tempfile.mkdtemp(prefix=os.path.basename(args.pkg_dir))
        cmd = f"git clone {args.pkg_dir} {git_clone_directory}".split()
        subprocess.check_call(cmd)
        args.pkg_dir = git_clone_directory
    else:
        args.pkg_dir = os.path.expanduser(args.pkg_dir)
        git_clone_directory = ""
        if not os.path.exists(args.pkg_dir):
            print(f"Project directory missing {args.pkg_dir}.")
            sys.exit(1)

    req_path = os.path.join(args.pkg_dir, args.requirements, "requirements.txt")
    if args.project is not None:
        project_name = args.project
    else:
        project_name = utils.get_project_name(Path(args.pkg_dir))
    local_wheels = os.path.join(project_name, "wheels")

    req_path = os.path.join(args.pkg_dir, args.requirements, "requirements.txt")
    poetry_lock_path = os.path.join(args.pkg_dir, "poetry.lock")

    use_poetry = False
    # Check if requirements.txt exists, if not check for poetry.lock and create a temporary requirements.txt
    if not os.path.exists(req_path):
        if os.path.exists(poetry_lock_path):
            use_poetry = True
            print(
                f"requirements.txt was not found at {req_path}, but poetry.lock was found at {poetry_lock_path}, using."
            )
        else:
            print(
                f"requirements.txt not found at {req_path} and poetry.lock not found at {poetry_lock_path}."
            )
            sys.exit(3)

    if os.path.exists(WHEEL_BUILD_DIR):
        shutil.rmtree(WHEEL_BUILD_DIR)
        os.mkdir(WHEEL_BUILD_DIR)
    else:
        os.mkdir(WHEEL_BUILD_DIR)

    with tempfile.TemporaryDirectory() as tmpdir:
        if use_poetry:
            poetry_reqs = utils.get_requirements_from_poetry(Path(args.pkg_dir) / 'poetry.lock', Path(args.pkg_dir) / 'pyproject.toml')
            req_path = os.path.join(tmpdir, 'requirements.txt')
            with open(req_path, 'w') as req_file:
                req_file.write(poetry_reqs)

        # The --require-hashes option will be used by default if there are
        # hashes in the requirements.txt file. We specify it anyway to guard
        # against use of a requirements.txt file without hashes.
        #
        # NOTE: Even with this invocation, pip may execute build steps, as
        # part of its metadata collection process. Switching to
        # manual downloading and hash verification may be preferable
        # to make this process more resilient.
        #
        # See https://github.com/pypa/pip/issues/1884 for background.
        cmd = [
            "pip3",
            "download",
            "--no-binary",
            ":all:",
            "--require-hashes",
            "--no-build-isolation",
            "--dest",
            tmpdir,
            "--requirement",
            req_path,
        ]
        subprocess.check_call(cmd)

        # Now we have all the source tarballs
        source_tar_balls = glob.glob(f"{tmpdir}/*.tar.gz")
        for source in source_tar_balls:
            cmd = ["tar", "-xvf", source, "-C", WHEEL_BUILD_DIR]
            subprocess.check_call(cmd)
        # Now we have the all source tarballs extracted in WHEEL_BUILD_DIR

        # Next step is to take each one of those and then execute the build command to build a wheel against them
        project_names = os.listdir(WHEEL_BUILD_DIR)
        for project in project_names:
            print(f"Building {project}")
            source_path = os.path.join(WHEEL_BUILD_DIR, project)
            cmd = [
                "python3",
                "-m",
                "build",
                "--wheel",
                source_path,
                "--no-isolation",
                "-o",
                tmpdir,
            ]
            subprocess.check_call(cmd)
            print(f"build command used: {' '.join(cmd)}")

        # Now find the names
        names = os.listdir(tmpdir)

        if os.path.exists(local_wheels):
            local_names = os.listdir(local_wheels)
        else:
            os.mkdir(local_wheels)
            local_names = []

        for name in names:
            if name == "requirements.txt":  # We don't need this
                continue
            if name in local_names:  # Means already there
                if not args.clobber:
                    continue

            # Else copy to local wheels
            filepath = os.path.join(tmpdir, name)
            shutil.copy(filepath, local_wheels, follow_symlinks=True)
            print(f"Copied {name} to {local_wheels}")

    if git_clone_directory:
        shutil.rmtree(git_clone_directory)


if __name__ == "__main__":
    main()