-
Notifications
You must be signed in to change notification settings - Fork 12.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Auto merge of #59926 - pietroalbini:android-sdk-manager, r=alexcrichton
ci: use a custom android sdk manager with pinning and mirroring Google's own sdkmanager has two issues that make it unsuitable for us: * Mirroring has to be done manually, which is annoying because we need to figure out on our own all the URLs to copy (I couldn't find any documentation when building this PR, had to use mitmproxy). * There is no support for pinning, which means an update on Google's side can break our CI, as it happened multiple times. This PR replaces all our usage of sdkmanager with a custom Python script which mimics its behavior, but with the two issues fixes. sdkmanager's logic for installing packages is thankfully very simple: the package name (like `system-images;android-18;default;armeabi-v7a`) is the directory where the package should live (with `;` replaced with `/`), so to install a package we only need to extract its contents in the right directory. r? @alexcrichton cc @kennytm fixes #59778
- Loading branch information
Showing
4 changed files
with
224 additions
and
68 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea | ||
| patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e | ||
| platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28 | ||
| platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb | ||
| system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd | ||
| tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,190 @@ | ||
| #!/usr/bin/env python3 | ||
| # Simpler reimplementation of Android's sdkmanager | ||
| # Extra features of this implementation are pinning and mirroring | ||
|
|
||
| # These URLs are the Google repositories containing the list of available | ||
| # packages and their versions. The list has been generated by listing the URLs | ||
| # fetched while executing `tools/bin/sdkmanager --list` | ||
| BASE_REPOSITORY = "https://dl.google.com/android/repository/" | ||
| REPOSITORIES = [ | ||
| "sys-img/android/sys-img2-1.xml", | ||
| "sys-img/android-wear/sys-img2-1.xml", | ||
| "sys-img/android-wear-cn/sys-img2-1.xml", | ||
| "sys-img/android-tv/sys-img2-1.xml", | ||
| "sys-img/google_apis/sys-img2-1.xml", | ||
| "sys-img/google_apis_playstore/sys-img2-1.xml", | ||
| "addon2-1.xml", | ||
| "glass/addon2-1.xml", | ||
| "extras/intel/addon2-1.xml", | ||
| "repository2-1.xml", | ||
| ] | ||
|
|
||
| # Available hosts: linux, macosx and windows | ||
| HOST_OS = "linux" | ||
|
|
||
| # Mirroring options | ||
| MIRROR_BUCKET = "rust-lang-ci2" | ||
| MIRROR_BASE_DIR = "rust-ci-mirror/android/" | ||
|
|
||
| import argparse | ||
| import hashlib | ||
| import os | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| import urllib.request | ||
| import xml.etree.ElementTree as ET | ||
|
|
||
| class Package: | ||
| def __init__(self, path, url, sha1, deps=None): | ||
| if deps is None: | ||
| deps = [] | ||
| self.path = path.strip() | ||
| self.url = url.strip() | ||
| self.sha1 = sha1.strip() | ||
| self.deps = deps | ||
|
|
||
| def download(self, base_url): | ||
| _, file = tempfile.mkstemp() | ||
| url = base_url + self.url | ||
| subprocess.run(["curl", "-o", file, url], check=True) | ||
| # Ensure there are no hash mismatches | ||
| with open(file, "rb") as f: | ||
| sha1 = hashlib.sha1(f.read()).hexdigest() | ||
| if sha1 != self.sha1: | ||
| raise RuntimeError( | ||
| "hash mismatch for package " + self.path + ": " + | ||
| sha1 + " vs " + self.sha1 + " (known good)" | ||
| ) | ||
| return file | ||
|
|
||
| def __repr__(self): | ||
| return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" | ||
|
|
||
| def fetch_url(url): | ||
| page = urllib.request.urlopen(url) | ||
| return page.read() | ||
|
|
||
| def fetch_repository(base, repo_url): | ||
| packages = {} | ||
| root = ET.fromstring(fetch_url(base + repo_url)) | ||
| for package in root: | ||
| if package.tag != "remotePackage": | ||
| continue | ||
| path = package.attrib["path"] | ||
|
|
||
| for archive in package.find("archives"): | ||
| host_os = archive.find("host-os") | ||
| if host_os is not None and host_os.text != HOST_OS: | ||
| continue | ||
| complete = archive.find("complete") | ||
| url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) | ||
| sha1 = complete.find("checksum").text | ||
|
|
||
| deps = [] | ||
| dependencies = package.find("dependencies") | ||
| if dependencies is not None: | ||
| for dep in dependencies: | ||
| deps.append(dep.attrib["path"]) | ||
|
|
||
| packages[path] = Package(path, url, sha1, deps) | ||
| break | ||
|
|
||
| return packages | ||
|
|
||
| def fetch_repositories(): | ||
| packages = {} | ||
| for repo in REPOSITORIES: | ||
| packages.update(fetch_repository(BASE_REPOSITORY, repo)) | ||
| return packages | ||
|
|
||
| class Lockfile: | ||
| def __init__(self, path): | ||
| self.path = path | ||
| self.packages = {} | ||
| if os.path.exists(path): | ||
| with open(path) as f: | ||
| for line in f: | ||
| path, url, sha1 = line.split(" ") | ||
| self.packages[path] = Package(path, url, sha1) | ||
|
|
||
| def add(self, packages, name, *, update=True): | ||
| if name not in packages: | ||
| raise NameError("package not found: " + name) | ||
| if not update and name in self.packages: | ||
| return | ||
| self.packages[name] = packages[name] | ||
| for dep in packages[name].deps: | ||
| self.add(packages, dep, update=False) | ||
|
|
||
| def save(self): | ||
| packages = list(sorted(self.packages.values(), key=lambda p: p.path)) | ||
| with open(self.path, "w") as f: | ||
| for package in packages: | ||
| f.write(package.path + " " + package.url + " " + package.sha1 + "\n") | ||
|
|
||
| def cli_add_to_lockfile(args): | ||
| lockfile = Lockfile(args.lockfile) | ||
| packages = fetch_repositories() | ||
| for package in args.packages: | ||
| lockfile.add(packages, package) | ||
| lockfile.save() | ||
|
|
||
| def cli_update_mirror(args): | ||
| lockfile = Lockfile(args.lockfile) | ||
| for package in lockfile.packages.values(): | ||
| path = package.download(BASE_REPOSITORY) | ||
| subprocess.run([ | ||
| "aws", "s3", "mv", path, | ||
| "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, | ||
| "--profile=" + args.awscli_profile, | ||
| ], check=True) | ||
|
|
||
| def cli_install(args): | ||
| lockfile = Lockfile(args.lockfile) | ||
| for package in lockfile.packages.values(): | ||
| # Download the file from the mirror into a temp file | ||
| url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR | ||
| downloaded = package.download(url) | ||
| # Extract the file in a temporary directory | ||
| extract_dir = tempfile.mkdtemp() | ||
| subprocess.run([ | ||
| "unzip", "-q", downloaded, "-d", extract_dir, | ||
| ], check=True) | ||
| # Figure out the prefix used in the zip | ||
| subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] | ||
| if len(subdirs) != 1: | ||
| raise RuntimeError("extracted directory contains more than one dir") | ||
| # Move the extracted files in the proper directory | ||
| dest = os.path.join(args.dest, package.path.replace(";", "/")) | ||
| os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) | ||
| os.rename(os.path.join(extract_dir, subdirs[0]), dest) | ||
| os.unlink(downloaded) | ||
|
|
||
| def cli(): | ||
| parser = argparse.ArgumentParser() | ||
| subparsers = parser.add_subparsers() | ||
|
|
||
| add_to_lockfile = subparsers.add_parser("add-to-lockfile") | ||
| add_to_lockfile.add_argument("lockfile") | ||
| add_to_lockfile.add_argument("packages", nargs="+") | ||
| add_to_lockfile.set_defaults(func=cli_add_to_lockfile) | ||
|
|
||
| update_mirror = subparsers.add_parser("update-mirror") | ||
| update_mirror.add_argument("lockfile") | ||
| update_mirror.add_argument("--awscli-profile", default="default") | ||
| update_mirror.set_defaults(func=cli_update_mirror) | ||
|
|
||
| install = subparsers.add_parser("install") | ||
| install.add_argument("lockfile") | ||
| install.add_argument("dest") | ||
| install.set_defaults(func=cli_install) | ||
|
|
||
| args = parser.parse_args() | ||
| if not hasattr(args, "func"): | ||
| print("error: a subcommand is required (see --help)") | ||
| exit(1) | ||
| args.func(args) | ||
|
|
||
| if __name__ == "__main__": | ||
| cli() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters