diff --git a/.taskcluster.yml b/.taskcluster.yml index 5f726e8a28e..a69c99ee924 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -7,7 +7,7 @@ tasks: - $let: taskgraph: branch: taskgraph - revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0 + revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57 trustDomain: mobile in: $let: @@ -249,7 +249,7 @@ tasks: else: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml && - taskcluster/scripts/install-sdk.sh && + taskcluster/scripts/decision-install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision --pushlog-id='0' diff --git a/taskcluster/ac_taskgraph/job.py b/taskcluster/ac_taskgraph/job.py index 3b47ed59787..4fdf119f9a8 100644 --- a/taskcluster/ac_taskgraph/job.py +++ b/taskcluster/ac_taskgraph/job.py @@ -38,9 +38,11 @@ def configure_gradlew(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"] - worker.setdefault("env", {}).update( - {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")} - ) + worker.setdefault("env", {}).update({ + "ANDROID_SDK_ROOT": path.join( + run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux" + ) + }) # defer to the run_task implementation run["command"] = _extract_command(run) diff --git a/taskcluster/ci/build-samples-browser/kind.yml b/taskcluster/ci/build-samples-browser/kind.yml index 9537c12dcdc..03050bb8576 100644 --- a/taskcluster/ci/build-samples-browser/kind.yml +++ b/taskcluster/ci/build-samples-browser/kind.yml @@ -10,6 +10,8 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain job-defaults: @@ -18,6 +20,9 @@ job-defaults: code-review: true component: samples-browser description: Execute Gradle tasks for component "{component}" + fetches: + toolchain: + - android-sdk-linux include-coverage: true run: using: gradlew diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index 413bf5ea303..0fe09833e3e 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -13,17 +13,15 @@ transforms: not-for-components: - samples-browser # Their configs are too different, so they're handled in a specific kind +kind-dependencies: + - toolchain + + job-defaults: artifact-template: type: file name: public/build/{artifact_file_name} path: '/builds/worker/checkouts/src/{component_path}/build/maven/org/mozilla/components/{component}/{version_with_snapshot}/{artifact_file_name}' - expose-artifacts: - by-build-type: - release: true - snapshot: true - nightly: true - default: false attributes: code-review: by-build-type: @@ -32,6 +30,15 @@ job-defaults: nightly: false default: true description: Execute Gradle tasks for component "{component}" + expose-artifacts: + by-build-type: + release: true + snapshot: true + nightly: true + default: false + fetches: + toolchain: + - android-sdk-linux include-coverage: by-build-type: release: false diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index b2ac0330517..e85d0646d2b 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -2,7 +2,10 @@ trust-domain: mobile treeherder: group-names: + Fetch: Fetch and store content I: Docker Image Builds + TL: Toolchain builds for Linux 64-bits + # Groups are generated off buildconfig.yml, except these ones: samples-browser-beta: samples-browser-beta samples-browser-release: samples-browser-release diff --git a/taskcluster/ci/fetch/kind.yml b/taskcluster/ci/fetch/kind.yml new file mode 100644 index 00000000000..f49abb5f3bb --- /dev/null +++ b/taskcluster/ci/fetch/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.fetch:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + docker-image: {in-tree: base} + +jobs: + android-sdk-3859397: + description: Android SDK + fetch: + type: static-url + url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip + artifact-name: sdk-tools-linux.zip + sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0 + size: 136964098 + artifact-prefix: mobile/android-sdk + fetch-alias: android-sdk diff --git a/taskcluster/ci/lint/kind.yml b/taskcluster/ci/lint/kind.yml index a0860f6b643..e553c8c34b8 100644 --- a/taskcluster/ci/lint/kind.yml +++ b/taskcluster/ci/lint/kind.yml @@ -9,13 +9,16 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain + + job-defaults: attributes: code-review: true - worker-type: b-android - worker: - docker-image: {in-tree: base} - max-run-time: 7200 + fetches: + toolchain: + - android-sdk-linux run: use-caches: false run-on-tasks-for: [github-pull-request, github-push] @@ -23,6 +26,10 @@ job-defaults: kind: test platform: 'lint/opt' tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 jobs: compare-locales: diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index 564df430729..13f0bbf3cf4 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -8,16 +8,23 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain + + job-defaults: attributes: build-type: debug code-review: true + fetches: + toolchain: + - android-sdk-linux run: # TODO Generate APKs in a build task instead gradlew: ['clean', 'assembleGeckoNightly', 'assembleAndroidTest', 'assembleDebug'] secrets: - name: project/mobile/android-components/firebase - key: firebaseToken + key: firebaseToken path: .firebase_token.json json: true using: gradlew @@ -39,6 +46,7 @@ job-defaults: GOOGLE_PROJECT: moz-android-components-230120 max-run-time: 7200 + jobs: unit-browser-engine-gecko-nightly: description: 'Run unit tests on device for browser component' diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml new file mode 100644 index 00000000000..e5d0a248d9a --- /dev/null +++ b/taskcluster/ci/toolchain/android.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: toolchain-script + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 1800 + + +linux64-android-sdk-linux-repack: + attributes: + artifact_prefix: mobile/android-sdk + description: "Android SDK (Linux) repack toolchain build" + fetches: + fetch: + - android-sdk + run: + script: repack-android-sdk-linux.sh + resources: [] + toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz + toolchain-alias: android-sdk-linux + treeherder: + symbol: TL(android-sdk-linux) diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml new file mode 100644 index 00000000000..ba9c02645ff --- /dev/null +++ b/taskcluster/ci/toolchain/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + + +jobs-from: + - android.yml diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile index 204d7f71445..cc210c1934e 100644 --- a/taskcluster/docker/base/Dockerfile +++ b/taskcluster/docker/base/Dockerfile @@ -19,13 +19,9 @@ WORKDIR /builds/worker/ #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV ANDROID_SDK_VERSION='3859397' \ - ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \ - CURL='curl --location --retry 5' \ - GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ +ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ LANG='en_US.UTF-8' \ - TERM='dumb' \ - SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" + TERM='dumb' #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -52,12 +48,6 @@ RUN pip install taskcluster RUN locale-gen "$LANG" -RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \ - && unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \ - && rm "$SDK_ZIP_LOCATION" \ - && yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \ - && chown -R worker:worker "$ANDROID_SDK_ROOT" - # %include-run-task diff --git a/taskcluster/docker/ui-tests/Dockerfile b/taskcluster/docker/ui-tests/Dockerfile index de743882de5..fcd8d3c079a 100644 --- a/taskcluster/docker/ui-tests/Dockerfile +++ b/taskcluster/docker/ui-tests/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get install -y jq \ USER worker:worker +ENV CURL 'curl --location --retry 5' ENV GOOGLE_SDK_DOWNLOAD ./gcloud.tar.gz ENV GOOGLE_SDK_VERSION 233 diff --git a/taskcluster/scripts/decision-install-sdk.sh b/taskcluster/scripts/decision-install-sdk.sh new file mode 100755 index 00000000000..9094d960910 --- /dev/null +++ b/taskcluster/scripts/decision-install-sdk.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +export CURL='curl --location --retry 5' + +ANDROID_SDK_VERSION='3859397' +ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' +SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" + +$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" +echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check +unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" +rm "$SDK_ZIP_LOCATION" +yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses diff --git a/taskcluster/scripts/install-sdk.sh b/taskcluster/scripts/install-sdk.sh deleted file mode 100755 index 7a431a34a85..00000000000 --- a/taskcluster/scripts/install-sdk.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set +x - -ANDROID_SDK_VERSION=3859397 - -curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" -unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip" -yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses diff --git a/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh new file mode 100755 index 00000000000..9eb10870901 --- /dev/null +++ b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR + +yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses + +# It's nice to have the build logs include the state of the world upon completion. +"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list + +tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"