release.py: Generate list of APKs dynamically and set indices.

.taskcluster.yml

@@ -121,6 +121,7 @@ tasks:
       - "project:mobile:focus:releng:signing:format:focus-jar"
       - "project:mobile:focus:releng:googleplay:product:focus"
       - "secrets:get:project/focus/tokens"
+      - "queue:route:index.project.mobile.focus.release.latest"
     payload:
       maxRunTime: 3600
       deadline: "{{ '2 hours' | $fromNow }}"
@@ -134,7 +135,13 @@ tasks:
           git fetch origin --tags
           && git config advice.detachedHead false
           && git checkout {{ event.version }}
-          && python tools/taskcluster/release.py --track alpha --commit --tag {{ event.version }}
+          && python tools/taskcluster/release.py \
+            --tag {{ event.version }} \
+            --track alpha \
+            --commit \
+            --output /opt/focus-android/app/build/outputs/apk \
+            --apk focusWebviewUniversal/release/app-focus-webview-universal-release-unsigned.apk \
+            --apk klarWebviewUniversal/release/app-klar-webview-universal-release-unsigned.apk
       features:
         taskclusterProxy: true
         chainOfTrust: true

tools/taskcluster/lib/tasks.py

@@ -18,7 +18,7 @@ def __init__(self, task_id, repo_url, branch, commit, owner, source):
 
     def build_task(self, name, description, command, dependencies = [], artifacts = {}, scopes = [], routes = [], features = {}, worker_type = 'github-worker'):
         created = datetime.datetime.now()
-        expires = taskcluster.fromNow('1 month')
+        expires = taskcluster.fromNow('1 year')
         deadline = taskcluster.fromNow('1 day')
 
         features = features.copy()
@@ -63,9 +63,9 @@ def build_task(self, name, description, command, dependencies = [], artifacts =
         }
 
 
-    def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[]):
+    def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[], routes=[]):
         created = datetime.datetime.now()
-        expires = taskcluster.fromNow('1 month')
+        expires = taskcluster.fromNow('1 year')
         deadline = taskcluster.fromNow('1 day')
 
         return {
@@ -79,7 +79,7 @@ def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[
             "schedulerId": "taskcluster-github",
             "deadline": taskcluster.stringDate(deadline),
             "dependencies": [ self.task_id, build_task_id],
-            "routes": [],
+            "routes": routes,
             "scopes": scopes,
             "requires": "all-completed",
             "payload": {
@@ -106,7 +106,7 @@ def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[
 
     def build_push_task(self, signing_task_id, name, description, apks=[], scopes=[], track='internal', commit=False):
         created = datetime.datetime.now()
-        expires = taskcluster.fromNow('1 month')
+        expires = taskcluster.fromNow('1 year')
         deadline = taskcluster.fromNow('1 day')
 
         return {

tools/taskcluster/release.py

@@ -25,68 +25,86 @@
     source="https://github.com/mozilla-mobile/focus-android/tree/master/tools/taskcluster"
 )
 
-def generate_build_task(tag):
+def generate_build_task(apks, tag):
+    artifacts = {}
+    for apk in apks:
+        artifact = {
+            "type": 'file',
+            "path": apk,
+            "expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
+        }
+        artifacts["public/%s" % os.path.basename(apk)] = artifact
+
     checkout = "git fetch origin && git reset --hard origin/master" if tag is None else "git fetch origin && git checkout %s" % (tag)
 
+    if tag:
+        # Non-tagged (nightly) builds should contain all languages
+        checkout = checkout + ' && python tools/l10n/filter-release-translations.py'
+
+
     return taskcluster.slugId(), BUILDER.build_task(
         name="(Focus for Android) Build task",
         description="Build Focus/Klar from source code.",
         command=(checkout +
-                 ' && python tools/l10n/filter-release-translations.py'
                  ' && python tools/taskcluster/get-adjust-token.py'
                  ' && ./gradlew --no-daemon clean test assembleRelease'),
         features = {
             "chainOfTrust": True
         },
-        artifacts = {
-			"public/focus.apk": {
-				"type": "file",
-				"path": "/opt/focus-android/app/build/outputs/apk/focusWebviewUniversal/release/app-focus-webview-universal-release-unsigned.apk",
-				"expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
-			},
-            "public/klar.apk": {
-				"type": "file",
-				"path": "/opt/focus-android/app/build/outputs/apk/klarWebviewUniversal/release/app-klar-webview-universal-release-unsigned.apk",
-				"expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
-			}
-		},
+        artifacts = artifacts,
         worker_type='gecko-focus',
         scopes=[
             "secrets:get:project/focus/tokens"
         ])
 
-def generate_signing_task(build_task_id):
+def generate_signing_task(build_task_id, apks, tag):
+    artifacts = []
+    for apk in apks:
+        artifacts.append("public/" + os.path.basename(apk))
+
+    routes = []
+    scopes = [
+        "project:mobile:focus:releng:signing:cert:release-signing",
+        "project:mobile:focus:releng:signing:format:focus-jar"
+    ]
+
+    if tag:
+        index = "index.project.mobile.focus.release.latest"
+        routes.append(index)
+        scopes.append("queue:route:" + index)
+    else:
+        index = "index.project.mobile.focus.nightly.latest"
+        routes.append(index)
+        scopes.append("queue:route:" + index)
+
     return taskcluster.slugId(), BUILDER.build_signing_task(
         build_task_id,
         name="(Focus for Android) Signing task",
         description="Sign release builds of Focus/Klar",
-        apks=[
-            "public/focus.apk",
-            "public/klar.apk"
-        ],
-        scopes = [
-            "project:mobile:focus:releng:signing:cert:release-signing",
-            "project:mobile:focus:releng:signing:format:focus-jar"
-        ]
+        apks=artifacts,
+        scopes=scopes,
+        routes=routes
     )
 
-def generate_push_task(signing_task_id, track, commit):
+def generate_push_task(signing_task_id, apks, track, commit):
+    artifacts = []
+    for apk in apks:
+        artifacts.append("public/" + os.path.basename(apk))
+
+    print artifacts
+
     return taskcluster.slugId(), BUILDER.build_push_task(
         signing_task_id,
         name="(Focus for Android) Push task",
         description="Upload signed release builds of Focus/Klar to Google Play",
-        apks=[
-            "public/focus.apk",
-            "public/klar.apk"
-        ],
+        apks=artifacts,
         scopes=[
             "project:mobile:focus:releng:googleplay:product:focus"
         ],
         track = track,
         commit = commit
     )
 
-
 def populate_chain_of_trust_required_but_unused_files():
     # Thoses files are needed to keep chainOfTrust happy. However, they have no need for Firefox
     # Focus, at the moment. For more details, see:
@@ -97,24 +115,24 @@ def populate_chain_of_trust_required_but_unused_files():
             json.dump({}, f)    # Yaml is a super-set of JSON.
 
 
-def release(track, commit, tag):
+def release(apks, track, commit, tag):
     queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' })
 
     task_graph = {}
 
-    build_task_id, build_task = generate_build_task(tag)
+    build_task_id, build_task = generate_build_task(apks, tag)
     lib.tasks.schedule_task(queue, build_task_id, build_task)
 
     task_graph[build_task_id] = {}
     task_graph[build_task_id]["task"] = queue.task(build_task_id)
 
-    sign_task_id, sign_task = generate_signing_task(build_task_id)
+    sign_task_id, sign_task = generate_signing_task(build_task_id, apks, tag)
     lib.tasks.schedule_task(queue, sign_task_id, sign_task)
 
     task_graph[sign_task_id] = {}
     task_graph[sign_task_id]["task"] = queue.task(sign_task_id)
 
-    push_task_id, push_task = generate_push_task(sign_task_id, track, commit)
+    push_task_id, push_task = generate_push_task(sign_task_id, apks, track, commit)
     lib.tasks.schedule_task(queue, push_task_id, push_task)
 
     task_graph[push_task_id] = {}
@@ -129,14 +147,19 @@ def release(track, commit, tag):
     populate_chain_of_trust_required_but_unused_files()
 
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description='Create a release pipeline (build, sign, publish) on taskcluster.')
 
     parser.add_argument('--track', dest="track", action="store", choices=['internal', 'alpha'], help="", required=True)
     parser.add_argument('--commit', dest="commit", action="store_true", help="commit the google play transaction")
     parser.add_argument('--tag', dest="tag", action="store", help="git tag to build from")
+    parser.add_argument('--apk', dest="apks", metavar="path", action="append", help="Path to APKs to sign and upload", required=True)
+    parser.add_argument('--output', dest="output", metavar="path", action="store", help="Path to the build output", required=True)
 
     result = parser.parse_args()
 
-    release(result.track, result.commit, result.tag)
+    apks = map(lambda x: result.output + '/' + x, result.apks)
+
+    release(apks, result.track, result.commit, result.tag)