Add Android Device Farm testing to CI (#681)

.github/workflows/ci.yml

@@ -17,6 +17,7 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_REGION: us-east-1
+  AWS_DEVICE_FARM_REGION: us-west-2 # Device Farm only available in us-west-2 region
 
 jobs:
   linux-compat:
@@ -79,7 +80,7 @@ jobs:
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream
-          
+
   linux-musl:
     runs-on: ubuntu-22.04 # latest
     strategy:
@@ -208,18 +209,49 @@ jobs:
           python3 codebuild/macos_compatibility_check.py
 
   android:
+      # ubuntu-20.04 comes with Android tooling, see:
+      # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android
     name: Android
-    # ubuntu-20.04 comes with Android tooling, see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android
     runs-on: ubuntu-20.04 # latest
+    permissions:
+      # These permissions needed to interact with GitHub's OIDC Token endpoint
+      id-token: write # This is required for requesting the JWT
     steps:
       - name: Checkout Sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
-          submodules: true
+            submodules: true
+      # Setup JDK 11
+      - name: set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: 'gradle'
+      # Build and publish locally for the test app to find the SNAPSHOT version
       - name: Build ${{ env.PACKAGE_NAME }}
         run: |
           ./gradlew :android:crt:build
-          ./gradlew test
+          ./gradlew :android:crt:publishToMavenLocal
+      # Setup files required by test app for Device Farm testing
+      - name: Setup Android Test Files
+        run: |
+          cd src/test/android/testapp/src/main/assets
+          python3 -m pip install boto3
+          python3 ./android_file_creation.py
+      - name: Build Test App
+        run: |
+          cd src/test/android/testapp
+          ../../../../gradlew assembledebug
+          ../../../../gradlew assembleAndroidTest
+      - name: Device Farm Tests
+        run: |
+          echo "Running Device Farm Python Script"
+          python3 ./.github/workflows/run_android_ci.py \
+          --run_id ${{ github.run_id }} \
+          --run_attempt ${{ github.run_attempt }} \
+          --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \
+          --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\')
 
   # check that docs can still build
   check-docs:

.github/workflows/run_android_ci.py

@@ -0,0 +1,183 @@
+import argparse
+import sys
+import os
+import time
+import datetime
+
+import requests # - for uploading files
+import boto3
+
+parser = argparse.ArgumentParser(description="Utility script to upload and run Android Device tests on AWS Device Farm for CI")
+parser.add_argument('--run_id', required=True, help="A unique number for each workflow run within a repository")
+parser.add_argument('--run_attempt', required=True, help="A unique number for each attempt of a particular workflow run in a repository")
+parser.add_argument('--project_arn', required=True, help="Arn for the Device Farm Project the apk will be tested on")
+parser.add_argument('--device_pool_arn', required=True, help="Arn for device pool of the Device Farm Project the apk will be tested on")
+
+current_working_directory = os.getcwd()
+build_file_location = current_working_directory + '/src/test/android/testapp/build/outputs/apk/debug/testapp-debug.apk'
+test_file_location = current_working_directory + '/src/test/android/testapp/build/outputs/apk/androidTest/debug/testapp-debug-androidTest.apk'
+test_spec_file_location = current_working_directory + '/src/test/android/testapp/instrumentedTestSpec.yml'
+
+def main():
+    args = parser.parse_args()
+    run_id = args.run_id
+    run_attempt = args.run_attempt
+    project_arn = args.project_arn
+    device_pool_arn = args.device_pool_arn
+
+    region = os.getenv('AWS_DEVICE_FARM_REGION')
+
+    print("Beginning Android Device Farm Setup \n")
+
+    # Create Boto3 client for Device Farm
+    try:
+        client = boto3.client('devicefarm', region_name=region)
+    except Exception:
+        print("Error - could not make Boto3 client. Credentials likely could not be sourced")
+        sys.exit(-1)
+    print("Boto3 client established")
+
+    # Upload the crt library shell app to Device Farm
+    upload_file_name = 'CI-' + run_id + '-' + run_attempt + '.apk'
+    print('Upload file name: ' + upload_file_name)
+
+    # Prepare upload to Device Farm project
+    create_upload_response = client.create_upload(
+        projectArn=project_arn,
+        name=upload_file_name,
+        type='ANDROID_APP'
+    )
+    device_farm_upload_arn = create_upload_response['upload']['arn']
+    device_farm_upload_url = create_upload_response['upload']['url']
+
+    # Upload crt library shell app apk
+    with open(build_file_location, 'rb') as f:
+        data = f.read()
+    r = requests.put(device_farm_upload_url, data=data)
+    print('File upload status code: ' + str(r.status_code) + ' reason: ' + r.reason)
+    device_farm_upload_status = client.get_upload(arn=device_farm_upload_arn)
+    while device_farm_upload_status['upload']['status'] != 'SUCCEEDED':
+        if device_farm_upload_status['upload']['status'] == 'FAILED':
+            print('Upload failed to process')
+            sys.exit(-1)
+        time.sleep(1)
+        device_farm_upload_status = client.get_upload(arn=device_farm_upload_arn)
+
+    # Upload the instrumentation test package to Device Farm
+    upload_test_file_name = 'CI-' + run_id + '-' + run_attempt + 'tests.apk'
+    print('Upload file name: ' + upload_test_file_name)
+
+    # Prepare upload to Device Farm project
+    create_upload_response = client.create_upload(
+        projectArn=project_arn,
+        name=upload_test_file_name,
+        type='INSTRUMENTATION_TEST_PACKAGE'
+    )
+    device_farm_instrumentation_upload_arn = create_upload_response['upload']['arn']
+    device_farm_instrumentation_upload_url = create_upload_response['upload']['url']
+
+    # Upload instrumentation test package
+    with open(test_file_location, 'rb') as f:
+        data_instrumentation = f.read()
+    r_instrumentation = requests.put(device_farm_instrumentation_upload_url, data=data_instrumentation)
+    print('File upload status code: ' + str(r_instrumentation.status_code) + ' reason: ' + r_instrumentation.reason)
+    device_farm_upload_status = client.get_upload(arn=device_farm_instrumentation_upload_arn)
+    while device_farm_upload_status['upload']['status'] != 'SUCCEEDED':
+        if device_farm_upload_status['upload']['status'] == 'FAILED':
+            print('Upload failed to process')
+            sys.exit(-1)
+        time.sleep(1)
+        device_farm_upload_status = client.get_upload(arn=device_farm_instrumentation_upload_arn)
+
+    # Upload the test spec file to Device Farm
+    upload_spec_file_name = 'CI-' + run_id + '-' + run_attempt + 'test-spec.yml'
+    print('Upload file name: ' + upload_spec_file_name)
+
+    # Prepare upload to Device Farm project
+    create_upload_response = client.create_upload(
+        projectArn=project_arn,
+        name=upload_spec_file_name,
+        type='INSTRUMENTATION_TEST_SPEC'
+    )
+    device_farm_test_spec_upload_arn = create_upload_response['upload']['arn']
+    device_farm_test_spec_upload_url = create_upload_response['upload']['url']
+
+    # Default Instrumentation tests run on Device Farm result in detailed individual test breakdowns but comes
+    # at the cost of the test suite running for up to two hours before completing. There is limited control for turning
+    # off unnecessary features which generates an immense amount of traffic resulting in hitting Device Farm rate limits
+    # A bare-bones test spec is used with instrumentation testing which will report a singular fail if any one test fails but
+    # the resulting Test spec output file contains information on each unit test, whether they passed, failed, or were skipped.
+    # Upload test spec yml
+    with open(test_spec_file_location, 'rb') as f:
+        data = f.read()
+    r = requests.put(device_farm_test_spec_upload_url, data=data)
+    print('File upload status code: ' + str(r.status_code) + ' reason: ' + r.reason)
+    device_farm_upload_status = client.get_upload(arn=device_farm_test_spec_upload_arn)
+    while device_farm_upload_status['upload']['status'] != 'SUCCEEDED':
+        if device_farm_upload_status['upload']['status'] == 'FAILED':
+            print('Upload failed to process')
+            sys.exit(-1)
+        time.sleep(1)
+        device_farm_upload_status = client.get_upload(arn=device_farm_test_spec_upload_arn)
+
+    print('scheduling run')
+    schedule_run_response = client.schedule_run(
+        projectArn=project_arn,
+        appArn=device_farm_upload_arn,
+        devicePoolArn=device_pool_arn,
+        name=upload_file_name,
+        test={
+            'type': 'INSTRUMENTATION',
+            'testPackageArn': device_farm_instrumentation_upload_arn,
+            'testSpecArn': device_farm_test_spec_upload_arn
+        },
+        executionConfiguration={
+            'jobTimeoutMinutes': 30
+        }
+    )
+
+    device_farm_run_arn = schedule_run_response['run']['arn']
+
+    run_start_time = schedule_run_response['run']['started']
+    run_start_date_time = run_start_time.strftime("%m/%d/%Y, %H:%M:%S")
+    print('run scheduled at ' + run_start_date_time)
+
+    get_run_response = client.get_run(arn=device_farm_run_arn)
+    while get_run_response['run']['result'] == 'PENDING':
+        time.sleep(10)
+        get_run_response = client.get_run(arn=device_farm_run_arn)
+
+    run_end_time = datetime.datetime.now()
+    run_end_date_time = run_end_time.strftime("%m/%d/%Y, %H:%M:%S")
+    print('Run ended at ' + run_end_date_time + ' with result: ' + get_run_response['run']['result'])
+
+    is_success = True
+    if get_run_response['run']['result'] != 'PASSED':
+        print('run has failed with result ' + get_run_response['run']['result'])
+        is_success = False
+
+    # If Clean up is not executed due to the job being cancelled in CI, the uploaded files will not be deleted
+    # from the Device Farm project and must be deleted manually.
+
+    # Clean up
+    print('Deleting ' + upload_file_name + ' from Device Farm project')
+    client.delete_upload(
+        arn=device_farm_upload_arn
+    )
+    print('Deleting ' + upload_test_file_name + ' from Device Farm project')
+    client.delete_upload(
+        arn=device_farm_instrumentation_upload_arn
+    )
+    print('Deleting ' + upload_spec_file_name + ' from Device Farm project')
+    client.delete_upload(
+        arn=device_farm_test_spec_upload_arn
+    )
+
+    if is_success == False:
+        print('Exiting with fail')
+        sys.exit(-1)
+
+    print('Exiting with success')
+
+if __name__ == "__main__":
+    main()

android/crt/build.gradle

@@ -127,7 +127,6 @@ build.dependsOn preBuild
 
 dependencies {
     coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
-
     androidTestImplementation 'org.mockito:mockito-core:3.11.2'
     androidTestImplementation 'androidx.appcompat:appcompat:1.3.1'
     androidTestImplementation 'junit:junit:4.13.2'
@@ -167,6 +166,7 @@ afterEvaluate {
     publishing {
         repositories {
             maven { name = "testLocal"; url = "$rootProject.buildDir/m2" }
+            mavenLocal()
         }
 
         publications {
@@ -185,13 +185,15 @@ afterEvaluate {
                             url.set("http://www.apache.org/licenses/LICENSE-2.0.txt")
                         }
                     }
+
                     developers {
                         developer {
                             id.set("aws-sdk-common-runtime")
                             name.set("AWS SDK Common Runtime Team")
                             email.set("[email protected]")
                         }
                     }
+
                     scm {
                         connection.set("scm:git:git://github.com/awslabs/aws-crt-java.git")
                         developerConnection.set("scm:git:ssh://github.com/awslabs/aws-crt-java.git")
@@ -200,7 +202,16 @@ afterEvaluate {
                 }
                 version = android.defaultConfig.versionName
             }
+
+            debug(MavenPublication) {
+                from components.release
+
+                groupId = 'software.amazon.awssdk.crt'
+                artifactId = 'aws-crt-android'
+                version = '1.0.0-SNAPSHOT'
+            }
         }
+
         repositories {
             maven {
                 def snapshotRepo = "https://aws.oss.sonatype.org/content/repositories/snapshots"

src/main/java/software/amazon/awssdk/crt/CRT.java

@@ -87,7 +87,6 @@ private static String normalize(String value) {
      * @return a string describing the detected platform the CRT is executing on
      */
     public static String getOSIdentifier() throws UnknownPlatformException {
-
         CrtPlatform platform = getPlatformImpl();
         String name = normalize(platform != null ? platform.getOSIdentifier() : System.getProperty("os.name"));
 
@@ -101,6 +100,8 @@ public static String getOSIdentifier() throws UnknownPlatformException {
             return "osx";
         } else if (name.contains("sun os") || name.contains("sunos") || name.contains("solaris")) {
             return "solaris";
+        } else if (name.contains("android")){
+            return "android";
         }
 
         throw new UnknownPlatformException("AWS CRT: OS not supported: " + name);
@@ -122,7 +123,6 @@ public static String getArchIdentifier() throws UnknownPlatformException {
 
         CrtPlatform platform = getPlatformImpl();
         String arch = normalize(platform != null ? platform.getArchIdentifier() : System.getProperty("os.arch"));
-
         if (arch.matches("^(x8664|amd64|ia32e|em64t|x64|x86_64)$")) {
             return "x86_64";
         } else if (arch.matches("^(x8632|x86|i[3-6]86|ia32|x32)$")) {
@@ -395,8 +395,11 @@ public boolean accept(File dir, String name) {
     private static CrtPlatform findPlatformImpl() {
         ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
         String[] platforms = new String[] {
-                // Search for test impl first, fall back to crt
+                // Search for OS specific test impl first
                 String.format("software.amazon.awssdk.crt.test.%s.CrtPlatformImpl", getOSIdentifier()),
+                // Search for android test impl specifically because getOSIdentifier will return "linux" on android
+                "software.amazon.awssdk.crt.test.android.CrtPlatformImpl",
+                // Fall back to crt
                 String.format("software.amazon.awssdk.crt.%s.CrtPlatformImpl", getOSIdentifier()), };
         for (String platformImpl : platforms) {
             try {

src/main/java/software/amazon/awssdk/crt/CrtPlatform.java

@@ -29,9 +29,12 @@ public String getArchIdentifier() {
         return System.getProperty("os.arch");
     }
 
+    // Called one and only one time during setup for testing
+    public void setupOnce() {}
+
     // Called before every JUnit test
     public void testSetup(Object context) {}
-    
+
     // Called after every JUnit test
     public void testTearDown(Object context) {}
 }

src/test/android/testapp/.gitignore

@@ -0,0 +1,3 @@
+/build
+.gradle
+/local.properties