Define separate profile for Azure cloud tests

Define separate profile for Azure cloud tests to keep how we handle tests which require external infrastructure consistent in the codebase. As a consequence this drops EnabledIfEnvironmentVariable annotation. The benefit is that with expicit profile and the way how we build condition guarding if tests should be run in ci.yml (use of CI_SKIP_SECRETS_PRESENCE_CHECKS) makes it much less probable to have false confidence that we run tests, when they are skipped. With EnabledIfEnvironmentVariable we could easily skip tests if we secrets propageted to env variables were not set in github. Remaining work item is to actually fill environment variables values in ci.yml.
trinodb · Sep 11, 2023 · cabc7c7 · cabc7c7
1 parent d06a3e0
commit cabc7c7
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 6 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -568,6 +568,7 @@ jobs:
             - { modules: core/trino-main }
             - { modules: core/trino-main, jdk: 20 }
             - { modules: lib/trino-filesystem-s3, profile: cloud-tests }
+            - { modules: lib/trino-filesystem-azure, profile: cloud-tests }
             - { modules: plugin/trino-accumulo }
             - { modules: plugin/trino-bigquery }
             - { modules: plugin/trino-bigquery, profile: cloud-tests-arrow }
@@ -652,6 +653,7 @@ jobs:
           && ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'cloud-tests'))
           && ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'fte-tests'))
           && ! (contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests'))
+          && ! (contains(matrix.modules, 'trino-filesystem-azure') && contains(matrix.profile, 'cloud-tests'))
         run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }}
       # Additional tests for selected modules
       - name: HDFS file system cache isolated JVM tests
@@ -669,6 +671,21 @@ jobs:
           (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
         run: |
           $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
+      - name: Azure FileSystem Cloud Tests
+        env:
+          ABFS_BLOB_ACCOUNT: todo
+          ABFS_BLOB_ACCESS_KEY: todo
+          ABFS_FLAT_ACCOUNT: todo
+          ABFS_FLAT_ACCESS_KEY: todo
+          ABFS_ACCOUNT: todo
+          ABFS_ACCESS_KEY: todo
+        # todo(https://github.com/trinodb/trino/issues/18998) Enable when we have env variables in place
+        if: >-
+          false &&
+          contains(matrix.modules, 'trino-filesystem-azure') && contains(matrix.profile, 'cloud-tests') &&
+          (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.ABFS_BLOB_ACCOUNT != '' || env.ABFS_BLOB_ACCESS_KEY != '' || env.ABFS_FLAT_ACCOUNT != '' || ABFS_FLAT_ACCESS_KEY != '' || ABFS_ACCOUNT != '' || ABFS_ACCESS_KEY != '')
+        run: |
+          $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
       - name: Cloud Delta Lake Tests
       # Cloud tests are separate because they are time intensive, requiring cross-cloud network communication
         env:

diff --git a/lib/trino-filesystem-azure/pom.xml b/lib/trino-filesystem-azure/pom.xml
@@ -192,4 +192,47 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
+
+    <profiles>
+        <profile>
+            <id>default</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes>
+                                <exclude>**/TestAzureFileSystemBlob.java</exclude>
+                                <exclude>**/TestAzureFileSystemGen2Flat.java</exclude>
+                                <exclude>**/TestAzureFileSystemGen2Hierarchical.java</exclude>
+                            </excludes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
+        <profile>
+            <id>cloud-tests</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <includes>
+                                <include>**/TestAzureFileSystemBlob.java</include>
+                                <include>**/TestAzureFileSystemGen2Flat.java</include>
+                                <include>**/TestAzureFileSystemGen2Hierarchical.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>
diff --git a/...ino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemBlob.java b/...ino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemBlob.java
@@ -16,13 +16,11 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 
 import java.io.IOException;
 
 import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.BLOB;
 
-@EnabledIfEnvironmentVariable(named = "ABFS_BLOB_ACCOUNT", matches = ".+")
 @TestInstance(Lifecycle.PER_CLASS)
 class TestAzureFileSystemBlob
         extends AbstractTestAzureFileSystem

diff --git a/...filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java b/...filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java
@@ -16,13 +16,11 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 
 import java.io.IOException;
 
 import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.FLAT;
 
-@EnabledIfEnvironmentVariable(named = "ABFS_FLAT_ACCOUNT", matches = ".+")
 @TestInstance(Lifecycle.PER_CLASS)
 class TestAzureFileSystemGen2Flat
         extends AbstractTestAzureFileSystem

diff --git a/...em-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java b/...em-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java
@@ -16,13 +16,11 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 
 import java.io.IOException;
 
 import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.HIERARCHICAL;
 
-@EnabledIfEnvironmentVariable(named = "ABFS_ACCOUNT", matches = ".+")
 @TestInstance(Lifecycle.PER_CLASS)
 class TestAzureFileSystemGen2Hierarchical
         extends AbstractTestAzureFileSystem