Run smoke tests on Windows as well (#2617)

* Run smoke tests on Windows as well * Allow long paths on windows * Allow long paths on windows * Polish * Add check that spans were generated by expected OS * Fix method name * Use proper extra tag * Use long paths on CI build on Windows * Don't run linux test images on Windows * Don't run linux test images on Windows * Typo * More debug * Don't use testcontainers on Windows * Polish * Update backend * Support for extra resources on linux * Trying Linux backend on Windows * Pull backend images if not found * We do use windows images for backend as well * Use correct windows images * Less logs * Polish * Better logs * Polish * Polish
open-telemetry · Mar 26, 2021 · 43f4ab4 · 43f4ab4
1 parent 1e50b72
commit 43f4ab4
Show file tree

Hide file tree

Showing 32 changed files with 1,091 additions and 222 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -51,6 +51,29 @@ jobs:
         run: ./gradlew build --stacktrace
         working-directory: examples/distro
 
+  smoke-test-windows:
+    runs-on: windows-latest
+    steps:
+      - name: Support longpaths
+        run: git config --system core.longpaths true
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up JDK 11 for running Gradle
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+
+      - name: Restore cache
+        uses: burrunan/[email protected]
+        with:
+          job-id: smokeTests
+
+      - name: Test
+        run: ./gradlew :smoke-tests:test
+
   test:
     runs-on: ubuntu-latest
     strategy:

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -96,12 +96,17 @@ jobs:
           if-no-files-found: ignore
 
   smoke-test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ windows-latest, ubuntu-latest]
         suite: ["glassfishAndJetty", "libertyAndTomcat", "tomeeAndWildfly", "other"]
       fail-fast: false
     steps:
+      - name: Support longpaths
+        run: git config --system core.longpaths true
+        if: matrix.os == 'windows-latest'
+
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0

diff --git a/smoke-tests/matrix/build.gradle b/smoke-tests/matrix/build.gradle
@@ -40,8 +40,8 @@ def linuxTargets = [
   ],
   "tomee": [
     [version: ["7.0.0"], vm: ["hotspot"], jdk: ["8"]],
-    [version: ["8.0.6"], vm: ["hotspot"], jdk: ["8", "11"]],
     [version: ["7.0.0"], vm: ["openj9"], jdk: ["8"], dockerfile: "tomee-custom"],
+    [version: ["8.0.6"], vm: ["hotspot"], jdk: ["8", "11"]],
     [version: ["8.0.6"], vm: ["openj9"], jdk: ["8", "11"], dockerfile: "tomee-custom"]
   ],
   "payara": [
@@ -59,6 +59,10 @@ def linuxTargets = [
 ]
 
 def windowsTargets = [
+  "jetty" : [
+    [version: ["9.4.35"], vm: ["hotspot", "openj9"], jdk: ["8", "11", "15"], args: [sourceVersion: "9.4.35.v20201120"]],
+    [version: ["10.0.0"], vm: ["hotspot", "openj9"], jdk: ["11", "15"], dockerfile: "jetty-split", args: [sourceVersion: "10.0.0.beta3"]]
+  ],
   "tomcat" : [
     [version: ["7.0.107"], vm: ["hotspot", "openj9"], jdk: ["8"], args: [majorVersion: "7"]],
     [version: ["8.5.60"], vm: ["hotspot", "openj9"], jdk: ["8", "11"], args: [majorVersion: "8"]],
@@ -68,10 +72,6 @@ def windowsTargets = [
     [version: ["7.0.0"], vm: ["hotspot", "openj9"], jdk: ["8"]],
     [version: ["8.0.6"], vm: ["hotspot", "openj9"], jdk: ["8", "11"]]
   ],
-  "jetty" : [
-    [version: ["9.4.35"], vm: ["hotspot", "openj9"], jdk: ["8", "11", "15"], args: [sourceVersion: "9.4.35.v20201120"]],
-    [version: ["10.0.0"], vm: ["hotspot", "openj9"], jdk: ["11", "15"], dockerfile: "jetty-split", args: [sourceVersion: "10.0.0.beta3"]]
-  ],
   "payara": [
     [version: ["5.2020.6"], vm: ["hotspot", "openj9"], jdk: ["8", "11"]]
   ],

diff --git a/smoke-tests/smoke-tests.gradle b/smoke-tests/smoke-tests.gradle
@@ -12,6 +12,7 @@ apply from: "$rootDir/gradle/java.gradle"
 
 description = 'smoke-tests'
 
+def dockerJavaVersion = "3.2.5"
 dependencies {
   api deps.spock
   api project(':testing-common')
@@ -26,12 +27,18 @@ dependencies {
   implementation 'io.grpc:grpc-netty-shaded'
   implementation 'io.grpc:grpc-protobuf'
   implementation 'io.grpc:grpc-stub'
+
+  testImplementation("com.github.docker-java:docker-java-core:$dockerJavaVersion")
+  testImplementation("com.github.docker-java:docker-java-transport-httpclient5:$dockerJavaVersion")
+
 }
 
 test {
   inputs.files(tasks.findByPath(':javaagent:shadowJar').outputs.files)
   maxParallelForks = 2
 
+  testLogging.showStandardStreams = true
+
   timeout.set(Duration.ofMinutes(60))
 
   //We enable/disable smoke tests based on the java version requests
@@ -57,7 +64,10 @@ test {
     }
   }
 
+  def shadowTask = project(":javaagent").tasks.named("shadowJar").get()
+  inputs.files(layout.files(shadowTask))
+
   doFirst {
-    jvmArgs "-Dio.opentelemetry.smoketest.agent.shadowJar.path=${project(':javaagent').tasks.shadowJar.archivePath}"
+    jvmArgs "-Dio.opentelemetry.smoketest.agent.shadowJar.path=${shadowTask.archiveFile.get()}"
   }
 }
diff --git a/smoke-tests/src/test/groovy/io/opentelemetry/smoketest/AppServerTest.groovy b/smoke-tests/src/test/groovy/io/opentelemetry/smoketest/AppServerTest.groovy
@@ -5,6 +5,7 @@
 
 package io.opentelemetry.smoketest
 
+import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.OS_TYPE
 import static org.junit.Assume.assumeTrue
 
 import io.opentelemetry.proto.trace.v1.Span
@@ -21,14 +22,33 @@ abstract class AppServerTest extends SmokeTest {
   String jdk
   @Shared
   String serverVersion
+  @Shared
+  boolean isWindows
 
   def setupSpec() {
     def appServer = AppServerTestRunner.currentAppServer(this.getClass())
     serverVersion = appServer.version()
     jdk = appServer.jdk()
-    startTarget(jdk, serverVersion)
+
+    isWindows = System.getProperty("os.name").toLowerCase().contains("windows")
+    startTarget(jdk, serverVersion, isWindows)
+  }
+
+  @Override
+  protected String getTargetImage(String jdk) {
+    throw new UnsupportedOperationException("App servers tests should use getTargetImagePrefix")
   }
 
+  @Override
+  protected String getTargetImage(String jdk, String serverVersion, boolean windows) {
+    String platformSuffix = windows ? "-windows" : ""
+    String extraTag = "20210316.657616194"
+    String fullSuffix = "-${serverVersion}-jdk$jdk$platformSuffix-$extraTag"
+    return getTargetImagePrefix() + fullSuffix
+  }
+
+  protected abstract String getTargetImagePrefix()
+
   def cleanupSpec() {
     stopTarget()
   }
@@ -51,10 +71,10 @@ abstract class AppServerTest extends SmokeTest {
 
   //TODO add assert that server spans were created by servers, not by servlets
   @Unroll
-  def "#appServer smoke test on JDK #jdk"(String appServer, String jdk) {
+  def "#appServer smoke test on JDK #jdk"(String appServer, String jdk, boolean isWindows) {
     assumeTrue(testSmoke())
 
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/greeting"
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/greeting"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -89,22 +109,19 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == 3
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == 3
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer test static file found on JDK #jdk"(String appServer, String jdk) {
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/hello.txt"
+  def "#appServer test static file found on JDK #jdk"(String appServer, String jdk, boolean isWindows) {
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/hello.txt"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -132,22 +149,19 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == 1
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == 1
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer test static file not found on JDK #jdk"(String appServer, String jdk) {
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/file-that-does-not-exist"
+  def "#appServer test static file not found on JDK #jdk"(String appServer, String jdk, boolean isWindows) {
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/file-that-does-not-exist"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -174,24 +188,21 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == traces.countSpans()
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == traces.countSpans()
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer test request for WEB-INF/web.xml on JDK #jdk"(String appServer, String jdk) {
+  def "#appServer test request for WEB-INF/web.xml on JDK #jdk"(String appServer, String jdk, boolean isWindows) {
     assumeTrue(testRequestWebInfWebXml())
 
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/WEB-INF/web.xml"
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/WEB-INF/web.xml"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -221,24 +232,21 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == traces.countSpans()
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == traces.countSpans()
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer test request with error JDK #jdk"(String appServer, String jdk) {
+  def "#appServer test request with error JDK #jdk"(String appServer, String jdk, boolean isWindows) {
     assumeTrue(testException())
 
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/exception"
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/exception"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -268,22 +276,19 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == 1
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == 1
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer test request outside deployed application JDK #jdk"(String appServer, String jdk) {
-    String url = "http://localhost:${target.getMappedPort(8080)}/this-is-definitely-not-there-but-there-should-be-a-trace-nevertheless"
+  def "#appServer test request outside deployed application JDK #jdk"(String appServer, String jdk, boolean isWindows) {
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/this-is-definitely-not-there-but-there-should-be-a-trace-nevertheless"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -313,24 +318,21 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == traces.countSpans()
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == traces.countSpans()
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   @Unroll
-  def "#appServer async smoke test on JDK #jdk"(String appServer, String jdk) {
+  def "#appServer async smoke test on JDK #jdk"(String appServer, String jdk, boolean isWindows) {
     assumeTrue(testAsyncSmoke())
 
-    String url = "http://localhost:${target.getMappedPort(8080)}/app/asyncgreeting"
+    String url = "http://localhost:${containerManager.getTargetMappedPort(8080)}/app/asyncgreeting"
     def request = new Request.Builder().url(url).get().build()
     def currentAgentVersion = new JarFile(agentPath).getManifest().getMainAttributes().get(Attributes.Name.IMPLEMENTATION_VERSION)
 
@@ -365,17 +367,14 @@ abstract class AppServerTest extends SmokeTest {
     and: "Number of spans tagged with current otel library version"
     traces.countFilteredResourceAttributes("telemetry.auto.version", currentAgentVersion) == 3
 
-    and:
-    traces.findResourceAttribute("os.type")
-      .map { it.stringValue }
-      .findAny()
-      .isPresent()
+    and: "Number of spans tagged with expected OS type"
+    traces.countFilteredResourceAttributes(OS_TYPE.key, isWindows ? "WINDOWS" : "LINUX") == 3
 
     cleanup:
     response?.close()
 
     where:
-    [appServer, jdk] << getTestParams()
+    [appServer, jdk, isWindows] << getTestParams()
   }
 
   protected String getSpanName(String path) {
@@ -394,7 +393,7 @@ abstract class AppServerTest extends SmokeTest {
 
   protected List<List<Object>> getTestParams() {
     return [
-      [serverVersion, jdk]
+      [serverVersion, jdk, isWindows]
     ]
   }
 }