Enable having integration tests run as a PR pre-merge check (gitpod-i…

…o#4362)

This is a step towards running integration tests on all PRs and have it be a requirement for merging.

Specifically this commit introduces the following changes:

- Trigger the integration test job without using the --follow-with-prefix - this allows the build job to succeed independently of the the result of the integration tests.
- The job is triggered in such a way that a ci/werft/run-integration-tests commit check will be created (this was made possible by making changes to Werft, thanks @csweichel)

After this is merged we can

- Configure Github to have ci/werft/run-integration-tests be a required check. That means people can't merge a PR unless the check succeed. This should only be done once we run integration tests by default.

.werft/build.ts

@@ -20,6 +20,11 @@ build(context, version)
         }
     });
 
+// Werft phases
+const phases = {
+    TRIGGER_INTEGRATION_TESTS: 'trigger integration tests'
+}
+
 export function parseVersion(context) {
     let buildConfig = context.Annotations || {};
     const explicitVersion = buildConfig.version;
@@ -195,11 +200,7 @@ export async function build(context, version) {
         analytics
     };
     await deployToDev(deploymentConfig, workspaceFeatureFlags, dynamicCPULimits, storage);
-
-    if (withIntegrationTests) {
-        exec(`git config --global user.name "${context.Owner}"`);
-        exec(`werft run --follow-with-prefix="int-tests: " --remote-job-path .werft/run-integration-tests.yaml -a version=${deploymentConfig.version} -a namespace=${deploymentConfig.namespace} github`);
-    }
+    await triggerIntegrationTests(deploymentConfig, !withIntegrationTests)
 }
 
 interface DeploymentConfig {
@@ -387,6 +388,33 @@ export async function deployToDev(deploymentConfig: DeploymentConfig, workspaceF
     }
 }
 
+/**
+ * Trigger integration tests
+ */
+export async function triggerIntegrationTests(deploymentConfig: DeploymentConfig, skip: boolean) {
+    werft.phase(phases.TRIGGER_INTEGRATION_TESTS, "Trigger integration tests");
+
+    if (skip) {
+        // If we're skipping integration tests we wont trigger the job, which in turn won't create the
+        // ci/werft/run-integration-tests Github Check. As ci/werft/run-integration-tests is a required
+        // check this means you can't merge your PR without override checks.
+        werft.log(phases.TRIGGER_INTEGRATION_TESTS, "Skipped integration tests")
+        werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+        return
+    }
+
+    exec(`git config --global user.name "${context.Owner}"`);
+    const annotations = [
+        `version=${deploymentConfig.version}`,
+        `namespace=${deploymentConfig.namespace}`,
+        `username=${context.Owner}`,
+        `updateGitHubStatus=gitpod-io/gitpod`
+    ].map(annotation => `-a ${annotation}`).join(' ')
+    const jobId = exec(`werft run --remote-job-path .werft/run-integration-tests.yaml ${annotations} github`, {slice: phases.TRIGGER_INTEGRATION_TESTS}).trim();
+    werft.log(phases.TRIGGER_INTEGRATION_TESTS, `Triggered job ${jobId} - https://werft.gitpod-dev.com/job/${jobId}/logs`)
+    werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+}
+
 interface PreviewWorkspaceClusterRef {
     shortname: string;
     subdomain: string;

.werft/run-integration-tests.yaml

@@ -76,4 +76,8 @@ pod:
       /entrypoint.sh -kubeconfig=/config/kubeconfig -namespace={{ .Annotations.namespace }} -username=$USERNAME 2>&1 | ts "[int-tests] "
 
       RC=${PIPESTATUS[0]}
-      if [ $RC -eq 1 ]; then echo "[int-tests|FAIL]"; else echo "[int-tests|DONE]"; fi
+      if [ $RC -eq 1 ]; then
+        echo "[int-tests|FAIL]"
+      else
+        echo "[int-tests|DONE]"
+      fi