feat: enable SSH access to users for debugging cloud instances

.ci/Jenkinsfile

@@ -38,6 +38,7 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'Run_As_Main_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on main branch.')
+    booleanParam(name: "DEVELOPER_MODE", defaultValue: false, description: "If checked, cloud resources won't be destroyed at the end of the pipeline. Default false")
     booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
     booleanParam(name: "NIGHTLY_SCENARIOS", defaultValue: false, description: "If it's needed to include the scenarios marked as @nightly in the test execution. Default false")
     string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
@@ -268,9 +269,14 @@ pipeline {
               script {
                 def stackWorkspace = "${env.WORKSPACE}/${env.BASE_DIR}"
                 def stackMachine = getMachineInfo(stackWorkspace, 'stack')
-                ansible(stackWorkspace,
+                if (params.DEVELOPER_MODE) {
+                  def stackRunner = getNodeIp(stackWorkspace, 'stack')
+                  log(level: 'DEBUG', text: "Stack instance won't be destroyed after the build. Please SSH into the stack machine on ${stackRunner.ip}")
+                } else {
+                  ansible(stackWorkspace,
                         env.RUN_ID.split('-')[0],
                         "-t destroy --extra-vars=\"nodeLabel=stack nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\"")
+                }
               }
             }
           }
@@ -530,9 +536,13 @@ def generateFunctionalTestStep(Map args = [:]){
              "e2e-testing/outputs/TEST-*${runId}*.xml",
              "outputs/${testRunner.ip}/.")
         sh "ls -l outputs/${testRunner.ip}"
-        ansible("${env.WORKSPACE}",
+        if (params.DEVELOPER_MODE) {
+          log(level: 'DEBUG', text: "Cloud instance won't be destroyed after the build. Please SSH into the test runner machine on ${testRunner.ip}. ")
+        } else {
+          ansible("${env.WORKSPACE}",
                 runId,
                 "-t destroy --extra-vars=\"nodeLabel=${platform} nodeImage=${machine.image} nodeInstanceType=${machine.instance_type}\"")
+        }
         junit allowEmptyResults: true,
           keepLongStdio: true,
           testResults: "outputs/${testRunner.ip}/TEST-*${runId}*.xml"

.ci/ansible/github-ssh-keys

@@ -0,0 +1 @@
+mdelapenya

.ci/ansible/playbook.yml

@@ -63,6 +63,11 @@
     tags:
       - setup-stack
 
+  - name: Add SSH keys to stack
+    include_tasks: tasks/install_ssh_keys.yml
+    tags:
+      - setup-stack
+
   - name: Start stack
     shell: |
       sed -i '' -e 's,http://elasticsearch,http://{{inventory_hostname}},g' /home/{{ansible_user}}/e2e-testing/cli/config/compose/profiles/fleet/default/kibana.config.yml
@@ -103,6 +108,11 @@
     tags:
       - setup-node
 
+  - name: Add SSH keys to runner instances
+    include_tasks: tasks/install_ssh_keys.yml
+    tags:
+      - setup-node
+
   - name: Setup source code
     include_tasks: tasks/copy_test_files.yml
     tags:

.ci/ansible/tasks/install_deps.yml

@@ -14,3 +14,7 @@
   register: package_install_res
   retries: 5
   until: package_install_res is success
+
+- name: Install ssh-import-id python package to copy public SSH keys from Github accounts
+  pip:
+    name: ssh-import-id

.ci/ansible/tasks/install_ssh_keys.yml

@@ -0,0 +1,4 @@
+---
+- name: Install SSH keys
+  shell: |
+    /home/{{ansible_user}}/e2e-testing/.ci/scripts/import-ssh-keys.sh

.ci/ansible/tasks/runners.yml

@@ -18,6 +18,7 @@
     image: '{{nodeImage}}'
     instance_type: '{{nodeInstanceType}}'
     instance_tags:
+      Kind: "e2e-testing-vm"
       Name: "e2e-{{nodeLabel}}-{{runId}}"
     count_tag:
       Name: "e2e-{{nodeLabel}}-{{runId}}"

.ci/aws-instances-reaper.groovy

@@ -0,0 +1,60 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-20' }
+  environment {
+    REPO = 'e2e-testing'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    HOME = "${env.WORKSPACE}"
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
+    AWS_PROVISIONER_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
+    AWS_EC2_INSTANCES_TAG= 'e2e-testing-vm'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron '0 0 * * 0'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "main",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+    stage('Reap AWS instances'){
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        withAWSEnv(secret: "${env.AWS_PROVISIONER_SECRET}") {
+          sh("aws ec2 terminate-instances --instance-ids `aws ec2 describe-instances --filters Name=tag:Kind,Values=${env.AWS_EC2_INSTANCES_TAG} --query Reservations[].Instances[].InstanceId --output text`")
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/jobs/aws-instances-reaper.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: e2e-tests/aws-instances-reaper
+    display-name: AWS Instances Reaper
+    description: Job to remove cloud resources on Sundays
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+      script-path: .ci/aws-instances-reaper.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/e2e-testing.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: true
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/e2e-testing.git
+            branches:
+              - main
+    triggers:
+      - timed: '0 0 * * 0'

.ci/scripts/import-ssh-keys.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+
+#
+# Imports public SSH keys from Github profiles
+#
+
+BASEDIR=$(dirname "$0")
+
+input="${BASEDIR}/../ansible/github-ssh-keys"
+while IFS= read -r line
+do
+    ssh-import-id "gh:$line"
+done < "$input"

e2e/TROUBLESHOOTING.md

@@ -5,6 +5,11 @@ The first step in determining the exact failure is to try and reproduce the test
 
 Each test suite's documentation should contain the specifics to run the tests, but it's summarises to executing `go test` or `godog` in the right directory.
 
+### SSH into the Cloud machines
+On CI, we are running the Elastic Stack and all test suites in AWS instances, so whenever a build failed we would need to access those machines and inspect the state of the machine: logs, files, containers... For that, we are enabling SSH access to those ephemeral machines, which will be kept for debugging purpose if and only if the DEVELOPER_MODE environment variable is set at the Jenkinsfile. In the UI of Jenkins, you can enable it using the DEVELOPMENT_MODE input argument, checking it to true (default is false). After the build finishes, the cloud instances won't be destroyed.
+
+To access the machines, you must be allowed to do so first, and for that, please submit a PR adding your Github username in alphabetical order to [this file](../.ci/ansible/github-ssh-keys), keeping a blank line as file ending.
+
 ### Tests fail because the product could not be configured or run correctly
 This type of failure usually indicates that code for these tests itself needs to be changed. See the sections on how to run the tests locally in the specific test suite.