Cherry-pick #18802 to 7.7: Add missing network.sent_packets_count metric into compute googlecloud

.ci/packaging.groovy

@@ -0,0 +1,238 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu && immutable' }
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    JOB_GCS_BUCKET = 'beats-ci-artifacts'
+    JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    SNAPSHOT = "true"
+  }
+  options {
+    timeout(time: 3, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/packaging$')
+    upstream('Beats/beats-beats-mbp/7.7')
+  }
+  parameters {
+    booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
+    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}")
+        setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+        //stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+    stage('Build Packages'){
+      matrix {
+        axes {
+          axis {
+            name 'BEATS_FOLDER'
+            values (
+              'auditbeat',
+              'filebeat',
+              'heartbeat',
+              'journalbeat',
+              'metricbeat',
+              'packetbeat',
+              'winlogbeat',
+              'x-pack/auditbeat',
+              'x-pack/dockerlogbeat',
+              'x-pack/filebeat',
+              'x-pack/functionbeat',
+              // 'x-pack/heartbeat',
+              // 'x-pack/journalbeat',
+              'x-pack/metricbeat',
+              // 'x-pack/packetbeat',
+              'x-pack/winlogbeat'
+            )
+          }
+        }
+        stages {
+          stage('Package Linux'){
+            agent { label 'ubuntu && immutable' }
+            options { skipDefaultCheckout() }
+            when {
+              beforeAgent true
+              expression {
+                return params.linux
+              }
+            }
+            environment {
+              HOME = "${env.WORKSPACE}"
+              PLATFORMS = [
+                '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                'linux/ppc64le',
+                'linux/mips64',
+                'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
+            }
+            steps {
+              release()
+              pushCIDockerImages()
+            }
+          }
+          stage('Package Mac OS'){
+            agent { label 'macosx' }
+            options { skipDefaultCheckout() }
+            when {
+              beforeAgent true
+              expression {
+                return params.macos
+              }
+            }
+            environment {
+              HOME = "${env.WORKSPACE}"
+              PLATFORMS = [
+                '+all',
+                'darwin/amd64',
+              ].join(' ')
+            }
+            steps {
+              withMacOSEnv(){
+                release()
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+def pushCIDockerImages(){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "auditbeat"){
+      tagAndPush('auditbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "filebeat") {
+      tagAndPush('filebeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
+      tagAndPush('heartbeat')
+      tagAndPush('heartbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
+      tagAndPush('journalbeat')
+      tagAndPush('journalbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat"){
+      tagAndPush('metricbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
+      tagAndPush('packetbeat')
+      tagAndPush('packetbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/auditbeat"){
+      tagAndPush('auditbeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      tagAndPush('elastic-agent')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
+      tagAndPush('filebeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
+      tagAndPush('metricbeat')
+    }
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  if("${env.SNAPSHOT}" == "true"){
+    libbetaVer += "-SNAPSHOT"
+  }
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
+}
+
+def release(){
+  withBeatsEnv(){
+    dir("${env.BEATS_FOLDER}") {
+      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    }
+    publishPackages("${env.BEATS_FOLDER}")
+  }
+}
+
+def withMacOSEnv(Closure body){
+  withEnvMask( vars: [
+      [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],
+      [var: "KEYCHAIN", password: "/var/lib/jenkins/Library/Keychains/Elastic.keychain-db"],
+      [var: "APPLE_SIGNING_ENABLED", password: "true"],
+  ]){
+    body()
+  }
+}
+
+def publishPackages(baseDir){
+  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
+    credentialsId: "${JOB_GCS_CREDENTIALS}",
+    pathPrefix: "${baseDir}/build/distributions/",
+    pattern: "${baseDir}/build/distributions/**/*",
+    sharedPublicly: true,
+    showInline: true
+  )
+}
+
+def withBeatsEnv(Closure body) {
+  def os = goos()
+  def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64"
+
+  withEnv([
+    "HOME=${env.WORKSPACE}",
+    "GOPATH=${env.WORKSPACE}",
+    "GOROOT=${goRoot}",
+    "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}",
+    "MAGEFILE_CACHE=${WORKSPACE}/.magefile",
+    "PYTHON_ENV=${WORKSPACE}/python-env"
+  ]) {
+    deleteDir()
+    //unstash 'source'
+    gitCheckout(basedir: "${BASE_DIR}")
+    dir("${env.BASE_DIR}"){
+      sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh")
+      sh(label: "Install Mage", script: "make mage")
+      body()
+    }
+  }
+}
+
+def goos(){
+  def labels = env.NODE_LABELS
+
+  if (labels.contains('linux')) {
+    return 'linux'
+  } else if (labels.contains('windows')) {
+    return 'windows'
+  } else if (labels.contains('darwin')) {
+    return 'darwin'
+  }
+
+  error("Unhandled OS name in NODE_LABELS: " + labels)
+}

.ci/scripts/install-docker-compose.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+DC_CMD="${HOME}/bin/docker-compose"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)"
+chmod +x "${DC_CMD}"

.ci/scripts/install-go.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+GO_VERSION=${GO_VERSION:?$MSG}
+PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
+HOME=${HOME:?$MSG}
+ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+GVM_CMD="${HOME}/bin/gvm"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"
+chmod +x "${GVM_CMD}"
+
+gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+
+eval $(gvm ${GO_VERSION})

.ci/scripts/install-tools.bat

@@ -0,0 +1,27 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+where /q curl
+IF ERRORLEVEL 1 (
+ choco install curl -y --no-progress --skipdownloadcache
+)
+mkdir %WORKSPACE%\bin
+where /q gvm
+IF ERRORLEVEL 1 (
+ curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe
+)
+FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+go install -mod=vendor github.com/magefile/mage
+mage -version
+where mage
+
+if not exist C:\Python38\python.exe (
+    REM Install python 3.8.
+    choco install python -y -r --no-progress --version 3.8.2
+)
+python --version
+where python
+

.ci/scripts/kind-setup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+KBC_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KBC_CMD}"
+
+GO111MODULE="on" go get sigs.k8s.io/[email protected]
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+export KUBECONFIG="$(kind get kubeconfig-path)"
+kubectl cluster-info

.editorconfig

@@ -29,6 +29,10 @@ indent_style = tab
 [*.mk]
 indent_style = tab
 
+[Jenkinsfile]
+indent_size = 2
+indent_style = space
+
 [Vagrantfile]
 indent_size = 2
 indent_style = space

.github/CODEOWNERS

@@ -21,20 +21,20 @@
 /x-pack/packetbeat/ @elastic/siem
 
 # Filebeat
-/filebeat/module/ @elastic/integrations
-/filebeat/module/elasticsearch/ @elastic/stack-monitoring
-/filebeat/module/kibana/ @elastic/stack-monitoring
-/filebeat/module/logstash/ @elastic/stack-monitoring
-/x-pack/filebeat/module/ @elastic/integrations
-/x-pack/filebeat/module/suricata/ @elastic/secops
+# /filebeat/module/ @elastic/integrations
+# /filebeat/module/elasticsearch/ @elastic/stack-monitoring
+# /filebeat/module/kibana/ @elastic/stack-monitoring
+# /filebeat/module/logstash/ @elastic/stack-monitoring
+# /x-pack/filebeat/module/ @elastic/integrations
+# /x-pack/filebeat/module/suricata/ @elastic/secops
 
 # Metricbeat
-/metricbeat/module/ @elastic/integrations
-/metricbeat/module/elasticsearch/ @elastic/stack-monitoring
-/metricbeat/module/kibana/ @elastic/stack-monitoring
-/metricbeat/module/logstash/ @elastic/stack-monitoring
-/metricbeat/module/beat/ @elastic/stack-monitoring
-/x-pack/metricbeat/module/ @elastic/integrations
+# /metricbeat/module/ @elastic/integrations
+# /metricbeat/module/elasticsearch/ @elastic/stack-monitoring
+# /metricbeat/module/kibana/ @elastic/stack-monitoring
+# /metricbeat/module/logstash/ @elastic/stack-monitoring
+# /metricbeat/module/beat/ @elastic/stack-monitoring
+# /x-pack/metricbeat/module/ @elastic/integrations
 
 # Heartbeat
 /heartbeat/ @elastic/uptime

.gitignore

@@ -24,6 +24,7 @@ beat.db
 mage_output_file.go
 x-pack/functionbeat/*/fields.yml
 x-pack/functionbeat/provider/*/functionbeat-*
+x-pack/dockerlogbeat/temproot.tar
 
 # Editor swap files
 *.swp

.go-version

@@ -1 +1 @@
-1.13.3
+1.13.9