Add more kubernetes integration tests (#18138)

* Performing testing on kubelet metrics.

* Add integration tests for kubelet, proxy, and scheduler.

* mage fmt

* Update to use GoVersion in pods create.

* Fix kubelet hosts string to connect with kind.

dev-tools/mage/kubectl.go → dev-tools/mage/kubernetes/kubectl.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package mage
+package kubernetes
 
 import (
 	"fmt"

dev-tools/mage/kubernetes/kuberemote.go

@@ -50,6 +50,8 @@ import (
 	"k8s.io/client-go/tools/portforward"
 	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/client-go/transport/spdy"
+
+	"github.com/elastic/beats/v7/dev-tools/mage"
 )
 
 const sshBitSize = 4096
@@ -72,8 +74,8 @@ type KubeRemote struct {
 	publicKey  []byte
 }
 
-// New creates a new kubernetes remote runner.
-func New(kubeconfig string, namespace string, name string, workDir string, destDir string, syncDir string) (*KubeRemote, error) {
+// NewKubeRemote creates a new kubernetes remote runner.
+func NewKubeRemote(kubeconfig string, namespace string, name string, workDir string, destDir string, syncDir string) (*KubeRemote, error) {
 	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
 		return nil, err
@@ -204,8 +206,13 @@ func (r *KubeRemote) syncServiceAccount() error {
 
 // createPod creates the pod.
 func (r *KubeRemote) createPod(env map[string]string, cmd ...string) (*apiv1.Pod, error) {
+	version, err := mage.GoVersion()
+	if err != nil {
+		return nil, err
+	}
+	image := fmt.Sprintf("golang:%s", version)
 	r.deletePod() // ensure it doesn't already exist
-	return r.cs.CoreV1().Pods(r.namespace).Create(createPodManifest(r.name, "golang:1.13.9", env, cmd, r.workDir, r.destDir, r.secretName, r.svcAccName))
+	return r.cs.CoreV1().Pods(r.namespace).Create(createPodManifest(r.name, image, env, cmd, r.workDir, r.destDir, r.secretName, r.svcAccName))
 }
 
 // deletePod deletes the pod.
@@ -426,6 +433,8 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 		},
 		Spec: apiv1.PodSpec{
 			ServiceAccountName: svcAccName,
+			HostNetwork:        true,
+			DNSPolicy:          apiv1.DNSClusterFirstWithHostNet,
 			RestartPolicy:      apiv1.RestartPolicyNever,
 			InitContainers: []apiv1.Container{
 				{
@@ -458,7 +467,7 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 				},
 			},
 			Containers: []apiv1.Container{
-				apiv1.Container{
+				{
 					Name:       "exec",
 					Image:      image,
 					Command:    cmd,

dev-tools/mage/kubernetes/kubernetes.go

@@ -102,14 +102,14 @@ func (d *KubernetesIntegrationTester) Test(dir string, mageTarget string, env ma
 
 	// Apply the manifest from the dir. This is the requirements for the tests that will
 	// run inside the cluster.
-	if err := mage.KubectlApply(env, stdOut, stdErr, manifestPath); err != nil {
+	if err := KubectlApply(env, stdOut, stdErr, manifestPath); err != nil {
 		return errors.Wrapf(err, "failed to apply manifest %s", manifestPath)
 	}
 	defer func() {
 		if mg.Verbose() {
 			fmt.Println(">> Deleting module manifest from cluster...")
 		}
-		if err := mage.KubectlDelete(env, stdOut, stdErr, manifestPath); err != nil {
+		if err := KubectlDelete(env, stdOut, stdErr, manifestPath); err != nil {
 			log.Printf("%s", errors.Wrapf(err, "failed to apply manifest %s", manifestPath))
 		}
 	}()
@@ -125,7 +125,7 @@ func (d *KubernetesIntegrationTester) Test(dir string, mageTarget string, env ma
 
 	destDir := filepath.Join("/go/src", repo.CanonicalRootImportPath)
 	workDir := filepath.Join(destDir, repo.SubDir)
-	remote, err := mage.NewKubeRemote(kubeConfig, "default", kubernetesPodName(), workDir, destDir, repo.RootDir)
+	remote, err := NewKubeRemote(kubeConfig, "default", kubernetesPodName(), workDir, destDir, repo.RootDir)
 	if err != nil {
 		return err
 	}

metricbeat/module/kubernetes/container/container_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package container
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetKubeletConfig(t, "container")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/node/node_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package node
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetKubeletConfig(t, "node")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/pod/pod_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package pod
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetKubeletConfig(t, "pod")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/proxy/proxy_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package proxy
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetKubeProxyConfig(t, "proxy")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/scheduler/scheduler_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package scheduler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetSchedulerConfig(t, "scheduler")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/system/system_integration_test.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration,linux
+
+package system
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/test"
+)
+
+func TestFetchMetricset(t *testing.T) {
+	config := test.GetKubeletConfig(t, "system")
+	metricSet := mbtest.NewFetcher(t, config)
+	events, errs := metricSet.FetchEvents()
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+}

metricbeat/module/kubernetes/test/integration.go

@@ -48,3 +48,40 @@ func GetKubeStateMetricsConfig(t *testing.T, metricSetName string) map[string]in
 		"hosts":      []string{"kube-state-metrics:8080"},
 	}
 }
+
+// GetKubeletConfig function returns configuration for talking to Kubelet API.
+func GetKubeletConfig(t *testing.T, metricSetName string) map[string]interface{} {
+	t.Helper()
+	return map[string]interface{}{
+		"module":            "kubernetes",
+		"metricsets":        []string{metricSetName},
+		"host":              "${NODE_NAME}",
+		"hosts":             []string{"https://localhost:10250"},
+		"bearer_token_file": "/var/run/secrets/kubernetes.io/serviceaccount/token",
+		"ssl": map[string]interface{}{
+			"verification_mode": "none",
+		},
+	}
+}
+
+// GetKubeProxyConfig function returns configuration for talking to kube-proxy.
+func GetKubeProxyConfig(t *testing.T, metricSetName string) map[string]interface{} {
+	t.Helper()
+	return map[string]interface{}{
+		"module":     "kubernetes",
+		"metricsets": []string{metricSetName},
+		"host":       "${NODE_NAME}",
+		"hosts":      []string{"localhost:10252"},
+	}
+}
+
+// GetSchedulerConfig function returns configuration for talking to kube-proxy.
+func GetSchedulerConfig(t *testing.T, metricSetName string) map[string]interface{} {
+	t.Helper()
+	return map[string]interface{}{
+		"module":     "kubernetes",
+		"metricsets": []string{metricSetName},
+		"host":       "${NODE_NAME}",
+		"hosts":      []string{"localhost:10251"},
+	}
+}