Skip to content

Commit

Permalink
Merge pull request #3262 from randomvariable/capture-metrics
Browse files Browse the repository at this point in the history
test framework: Enable metrics collection
  • Loading branch information
k8s-ci-robot authored Jun 26, 2020
2 parents cc327c8 + f429fb5 commit 8e3598b
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 4 deletions.
12 changes: 12 additions & 0 deletions test/e2e/config/docker-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,39 @@ providers:
- name: v0.3.0
# Use manifest from source files
value: ../../../config
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: BootstrapProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../bootstrap/kubeadm/config
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: ControlPlaneProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../controlplane/kubeadm/config
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: docker
type: InfrastructureProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../test/infrastructure/docker/config
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
# Add a metadata for docker provider
- sourcePath: "../data/infrastructure-docker/metadata.yaml"
Expand Down
8 changes: 8 additions & 0 deletions test/e2e/config/docker-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ providers:
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: BootstrapProvider
Expand All @@ -60,6 +62,8 @@ providers:
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: ControlPlaneProvider
Expand All @@ -72,6 +76,8 @@ providers:
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: docker
type: InfrastructureProvider
Expand All @@ -84,6 +90,8 @@ providers:
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
# Add a metadata for docker provider
- sourcePath: "../data/infrastructure-docker/metadata.yaml"
Expand Down
19 changes: 15 additions & 4 deletions test/framework/clusterctl/clusterctl_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ import (

// InitManagementClusterAndWatchControllerLogsInput is the input type for InitManagementClusterAndWatchControllerLogs.
type InitManagementClusterAndWatchControllerLogsInput struct {
ClusterProxy framework.ClusterProxy
ClusterctlConfigPath string
InfrastructureProviders []string
LogFolder string
ClusterProxy framework.ClusterProxy
ClusterctlConfigPath string
InfrastructureProviders []string
LogFolder string
DisableMetricsCollection bool
}

// InitManagementClusterAndWatchControllerLogs initializes a management using clusterctl and setup watches for controller logs.
Expand Down Expand Up @@ -87,6 +88,16 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
Deployment: deployment,
LogPath: filepath.Join(input.LogFolder, "controllers"),
})

if input.DisableMetricsCollection {
return
}
framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{
GetLister: client,
ClientSet: input.ClusterProxy.GetClientSet(),
Deployment: deployment,
MetricsPath: filepath.Join(input.LogFolder, "controllers"),
})
}
}

Expand Down
77 changes: 77 additions & 0 deletions test/framework/deployment_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"time"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
Expand Down Expand Up @@ -134,6 +136,81 @@ func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) {
}
}

type WatchPodMetricsInput struct {
GetLister GetLister
ClientSet *kubernetes.Clientset
Deployment *appsv1.Deployment
MetricsPath string
}

// WatchPodMetrics captures metrics from all pods every 5s. It expects to find port 8080 open on the controller.
// Use replacements in an e2econfig to enable metrics scraping without kube-rbac-proxy, e.g:
// - new: --metrics-addr=:8080
// old: --metrics-addr=127.0.0.1:8080
func WatchPodMetrics(ctx context.Context, input WatchPodMetricsInput) {
// Dump machine metrics every 5 seconds
ticker := time.NewTicker(time.Second * 5)
Expect(ctx).NotTo(BeNil(), "ctx is required for dumpContainerMetrics")
Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for dumpContainerMetrics")
Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for dumpContainerMetrics")

deployment := &appsv1.Deployment{}
key, err := client.ObjectKeyFromObject(input.Deployment)
Expect(err).NotTo(HaveOccurred(), "Failed to get key for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)

selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)

pods := &corev1.PodList{}
Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)

go func() {
defer GinkgoRecover()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
dumpPodMetrics(input.ClientSet, input.MetricsPath, deployment.Name, pods)
}
}
}()
}

// dumpPodMetrics captures metrics from all pods. It expects to find port 8080 open on the controller.
// Use replacements in an e2econfig to enable metrics scraping without kube-rbac-proxy, e.g:
// - new: --metrics-addr=:8080
// old: --metrics-addr=127.0.0.1:8080
func dumpPodMetrics(client *kubernetes.Clientset, metricsPath string, deploymentName string, pods *corev1.PodList) {
for _, pod := range pods.Items {

metricsDir := path.Join(metricsPath, deploymentName, pod.Name)
metricsFile := path.Join(metricsDir, "metrics.txt")
Expect(os.MkdirAll(metricsDir, 0750)).To(Succeed())

res := client.CoreV1().RESTClient().Get().
Namespace(pod.Namespace).
Resource("pods").
Name(fmt.Sprintf("%s:8080", pod.Name)).
SubResource("proxy").
Suffix("metrics").
Do()
data, err := res.Raw()

if err != nil {
// Failing to dump metrics should not cause the test to fail
data = []byte(fmt.Sprintf("Error retrieving metrics for pod %s/%s: %v\n%s", pod.Namespace, pod.Name, err, string(data)))
metricsFile = path.Join(metricsDir, "metrics-error.txt")
}

if err := ioutil.WriteFile(metricsFile, data, 0600); err != nil {
// Failing to dump metrics should not cause the test to fail
log.Logf("Error writing metrics for pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
}
}

// WaitForDNSUpgradeInput is the input for WaitForDNSUpgrade.
type WaitForDNSUpgradeInput struct {
Getter Getter
Expand Down

0 comments on commit 8e3598b

Please sign in to comment.