From 83aee7618d024ea7165ae166c40895a0509bc194 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 7 Dec 2016 10:13:46 +0100 Subject: [PATCH] Fetch container stats in parallel (#3127) (#3135) Currently fetching container stats is very slow as each request takes up to 2 seconds. To improve the fetching time if lots of containers are around, this creates the rrequests in parallel. The main downside is that this opens lots of connections. This fix should only temporary until the bulk api is available: https://github.com/docker/docker/pull/25361 (cherry picked from commit 78f547f235a177e23454c910f77e87a810a4a2ac) --- metricbeat/module/docker/docker.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/docker/docker.go b/metricbeat/module/docker/docker.go index 9a3106f003a4..67437eef1b8a 100644 --- a/metricbeat/module/docker/docker.go +++ b/metricbeat/module/docker/docker.go @@ -64,17 +64,35 @@ func FetchStats(client *docker.Client) ([]Stat, error) { return nil, err } + var wg sync.WaitGroup + containersList := []Stat{} + queue := make(chan Stat, 1) + wg.Add(len(containers)) + for _, container := range containers { - // This is currently very inefficient as docker calculates the average for each request, - // means each request will take at least 2s: https://github.com/docker/docker/blob/master/cli/command/container/stats_helpers.go#L148 - // Getting all stats at once is implemented here: https://github.com/docker/docker/pull/25361 - containersList = append(containersList, exportContainerStats(client, &container)) + go func(container docker.APIContainers) { + queue <- exportContainerStats(client, &container) + }(container) } + go func() { + for container := range queue { + containersList = append(containersList, container) + wg.Done() + } + }() + + wg.Wait() + return containersList, err } +// exportContainerStats loads stats for the given container +// +// This is currently very inefficient as docker calculates the average for each request, +// means each request will take at least 2s: https://github.com/docker/docker/blob/master/cli/command/container/stats_helpers.go#L148 +// Getting all stats at once is implemented here: https://github.com/docker/docker/pull/25361 func exportContainerStats(client *docker.Client, container *docker.APIContainers) Stat { var wg sync.WaitGroup var event Stat