From c1132fa6a7d6a91ed83fc8ca54cc92e6b48de0b4 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 4 May 2017 12:57:41 +0200 Subject: [PATCH] Http module improvements (#4170) This is a follow up PR for https://github.com/elastic/beats/pull/4156 * Add environment with small golang web service * Remove not needed body fields * Add defaults to configs * Add system tests * Update config with all config options --- metricbeat/docker-compose.yml | 5 ++ metricbeat/docs/fields.asciidoc | 11 +---- metricbeat/docs/modules/http.asciidoc | 17 ++++--- metricbeat/docs/modules_list.asciidoc | 2 +- metricbeat/metricbeat.full.yml | 12 +++-- metricbeat/module/http/_meta/Dockerfile | 15 +++--- metricbeat/module/http/_meta/config.yml | 10 ++-- metricbeat/module/http/_meta/docs.asciidoc | 5 +- metricbeat/module/http/_meta/env | 4 +- metricbeat/module/http/_meta/fields.yml | 2 +- metricbeat/module/http/_meta/test/main.go | 19 ++++++++ .../module/http/json/_meta/docs.asciidoc | 23 ++++++---- metricbeat/module/http/json/_meta/fields.yml | 4 -- metricbeat/module/http/json/json.go | 7 ++- .../module/http/json/json_integration_test.go | 8 ++-- metricbeat/tests/system/test_http.py | 46 +++++++++++++++++++ 16 files changed, 136 insertions(+), 54 deletions(-) create mode 100644 metricbeat/module/http/_meta/test/main.go create mode 100644 metricbeat/tests/system/test_http.py diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index dd256ca72a4..467c066e617 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -22,6 +22,7 @@ services: - ${PWD}/module/dropwizard/_meta/env - ${PWD}/module/elasticsearch/_meta/env - ${PWD}/module/haproxy/_meta/env + - ${PWD}/module/http/_meta/env - ${PWD}/module/jolokia/_meta/env - ${PWD}/module/kafka/_meta/env - ${PWD}/module/kibana/_meta/env @@ -46,6 +47,7 @@ services: dropwizard: { condition: service_healthy } elasticsearch: { condition: service_healthy } haproxy: { condition: service_healthy } + http: { condition: service_healthy } jolokia: { condition: service_healthy } kafka: { condition: service_healthy } kibana: { condition: service_healthy } @@ -80,6 +82,9 @@ services: haproxy: build: ${PWD}/module/haproxy/_meta + http: + build: ${PWD}/module/http/_meta + jolokia: build: ${PWD}/module/jolokia/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 574e928baab..50e2406d153 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3446,7 +3446,7 @@ The average queue time in ms over the last 1024 requests. [[exported-fields-http]] -== http Fields +== HTTP Fields http Module @@ -3526,15 +3526,6 @@ The HTTP payload received json metricset - -[float] -=== http.json.body - -type: keyword - -The HTTP payload received - - [[exported-fields-jolokia]] == Jolokia Fields diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index 0722fd7c94a..28c0f2f34e0 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -5,8 +5,9 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-module-http]] == http Module -Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available. -Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. +Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) module used to call arbitrary HTTP endpoints for which a dedicated metricbeat module is not available. + +Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. It is recommended to install a metricbeat instance on each host from which data should be fetched. Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch. This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. @@ -15,7 +16,7 @@ This is often necessary in security restricted network setups, where Logstash is [float] === Example Configuration -The http module supports the standard configuration options that are described +The HTTP module supports the standard configuration options that are described in <>. Here is an example configuration: [source,yaml] @@ -25,9 +26,13 @@ metricbeat.modules: metricsets: ["json"] enabled: false period: 10s - hosts: ["httpbin.org"] - namespace: "http_json_namespace" - path: "/headers" + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #request.enabled: false + #response.enabled: false ---- [float] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index dcab36e12ab..9c9e4d8dde4 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> - * <> + * <> * <> * <> * <> diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 04212daacac..f0c0cc8214f 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -177,14 +177,18 @@ metricbeat.modules: period: 10s hosts: ["tcp://127.0.0.1:14567"] -#-------------------------------- http Module -------------------------------- +#-------------------------------- HTTP Module -------------------------------- - module: http metricsets: ["json"] enabled: false period: 10s - hosts: ["httpbin.org"] - namespace: "http_json_namespace" - path: "/headers" + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #request.enabled: false + #response.enabled: false #------------------------------- Jolokia Module ------------------------------ - module: jolokia diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 7d0fa83cfe6..b99ca4a7734 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,11 +1,10 @@ # Tomcat is started to fetch Jolokia metrics from it -FROM jolokia/java-jolokia:7 -ENV TOMCAT_VERSION 7.0.55 -ENV TC apache-tomcat-${TOMCAT_VERSION} +FROM golang:1.8.1 -HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/ -EXPOSE 8778 -RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz -RUN tar xzf ${TC}.tar.gz -C /opt +COPY test/main.go main.go -CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run +EXPOSE 8080 + +HEALTHCHECK CMD curl -f curl localhost:8080/ + +CMD go run main.go diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml index 228730ff7ff..936c435b964 100644 --- a/metricbeat/module/http/_meta/config.yml +++ b/metricbeat/module/http/_meta/config.yml @@ -2,6 +2,10 @@ metricsets: ["json"] enabled: false period: 10s - hosts: ["httpbin.org"] - namespace: "http_json_namespace" - path: "/headers" + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #request.enabled: false + #response.enabled: false diff --git a/metricbeat/module/http/_meta/docs.asciidoc b/metricbeat/module/http/_meta/docs.asciidoc index 2ed17ff0962..a76af6cba9e 100644 --- a/metricbeat/module/http/_meta/docs.asciidoc +++ b/metricbeat/module/http/_meta/docs.asciidoc @@ -1,7 +1,8 @@ == http Module -Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call arbitrary HTTP endpoints for which not a dedicated metricbeat is available. -Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. +Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) module used to call arbitrary HTTP endpoints for which a dedicated metricbeat module is not available. + +Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. It is recommended to install a metricbeat instance on each host from which data should be fetched. Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch. This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. diff --git a/metricbeat/module/http/_meta/env b/metricbeat/module/http/_meta/env index 9c0340b6f3c..dfe04431eca 100644 --- a/metricbeat/module/http/_meta/env +++ b/metricbeat/module/http/_meta/env @@ -1,2 +1,2 @@ -JOLOKIA_HOST=jolokia -JOLOKIA_PORT=8778 +HTTP_HOST=http +HTTP_PORT=8080 diff --git a/metricbeat/module/http/_meta/fields.yml b/metricbeat/module/http/_meta/fields.yml index e3799fb3c94..93d6bd89110 100644 --- a/metricbeat/module/http/_meta/fields.yml +++ b/metricbeat/module/http/_meta/fields.yml @@ -1,5 +1,5 @@ - key: http - title: "http" + title: "HTTP" description: > http Module fields: diff --git a/metricbeat/module/http/_meta/test/main.go b/metricbeat/module/http/_meta/test/main.go new file mode 100644 index 00000000000..a2a7cad313a --- /dev/null +++ b/metricbeat/module/http/_meta/test/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + "log" + "net/http" +) + +func main() { + http.HandleFunc("/", serve) + err := http.ListenAndServe(":8080", nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} + +func serve(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"hello":"world"}`) +} diff --git a/metricbeat/module/http/json/_meta/docs.asciidoc b/metricbeat/module/http/json/_meta/docs.asciidoc index 0605f49389e..033ebb90b8a 100644 --- a/metricbeat/module/http/json/_meta/docs.asciidoc +++ b/metricbeat/module/http/json/_meta/docs.asciidoc @@ -7,7 +7,8 @@ This is the json metricset of the module http. The JSON structure returned by the HTTP endpoint will be added to the provided `namespace` field as shown in the following example: -```json +[source,json] +---- { "@timestamp": "2017-05-01T13:00:24.745Z", "beat": { @@ -31,16 +32,18 @@ The JSON structure returned by the HTTP endpoint will be added to the provided ` }, "type": "metricsets" } -``` +---- Here the response from `date.jsontest.com` is returned in the configured `http_json_namespace` namespace: -```json + +[source,json] +---- { "date": "05-01-2017", "milliseconds_since_epoch": 1493643625474.000000, "time": "01:00:25 PM" } -``` +---- It is required to set a namespace in the general module config section. @@ -53,7 +56,9 @@ With this configuration enabled additional information about the request are inc * Body/Payload Example: -```json + +[source,json] +---- { "@timestamp": "2017-05-01T13:00:24.745Z", "beat": { @@ -84,7 +89,7 @@ Example: }, "type": "metricsets" } -``` +---- [float] ==== response.enabled @@ -94,7 +99,9 @@ With this configuration enabled additional information about the response are in * HTTP Status Code Example: -```json + +[source,json] +---- { "@timestamp": "2017-05-01T13:00:24.745Z", "beat": { @@ -129,7 +136,7 @@ Example: }, "type": "metricsets" } -``` +---- [float] diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index b1683407a89..70eccef833d 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -3,7 +3,3 @@ description: > json metricset fields: - - name: body - type: keyword - description: > - The HTTP payload received diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 6aee44852e1..26cd995cbbe 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -64,7 +64,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { Body string `config:"body"` RequestEnabled bool `config:"request.enabled"` ResponseEnabled bool `config:"response.enabled"` - }{} + }{ + Method: "GET", + Body: "", + RequestEnabled: false, + ResponseEnabled: false, + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go index 0f1698d7f6e..b58ec825909 100644 --- a/metricbeat/module/http/json/json_integration_test.go +++ b/metricbeat/module/http/json/json_integration_test.go @@ -33,13 +33,13 @@ func getConfig() map[string]interface{} { "module": "http", "metricsets": []string{"json"}, "hosts": []string{getEnvHost() + ":" + getEnvPort()}, - "path": "/jolokia/?ignoreErrors=true&canonicalNaming=false", + "path": "/", "namespace": "testnamespace", } } func getEnvHost() string { - host := os.Getenv("JOLOKIA_HOST") + host := os.Getenv("HTTP_HOST") if len(host) == 0 { host = "127.0.0.1" @@ -48,10 +48,10 @@ func getEnvHost() string { } func getEnvPort() string { - port := os.Getenv("JOLOKIA_PORT") + port := os.Getenv("HTTP_PORT") if len(port) == 0 { - port = "8778" + port = "8080" } return port } diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py new file mode 100644 index 00000000000..8c2a3a33894 --- /dev/null +++ b/metricbeat/tests/system/test_http.py @@ -0,0 +1,46 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + +HTTP_FIELDS = metricbeat.COMMON_FIELDS + ["http"] + + +class Test(metricbeat.BaseTest): + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_json(self): + """ + http json metricset test + """ + self.render_config_template(modules=[{ + "name": "http", + "metricsets": ["json"], + "hosts": self.get_hosts(), + "period": "5s", + "namespace": "test", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log.replace("WARN The http json metricset is in beta.", ""), "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + assert evt["http"]["test"]["hello"] == "world" + + # Delete dynamic namespace part for fields comparison + del evt["http"]["test"] + + self.assertItemsEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return ["http://" + os.getenv('HTTP_HOST', 'localhost') + ':' + + os.getenv('HTTP_PORT', '8080')]