From 2f19805003883b8af74e3c225404ee102047cbe2 Mon Sep 17 00:00:00 2001 From: Calvin Date: Thu, 11 Jul 2024 01:15:48 +0800 Subject: [PATCH 1/3] [feature] support collect data from header --- .../collect/http/HttpCollectImpl.java | 59 ++++++++++++------- manager/src/main/resources/define/app-api.yml | 45 ++++++++++++++ .../src/main/resources/define/app-website.yml | 45 ++++++++++++++ 3 files changed, 127 insertions(+), 22 deletions(-) diff --git a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java index 7c1db0a60f4..c028f522179 100644 --- a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java +++ b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java @@ -59,6 +59,7 @@ import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.CommonUtil; import org.apache.hertzbeat.common.util.IpDomainUtil; +import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; @@ -149,7 +150,7 @@ public void collect(CollectRep.MetricsData.Builder builder, case DispatchConstants.PARSE_XML_PATH -> parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); case DispatchConstants.PARSE_WEBSITE -> - parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); + parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, response); case DispatchConstants.PARSE_SITE_MAP -> parseResponseBySiteMap(resp, metrics.getAliasFields(), builder); default -> @@ -197,22 +198,46 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_HTTP; } - private void parseResponseByWebsite(String resp, List aliasFields, HttpProtocol http, - CollectRep.MetricsData.Builder builder, Long responseTime) { + private void parseResponseByWebsite(String resp, Metrics metrics, HttpProtocol http, + CollectRep.MetricsData.Builder builder, Long responseTime, + CloseableHttpResponse response) { CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword()); - for (String alias : aliasFields) { - if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(responseTime.toString()); - } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(Integer.toString(keywordNum)); - } else { - valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + for (String alias : metrics.getAliasFields()) { + if (metrics.getPriority() == 0) { + addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); + }else if (metrics.getPriority() == 1) { + addColumnFromHeader(valueRowBuilder, alias, response); } } builder.addValues(valueRowBuilder.build()); } + private void addColumnFromHeader(CollectRep.ValueRow.Builder valueRowBuilder, String alias, CloseableHttpResponse response) { + if (!StringUtils.hasText(alias)) { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + return; + } + + final Header firstHeader = response.getFirstHeader(alias); + if (Objects.isNull(firstHeader)) { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + return; + } + + valueRowBuilder.addColumns(firstHeader.getValue()); + } + + private void addColumnForSummary(Long responseTime, CollectRep.ValueRow.Builder valueRowBuilder, int keywordNum, String alias) { + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(responseTime.toString()); + } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(Integer.toString(keywordNum)); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } + } + private void parseResponseBySiteMap(String resp, List aliasFields, CollectRep.MetricsData.Builder builder) { List siteUrls = new LinkedList<>(); @@ -326,12 +351,8 @@ private void parseResponseByJsonPath(String resp, List aliasFields, Http } else { valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); } - } else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(responseTime.toString()); - } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(Integer.toString(keywordNum)); } else { - valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); } } } @@ -422,13 +443,7 @@ private void getValueFromJson(List aliasFields, CollectRep.MetricsData.B String value = valueElement.getAsString(); valueRowBuilder.addColumns(value); } else { - if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(responseTime.toString()); - } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { - valueRowBuilder.addColumns(Integer.toString(keywordNum)); - } else { - valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); - } + addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); } } builder.addValues(valueRowBuilder.build()); diff --git a/manager/src/main/resources/define/app-api.yml b/manager/src/main/resources/define/app-api.yml index add08ab5254..e3eaa9b631f 100644 --- a/manager/src/main/resources/define/app-api.yml +++ b/manager/src/main/resources/define/app-api.yml @@ -309,3 +309,48 @@ metrics: # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule parseType: website keyword: ^_^keyword^_^ + - name: header + i18n: + zh-CN: 请求头 + en-US: Header + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + priority: 1 + # collect metrics content + fields: + # field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field + - field: Content-Type + type: 1 + i18n: + zh-CN: Content Type + en-US: Content Type + - field: Content-Length + type: 1 + i18n: + zh-CN: 响应内容长度 + en-US: Content Length + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # http port + port: ^_^port^_^ + # http url + url: ^_^uri^_^ + timeout: ^_^timeout^_^ + # http method: GET POST PUT DELETE PATCH + method: GET + # if enabled https + ssl: ^_^ssl^_^ + authorization: + # http auth type: Basic Auth, Digest Auth, Bearer Token + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule + parseType: website + keyword: ^_^keyword^_^ \ No newline at end of file diff --git a/manager/src/main/resources/define/app-website.yml b/manager/src/main/resources/define/app-website.yml index b14547b9889..fd19bf25bd5 100644 --- a/manager/src/main/resources/define/app-website.yml +++ b/manager/src/main/resources/define/app-website.yml @@ -197,3 +197,48 @@ metrics: # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule parseType: website keyword: ^_^keyword^_^ + - name: header + i18n: + zh-CN: 请求头 + en-US: Header + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + priority: 1 + # collect metrics content + fields: + # field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field + - field: Content-Type + type: 1 + i18n: + zh-CN: Content Type + en-US: Content Type + - field: Content-Length + type: 1 + i18n: + zh-CN: 响应内容长度 + en-US: Content Length + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # http port + port: ^_^port^_^ + # http url + url: ^_^uri^_^ + timeout: ^_^timeout^_^ + # http method: GET POST PUT DELETE PATCH + method: GET + # if enabled https + ssl: ^_^ssl^_^ + authorization: + # http auth type: Basic Auth, Digest Auth, Bearer Token + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule + parseType: website + keyword: ^_^keyword^_^ From 0d4c50d63fbdaf9ecd9aa5404c65ff551458f65c Mon Sep 17 00:00:00 2001 From: Calvin Date: Thu, 11 Jul 2024 01:21:48 +0800 Subject: [PATCH 2/3] [feature] support collect data from header --- .../hertzbeat/collector/collect/http/HttpCollectImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java index c028f522179..b267714a938 100644 --- a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java +++ b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java @@ -204,9 +204,9 @@ private void parseResponseByWebsite(String resp, Metrics metrics, HttpProtocol h CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword()); for (String alias : metrics.getAliasFields()) { - if (metrics.getPriority() == 0) { + if ("summary".equalsIgnoreCase(metrics.getName())) { addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); - }else if (metrics.getPriority() == 1) { + }else if ("header".equalsIgnoreCase(metrics.getName())) { addColumnFromHeader(valueRowBuilder, alias, response); } } From adbabbbcaf297daf94d581116967e58e0ae019e5 Mon Sep 17 00:00:00 2001 From: Calvin Date: Thu, 11 Jul 2024 20:20:37 +0800 Subject: [PATCH 3/3] [refactor] fix backend ci failed --- .../hertzbeat/collector/collect/http/HttpCollectImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java index b267714a938..193dd1a38b2 100644 --- a/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java +++ b/collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java @@ -206,7 +206,7 @@ private void parseResponseByWebsite(String resp, Metrics metrics, HttpProtocol h for (String alias : metrics.getAliasFields()) { if ("summary".equalsIgnoreCase(metrics.getName())) { addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); - }else if ("header".equalsIgnoreCase(metrics.getName())) { + } else if ("header".equalsIgnoreCase(metrics.getName())) { addColumnFromHeader(valueRowBuilder, alias, response); } }