Add Hadoop metrics receiver

apps/hadoop.go

@@ -0,0 +1,58 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apps
+
+import (
+	"fmt"
+
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
+)
+
+type MetricsReceiverHadoop struct {
+	confgenerator.ConfigComponent                 `yaml:",inline"`
+	confgenerator.MetricsReceiverShared           `yaml:",inline"`
+	confgenerator.MetricsReceiverSharedJVM        `yaml:",inline"`
+	confgenerator.MetricsReceiverSharedCollectJVM `yaml:",inline"`
+}
+
+const defaultHadoopEndpoint = "localhost:8004"
+
+func (r MetricsReceiverHadoop) Type() string {
+	return "hadoop"
+}
+
+func (r MetricsReceiverHadoop) Pipelines() []otel.Pipeline {
+	targetSystem := "hadoop"
+	if r.CollectJVMMetrics == nil || *r.CollectJVMMetrics {
+		targetSystem = fmt.Sprintf("%s,%s", targetSystem, "jvm")
+	}
+
+	return r.MetricsReceiverSharedJVM.JVMConfig(
+		targetSystem,
+		defaultHadoopEndpoint,
+		r.CollectionIntervalString(),
+		[]otel.Component{
+			otel.NormalizeSums(),
+			otel.MetricsTransform(
+				otel.AddPrefix("workload.googleapis.com"),
+			),
+		},
+	)
+}
+
+func init() {
+	confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.Component { return &MetricsReceiverHadoop{} })
+}

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_malformed_url/golden_error

@@ -0,0 +1,9 @@
+[19:17] Key: 'MetricsReceiverSharedJVM.endpoint' Error:Field validation for 'endpoint' failed on the 'hostname_port|startswith=service:jmx:' tag
+  16 |   receivers:
+  17 |     hadoop:
+  18 |       type: hadoop
+> 19 |       endpoint: localhost
+                       ^
+  20 |       collection_interval: 30s
+  21 |   service:
+  22 |     pipelines:

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_malformed_url/input.yaml

@@ -0,0 +1,25 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metrics:
+  receivers:
+    hadoop:
+      type: hadoop
+      endpoint: localhost
+      collection_interval: 30s
+  service:
+    pipelines:
+      hadoop:
+        receivers:
+          - hadoop

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_not_service_url/golden_error

@@ -0,0 +1,9 @@
+[19:17] Key: 'MetricsReceiverSharedJVM.endpoint' Error:Field validation for 'endpoint' failed on the 'hostname_port|startswith=service:jmx:' tag
+  16 |   receivers:
+  17 |     hadoop:
+  18 |       type: hadoop
+> 19 |       endpoint: http://localhost:80/forbidden.html
+                       ^
+  20 |       collection_interval: 30s
+  21 |   service:
+  22 |     pipelines:

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_not_service_url/input.yaml

@@ -0,0 +1,25 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metrics:
+  receivers:
+    hadoop:
+      type: hadoop
+      endpoint: http://localhost:80/forbidden.html
+      collection_interval: 30s
+  service:
+    pipelines:
+      hadoop:
+        receivers:
+          - hadoop

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_username_no_password/golden_error

@@ -0,0 +1 @@
+"password" is required when "Username" is set

confgenerator/testdata/invalid/linux/metrics-receiver_hadoop_username_no_password/input.yaml

@@ -0,0 +1,25 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metrics:
+  receivers:
+    hadoop:
+      type: hadoop
+      username: usr
+      collection_interval: 30s
+  service:
+    pipelines:
+      hadoop:
+        receivers:
+          - hadoop

confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/golden_error

@@ -1 +1 @@
-metrics receiver with type "iis" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].
+metrics receiver with type "iis" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hadoop, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].

confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/golden_error

@@ -1 +1 @@
-metrics receiver with type "mssql" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].
+metrics receiver with type "mssql" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hadoop, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].

confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error

@@ -1 +1 @@
-metrics receiver with type "unsupported_type" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].
+metrics receiver with type "unsupported_type" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hadoop, hostmetrics, jvm, kafka, memcached, mongodb, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].

confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error

@@ -1 +1 @@
-metrics receiver with type "unsupported_type" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hostmetrics, iis, jvm, kafka, memcached, mongodb, mssql, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].
+metrics receiver with type "unsupported_type" is not supported. Supported metrics receiver types: [activemq, apache, cassandra, couchdb, elasticsearch, hadoop, hostmetrics, iis, jvm, kafka, memcached, mongodb, mssql, mysql, nginx, postgresql, redis, solr, tomcat, zookeeper].

confgenerator/testdata/valid/linux/metrics-receiver_hadoop/golden_fluent_bit_main.conf

@@ -0,0 +1,79 @@
+@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers
+@SET logs_dir=/var/log/google-cloud-ops-agent/subagents
+
+[SERVICE]
+    Daemon                    off
+    Flush                     1
+    Log_Level                 info
+    dns.resolver              legacy
+    storage.backlog.mem_limit 50M
+    storage.checksum          on
+    storage.max_chunks_up     128
+    storage.metrics           on
+    storage.sync              normal
+
+[INPUT]
+    Name            fluentbit_metrics
+    Scrape_Interval 60
+    Scrape_On_Start True
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/default_pipeline_syslog
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              /var/log/messages,/var/log/syslog
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               default_pipeline.syslog
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/ops-agent-fluent-bit
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/logging-module.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-fluent-bit
+    storage.type      filesystem
+
+[FILTER]
+    Add   logging.googleapis.com/logName syslog
+    Match default_pipeline.syslog
+    Name  modify
+
+[OUTPUT]
+    Match_Regex                   ^(default_pipeline\.syslog)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match_Regex                   ^(ops-agent-fluent-bit)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match *
+    Name  prometheus_exporter
+    host  0.0.0.0
+    port  20202