elastic · mtojek · Feb 5, 2020 · Feb 6, 2020 · Feb 6, 2020 · Feb 7, 2020
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3932,6 +3932,42 @@ type: long
 
 --
 
+[float]
+=== mgr_cluster_disk
+
+Disk metrics of Ceph cluster
+
+
+[float]
+=== mgr_cluster_health
+
+Status metrics of Ceph cluster
+
+
+[float]
+=== mgr_osd_disk
+
+OSD disk metrics of Ceph cluster
+
+
+[float]
+=== mgr_osd_perf
+
+OSD performance metrics of Ceph cluster
+
+
+[float]
+=== mgr_osd_pool_stats
+
+OSD pool stats of Ceph cluster
+
+
+[float]
+=== mgr_osd_tree
+
+OSD tree stats of Ceph cluster
+
+
 [float]
 === monitor_health
 

diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc
@@ -11,7 +11,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de
 [float]
 === Compatibility
 
-The Ceph module is tested with Ceph Jewel (10.2.10).
+The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7).
+
+Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon.
 
 [float]
 === Dashboard
@@ -52,6 +54,18 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-ceph-cluster_status,cluster_status>>
 
+* <<metricbeat-metricset-ceph-mgr_cluster_disk,mgr_cluster_disk>>
+
+* <<metricbeat-metricset-ceph-mgr_cluster_health,mgr_cluster_health>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_disk,mgr_osd_disk>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_perf,mgr_osd_perf>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_pool_stats,mgr_osd_pool_stats>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_tree,mgr_osd_tree>>
+
 * <<metricbeat-metricset-ceph-monitor_health,monitor_health>>
 
 * <<metricbeat-metricset-ceph-osd_df,osd_df>>
@@ -66,6 +80,18 @@ include::ceph/cluster_health.asciidoc[]
 
 include::ceph/cluster_status.asciidoc[]
 
+include::ceph/mgr_cluster_disk.asciidoc[]
+
+include::ceph/mgr_cluster_health.asciidoc[]
+
+include::ceph/mgr_osd_disk.asciidoc[]
+
+include::ceph/mgr_osd_perf.asciidoc[]
+
+include::ceph/mgr_osd_pool_stats.asciidoc[]
+
+include::ceph/mgr_osd_tree.asciidoc[]
+
 include::ceph/monitor_health.asciidoc[]
 
 include::ceph/osd_df.asciidoc[]

diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_cluster_disk]]
+=== Ceph mgr_cluster_disk metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_cluster_disk/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_cluster_health]]
+=== Ceph mgr_cluster_health metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_cluster_health/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_disk]]
+=== Ceph mgr_osd_disk metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_disk/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_perf]]
+=== Ceph mgr_osd_perf metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_perf/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_pool_stats]]
+=== Ceph mgr_osd_pool_stats metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc
@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_tree]]
+=== Ceph mgr_osd_tree metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_tree/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -38,9 +38,15 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-beat-state,state>>   
 |<<metricbeat-metricset-beat-stats,stats>>   
 |<<metricbeat-module-ceph,Ceph>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.7+| .7+|  |<<metricbeat-metricset-ceph-cluster_disk,cluster_disk>>   
+.13+| .13+|  |<<metricbeat-metricset-ceph-cluster_disk,cluster_disk>>   
 |<<metricbeat-metricset-ceph-cluster_health,cluster_health>>   
 |<<metricbeat-metricset-ceph-cluster_status,cluster_status>>   
+|<<metricbeat-metricset-ceph-mgr_cluster_disk,mgr_cluster_disk>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_cluster_health,mgr_cluster_health>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_disk,mgr_osd_disk>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_perf,mgr_osd_perf>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_pool_stats,mgr_osd_pool_stats>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_tree,mgr_osd_tree>> beta[]  
 |<<metricbeat-metricset-ceph-monitor_health,monitor_health>>   
 |<<metricbeat-metricset-ceph-osd_df,osd_df>>   
 |<<metricbeat-metricset-ceph-osd_tree,osd_tree>>   

diff --git a/metricbeat/module/ceph/_meta/Dockerfile b/metricbeat/module/ceph/_meta/Dockerfile
@@ -8,7 +8,6 @@ HEALTHCHECK --interval=1s --retries=300 \
   CMD curl -s -H "Accept: application/json" localhost:5000/api/v0.1/health \
         | jq .output.health.health_services[0].mons[0] \
         | grep health
-EXPOSE 5000
 
 ENV NETWORK_AUTO_DETECT 4
 ENV DEMO_DAEMONS osd,rest_api

diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus
@@ -0,0 +1,19 @@
+ARG CEPH_VERSION
+FROM ceph/daemon:${CEPH_VERSION}
+
+RUN yum -q install -y jq && yum clean all && rm -fr /var/cache/yum
+
+# Wait for the health endpoint to have monitors information
+ADD healthcheck.sh /
+HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh
+
+EXPOSE 5000 8003 9283
+
+ENV NETWORK_AUTO_DETECT 4
+ENV CEPH_DAEMON demo
+ENV CEPH_DEMO_UID beats
+ENV CEPH_DEMO_BUCKET beats
+ENV CEPH_DEMO_ACCESS_KEY demo
+ENV CEPH_DEMO_SECRET_KEY demo
+
+CMD ["demo"]
diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml
@@ -16,3 +16,23 @@
   #  - pool_disk
   period: 1m
   hosts: ["localhost:5000"]
+
+- module: ceph
+  #metricsets:
+  #  - mgr_cluster_health
+  period: 10s
+  hosts: [ "https://localhost:8003" ]
+  #username: "user"
+  #password: "secret"
+
+- module: ceph
+  #metricsets:
+  #  - mgr_cluster_disk
+  #  - mgr_osd_disk
+  #  - mgr_osd_perf
+  #  - mgr_osd_pool_stats
+  #  - mgr_osd_tree
+  period: 1m
+  hosts: [ "https://localhost:8003" ]
+  #username: "user"
+  #password: "secret"
diff --git a/metricbeat/module/ceph/_meta/docs.asciidoc b/metricbeat/module/ceph/_meta/docs.asciidoc
@@ -4,7 +4,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de
 [float]
 === Compatibility
 
-The Ceph module is tested with Ceph Jewel (10.2.10).
+The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7).
+
+Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon.
 
 [float]
 === Dashboard

diff --git a/metricbeat/module/ceph/_meta/healthcheck.sh b/metricbeat/module/ceph/_meta/healthcheck.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+KEYS_JSON=/opt/ceph-container/sree/static/restful-list-keys.json
+
+if [[ ! -f "${KEYS_JSON}" ]]; then
+  ceph restful list-keys | grep demo >/dev/null
+  if [[ $? -eq 0 ]]; then
+    ceph restful list-keys > ${KEYS_JSON}
+  else
+    exit 1
+  fi
+fi
+
+ceph health | grep HEALTH_OK && curl -s localhost:5000 >/dev/null
diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml
@@ -1,11 +1,23 @@
-version: '2.3'
+version: '3.7'
 
 services:
   ceph:
     image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}-1
     build:
       context: ./_meta
+      dockerfile: Dockerfile
       args:
         CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}
     ports:
       - 5000
+  ceph-mgr:
+    image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1
+    build:
+      context: ./_meta
+      dockerfile: Dockerfile.nautilus
+      args:
+        CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}
+    ports:
+      - 5000
+      - 8003
+      - 8080
diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go
diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc
@@ -0,0 +1 @@
+This is the `mgr_cluster_disk` metricset of the Ceph module.
diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml
@@ -0,0 +1,6 @@
+- name: mgr_cluster_disk
+  type: group
+  description: >
+    Disk metrics of Ceph cluster
+  release: beta
+  fields:
diff --git a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml
@@ -0,0 +1,25 @@
+default: true
+input:
+  module: http
+  metricset: json
+  defaults:
+    method: POST
+    namespace: "json_namespace"
+    basepath: "/request"
+    headers:
+      Content-Type: application/json
+    query:
+      wait: "1"
+    body: '{"prefix": "df", "format": "json"}'
+
+processors:
+  - extract_array:
+      field: http.json_namespace.finished
+      mappings:
+        ceph.request: 0
+  - decode_json_fields:
+      fields: ["ceph.request.outb"]
+      process_array: true
+      target: "ceph.mgr_cluster_disk"
+  - drop_fields:
+      fields: ["http", "ceph.request"]
diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc
@@ -0,0 +1 @@
+This is the `mgr_cluster_health` metricset of the Ceph module.
diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml
@@ -0,0 +1,6 @@
+- name: mgr_cluster_health
+  type: group
+  description: >
+    Status metrics of Ceph cluster
+  release: beta
+  fields:
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		This is the `mgr_cluster_disk` metricset of the Ceph module.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		This is the `mgr_cluster_health` metricset of the Ceph module.