From 1a1539a30a5635c4d58046cf88eb5e73648425b2 Mon Sep 17 00:00:00 2001 From: Enzo Hamelin Date: Tue, 10 Nov 2020 13:34:52 +0100 Subject: [PATCH 1/3] Add dataset metrics to zfs input --- plugins/inputs/zfs/README.md | 23 ++++++++++- plugins/inputs/zfs/zfs.go | 17 +++++--- plugins/inputs/zfs/zfs_freebsd.go | 54 +++++++++++++++++++++++++- plugins/inputs/zfs/zfs_freebsd_test.go | 54 ++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/zfs/README.md b/plugins/inputs/zfs/README.md index f0e71a47d714c..1f3f125d391ec 100644 --- a/plugins/inputs/zfs/README.md +++ b/plugins/inputs/zfs/README.md @@ -2,7 +2,7 @@ This ZFS plugin provides metrics from your ZFS filesystems. It supports ZFS on Linux and FreeBSD. It gets ZFS stat from `/proc/spl/kstat/zfs` on Linux and -from `sysctl` and `zpool` on FreeBSD. +from `sysctl`, 'zfs' and `zpool` on FreeBSD. ### Configuration: @@ -22,11 +22,14 @@ from `sysctl` and `zpool` on FreeBSD. ## By default, don't gather zpool stats # poolMetrics = false + + ## By default, don't gather dataset stats + # datasetMetrics = false ``` ### Measurements & Fields: -By default this plugin collects metrics about ZFS internals and pool. +By default this plugin collects metrics about ZFS internals pool and dataset. These metrics are either counters or measure sizes in bytes. These metrics will be in the `zfs` measurement with the field names listed bellow. @@ -34,6 +37,9 @@ names listed bellow. If `poolMetrics` is enabled then additional metrics will be gathered for each pool. +If `datasetMetrics` is enabled then additional metrics will be gathered for +each dataset. + - zfs With fields listed bellow. @@ -206,21 +212,34 @@ On FreeBSD: - size (integer, bytes) - fragmentation (integer, percent) +#### Dataset Metrics (optional, only on FreeBSD) + +- zfs_dataset + - avail (integer, bytes) + - used (integer, bytes) + - usedsnap (integer, bytes + - usedds (integer, bytes) + ### Tags: - ZFS stats (`zfs`) will have the following tag: - pools - A `::` concatenated list of all ZFS pools on the machine. + - datasets - A `::` concatenated list of all ZFS datasets on the machine. - Pool metrics (`zfs_pool`) will have the following tag: - pool - with the name of the pool which the metrics are for. - health - the health status of the pool. (FreeBSD only) +- Dataset metrics (`zfs_dataset`) will have the following tag: + - dataset - with the name of the dataset which the metrics are for. + ### Example Output: ``` $ ./telegraf --config telegraf.conf --input-filter zfs --test * Plugin: zfs, Collection 1 > zfs_pool,health=ONLINE,pool=zroot allocated=1578590208i,capacity=2i,dedupratio=1,fragmentation=1i,free=64456531968i,size=66035122176i 1464473103625653908 +> zfs_dataset,dataset=zata avail=10741741326336,used=8564135526400,usedsnap=0,usedds=90112 > zfs,pools=zroot arcstats_allocated=4167764i,arcstats_anon_evictable_data=0i,arcstats_anon_evictable_metadata=0i,arcstats_anon_size=16896i,arcstats_arc_meta_limit=10485760i,arcstats_arc_meta_max=115269568i,arcstats_arc_meta_min=8388608i,arcstats_arc_meta_used=51977456i,arcstats_c=16777216i,arcstats_c_max=41943040i,arcstats_c_min=16777216i,arcstats_data_size=0i,arcstats_deleted=1699340i,arcstats_demand_data_hits=14836131i,arcstats_demand_data_misses=2842945i,arcstats_demand_hit_predictive_prefetch=0i,arcstats_demand_metadata_hits=1655006i,arcstats_demand_metadata_misses=830074i,arcstats_duplicate_buffers=0i,arcstats_duplicate_buffers_size=0i,arcstats_duplicate_reads=123i,arcstats_evict_l2_cached=0i,arcstats_evict_l2_eligible=332172623872i,arcstats_evict_l2_ineligible=6168576i,arcstats_evict_l2_skip=0i,arcstats_evict_not_enough=12189444i,arcstats_evict_skip=195190764i,arcstats_hash_chain_max=2i,arcstats_hash_chains=10i,arcstats_hash_collisions=43134i,arcstats_hash_elements=2268i,arcstats_hash_elements_max=6136i,arcstats_hdr_size=565632i,arcstats_hits=16515778i,arcstats_l2_abort_lowmem=0i,arcstats_l2_asize=0i,arcstats_l2_cdata_free_on_write=0i,arcstats_l2_cksum_bad=0i,arcstats_l2_compress_failures=0i,arcstats_l2_compress_successes=0i,arcstats_l2_compress_zeros=0i,arcstats_l2_evict_l1cached=0i,arcstats_l2_evict_lock_retry=0i,arcstats_l2_evict_reading=0i,arcstats_l2_feeds=0i,arcstats_l2_free_on_write=0i,arcstats_l2_hdr_size=0i,arcstats_l2_hits=0i,arcstats_l2_io_error=0i,arcstats_l2_misses=0i,arcstats_l2_read_bytes=0i,arcstats_l2_rw_clash=0i,arcstats_l2_size=0i,arcstats_l2_write_buffer_bytes_scanned=0i,arcstats_l2_write_buffer_iter=0i,arcstats_l2_write_buffer_list_iter=0i,arcstats_l2_write_buffer_list_null_iter=0i,arcstats_l2_write_bytes=0i,arcstats_l2_write_full=0i,arcstats_l2_write_in_l2=0i,arcstats_l2_write_io_in_progress=0i,arcstats_l2_write_not_cacheable=380i,arcstats_l2_write_passed_headroom=0i,arcstats_l2_write_pios=0i,arcstats_l2_write_spa_mismatch=0i,arcstats_l2_write_trylock_fail=0i,arcstats_l2_writes_done=0i,arcstats_l2_writes_error=0i,arcstats_l2_writes_lock_retry=0i,arcstats_l2_writes_sent=0i,arcstats_memory_throttle_count=0i,arcstats_metadata_size=17014784i,arcstats_mfu_evictable_data=0i,arcstats_mfu_evictable_metadata=16384i,arcstats_mfu_ghost_evictable_data=5723648i,arcstats_mfu_ghost_evictable_metadata=10709504i,arcstats_mfu_ghost_hits=1315619i,arcstats_mfu_ghost_size=16433152i,arcstats_mfu_hits=7646611i,arcstats_mfu_size=305152i,arcstats_misses=3676993i,arcstats_mru_evictable_data=0i,arcstats_mru_evictable_metadata=0i,arcstats_mru_ghost_evictable_data=0i,arcstats_mru_ghost_evictable_metadata=80896i,arcstats_mru_ghost_hits=324250i,arcstats_mru_ghost_size=80896i,arcstats_mru_hits=8844526i,arcstats_mru_size=16693248i,arcstats_mutex_miss=354023i,arcstats_other_size=34397040i,arcstats_p=4172800i,arcstats_prefetch_data_hits=0i,arcstats_prefetch_data_misses=0i,arcstats_prefetch_metadata_hits=24641i,arcstats_prefetch_metadata_misses=3974i,arcstats_size=51977456i,arcstats_sync_wait_for_async=0i,vdev_cache_stats_delegations=779i,vdev_cache_stats_hits=323123i,vdev_cache_stats_misses=59929i,zfetchstats_hits=0i,zfetchstats_max_streams=0i,zfetchstats_misses=0i 1464473103634124908 ``` diff --git a/plugins/inputs/zfs/zfs.go b/plugins/inputs/zfs/zfs.go index 8e6bec4644932..0fde1950bc13c 100644 --- a/plugins/inputs/zfs/zfs.go +++ b/plugins/inputs/zfs/zfs.go @@ -2,13 +2,16 @@ package zfs type Sysctl func(metric string) ([]string, error) type Zpool func() ([]string, error) +type Zdataset func(properties []string) ([]string, error) type Zfs struct { - KstatPath string - KstatMetrics []string - PoolMetrics bool - sysctl Sysctl - zpool Zpool + KstatPath string + KstatMetrics []string + PoolMetrics bool + DatasetMetrics bool + sysctl Sysctl + zpool Zpool + zdataset Zdataset } var sampleConfig = ` @@ -24,6 +27,8 @@ var sampleConfig = ` # "dmu_tx", "fm", "vdev_mirror_stats", "zfetchstats", "zil"] ## By default, don't gather zpool stats # poolMetrics = false + ## By default, don't gather zdataset stats + # datasetMetrics = false ` func (z *Zfs) SampleConfig() string { @@ -31,5 +36,5 @@ func (z *Zfs) SampleConfig() string { } func (z *Zfs) Description() string { - return "Read metrics of ZFS from arcstats, zfetchstats, vdev_cache_stats, and pools" + return "Read metrics of ZFS from arcstats, zfetchstats, vdev_cache_stats, pools and datasets" } diff --git a/plugins/inputs/zfs/zfs_freebsd.go b/plugins/inputs/zfs/zfs_freebsd.go index 51c20682e832b..1a7455bf0b568 100644 --- a/plugins/inputs/zfs/zfs_freebsd.go +++ b/plugins/inputs/zfs/zfs_freebsd.go @@ -87,6 +87,46 @@ func (z *Zfs) gatherPoolStats(acc telegraf.Accumulator) (string, error) { return strings.Join(pools, "::"), nil } +func (z *Zfs) gatherDatasetStats(acc telegraf.Accumulator) (string, error) { + properties := []string{"name", "avail", "used", "usedsnap", "usedds"} + + lines, err := z.zdataset(properties) + if err != nil { + return "", err + } + + datasets := []string{} + for _, line := range lines { + col := strings.Split(line, "\t") + + datasets = append(datasets, col[0]) + } + + if z.DatasetMetrics { + for _, line := range lines { + col := strings.Split(line, "\t") + if len(col) != len(properties) { + continue + } + + tags := map[string]string{"dataset": col[0]} + fields := map[string]interface{}{} + + for i, key := range properties[1:] { + value, err := strconv.ParseInt(col[i+1], 10, 64) + if err != nil { + return "", fmt.Errorf("Error parsing %s: %s", key, err) + } + fields[key] = value + } + + acc.AddFields("zfs_dataset", fields, tags) + } + } + + return strings.Join(datasets, "::"), nil +} + func (z *Zfs) Gather(acc telegraf.Accumulator) error { kstatMetrics := z.KstatMetrics if len(kstatMetrics) == 0 { @@ -99,6 +139,11 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { return err } tags["pools"] = poolNames + datasetNames, err := z.gatherDatasetStats(acc) + if err != nil { + return err + } + tags["datasets"] = datasetNames fields := make(map[string]interface{}) for _, metric := range kstatMetrics { @@ -137,6 +182,10 @@ func zpool() ([]string, error) { return run("zpool", []string{"list", "-Hp", "-o", "name,health,size,alloc,free,fragmentation,capacity,dedupratio"}...) } +func zdataset(properties []string) ([]string, error) { + return run("zfs", []string{"list", "-Hp", "-o", strings.join(properties, ',')}...) +} + func sysctl(metric string) ([]string, error) { return run("sysctl", []string{"-q", fmt.Sprintf("kstat.zfs.misc.%s", metric)}...) } @@ -144,8 +193,9 @@ func sysctl(metric string) ([]string, error) { func init() { inputs.Add("zfs", func() telegraf.Input { return &Zfs{ - sysctl: sysctl, - zpool: zpool, + sysctl: sysctl, + zpool: zpool, + zdataset: zdataset, } }) } diff --git a/plugins/inputs/zfs/zfs_freebsd_test.go b/plugins/inputs/zfs/zfs_freebsd_test.go index 87f21f67245f4..4d1fea0ae483a 100644 --- a/plugins/inputs/zfs/zfs_freebsd_test.go +++ b/plugins/inputs/zfs/zfs_freebsd_test.go @@ -31,6 +31,18 @@ func mock_zpool_unavail() ([]string, error) { return zpool_output_unavail, nil } +// $ zfs list -Hp -o name,avail,used,usedsnap,usedds +var zdataset_output = []string{ + "zata 10741741326336 8564135526400 0 90112", + "zata/home 10741741326336 2498560 212992 2285568", + "zata/import 10741741326336 196608 81920 114688", + "zata/storage 10741741326336 8556084379648 3601138999296 4954945380352", +} + +func mock_zdataset() ([]string, error) { + return zdataset_output, nil +} + // sysctl -q kstat.zfs.misc.arcstats // sysctl -q kstat.zfs.misc.vdev_cache_stats @@ -126,6 +138,39 @@ func TestZfsPoolMetrics_unavail(t *testing.T) { acc.AssertContainsTaggedFields(t, "zfs_pool", poolMetrics, tags) } +func TestZfsDatasetMetrics(t *testing.T) { + var acc testutil.Accumulator + + z := &Zfs{ + KstatMetrics: []string{"vdev_cache_stats"}, + sysctl: mock_sysctl, + zdataset: mock_zdataset, + } + err := z.Gather(&acc) + require.NoError(t, err) + + require.False(t, acc.HasMeasurement("zfs_dataset")) + acc.Metrics = nil + + z = &Zfs{ + KstatMetrics: []string{"vdev_cache_stats"}, + DatasetMetrics: true, + sysctl: mock_sysctl, + zdataset: mock_zdataset, + } + err = z.Gather(&acc) + require.NoError(t, err) + + //one pool, all metrics + tags := map[string]string{ + "dataset": "zata", + } + + datasetMetrics := getZataDatasetMetrics() + + acc.AssertContainsTaggedFields(t, "zfs_dataset", datasetMetrics, tags) +} + func TestZfsGeneratesMetrics(t *testing.T) { var acc testutil.Accumulator @@ -178,6 +223,15 @@ func getTemp2PoolMetrics() map[string]interface{} { } } +func getZataDatasetMetrics() map[string]interface{} { + return map[string]interface{}{ + "avail": int64(10741741326336), + "used": int64(8564135526400), + "usedsnap": int64(0), + "usedds": int64(90112), + } +} + func getKstatMetricsVdevOnly() map[string]interface{} { return map[string]interface{}{ "vdev_cache_stats_misses": int64(87789), From b65d758bbd8f2b2738dd2c16a626f85bebde5a63 Mon Sep 17 00:00:00 2001 From: Enzo Hamelin Date: Mon, 23 Nov 2020 09:55:00 +0100 Subject: [PATCH 2/3] Add log --- plugins/inputs/zfs/zfs.go | 5 +++++ plugins/inputs/zfs/zfs_freebsd.go | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/zfs/zfs.go b/plugins/inputs/zfs/zfs.go index 0fde1950bc13c..297e3cc07ec42 100644 --- a/plugins/inputs/zfs/zfs.go +++ b/plugins/inputs/zfs/zfs.go @@ -1,5 +1,9 @@ package zfs +import ( + "github.com/influxdata/telegraf" +) + type Sysctl func(metric string) ([]string, error) type Zpool func() ([]string, error) type Zdataset func(properties []string) ([]string, error) @@ -12,6 +16,7 @@ type Zfs struct { sysctl Sysctl zpool Zpool zdataset Zdataset + Log telegraf.Logger `toml:"-"` } var sampleConfig = ` diff --git a/plugins/inputs/zfs/zfs_freebsd.go b/plugins/inputs/zfs/zfs_freebsd.go index 1a7455bf0b568..91da244d14946 100644 --- a/plugins/inputs/zfs/zfs_freebsd.go +++ b/plugins/inputs/zfs/zfs_freebsd.go @@ -106,6 +106,7 @@ func (z *Zfs) gatherDatasetStats(acc telegraf.Accumulator) (string, error) { for _, line := range lines { col := strings.Split(line, "\t") if len(col) != len(properties) { + z.Log.Warnf("Invalid number of columns for line: %s", line) continue } @@ -183,7 +184,7 @@ func zpool() ([]string, error) { } func zdataset(properties []string) ([]string, error) { - return run("zfs", []string{"list", "-Hp", "-o", strings.join(properties, ',')}...) + return run("zfs", []string{"list", "-Hp", "-o", strings.Join(properties, ",")}...) } func sysctl(metric string) ([]string, error) { From 56d55f91dffa78a345f084dc61842c5c2e129039 Mon Sep 17 00:00:00 2001 From: Steven Soroka Date: Fri, 27 Nov 2020 13:27:20 -0500 Subject: [PATCH 3/3] Update plugins/inputs/zfs/zfs_freebsd.go --- plugins/inputs/zfs/zfs_freebsd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/zfs/zfs_freebsd.go b/plugins/inputs/zfs/zfs_freebsd.go index 91da244d14946..491388147d93c 100644 --- a/plugins/inputs/zfs/zfs_freebsd.go +++ b/plugins/inputs/zfs/zfs_freebsd.go @@ -116,7 +116,7 @@ func (z *Zfs) gatherDatasetStats(acc telegraf.Accumulator) (string, error) { for i, key := range properties[1:] { value, err := strconv.ParseInt(col[i+1], 10, 64) if err != nil { - return "", fmt.Errorf("Error parsing %s: %s", key, err) + return "", fmt.Errorf("Error parsing %s %q: %s", key, col[i+1], err) } fields[key] = value }