From ac586c758005de583033dedc56d7bb5855014aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:01:18 +0100 Subject: [PATCH 01/24] PMM-12805 Func to list collections without views. --- exporter/common.go | 25 +++++++++++++++++++++++++ exporter/common_test.go | 21 ++++++++++++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index ee36ff900..84ebd74f1 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -17,6 +17,7 @@ package exporter import ( "context" + "fmt" "sort" "strings" @@ -153,6 +154,30 @@ func unique(slice []string) []string { return list } +func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) ([]string, error) { + dbs, err := databases(ctx, client, nil, nil) + if err != nil { + return nil, errors.Wrap(err, "cannot make the list of databases to list all collections") + } + + var res []string + for _, db := range dbs { + if db == "" { + continue + } + + collections, err := client.Database(db).ListCollectionNames(ctx, bson.M{"type": "collection"}) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("cannot get the list of collections from database %s", db)) + } + for _, collection := range collections { + res = append(res, fmt.Sprintf("%s.%s", db, collection)) + } + } + + return res, nil +} + func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) { namespaces := make(map[string][]string) diff --git a/exporter/common_test.go b/exporter/common_test.go index 12f25304f..d40496899 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -32,6 +32,7 @@ import ( var ( testDBs = []string{"testdb01", "testdb02"} testColls = []string{"col01", "col02", "colxx", "colyy"} + testViews = []string{"view01", "view02"} ) func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) { @@ -45,6 +46,10 @@ func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) { } } } + for _, view := range testViews { + err := client.Database(testDBs[0]).CreateView(ctx, view, testColls[0], mongo.Pipeline{}) + assert.NoError(t, err) + } } func cleanupDB(ctx context.Context, client *mongo.Client) { @@ -115,7 +120,7 @@ func TestListCollections(t *testing.T) { t.Run("With namespaces list", func(t *testing.T) { // Advanced filtering test wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, "testdb02": {"col01", "col02"}, } // List all collections in testdb01 (inDBs[0]) but only col01 and col02 from testdb02. @@ -127,7 +132,7 @@ func TestListCollections(t *testing.T) { t.Run("Empty namespaces list", func(t *testing.T) { wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, "testdb02": {"col01", "col02", "colxx", "colyy"}, } namespaces, err := listAllCollections(ctx, client, nil, systemDBs) @@ -135,10 +140,20 @@ func TestListCollections(t *testing.T) { assert.Equal(t, wantNS, namespaces) }) + t.Run("Collections without views", func(t *testing.T) { + expected := []string{"testdb01.system.views", "testdb01.col01", "testdb01.colxx", "testdb01.colyy", "testdb02.colxx", "testdb02.colyy", "testdb02.col02", "testdb02.col01"} + collections, err := listCollectionsWithoutViews(ctx, client) + assert.NoError(t, err) + //assert.Equal(t, collections, expected) + for _, expectedCollection := range expected { + assert.Contains(t, collections, expectedCollection) + } + }) + t.Run("Count basic", func(t *testing.T) { count, err := nonSystemCollectionsCount(ctx, client, nil, nil) assert.NoError(t, err) - assert.Equal(t, 8, count) + assert.Equal(t, 11, count) }) t.Run("Filtered count", func(t *testing.T) { From 4656b4ff843d2172a7506289a7bbaf5cf345ae9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:25:42 +0100 Subject: [PATCH 02/24] PMM-12805 Filter for real collections. --- exporter/collstats_collector.go | 9 ++++++++- exporter/common.go | 29 +++++++++++++++++++++++++++++ exporter/common_test.go | 18 ++++++++++++++++-- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 3ad56b42d..ffe449f61 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -77,7 +77,14 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { collections = fromMapToSlice(namespaces) } - for _, dbCollection := range collections { + onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } + + for _, dbCollection := range onlyCollections { parts := strings.Split(dbCollection, ".") if len(parts) < 2 { //nolint:gomnd continue diff --git a/exporter/common.go b/exporter/common.go index 84ebd74f1..653aad6a4 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -178,6 +178,35 @@ func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) ([]s return res, nil } +func isInArray(array []string, item string) bool { + for _, i := range array { + if item == i { + return true + } + } + + return false +} + +func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { + var filteredCollections []string + onlyCollections, err := listCollectionsWithoutViews(ctx, client) + if err != nil { + return nil, err + } + + for _, collection := range collections { + fmt.Println(collection) + if !isInArray(onlyCollections, collection) { + continue + } + + filteredCollections = append(filteredCollections, collection) + } + + return filteredCollections, nil +} + func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) { namespaces := make(map[string][]string) diff --git a/exporter/common_test.go b/exporter/common_test.go index d40496899..750e9d937 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -21,11 +21,10 @@ import ( "testing" "time" + "github.com/percona/mongodb_exporter/internal/tu" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" - - "github.com/percona/mongodb_exporter/internal/tu" ) //nolint:gochecknoglobals @@ -187,3 +186,18 @@ func TestSplitNamespace(t *testing.T) { assert.Equal(t, tc.wantCollection, coll) } } + +func TestFilterCollectionsWithoutViews(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + setupDB(ctx, t, client) + defer cleanupDB(ctx, client) + + expected := []string{"testdb01.col01", "testdb01.system.views"} + filtered, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) + assert.NoError(t, err) + assert.Equal(t, expected, filtered) +} From 1f02217e55fc688deb1a75c0edfeef1e2febe349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:29:32 +0100 Subject: [PATCH 03/24] PMM-12805 Remove print. --- exporter/common.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/common.go b/exporter/common.go index 653aad6a4..1b4d2cce5 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -196,7 +196,6 @@ func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, co } for _, collection := range collections { - fmt.Println(collection) if !isInArray(onlyCollections, collection) { continue } From e76cf314e8839051bf7887529dfec39ad7b08af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:30:37 +0100 Subject: [PATCH 04/24] PMM-12805 Remove comment. --- exporter/common_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/common_test.go b/exporter/common_test.go index 750e9d937..2da127583 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -143,7 +143,6 @@ func TestListCollections(t *testing.T) { expected := []string{"testdb01.system.views", "testdb01.col01", "testdb01.colxx", "testdb01.colyy", "testdb02.colxx", "testdb02.colyy", "testdb02.col02", "testdb02.col01"} collections, err := listCollectionsWithoutViews(ctx, client) assert.NoError(t, err) - //assert.Equal(t, collections, expected) for _, expectedCollection := range expected { assert.Contains(t, collections, expectedCollection) } From 257c3b5bbfe1b022365a4697dd4ac0dba774d295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:35:49 +0100 Subject: [PATCH 05/24] PMM-12805 Format. --- exporter/common_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/common_test.go b/exporter/common_test.go index 2da127583..002993bf6 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -21,10 +21,11 @@ import ( "testing" "time" - "github.com/percona/mongodb_exporter/internal/tu" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + + "github.com/percona/mongodb_exporter/internal/tu" ) //nolint:gochecknoglobals From 2189b7880eafe2e4046ea2743a817fdce9097c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:48:07 +0100 Subject: [PATCH 06/24] PMM-12805 Lint. --- exporter/common.go | 2 +- exporter/common_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/exporter/common.go b/exporter/common.go index 1b4d2cce5..39ec88536 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -189,12 +189,12 @@ func isInArray(array []string, item string) bool { } func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { - var filteredCollections []string onlyCollections, err := listCollectionsWithoutViews(ctx, client) if err != nil { return nil, err } + filteredCollections := []string{} for _, collection := range collections { if !isInArray(onlyCollections, collection) { continue diff --git a/exporter/common_test.go b/exporter/common_test.go index 002993bf6..008ad80ce 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -188,6 +188,8 @@ func TestSplitNamespace(t *testing.T) { } func TestFilterCollectionsWithoutViews(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() From 7e90ef4f955bf7c6f72e009675afef471aede341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 31 Jan 2024 12:56:30 +0100 Subject: [PATCH 07/24] PMM-12805 Lint. --- exporter/common_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exporter/common_test.go b/exporter/common_test.go index 008ad80ce..bf0799249 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -187,9 +187,8 @@ func TestSplitNamespace(t *testing.T) { } } +//nolint:paralleltest func TestFilterCollectionsWithoutViews(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() From 107f41f520faed53ba9bcd17ac99a6bdbf377f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 14:40:12 +0100 Subject: [PATCH 08/24] PMM-12805 Required changes. --- exporter/collstats_collector.go | 13 +++---------- exporter/common.go | 20 +++++--------------- exporter/indexstats_collector.go | 6 +++--- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index ffe449f61..986bcc8ab 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -67,24 +67,17 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { logger := d.base.logger if d.discoveringMode { - namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs) + onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = fromMapToSlice(namespaces) + collections = onlyCollections } - onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) - if err != nil { - logger.Errorf("cannot list collections: %s", err.Error()) - - return - } - - for _, dbCollection := range onlyCollections { + for _, dbCollection := range collections { parts := strings.Split(dbCollection, ".") if len(parts) < 2 { //nolint:gomnd continue diff --git a/exporter/common.go b/exporter/common.go index 39ec88536..de263e3d3 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -154,13 +154,13 @@ func unique(slice []string) []string { return list } -func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) ([]string, error) { +func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) (map[string]struct{}, error) { dbs, err := databases(ctx, client, nil, nil) if err != nil { return nil, errors.Wrap(err, "cannot make the list of databases to list all collections") } - var res []string + res := make(map[string]struct{}) for _, db := range dbs { if db == "" { continue @@ -171,23 +171,13 @@ func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) ([]s return nil, errors.Wrap(err, fmt.Sprintf("cannot get the list of collections from database %s", db)) } for _, collection := range collections { - res = append(res, fmt.Sprintf("%s.%s", db, collection)) + res[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} } } return res, nil } -func isInArray(array []string, item string) bool { - for _, i := range array { - if item == i { - return true - } - } - - return false -} - func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { onlyCollections, err := listCollectionsWithoutViews(ctx, client) if err != nil { @@ -196,8 +186,8 @@ func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, co filteredCollections := []string{} for _, collection := range collections { - if !isInArray(onlyCollections, collection) { - continue + if _, ok := onlyCollections[collection]; !ok { + return nil, fmt.Errorf("collection/namespace %s is view. Cannot be used for collstats/indexstats", collection) } filteredCollections = append(filteredCollections, collection) diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index 9f75bf2db..1e06de529 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -68,14 +68,14 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client if d.discoveringMode { - namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs) + onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) if err != nil { - logger.Errorf("cannot auto discover databases and collections") + logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = fromMapToSlice(namespaces) + collections = onlyCollections } for _, dbCollection := range collections { From 4afd50738eb6e6ca90c9c573c17d6ab5a25ac099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 14:47:12 +0100 Subject: [PATCH 09/24] PMM-12805 Lint. --- exporter/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/common.go b/exporter/common.go index de263e3d3..b033b5215 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -187,7 +187,7 @@ func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, co filteredCollections := []string{} for _, collection := range collections { if _, ok := onlyCollections[collection]; !ok { - return nil, fmt.Errorf("collection/namespace %s is view. Cannot be used for collstats/indexstats", collection) + return nil, errors.Errorf("collection/namespace %s is view. Cannot be used for collstats/indexstats", collection) } filteredCollections = append(filteredCollections, collection) From cc3999004998b06ce8b0584ac97bce98da1bd6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 14:58:51 +0100 Subject: [PATCH 10/24] PMM-12805 Fix test. --- exporter/common.go | 2 +- exporter/common_test.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index b033b5215..f03b7939d 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -187,7 +187,7 @@ func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, co filteredCollections := []string{} for _, collection := range collections { if _, ok := onlyCollections[collection]; !ok { - return nil, errors.Errorf("collection/namespace %s is view. Cannot be used for collstats/indexstats", collection) + return nil, errors.Errorf("collection/namespace %s is view and cannot be used for collstats/indexstats", collection) } filteredCollections = append(filteredCollections, collection) diff --git a/exporter/common_test.go b/exporter/common_test.go index bf0799249..5510b0a72 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -197,8 +197,14 @@ func TestFilterCollectionsWithoutViews(t *testing.T) { setupDB(ctx, t, client) defer cleanupDB(ctx, client) - expected := []string{"testdb01.col01", "testdb01.system.views"} - filtered, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) - assert.NoError(t, err) - assert.Equal(t, expected, filtered) + t.Run("Views in provided collection list (should fail)", func(t *testing.T) { + _, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) + assert.Error(t, err, "collection/namespace testdb01.view01 is view and annot be used for collstats/indexstats") + }) + + t.Run("No Views in provided collection list", func(t *testing.T) { + filtered, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) + assert.NoError(t, err) + assert.Equal(t, []string{"testdb01.col01", "testdb01.system.views"}, filtered) + }) } From e52fb67700b1df85005c525f2749c79f057e3b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 15:17:34 +0100 Subject: [PATCH 11/24] PMM-12805 Change in logic. --- exporter/collstats_collector.go | 26 +++++++++++--------------- exporter/indexstats_collector.go | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 986bcc8ab..f90f5cd7b 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -61,20 +61,27 @@ func (d *collstatsCollector) Collect(ch chan<- prometheus.Metric) { func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { defer measureCollectTime(ch, "mongodb", "collstats")() - collections := d.collections - client := d.base.client logger := d.base.logger + collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } + if d.discoveringMode { - onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) + onlyCollections, err := listCollectionsWithoutViews(d.ctx, client) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = onlyCollections + for collection := range onlyCollections { + collections = append(collections, collection) + } } for _, dbCollection := range collections { @@ -134,15 +141,4 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } } -func fromMapToSlice(databases map[string][]string) []string { - var collections []string - for db, cols := range databases { - for _, value := range cols { - collections = append(collections, db+"."+value) - } - } - - return collections -} - var _ prometheus.Collector = (*collstatsCollector)(nil) diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index 1e06de529..f59954ddd 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -62,20 +62,27 @@ func (d *indexstatsCollector) Collect(ch chan<- prometheus.Metric) { func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { defer measureCollectTime(ch, "mongodb", "indexstats")() - collections := d.collections - - logger := d.base.logger client := d.base.client + logger := d.base.logger + + collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } if d.discoveringMode { - onlyCollections, err := filterCollectionsWithoutViews(d.ctx, client, collections) + onlyCollections, err := listCollectionsWithoutViews(d.ctx, client) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = onlyCollections + for collection := range onlyCollections { + collections = append(collections, collection) + } } for _, dbCollection := range collections { From 43ab36862b94875297524f0cf282f2500dad5427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 15:22:22 +0100 Subject: [PATCH 12/24] PMM-12805 Better name for func. --- exporter/collstats_collector.go | 2 +- exporter/common.go | 2 +- exporter/common_test.go | 6 +++--- exporter/indexstats_collector.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index f90f5cd7b..0ba38494a 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -64,7 +64,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections) + collections, err := checkCollectionsForViews(d.ctx, client, d.collections) if err != nil { logger.Errorf("cannot list collections: %s", err.Error()) diff --git a/exporter/common.go b/exporter/common.go index f03b7939d..df90896da 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -178,7 +178,7 @@ func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) (map return res, nil } -func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { +func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { onlyCollections, err := listCollectionsWithoutViews(ctx, client) if err != nil { return nil, err diff --git a/exporter/common_test.go b/exporter/common_test.go index 5510b0a72..e9e028cad 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -188,7 +188,7 @@ func TestSplitNamespace(t *testing.T) { } //nolint:paralleltest -func TestFilterCollectionsWithoutViews(t *testing.T) { +func TestCheckCollectionsForViews(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -198,12 +198,12 @@ func TestFilterCollectionsWithoutViews(t *testing.T) { defer cleanupDB(ctx, client) t.Run("Views in provided collection list (should fail)", func(t *testing.T) { - _, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) + _, err := checkCollectionsForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) assert.Error(t, err, "collection/namespace testdb01.view01 is view and annot be used for collstats/indexstats") }) t.Run("No Views in provided collection list", func(t *testing.T) { - filtered, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) + filtered, err := checkCollectionsForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) assert.NoError(t, err) assert.Equal(t, []string{"testdb01.col01", "testdb01.system.views"}, filtered) }) diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index f59954ddd..ac5236252 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -65,7 +65,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections) + collections, err := checkCollectionsForViews(d.ctx, client, d.collections) if err != nil { logger.Errorf("cannot list collections: %s", err.Error()) From 648f7548f94a9408f3b6de00df5a23777622a1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 17:29:45 +0100 Subject: [PATCH 13/24] PMM-12805 Refactor to get collections with/without views. --- exporter/collstats_collector.go | 2 +- exporter/common.go | 39 ++++++++------------------------ exporter/common_test.go | 25 ++++++++++---------- exporter/indexstats_collector.go | 2 +- 4 files changed, 24 insertions(+), 44 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 0ba38494a..04e1229e4 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -72,7 +72,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } if d.discoveringMode { - onlyCollections, err := listCollectionsWithoutViews(d.ctx, client) + onlyCollections, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) diff --git a/exporter/common.go b/exporter/common.go index df90896da..b5d4827f9 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -17,7 +17,6 @@ package exporter import ( "context" - "fmt" "sort" "strings" @@ -31,7 +30,7 @@ import ( var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals -func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string) ([]string, error) { +func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string, skipViews bool) ([]string, error) { filter := bson.D{} // Default=empty -> list all collections // if there is a filter with the list of collections we want, create a filter like @@ -58,6 +57,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, } } + if skipViews { + filter = append(filter, primitive.E{Key: "type", Value: "collection"}) + } + collections, err := client.Database(database).ListCollectionNames(ctx, filter) if err != nil { return nil, errors.Wrap(err, "cannot get the list of collections for discovery") @@ -154,32 +157,8 @@ func unique(slice []string) []string { return list } -func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) (map[string]struct{}, error) { - dbs, err := databases(ctx, client, nil, nil) - if err != nil { - return nil, errors.Wrap(err, "cannot make the list of databases to list all collections") - } - - res := make(map[string]struct{}) - for _, db := range dbs { - if db == "" { - continue - } - - collections, err := client.Database(db).ListCollectionNames(ctx, bson.M{"type": "collection"}) - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("cannot get the list of collections from database %s", db)) - } - for _, collection := range collections { - res[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} - } - } - - return res, nil -} - func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { - onlyCollections, err := listCollectionsWithoutViews(ctx, client) + onlyCollections, err := listAllCollections(ctx, client, nil, nil, true) if err != nil { return nil, err } @@ -196,7 +175,7 @@ func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collect return filteredCollections, nil } -func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) { +func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string, skipViews bool) (map[string][]string, error) { namespaces := make(map[string][]string) dbs, err := databases(ctx, client, filterInNamespaces, excludeDBs) @@ -220,7 +199,7 @@ func listAllCollections(ctx context.Context, client *mongo.Client, filterInNames continue } - colls, err := listCollections(ctx, client, db, []string{namespace}) + colls, err := listCollections(ctx, client, db, []string{namespace}, skipViews) if err != nil { return nil, errors.Wrapf(err, "cannot list the collections for %q", db) } @@ -252,7 +231,7 @@ func nonSystemCollectionsCount(ctx context.Context, client *mongo.Client, includ var count int for _, dbname := range databases { - colls, err := listCollections(ctx, client, dbname, filterInCollections) + colls, err := listCollections(ctx, client, dbname, filterInCollections, true) if err != nil { return 0, errors.Wrap(err, "cannot get collections count") } diff --git a/exporter/common_test.go b/exporter/common_test.go index e9e028cad..de2d6410b 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -110,7 +110,7 @@ func TestListCollections(t *testing.T) { t.Run("Filter in databases", func(t *testing.T) { want := []string{"col01", "col02", "colxx"} inNameSpaces := []string{testDBs[0] + ".col0", testDBs[0] + ".colx"} - colls, err := listCollections(ctx, client, testDBs[0], inNameSpaces) + colls, err := listCollections(ctx, client, testDBs[0], inNameSpaces, true) sort.Strings(colls) assert.NoError(t, err) @@ -120,39 +120,40 @@ func TestListCollections(t *testing.T) { t.Run("With namespaces list", func(t *testing.T) { // Advanced filtering test wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views"}, "testdb02": {"col01", "col02"}, } // List all collections in testdb01 (inDBs[0]) but only col01 and col02 from testdb02. filterInNameSpaces := []string{testDBs[0], testDBs[1] + ".col01", testDBs[1] + ".col02"} - namespaces, err := listAllCollections(ctx, client, filterInNameSpaces, systemDBs) + namespaces, err := listAllCollections(ctx, client, filterInNameSpaces, systemDBs, true) assert.NoError(t, err) assert.Equal(t, wantNS, namespaces) }) t.Run("Empty namespaces list", func(t *testing.T) { wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views"}, "testdb02": {"col01", "col02", "colxx", "colyy"}, } - namespaces, err := listAllCollections(ctx, client, nil, systemDBs) + namespaces, err := listAllCollections(ctx, client, nil, systemDBs, true) assert.NoError(t, err) assert.Equal(t, wantNS, namespaces) }) - t.Run("Collections without views", func(t *testing.T) { - expected := []string{"testdb01.system.views", "testdb01.col01", "testdb01.colxx", "testdb01.colyy", "testdb02.colxx", "testdb02.colyy", "testdb02.col02", "testdb02.col01"} - collections, err := listCollectionsWithoutViews(ctx, client) - assert.NoError(t, err) - for _, expectedCollection := range expected { - assert.Contains(t, collections, expectedCollection) + t.Run("Collections with views", func(t *testing.T) { + wantNS := map[string][]string{ + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, + "testdb02": {"col01", "col02", "colxx", "colyy"}, } + namespaces, err := listAllCollections(ctx, client, nil, systemDBs, false) + assert.NoError(t, err) + assert.Equal(t, wantNS, namespaces) }) t.Run("Count basic", func(t *testing.T) { count, err := nonSystemCollectionsCount(ctx, client, nil, nil) assert.NoError(t, err) - assert.Equal(t, 11, count) + assert.Equal(t, 9, count) }) t.Run("Filtered count", func(t *testing.T) { diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index ac5236252..b6d9cd7fa 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -73,7 +73,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { } if d.discoveringMode { - onlyCollections, err := listCollectionsWithoutViews(d.ctx, client) + onlyCollections, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) From b48fdd720d158291cfa798148fbd41942d20c88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:32:02 +0100 Subject: [PATCH 14/24] Update exporter/common.go Co-authored-by: Alex Demidoff --- exporter/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/common.go b/exporter/common.go index b5d4827f9..09ea6e9cc 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -166,7 +166,7 @@ func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collect filteredCollections := []string{} for _, collection := range collections { if _, ok := onlyCollections[collection]; !ok { - return nil, errors.Errorf("collection/namespace %s is view and cannot be used for collstats/indexstats", collection) + return nil, errors.Errorf("collection/namespace %s is a view and cannot be used for collstats/indexstats", collection) } filteredCollections = append(filteredCollections, collection) From dcaa9dddf6bb894e5aa41812d17c69dd0b844415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 17:52:21 +0100 Subject: [PATCH 15/24] PMM-12805 Fix tests after refactor. --- exporter/common.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/exporter/common.go b/exporter/common.go index 09ea6e9cc..af7032eb3 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -17,6 +17,7 @@ package exporter import ( "context" + "fmt" "sort" "strings" @@ -163,6 +164,13 @@ func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collect return nil, err } + converted := make(map[string]struct{}) + for db, collections := range onlyCollections { + for _, collection := range collections { + converted[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} + } + } + filteredCollections := []string{} for _, collection := range collections { if _, ok := onlyCollections[collection]; !ok { From 6a01aa15c6962c1d9982e3fc73897279514a628b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 17:54:25 +0100 Subject: [PATCH 16/24] PMM-12805 Fix. --- exporter/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/common.go b/exporter/common.go index af7032eb3..2a4b62772 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -173,7 +173,7 @@ func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collect filteredCollections := []string{} for _, collection := range collections { - if _, ok := onlyCollections[collection]; !ok { + if _, ok := converted[collection]; !ok { return nil, errors.Errorf("collection/namespace %s is a view and cannot be used for collstats/indexstats", collection) } From 5c694dd29fa3d926cfdd5530e792e0de5b050212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 18:02:30 +0100 Subject: [PATCH 17/24] PMM-12805 Better naming. --- exporter/collstats_collector.go | 8 ++++---- exporter/common.go | 8 ++++---- exporter/common_test.go | 8 ++++---- exporter/indexstats_collector.go | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 04e1229e4..1db14db58 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -64,7 +64,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := checkCollectionsForViews(d.ctx, client, d.collections) + collections, err := checkNamespacesForViews(d.ctx, client, d.collections) if err != nil { logger.Errorf("cannot list collections: %s", err.Error()) @@ -72,15 +72,15 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } if d.discoveringMode { - onlyCollections, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) + onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - for collection := range onlyCollections { - collections = append(collections, collection) + for collectionNamespace := range onlyCollectionsNamespaces { + collections = append(collections, collectionNamespace) } } diff --git a/exporter/common.go b/exporter/common.go index 2a4b62772..1f877f628 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -158,14 +158,14 @@ func unique(slice []string) []string { return list } -func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { - onlyCollections, err := listAllCollections(ctx, client, nil, nil, true) +func checkNamespacesForViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { + onlyCollectionsNamespaces, err := listAllCollections(ctx, client, nil, nil, true) if err != nil { return nil, err } converted := make(map[string]struct{}) - for db, collections := range onlyCollections { + for db, collections := range onlyCollectionsNamespaces { for _, collection := range collections { converted[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} } @@ -174,7 +174,7 @@ func checkCollectionsForViews(ctx context.Context, client *mongo.Client, collect filteredCollections := []string{} for _, collection := range collections { if _, ok := converted[collection]; !ok { - return nil, errors.Errorf("collection/namespace %s is a view and cannot be used for collstats/indexstats", collection) + return nil, errors.Errorf("namespace %s is a view and cannot be used for collstats/indexstats", collection) } filteredCollections = append(filteredCollections, collection) diff --git a/exporter/common_test.go b/exporter/common_test.go index de2d6410b..e1c59fd0f 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -189,7 +189,7 @@ func TestSplitNamespace(t *testing.T) { } //nolint:paralleltest -func TestCheckCollectionsForViews(t *testing.T) { +func TestCheckNamespacesForViews(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -199,12 +199,12 @@ func TestCheckCollectionsForViews(t *testing.T) { defer cleanupDB(ctx, client) t.Run("Views in provided collection list (should fail)", func(t *testing.T) { - _, err := checkCollectionsForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) - assert.Error(t, err, "collection/namespace testdb01.view01 is view and annot be used for collstats/indexstats") + _, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) + assert.Error(t, err, "namespace testdb01.view01 is view and annot be used for collstats/indexstats") }) t.Run("No Views in provided collection list", func(t *testing.T) { - filtered, err := checkCollectionsForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) + filtered, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) assert.NoError(t, err) assert.Equal(t, []string{"testdb01.col01", "testdb01.system.views"}, filtered) }) diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index b6d9cd7fa..a3dc243d7 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -65,7 +65,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := checkCollectionsForViews(d.ctx, client, d.collections) + collections, err := checkNamespacesForViews(d.ctx, client, d.collections) if err != nil { logger.Errorf("cannot list collections: %s", err.Error()) @@ -73,15 +73,15 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { } if d.discoveringMode { - onlyCollections, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) + onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - for collection := range onlyCollections { - collections = append(collections, collection) + for collectionNamespace := range onlyCollectionsNamespaces { + collections = append(collections, collectionNamespace) } } From 67297ab4beb0593647ba391d3247212ecb696ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 6 Feb 2024 18:04:23 +0100 Subject: [PATCH 18/24] PMM-12805 Better naming for converted map. --- exporter/common.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index 1f877f628..0f0820563 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -164,16 +164,16 @@ func checkNamespacesForViews(ctx context.Context, client *mongo.Client, collecti return nil, err } - converted := make(map[string]struct{}) + namespaces := make(map[string]struct{}) for db, collections := range onlyCollectionsNamespaces { for _, collection := range collections { - converted[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} + namespaces[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} } } filteredCollections := []string{} for _, collection := range collections { - if _, ok := converted[collection]; !ok { + if _, ok := namespaces[collection]; !ok { return nil, errors.Errorf("namespace %s is a view and cannot be used for collstats/indexstats", collection) } From cc2a07950b623876ce3137f1808b129709773b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:39:13 +0100 Subject: [PATCH 19/24] Update exporter/common_test.go Co-authored-by: Michael Okoko <10512379+idoqo@users.noreply.github.com> --- exporter/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/common_test.go b/exporter/common_test.go index e1c59fd0f..4e998b62f 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -200,7 +200,7 @@ func TestCheckNamespacesForViews(t *testing.T) { t.Run("Views in provided collection list (should fail)", func(t *testing.T) { _, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) - assert.Error(t, err, "namespace testdb01.view01 is view and annot be used for collstats/indexstats") + assert.Error(t, err, "namespace testdb01.view01 is view and cannot be used for collstats/indexstats") }) t.Run("No Views in provided collection list", func(t *testing.T) { From f2cd323e42ddb83c106722ff96f8433873d9c820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 7 Feb 2024 09:40:20 +0100 Subject: [PATCH 20/24] PMM-12805 Small change to check exact error message. --- exporter/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/common_test.go b/exporter/common_test.go index 4e998b62f..eacd6469a 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -200,7 +200,7 @@ func TestCheckNamespacesForViews(t *testing.T) { t.Run("Views in provided collection list (should fail)", func(t *testing.T) { _, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) - assert.Error(t, err, "namespace testdb01.view01 is view and cannot be used for collstats/indexstats") + assert.EqualError(t, err, "namespace testdb01.view01 is a view and cannot be used for collstats/indexstats") }) t.Run("No Views in provided collection list", func(t *testing.T) { From 20242d8b8d99be6acb9be7f5afc7e23fa30aa457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 8 Feb 2024 15:22:34 +0100 Subject: [PATCH 21/24] PMM-12805 Fix for empty namaspace. --- exporter/common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index 0f0820563..8114266ac 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -166,13 +166,13 @@ func checkNamespacesForViews(ctx context.Context, client *mongo.Client, collecti namespaces := make(map[string]struct{}) for db, collections := range onlyCollectionsNamespaces { - for _, collection := range collections { + for _, collection := range removeEmptyStrings(collections) { namespaces[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} } } filteredCollections := []string{} - for _, collection := range collections { + for _, collection := range removeEmptyStrings(collections) { if _, ok := namespaces[collection]; !ok { return nil, errors.Errorf("namespace %s is a view and cannot be used for collstats/indexstats", collection) } From 5ba2cb98b6869cf6b8d5a98b2ea9887a59c2f91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 8 Feb 2024 16:47:17 +0100 Subject: [PATCH 22/24] PMM-12805 Fix for namespaces. --- exporter/collstats_collector.go | 4 +--- exporter/common.go | 11 +++++++++++ exporter/indexstats_collector.go | 4 +--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 1db14db58..fe7c743eb 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -79,9 +79,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { return } - for collectionNamespace := range onlyCollectionsNamespaces { - collections = append(collections, collectionNamespace) - } + collections = fromMapToSlice(onlyCollectionsNamespaces) } for _, dbCollection := range collections { diff --git a/exporter/common.go b/exporter/common.go index 8114266ac..46ab9d2b1 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -257,3 +257,14 @@ func splitNamespace(ns string) (database, collection string) { return parts[0], strings.Join(parts[1:], ".") } + +func fromMapToSlice(databases map[string][]string) []string { + var collections []string + for db, cols := range databases { + for _, value := range cols { + collections = append(collections, db+"."+value) + } + } + + return collections +} diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index a3dc243d7..155df20bd 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -80,9 +80,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { return } - for collectionNamespace := range onlyCollectionsNamespaces { - collections = append(collections, collectionNamespace) - } + collections = fromMapToSlice(onlyCollectionsNamespaces) } for _, dbCollection := range collections { From 7d03f95d1640fe785f025d454bf8adbe23e3cc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 9 Feb 2024 09:55:55 +0100 Subject: [PATCH 23/24] PMM-12805 Skip not complete namespaces. --- exporter/common.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/exporter/common.go b/exporter/common.go index 46ab9d2b1..cafe243d3 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -173,6 +173,10 @@ func checkNamespacesForViews(ctx context.Context, client *mongo.Client, collecti filteredCollections := []string{} for _, collection := range removeEmptyStrings(collections) { + if len(strings.Split(collection, ".")) < 2 { //nolint:gomnd + continue + } + if _, ok := namespaces[collection]; !ok { return nil, errors.Errorf("namespace %s is a view and cannot be used for collstats/indexstats", collection) } From e97c9cb982d8f1a9fa3ec144fc1b07c263501ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 9 Feb 2024 10:35:21 +0100 Subject: [PATCH 24/24] PMM-12805 Better performance for discovery mode. --- exporter/collstats_collector.go | 16 +++++++++------- exporter/indexstats_collector.go | 16 +++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index fe7c743eb..2f084010d 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -64,13 +64,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := checkNamespacesForViews(d.ctx, client, d.collections) - if err != nil { - logger.Errorf("cannot list collections: %s", err.Error()) - - return - } - + var collections []string if d.discoveringMode { onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { @@ -80,6 +74,14 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } collections = fromMapToSlice(onlyCollectionsNamespaces) + } else { + var err error + collections, err = checkNamespacesForViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } } for _, dbCollection := range collections { diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index 155df20bd..132343d93 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -65,13 +65,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { client := d.base.client logger := d.base.logger - collections, err := checkNamespacesForViews(d.ctx, client, d.collections) - if err != nil { - logger.Errorf("cannot list collections: %s", err.Error()) - - return - } - + var collections []string if d.discoveringMode { onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { @@ -81,6 +75,14 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { } collections = fromMapToSlice(onlyCollectionsNamespaces) + } else { + var err error + collections, err = checkNamespacesForViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } } for _, dbCollection := range collections {