PMM-12805 Collstats, indexstats iterate only over collections, not views.

exporter/collstats_collector.go

@@ -61,20 +61,27 @@ func (d *collstatsCollector) Collect(ch chan<- prometheus.Metric) {
 func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
 	defer measureCollectTime(ch, "mongodb", "collstats")()
 
-	collections := d.collections
-
 	client := d.base.client
 	logger := d.base.logger
 
+	collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections)
+	if err != nil {
+		logger.Errorf("cannot list collections: %s", err.Error())
+
+		return
+	}
+
 	if d.discoveringMode {
-		namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs)
+		onlyCollections, err := listCollectionsWithoutViews(d.ctx, client)
 		if err != nil {
 			logger.Errorf("cannot auto discover databases and collections: %s", err.Error())
 
 			return
 		}
 
-		collections = fromMapToSlice(namespaces)
+		for collection := range onlyCollections {
+			collections = append(collections, collection)
+		}
 	}
 
 	for _, dbCollection := range collections {
@@ -134,15 +141,4 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
 	}
 }
 
-func fromMapToSlice(databases map[string][]string) []string {
-	var collections []string
-	for db, cols := range databases {
-		for _, value := range cols {
-			collections = append(collections, db+"."+value)
-		}
-	}
-
-	return collections
-}
-
 var _ prometheus.Collector = (*collstatsCollector)(nil)

exporter/common.go

@@ -17,6 +17,7 @@ package exporter
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"strings"
 
@@ -153,6 +154,48 @@ func unique(slice []string) []string {
 	return list
 }
 
+func listCollectionsWithoutViews(ctx context.Context, client *mongo.Client) (map[string]struct{}, error) {
+	dbs, err := databases(ctx, client, nil, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot make the list of databases to list all collections")
+	}
+
+	res := make(map[string]struct{})
+	for _, db := range dbs {
+		if db == "" {
+			continue
+		}
+
+		collections, err := client.Database(db).ListCollectionNames(ctx, bson.M{"type": "collection"})
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("cannot get the list of collections from database %s", db))
+		}
+		for _, collection := range collections {
+			res[fmt.Sprintf("%s.%s", db, collection)] = struct{}{}
+		}
+	}
+
+	return res, nil
+}
+
+func filterCollectionsWithoutViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) {
+	onlyCollections, err := listCollectionsWithoutViews(ctx, client)
+	if err != nil {
+		return nil, err
+	}
+
+	filteredCollections := []string{}
+	for _, collection := range collections {
+		if _, ok := onlyCollections[collection]; !ok {
+			return nil, errors.Errorf("collection/namespace %s is view and cannot be used for collstats/indexstats", collection)
+		}
+
+		filteredCollections = append(filteredCollections, collection)
+	}
+
+	return filteredCollections, nil
+}
+
 func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) {
 	namespaces := make(map[string][]string)
 

exporter/common_test.go

@@ -32,6 +32,7 @@ import (
 var (
 	testDBs   = []string{"testdb01", "testdb02"}
 	testColls = []string{"col01", "col02", "colxx", "colyy"}
+	testViews = []string{"view01", "view02"}
 )
 
 func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) {
@@ -45,6 +46,10 @@ func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) {
 			}
 		}
 	}
+	for _, view := range testViews {
+		err := client.Database(testDBs[0]).CreateView(ctx, view, testColls[0], mongo.Pipeline{})
+		assert.NoError(t, err)
+	}
 }
 
 func cleanupDB(ctx context.Context, client *mongo.Client) {
@@ -115,7 +120,7 @@ func TestListCollections(t *testing.T) {
 	t.Run("With namespaces list", func(t *testing.T) {
 		// Advanced filtering test
 		wantNS := map[string][]string{
-			"testdb01": {"col01", "col02", "colxx", "colyy"},
+			"testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"},
 			"testdb02": {"col01", "col02"},
 		}
 		// List all collections in testdb01 (inDBs[0]) but only col01 and col02 from testdb02.
@@ -127,18 +132,27 @@ func TestListCollections(t *testing.T) {
 
 	t.Run("Empty namespaces list", func(t *testing.T) {
 		wantNS := map[string][]string{
-			"testdb01": {"col01", "col02", "colxx", "colyy"},
+			"testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"},
 			"testdb02": {"col01", "col02", "colxx", "colyy"},
 		}
 		namespaces, err := listAllCollections(ctx, client, nil, systemDBs)
 		assert.NoError(t, err)
 		assert.Equal(t, wantNS, namespaces)
 	})
 
+	t.Run("Collections without views", func(t *testing.T) {
+		expected := []string{"testdb01.system.views", "testdb01.col01", "testdb01.colxx", "testdb01.colyy", "testdb02.colxx", "testdb02.colyy", "testdb02.col02", "testdb02.col01"}
+		collections, err := listCollectionsWithoutViews(ctx, client)
+		assert.NoError(t, err)
+		for _, expectedCollection := range expected {
+			assert.Contains(t, collections, expectedCollection)
+		}
+	})
+
 	t.Run("Count basic", func(t *testing.T) {
 		count, err := nonSystemCollectionsCount(ctx, client, nil, nil)
 		assert.NoError(t, err)
-		assert.Equal(t, 8, count)
+		assert.Equal(t, 11, count)
 	})
 
 	t.Run("Filtered count", func(t *testing.T) {
@@ -172,3 +186,25 @@ func TestSplitNamespace(t *testing.T) {
 		assert.Equal(t, tc.wantCollection, coll)
 	}
 }
+
+//nolint:paralleltest
+func TestFilterCollectionsWithoutViews(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	client := tu.DefaultTestClient(ctx, t)
+
+	setupDB(ctx, t, client)
+	defer cleanupDB(ctx, client)
+
+	t.Run("Views in provided collection list (should fail)", func(t *testing.T) {
+		_, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"})
+		assert.Error(t, err, "collection/namespace testdb01.view01 is view and annot be used for collstats/indexstats")
+	})
+
+	t.Run("No Views in provided collection list", func(t *testing.T) {
+		filtered, err := filterCollectionsWithoutViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"})
+		assert.NoError(t, err)
+		assert.Equal(t, []string{"testdb01.col01", "testdb01.system.views"}, filtered)
+	})
+}

exporter/indexstats_collector.go

@@ -62,20 +62,27 @@ func (d *indexstatsCollector) Collect(ch chan<- prometheus.Metric) {
 func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) {
 	defer measureCollectTime(ch, "mongodb", "indexstats")()
 
-	collections := d.collections
-
-	logger := d.base.logger
 	client := d.base.client
+	logger := d.base.logger
+
+	collections, err := filterCollectionsWithoutViews(d.ctx, client, d.collections)
+	if err != nil {
+		logger.Errorf("cannot list collections: %s", err.Error())
+
+		return
+	}
 
 	if d.discoveringMode {
-		namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs)
+		onlyCollections, err := listCollectionsWithoutViews(d.ctx, client)
 		if err != nil {
-			logger.Errorf("cannot auto discover databases and collections")
+			logger.Errorf("cannot auto discover databases and collections: %s", err.Error())
 
 			return
 		}
 
-		collections = fromMapToSlice(namespaces)
+		for collection := range onlyCollections {
+			collections = append(collections, collection)
+		}
 	}
 
 	for _, dbCollection := range collections {