diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 3ad56b42d..2f084010d 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -61,20 +61,27 @@ func (d *collstatsCollector) Collect(ch chan<- prometheus.Metric) { func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { defer measureCollectTime(ch, "mongodb", "collstats")() - collections := d.collections - client := d.base.client logger := d.base.logger + var collections []string if d.discoveringMode { - namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs) + onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = fromMapToSlice(namespaces) + collections = fromMapToSlice(onlyCollectionsNamespaces) + } else { + var err error + collections, err = checkNamespacesForViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } } for _, dbCollection := range collections { @@ -134,15 +141,4 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } } -func fromMapToSlice(databases map[string][]string) []string { - var collections []string - for db, cols := range databases { - for _, value := range cols { - collections = append(collections, db+"."+value) - } - } - - return collections -} - var _ prometheus.Collector = (*collstatsCollector)(nil) diff --git a/exporter/common.go b/exporter/common.go index ee36ff900..cafe243d3 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -17,6 +17,7 @@ package exporter import ( "context" + "fmt" "sort" "strings" @@ -30,7 +31,7 @@ import ( var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals -func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string) ([]string, error) { +func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string, skipViews bool) ([]string, error) { filter := bson.D{} // Default=empty -> list all collections // if there is a filter with the list of collections we want, create a filter like @@ -57,6 +58,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, } } + if skipViews { + filter = append(filter, primitive.E{Key: "type", Value: "collection"}) + } + collections, err := client.Database(database).ListCollectionNames(ctx, filter) if err != nil { return nil, errors.Wrap(err, "cannot get the list of collections for discovery") @@ -153,7 +158,36 @@ func unique(slice []string) []string { return list } -func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) { +func checkNamespacesForViews(ctx context.Context, client *mongo.Client, collections []string) ([]string, error) { + onlyCollectionsNamespaces, err := listAllCollections(ctx, client, nil, nil, true) + if err != nil { + return nil, err + } + + namespaces := make(map[string]struct{}) + for db, collections := range onlyCollectionsNamespaces { + for _, collection := range removeEmptyStrings(collections) { + namespaces[fmt.Sprintf("%s.%s", db, collection)] = struct{}{} + } + } + + filteredCollections := []string{} + for _, collection := range removeEmptyStrings(collections) { + if len(strings.Split(collection, ".")) < 2 { //nolint:gomnd + continue + } + + if _, ok := namespaces[collection]; !ok { + return nil, errors.Errorf("namespace %s is a view and cannot be used for collstats/indexstats", collection) + } + + filteredCollections = append(filteredCollections, collection) + } + + return filteredCollections, nil +} + +func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string, skipViews bool) (map[string][]string, error) { namespaces := make(map[string][]string) dbs, err := databases(ctx, client, filterInNamespaces, excludeDBs) @@ -177,7 +211,7 @@ func listAllCollections(ctx context.Context, client *mongo.Client, filterInNames continue } - colls, err := listCollections(ctx, client, db, []string{namespace}) + colls, err := listCollections(ctx, client, db, []string{namespace}, skipViews) if err != nil { return nil, errors.Wrapf(err, "cannot list the collections for %q", db) } @@ -209,7 +243,7 @@ func nonSystemCollectionsCount(ctx context.Context, client *mongo.Client, includ var count int for _, dbname := range databases { - colls, err := listCollections(ctx, client, dbname, filterInCollections) + colls, err := listCollections(ctx, client, dbname, filterInCollections, true) if err != nil { return 0, errors.Wrap(err, "cannot get collections count") } @@ -227,3 +261,14 @@ func splitNamespace(ns string) (database, collection string) { return parts[0], strings.Join(parts[1:], ".") } + +func fromMapToSlice(databases map[string][]string) []string { + var collections []string + for db, cols := range databases { + for _, value := range cols { + collections = append(collections, db+"."+value) + } + } + + return collections +} diff --git a/exporter/common_test.go b/exporter/common_test.go index 12f25304f..eacd6469a 100644 --- a/exporter/common_test.go +++ b/exporter/common_test.go @@ -32,6 +32,7 @@ import ( var ( testDBs = []string{"testdb01", "testdb02"} testColls = []string{"col01", "col02", "colxx", "colyy"} + testViews = []string{"view01", "view02"} ) func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) { @@ -45,6 +46,10 @@ func setupDB(ctx context.Context, t *testing.T, client *mongo.Client) { } } } + for _, view := range testViews { + err := client.Database(testDBs[0]).CreateView(ctx, view, testColls[0], mongo.Pipeline{}) + assert.NoError(t, err) + } } func cleanupDB(ctx context.Context, client *mongo.Client) { @@ -105,7 +110,7 @@ func TestListCollections(t *testing.T) { t.Run("Filter in databases", func(t *testing.T) { want := []string{"col01", "col02", "colxx"} inNameSpaces := []string{testDBs[0] + ".col0", testDBs[0] + ".colx"} - colls, err := listCollections(ctx, client, testDBs[0], inNameSpaces) + colls, err := listCollections(ctx, client, testDBs[0], inNameSpaces, true) sort.Strings(colls) assert.NoError(t, err) @@ -115,22 +120,32 @@ func TestListCollections(t *testing.T) { t.Run("With namespaces list", func(t *testing.T) { // Advanced filtering test wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views"}, "testdb02": {"col01", "col02"}, } // List all collections in testdb01 (inDBs[0]) but only col01 and col02 from testdb02. filterInNameSpaces := []string{testDBs[0], testDBs[1] + ".col01", testDBs[1] + ".col02"} - namespaces, err := listAllCollections(ctx, client, filterInNameSpaces, systemDBs) + namespaces, err := listAllCollections(ctx, client, filterInNameSpaces, systemDBs, true) assert.NoError(t, err) assert.Equal(t, wantNS, namespaces) }) t.Run("Empty namespaces list", func(t *testing.T) { wantNS := map[string][]string{ - "testdb01": {"col01", "col02", "colxx", "colyy"}, + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views"}, + "testdb02": {"col01", "col02", "colxx", "colyy"}, + } + namespaces, err := listAllCollections(ctx, client, nil, systemDBs, true) + assert.NoError(t, err) + assert.Equal(t, wantNS, namespaces) + }) + + t.Run("Collections with views", func(t *testing.T) { + wantNS := map[string][]string{ + "testdb01": {"col01", "col02", "colxx", "colyy", "system.views", "view01", "view02"}, "testdb02": {"col01", "col02", "colxx", "colyy"}, } - namespaces, err := listAllCollections(ctx, client, nil, systemDBs) + namespaces, err := listAllCollections(ctx, client, nil, systemDBs, false) assert.NoError(t, err) assert.Equal(t, wantNS, namespaces) }) @@ -138,7 +153,7 @@ func TestListCollections(t *testing.T) { t.Run("Count basic", func(t *testing.T) { count, err := nonSystemCollectionsCount(ctx, client, nil, nil) assert.NoError(t, err) - assert.Equal(t, 8, count) + assert.Equal(t, 9, count) }) t.Run("Filtered count", func(t *testing.T) { @@ -172,3 +187,25 @@ func TestSplitNamespace(t *testing.T) { assert.Equal(t, tc.wantCollection, coll) } } + +//nolint:paralleltest +func TestCheckNamespacesForViews(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + setupDB(ctx, t, client) + defer cleanupDB(ctx, client) + + t.Run("Views in provided collection list (should fail)", func(t *testing.T) { + _, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views", "testdb01.view01"}) + assert.EqualError(t, err, "namespace testdb01.view01 is a view and cannot be used for collstats/indexstats") + }) + + t.Run("No Views in provided collection list", func(t *testing.T) { + filtered, err := checkNamespacesForViews(ctx, client, []string{"testdb01.col01", "testdb01.system.views"}) + assert.NoError(t, err) + assert.Equal(t, []string{"testdb01.col01", "testdb01.system.views"}, filtered) + }) +} diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index 9f75bf2db..132343d93 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -62,20 +62,27 @@ func (d *indexstatsCollector) Collect(ch chan<- prometheus.Metric) { func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { defer measureCollectTime(ch, "mongodb", "indexstats")() - collections := d.collections - - logger := d.base.logger client := d.base.client + logger := d.base.logger + var collections []string if d.discoveringMode { - namespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs) + onlyCollectionsNamespaces, err := listAllCollections(d.ctx, client, d.collections, systemDBs, true) if err != nil { - logger.Errorf("cannot auto discover databases and collections") + logger.Errorf("cannot auto discover databases and collections: %s", err.Error()) return } - collections = fromMapToSlice(namespaces) + collections = fromMapToSlice(onlyCollectionsNamespaces) + } else { + var err error + collections, err = checkNamespacesForViews(d.ctx, client, d.collections) + if err != nil { + logger.Errorf("cannot list collections: %s", err.Error()) + + return + } } for _, dbCollection := range collections {