Compare event by event in testdata framework to avoid sorting problems

CHANGELOG.next.asciidoc

@@ -255,7 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add autodetection mode for add_docker_metadata and enable it by default in included configuration files{pull}13374[13374]
 - Added `monitoring.cluster_uuid` setting to associate Beat data with specified ES cluster in Stack Monitoring UI. {pull}13182[13182]
 - Add autodetection mode for add_kubernetes_metadata and enable it by default in included configuration files. {pull}13473[13473]
-
+- Compare event by event in `testadata` framework to avoid sorting problems {pull}13747[13747]
 
 *Auditbeat*
 

metricbeat/mb/testing/testdata.go

@@ -31,7 +31,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/mitchellh/hashstructure"
-	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 
 	"github.com/elastic/beats/libbeat/asset"
@@ -263,18 +262,30 @@ func runTest(t *testing.T, file string, module, metricSetName string, config Dat
 		}
 	}
 
-	output, err := json.Marshal(&data)
-	if err != nil {
-		t.Fatal(err)
+	for _, event := range data {
+		// ensure the event is in expected list
+		found := -1
+		for i, expectedEvent := range expectedMap {
+			if event.String() == expectedEvent.String() {
+				found = i
+				break
+			}
+		}
+		if found > -1 {
+			expectedMap = append(expectedMap[:found], expectedMap[found+1:]...)
+		} else {
+			t.Errorf("Event was not expected: %+v", event)
+		}
 	}
 
-	expectedJSON, err := json.Marshal(&expectedMap)
-	if err != nil {
-		t.Fatal(err)
+	if len(expectedMap) > 0 {
+		t.Error("Some events were missing:")
+		for _, e := range expectedMap {
+			t.Error(e)
+		}
+		t.Fatal()
 	}
 
-	assert.Equal(t, string(expectedJSON), string(output))
-
 	if strings.HasSuffix(file, "docs."+config.Suffix) {
 		writeDataJSON(t, data[0], filepath.Join(config.WritePath, "data.json"))
 	}