Add tests

cmd/flakechecker/flake_config_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+func TestLoadFlakeConfigFile(t *testing.T) {
+	samples := []struct {
+		name     string
+		fileName string
+
+		expectError  bool
+		expectConfig []*flakeCheckerRecord
+	}{
+		{
+			name:         "no config file",
+			fileName:     "no_config.json",
+			expectError:  true,
+			expectConfig: nil,
+		},
+		{
+			name:        "valid config file",
+			fileName:    "testdata/flake-config.json",
+			expectError: false,
+			expectConfig: []*flakeCheckerRecord{
+				{
+					MatchJobName:       "pr-.*",
+					RatioJobName:       "main-branch-tests",
+					TestNameRegex:      "TestLoadFlakeConf.*",
+					Classname:          "TestLoadFlakeConfigFile",
+					RatioThreshold:     5,
+					regexMatchJobName:  regexp.MustCompile("^pr-.*$"),
+					regexTestNameRegex: regexp.MustCompile("^TestLoadFlakeConf.*$"),
+				},
+				{
+					MatchJobName:       "pull-request-tests",
+					RatioJobName:       "main-branch-tests",
+					TestNameRegex:      "TestLoadFlakeConfigFile",
+					Classname:          "TestLoadFlakeConfigFile",
+					RatioThreshold:     10,
+					regexMatchJobName:  regexp.MustCompile("^pull-request-tests$"),
+					regexTestNameRegex: regexp.MustCompile("^TestLoadFlakeConfigFile$"),
+				},
+			},
+		},
+	}
+
+	for _, sample := range samples {
+		t.Run(sample.name, func(tt *testing.T) {
+			config, err := loadFlakeConfigFile(sample.fileName)
+
+			assert.Equal(tt, sample.expectError, err != nil)
+			assert.Equal(tt, sample.expectConfig, config)
+		})
+	}
+}

cmd/flakechecker/main_test.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"github.com/pkg/errors"
+	"github.com/stackrox/junit2jira/pkg/testcase"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+type biqQueryClientMock struct {
+	getRatioForTest func(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error)
+}
+
+func (c *biqQueryClientMock) GetRatioForTest(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error) {
+	if c.getRatioForTest != nil {
+		return c.getRatioForTest(flakeTestConfig, testName)
+	}
+
+	// By default, fail. In most cases, we will not reach this part in tests.
+	return 0, 0, errors.New("fail")
+}
+
+func getRatioForTestNoFailures(_ *flakeCheckerRecord, _ string) (int, int, error) {
+	return totalRunsLimit, 0, nil
+}
+
+func getRatioForTestAllFailures(_ *flakeCheckerRecord, _ string) (int, int, error) {
+	return totalRunsLimit, 50, nil
+}
+
+func TestCheckFailedTests(t *testing.T) {
+	p := flakeCheckerParams{jobName: "test-job"}
+
+	samples := map[string]struct {
+		bqClient     biqQueryClient
+		failedTests  []testcase.TestCase
+		flakeConfigs []*flakeCheckerRecord
+		expectError  bool
+	}{
+		"no failed tests": {
+			bqClient:    &biqQueryClientMock{},
+			failedTests: []testcase.TestCase{},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job1-", TestNameRegex: "test name"},
+			},
+			expectError: false,
+		},
+		"no config match - job name": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job-1", TestNameRegex: "test", Classname: "class"},
+			},
+			expectError: true,
+		},
+		"no config match - test name": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "wrong-test", Classname: "class"},
+			},
+			expectError: true,
+		},
+		"no config match - classname": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "wrong-class"},
+			},
+			expectError: true,
+		},
+		"unable to fetch ratio": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: func(_ *flakeCheckerRecord, _ string) (int, int, error) {
+					return 0, 0, errors.New("fail")
+				},
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1},
+			},
+			expectError: true,
+		},
+		"total runs below limit": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: func(_ *flakeCheckerRecord, _ string) (int, int, error) {
+					return totalRunsLimit - 1, 0, nil
+				},
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1},
+			},
+			expectError: true,
+		},
+		"fail ratio below threshold": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1},
+			},
+			expectError: false,
+		},
+		"fail ratio above threshold": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestAllFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1},
+			},
+			expectError: true,
+		},
+		"fail ratio below threshold - multiple tests": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{
+				{Name: "test", Classname: "class"},
+				{Name: "test-1", Classname: "class-1"},
+			},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1},
+				{MatchJobName: "test-job", TestNameRegex: "test-1", Classname: "class-1", RatioThreshold: 1},
+			},
+			expectError: false,
+		},
+		"fail ratio above threshold - multiple tests": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestAllFailures,
+			},
+			failedTests: []testcase.TestCase{
+				{Name: "test-ratio-below", Classname: "class"},
+				{Name: "test-ratio-above", Classname: "class"},
+			},
+			flakeConfigs: []*flakeCheckerRecord{
+				{MatchJobName: "test-job", TestNameRegex: "test-ratio-below", Classname: "class", RatioThreshold: 90},
+				{MatchJobName: "test-job", TestNameRegex: "test-ratio-above", Classname: "class", RatioThreshold: 10},
+			},
+			expectError: true,
+		},
+	}
+
+	for sampleName, sample := range samples {
+		t.Run(sampleName, func(tt *testing.T) {
+			err := p.checkFailedTests(sample.bqClient, sample.failedTests, sample.flakeConfigs)
+
+			assert.Equal(tt, sample.expectError, err != nil)
+		})
+	}
+}

cmd/flakechecker/testdata/flake-config.json

@@ -0,0 +1,18 @@
+[
+  {
+    "_comment": "Config with regex",
+    "match_job_name": "pr-.*",
+    "ratio_job_name": "main-branch-tests",
+    "test_name_regex": "TestLoadFlakeConf.*",
+    "classname": "TestLoadFlakeConfigFile",
+    "ratio_threshold": 5
+  },
+  {
+    "_comment": "Config with flat values",
+    "match_job_name": "pull-request-tests",
+    "ratio_job_name": "main-branch-tests",
+    "test_name_regex": "TestLoadFlakeConfigFile",
+    "classname": "TestLoadFlakeConfigFile",
+    "ratio_threshold": 10
+  }
+]