Add tests

cmd/flakechecker/flake_config_test.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestLoadFlakeConfigFile(t *testing.T) {
+	samples := []struct {
+		name     string
+		fileName string
+
+		expectError  bool
+		expectConfig []*flakeDetectionPolicy
+	}{
+		{
+			name:         "no config file",
+			fileName:     "no_config.yml",
+			expectError:  true,
+			expectConfig: nil,
+		},
+		{
+			name:        "valid config file",
+			fileName:    "testdata/flake-config.yml",
+			expectError: false,
+			expectConfig: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
+					MatchJobName:   "pr-.*",
+					RatioJobName:   "main-branch-tests",
+					TestNameRegex:  "TestLoadFlakeConf.*",
+					Classname:      "TestLoadFlakeConfigFile",
+					RatioThreshold: 5,
+				}),
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{
+					MatchJobName:   "pull-request-tests",
+					RatioJobName:   "main-branch-tests",
+					TestNameRegex:  "TestLoadFlakeConfigFile",
+					Classname:      "TestLoadFlakeConfigFile",
+					RatioThreshold: 10,
+				}),
+			},
+		},
+	}
+
+	for _, sample := range samples {
+		t.Run(sample.name, func(tt *testing.T) {
+			config, err := loadFlakeConfigFile(sample.fileName)
+
+			assert.Equal(tt, sample.expectError, err != nil)
+			assert.Equal(tt, sample.expectConfig, config)
+		})
+	}
+}

cmd/flakechecker/main_test.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"github.com/pkg/errors"
+	"github.com/stackrox/junit2jira/pkg/testcase"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+type biqQueryClientMock struct {
+	getRatioForTest func(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error)
+}
+
+func (c *biqQueryClientMock) GetRatioForTest(flakeTestConfig *flakeDetectionPolicy, testName string) (int, int, error) {
+	if c.getRatioForTest != nil {
+		return c.getRatioForTest(flakeTestConfig, testName)
+	}
+
+	// By default, fail. In most cases, we will not reach this part in tests.
+	return 0, 0, errors.New("fail")
+}
+
+func getRatioForTestNoFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
+	return totalRunsLimit, 0, nil
+}
+
+func getRatioForTestAllFailures(_ *flakeDetectionPolicy, _ string) (int, int, error) {
+	return totalRunsLimit, 50, nil
+}
+
+func TestCheckFailedTests(t *testing.T) {
+	p := flakeCheckerParams{jobName: "test-job"}
+
+	samples := map[string]struct {
+		bqClient               biqQueryClient
+		failedTests            []testcase.TestCase
+		flakeDetectionPolicies []*flakeDetectionPolicy
+		expectError            bool
+		expectErrorStr         string
+	}{
+		"no failed tests": {
+			bqClient:    &biqQueryClientMock{},
+			failedTests: []testcase.TestCase{},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job1-", TestNameRegex: "test name"}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescNoFailedTests,
+		},
+		"no config match - job name": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job-1", TestNameRegex: "test", Classname: "class"}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescNoMatch,
+		},
+		"no config match - test name": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "wrong-test", Classname: "class"}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescNoMatch,
+		},
+		"no config match - classname": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "wrong-class"}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescNoMatch,
+		},
+		"unable to fetch ratio": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
+					return 0, 0, errors.New("fail")
+				},
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescGetRatio,
+		},
+		"total runs below limit": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: func(_ *flakeDetectionPolicy, _ string) (int, int, error) {
+					return totalRunsLimit - 1, 0, nil
+				},
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescShortHistory,
+		},
+		"fail ratio below threshold": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
+			},
+			expectError: false,
+		},
+		"fail ratio above threshold": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestAllFailures,
+			},
+			failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescAboveThreshold,
+		},
+		"fail ratio below threshold - multiple tests": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestNoFailures,
+			},
+			failedTests: []testcase.TestCase{
+				{Name: "test", Classname: "class"},
+				{Name: "test-1", Classname: "class-1"},
+			},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-1", Classname: "class-1", RatioThreshold: 1}),
+			},
+			expectError: false,
+		},
+		"fail ratio above threshold - multiple tests": {
+			bqClient: &biqQueryClientMock{
+				getRatioForTest: getRatioForTestAllFailures,
+			},
+			failedTests: []testcase.TestCase{
+				{Name: "test-ratio-below", Classname: "class"},
+				{Name: "test-ratio-above", Classname: "class"},
+			},
+			flakeDetectionPolicies: []*flakeDetectionPolicy{
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-below", Classname: "class", RatioThreshold: 90}),
+				newFlakeDetectionPolicyMust(&flakeDetectionPolicyConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-above", Classname: "class", RatioThreshold: 10}),
+			},
+			expectError:    true,
+			expectErrorStr: errDescAboveThreshold,
+		},
+	}
+
+	for sampleName, sample := range samples {
+		t.Run(sampleName, func(tt *testing.T) {
+			err := p.checkFailedTests(sample.bqClient, sample.failedTests, sample.flakeDetectionPolicies)
+
+			if sample.expectError {
+				assert.ErrorContains(tt, err, sample.expectErrorStr)
+			} else {
+				assert.NoError(tt, err)
+			}
+		})
+	}
+}

cmd/flakechecker/testdata/flake-config.yml

@@ -0,0 +1,13 @@
+# Config with regex
+- matchJobName: 'pr-.*'
+  ratioJobName: main-branch-tests
+  testNameRegex: 'TestLoadFlakeConf.*'
+  classname: TestLoadFlakeConfigFile
+  ratioThreshold: 5
+
+# Config with flat values
+- matchJobName: pull-request-tests
+  ratioJobName: main-branch-tests
+  testNameRegex: TestLoadFlakeConfigFile
+  classname: TestLoadFlakeConfigFile
+  ratioThreshold: 10