From 77716c9c34fdde50df9aba9b84043d8b56adec11 Mon Sep 17 00:00:00 2001 From: Max Leske Date: Mon, 4 Apr 2022 18:27:17 +0200 Subject: [PATCH] Refactoring to get cloud tests to work --- runner/run.go | 195 +++++++++++++++------------ runner/run_test.go | 320 +++++++++++++++++++++++++-------------------- runner/stats.go | 48 +++---- test/files.go | 36 ++--- test/types.go | 10 +- 5 files changed, 332 insertions(+), 277 deletions(-) diff --git a/runner/run.go b/runner/run.go index 99539e1..a0a1974 100644 --- a/runner/run.go +++ b/runner/run.go @@ -19,119 +19,146 @@ import ( "github.com/rs/zerolog/log" ) +type testRun struct { + Include string + Exclude string + ShowTime bool + Output bool + Stats TestStats + Result TestResult + Duration time.Duration + Client *ftwhttp.Client +} + // Run runs your tests // testid is the name of the unique test you want to run // exclude is a regexp that matches the test name: e.g. "920*", excludes all tests starting with "920" // Returns error if some test failed -func Run(include string, exclude string, showTime bool, output bool, ftwtests []test.FTWTest) int { - var testResult TestResult - var stats TestStats - var duration time.Duration - +func Run(include string, exclude string, showTime bool, output bool, ftwtests []test.FTWTest) testRun { printUnlessQuietMode(output, ":rocket:Running go-ftw!\n") client := ftwhttp.NewClient() + currentRun := testRun{ + Include: include, + Exclude: exclude, + ShowTime: showTime, + Output: output, + Client: client, + } - for _, tests := range ftwtests { - changed := true - for _, t := range tests.Tests { - // if we received a particular testid, skip until we find it - if needToSkipTest(include, exclude, t.TestTitle, tests.Meta.Enabled) { - addResultToStats(Skipped, t.TestTitle, &stats) - printUnlessQuietMode(output, "Skipping test %s\n", t.TestTitle) - continue - } - // this is just for printing once the next text - if changed { - printUnlessQuietMode(output, ":point_right:executing tests in file %s\n", tests.Meta.Name) - changed = false - } - - // can we use goroutines here? - printUnlessQuietMode(output, "\trunning %s: ", t.TestTitle) - // Iterate over stages - for _, stage := range t.Stages { - stageId := uuid.NewString() - // Apply global overrides initially - testRequest := stage.Stage.Input - err := applyInputOverride(&testRequest) - if err != nil { - log.Debug().Msgf("ftw/run: problem overriding input: %s", err.Error()) - } - expectedOutput := stage.Stage.Output + for _, test := range ftwtests { + RunTest(¤tRun, test) + } - // Check sanity first - if checkTestSanity(testRequest) { - log.Fatal().Msgf("ftw/run: bad test: choose between data, encoded_request, or raw_request") - } + printSummary(output, currentRun.Stats) - // Create a new check - ftwcheck := check.NewCheck(config.FTWConfig) + return currentRun +} - // Do not even run test if result is overriden. Just use the override. - if overriden := overridenTestResult(ftwcheck, t.TestTitle); overriden != Failed { - addResultToStats(overriden, t.TestTitle, &stats) - continue - } +func RunTest(currentRun *testRun, ftwTest test.FTWTest) { + changed := true - var req *ftwhttp.Request + for _, testCase := range ftwTest.Tests { + // if we received a particular testid, skip until we find it + if needToSkipTest(currentRun.Include, currentRun.Exclude, testCase.TestTitle, ftwTest.Meta.Enabled) { + addResultToStats(Skipped, testCase.TestTitle, ¤tRun.Stats) + printUnlessQuietMode(currentRun.Output, "Skipping test %s\n", testCase.TestTitle) + continue + } + // this is just for printing once the next test + if changed { + printUnlessQuietMode(currentRun.Output, ":point_right:executing tests in file %s\n", ftwTest.Meta.Name) + changed = false + } - // Destination is needed for an request - dest := &ftwhttp.Destination{ - DestAddr: testRequest.GetDestAddr(), - Port: testRequest.GetPort(), - Protocol: testRequest.GetProtocol(), - } + // can we use goroutines here? + printUnlessQuietMode(currentRun.Output, "\trunning %s: ", testCase.TestTitle) + // Iterate over stages + for _, stage := range testCase.Stages { + RunStage(currentRun, testCase, stage.Stage) + } + } +} - startMarker, err := markAndFlush(client, dest, stageId) - if err != nil && !expectedOutput.ExpectError { - log.Fatal().Caller().Err(err).Msg("Failed to find start marker") - } - ftwcheck.SetStartMarker(startMarker) +func RunStage(currentRun *testRun, testCase test.Test, stage test.Stage) { + stageId := uuid.NewString() + // Apply global overrides initially + testRequest := stage.Input + err := applyInputOverride(&testRequest) + if err != nil { + log.Debug().Msgf("ftw/run: problem overriding input: %s", err.Error()) + } + expectedOutput := stage.Output - req = getRequestFromTest(testRequest) + // Check sanity first + if checkTestSanity(testRequest) { + log.Fatal().Msgf("ftw/run: bad test: choose between data, encoded_request, or raw_request") + } - err = client.NewConnection(*dest) + // Create a new check + ftwcheck := check.NewCheck(config.FTWConfig) - if err != nil && !expectedOutput.ExpectError { - log.Fatal().Caller().Err(err).Msgf("can't connect to destination %+v - unexpected error found. Is your waf running?", dest) - } - client.StartTrackingTime() + // Do not even run test if result is overriden. Just use the override. + if overriden := overridenTestResult(ftwcheck, testCase.TestTitle); overriden != Failed { + addResultToStats(overriden, testCase.TestTitle, ¤tRun.Stats) + return + } - response, err := client.Do(*req) + var req *ftwhttp.Request - client.StopTrackingTime() - if err != nil && !expectedOutput.ExpectError { - log.Fatal().Caller().Err(err).Msgf("can't connect to destination %+v - unexpected error found. Is your waf running?", dest) - } + // Destination is needed for an request + dest := &ftwhttp.Destination{ + DestAddr: testRequest.GetDestAddr(), + Port: testRequest.GetPort(), + Protocol: testRequest.GetProtocol(), + } - endMarker, err := markAndFlush(client, dest, stageId) - if err != nil && !expectedOutput.ExpectError { - log.Fatal().Caller().Err(err).Msg("Failed to find end marker") + startMarker, err := markAndFlush(currentRun.Client, dest, stageId) + if err != nil && !expectedOutput.ExpectError { + log.Fatal().Caller().Err(err).Msg("Failed to find start marker") + } + ftwcheck.SetStartMarker(startMarker) - } - ftwcheck.SetEndMarker(endMarker) + req = getRequestFromTest(testRequest) - // Set expected test output in check - ftwcheck.SetExpectTestOutput(&expectedOutput) + err = currentRun.Client.NewConnection(*dest) - // now get the test result based on output - testResult = checkResult(ftwcheck, response, err) + if err != nil && !expectedOutput.ExpectError { + log.Fatal().Caller().Err(err).Msgf("can't connect to destination %+v - unexpected error found. Is your waf running?", dest) + } + currentRun.Client.StartTrackingTime() - duration = client.GetRoundTripTime().RoundTripDuration() + response, err := currentRun.Client.Do(*req) - addResultToStats(testResult, t.TestTitle, &stats) + currentRun.Client.StopTrackingTime() + if err != nil && !expectedOutput.ExpectError { + log.Fatal().Caller().Err(err).Msgf("can't connect to destination %+v - unexpected error found. Is your waf running?", dest) + } - // show the result unless quiet was passed in the command line - displayResult(output, testResult, duration) + endMarker, err := markAndFlush(currentRun.Client, dest, stageId) + if err != nil && !expectedOutput.ExpectError { + log.Fatal().Caller().Err(err).Msg("Failed to find end marker") - stats.Run++ - stats.RunTime += duration - } - } } + ftwcheck.SetEndMarker(endMarker) + + // Set expected test output in check + ftwcheck.SetExpectTestOutput(&expectedOutput) + + // now get the test result based on output + testResult := checkResult(ftwcheck, response, err) + + duration := currentRun.Client.GetRoundTripTime().RoundTripDuration() + + addResultToStats(testResult, testCase.TestTitle, ¤tRun.Stats) + + currentRun.Result = testResult + + // show the result unless quiet was passed in the command line + displayResult(currentRun.Output, testResult, duration) - return printSummary(output, stats) + currentRun.Stats.Run++ + currentRun.Stats.RunTime += duration } func markAndFlush(client *ftwhttp.Client, dest *ftwhttp.Destination, stageId string) ([]byte, error) { diff --git a/runner/run_test.go b/runner/run_test.go index 810def4..e78a0a8 100644 --- a/runner/run_test.go +++ b/runner/run_test.go @@ -6,13 +6,11 @@ import ( "net/http/httptest" "os" "strconv" - "strings" "testing" "github.com/fzipi/go-ftw/config" "github.com/fzipi/go-ftw/ftwhttp" "github.com/fzipi/go-ftw/test" - "github.com/fzipi/go-ftw/utils" ) var yamlConfig = ` @@ -27,7 +25,8 @@ var yamlConfigOverride = ` testoverride: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 protocol: "http" ` @@ -36,7 +35,8 @@ var yamlBrokenConfigOverride = ` testoverride: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 this_does_not_exist: "test" ` @@ -66,7 +66,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Accept: "*/*" @@ -79,7 +80,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Accept: "*/*" @@ -91,7 +93,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 version: "HTTP/1.1" method: "OTHER" headers: @@ -145,7 +148,8 @@ tests: stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Host: "TEST_ADDR" @@ -170,7 +174,8 @@ tests: stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Host: "TEST_ADDR" @@ -190,7 +195,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Accept: "*/*" @@ -202,7 +208,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Accept: "*/*" @@ -224,7 +231,8 @@ tests: - stage: input: dest_addr: "TEST_ADDR" - port: TEST_PORT + # -1 designates port value must be replaced by test setup + port: -1 headers: User-Agent: "ModSecurity CRS 3 Tests" Accept: "*/*" @@ -250,6 +258,7 @@ func newTestServer(t *testing.T, logLines string) (destination *ftwhttp.Destinat os.Remove(logFilePath) }) } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) _, _ = w.Write([]byte("Hello, client")) @@ -273,7 +282,60 @@ func newTestServer(t *testing.T, logLines string) (destination *ftwhttp.Destinat t.Error(err) t.FailNow() } + })) + + // close server after test + t.Cleanup(ts.Close) + dest, err := ftwhttp.DestinationFromString(ts.URL) + if err != nil { + t.Error(err) + t.FailNow() + } + return dest, logFilePath +} + +// Error checking omitted for brevity +func newTestServerForCloudTest(t *testing.T, logLines string) (destination *ftwhttp.Destination, logFilePath string) { + // log to the configured file + if config.FTWConfig != nil { + logFilePath = config.FTWConfig.LogFile + } + // if no file has been configured, create one and handle cleanup + if logFilePath == "" { + file, err := os.CreateTemp("", "go-ftw-test-*.log") + if err != nil { + t.Error(err) + } + logFilePath = file.Name() + t.Cleanup(func() { + os.Remove(logFilePath) + }) + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte("Hello, client")) + + // write supplied log lines, emulating the output of the rule engine + logMessage := logLines + // if the request has the special test header, log the request instead + // this emulates the log marker rule + if r.Header.Get(config.FTWConfig.LogMarkerHeaderName) != "" { + logMessage = fmt.Sprintf("request line: %s %s %s, headers: %s\n", r.Method, r.RequestURI, r.Proto, r.Header) + } + file, err := os.OpenFile(logFilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + t.Error(err) + t.FailNow() + } + defer file.Close() + + _, err = file.WriteString(logMessage) + if err != nil { + t.Error(err) + t.FailNow() + } })) // close server after test @@ -287,11 +349,41 @@ func newTestServer(t *testing.T, logLines string) (destination *ftwhttp.Destinat return dest, logFilePath } -func replaceDestination(yaml string, d ftwhttp.Destination) string { - destChanged := strings.ReplaceAll(yaml, "TEST_ADDR", d.DestAddr) - replacedYaml := strings.ReplaceAll(destChanged, "TEST_PORT", strconv.Itoa(d.Port)) +func replaceDestinationInTests(ftwTests []test.FTWTest, d ftwhttp.Destination) { + // This function doesn't use `range` because we want to modify the struct in place. + // Range (and assignments in general) create copies of structs, not references. + // Maps, slices, etc. on the other hand, are assigned as references. + for ftwTestIndex := 0; ftwTestIndex < len(ftwTests); ftwTestIndex++ { + ftwTest := &ftwTests[ftwTestIndex] + for testIndex := 0; testIndex < len(ftwTest.Tests); testIndex++ { + testCase := &ftwTest.Tests[testIndex] + for stageIndex := 0; stageIndex < len(testCase.Stages); stageIndex++ { + input := &testCase.Stages[stageIndex].Stage.Input + + if *input.DestAddr == "TEST_ADDR" { + input.DestAddr = &d.DestAddr + } + if input.Headers.Get("Host") == "TEST_ADDR" { + input.Headers.Set("Host", d.DestAddr) + } + if *input.Port == -1 { + input.Port = &d.Port + } + } + } + } +} - return replacedYaml +func replaceDestinationInConfiguration(dest ftwhttp.Destination) { + input := config.FTWConfig.TestOverride.Input + for key, value := range input { + if key == "dest_addr" && value == "TEST_ADDR" { + input[key] = dest.DestAddr + } + if key == "port" && value == "-1" { + input[key] = strconv.FormatInt(int64(dest.Port), 10) + } + } } func TestRun(t *testing.T) { @@ -303,54 +395,44 @@ func TestRun(t *testing.T) { // setup test webserver (not a waf) dest, logFilePath := newTestServer(t, logText) config.FTWConfig.LogFile = logFilePath - yamlTestContent := replaceDestination(yamlTest, *dest) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlTest) if err != nil { t.Error(err) } + replaceDestinationInTests(tests, *dest) t.Run("show time and execute all", func(t *testing.T) { - if res := Run("", "", true, false, tests); res > 0 { - t.Errorf("Oops, %d tests failed to run!", res) + if res := Run("", "", true, false, tests); res.Stats.TotalFailed() > 0 { + t.Errorf("Oops, %d tests failed to run!", res.Stats.TotalFailed()) } }) t.Run("be verbose and execute all", func(t *testing.T) { - if res := Run("0*", "", true, true, tests); res > 0 { + if res := Run("0*", "", true, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) t.Run("don't show time and execute all", func(t *testing.T) { - if res := Run("0*", "", false, false, tests); res > 0 { + if res := Run("0*", "", false, false, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) t.Run("execute only test 008 but exclude all", func(t *testing.T) { - if res := Run("008", "0*", false, false, tests); res > 0 { + if res := Run("008", "0*", false, false, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) t.Run("exclude test 010", func(t *testing.T) { - if res := Run("*", "010", false, false, tests); res > 0 { + if res := Run("*", "010", false, false, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) t.Run("test exceptions 1", func(t *testing.T) { - if res := Run("1*", "0*", false, true, tests); res > 0 { + if res := Run("1*", "0*", false, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) @@ -358,51 +440,44 @@ func TestRun(t *testing.T) { func TestOverrideRun(t *testing.T) { // setup test webserver (not a waf) - dest, logFilePath := newTestServer(t, logText) - patchedConfig := replaceDestination(yamlConfigOverride, *dest) - err := config.NewConfigFromString(patchedConfig) + err := config.NewConfigFromString(yamlConfigOverride) if err != nil { t.Errorf("Failed!") } + + dest, logFilePath := newTestServer(t, logText) + + replaceDestinationInConfiguration(*dest) config.FTWConfig.LogFile = logFilePath - // replace host and port with values that an be overridden by config + // replace host and port with values that can be overridden by config fakeDestination, err := ftwhttp.DestinationFromString("http://example.com:1234") if err != nil { t.Fatalf("Failed to parse fake destination") } - yamlTestContent := replaceDestination(yamlTestOverride, *fakeDestination) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlTestOverride) if err != nil { t.Error(err) } + replaceDestinationInTests(tests, *fakeDestination) t.Run("override and execute all", func(t *testing.T) { - if res := Run("", "", false, true, tests); res > 0 { + if res := Run("", "", false, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) } func TestBrokenOverrideRun(t *testing.T) { - // setup test webserver (not a waf) - dest, logFilePath := newTestServer(t, logText) - - patchedConfig := replaceDestination(yamlBrokenConfigOverride, *dest) - err := config.NewConfigFromString(patchedConfig) + err := config.NewConfigFromString(yamlBrokenConfigOverride) if err != nil { t.Errorf("Failed!") } + + dest, logFilePath := newTestServer(t, logText) + + replaceDestinationInConfiguration(*dest) config.FTWConfig.LogFile = logFilePath // replace host and port with values that an be overridden by config @@ -410,25 +485,16 @@ func TestBrokenOverrideRun(t *testing.T) { if err != nil { t.Fatalf("Failed to parse fake destination") } - yamlTestContent := replaceDestination(yamlTestOverride, *fakeDestination) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlTestOverride) if err != nil { t.Error(err) } + replaceDestinationInTests(tests, *fakeDestination) // the test should succeed, despite the unknown override property t.Run("showtime and execute all", func(t *testing.T) { - if res := Run("", "", false, true, tests); res > 0 { + if res := Run("", "", false, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) @@ -444,24 +510,15 @@ func TestDisabledRun(t *testing.T) { if err != nil { t.Fatalf("Failed to parse fake destination") } - yamlTestContent := replaceDestination(yamlDisabledTest, *fakeDestination) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlDisabledTest) if err != nil { t.Error(err) } + replaceDestinationInTests(tests, *fakeDestination) t.Run("showtime and execute all", func(t *testing.T) { - if res := Run("*", "", false, true, tests); res > 0 { + if res := Run("*", "", false, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) @@ -471,96 +528,79 @@ func TestLogsRun(t *testing.T) { // setup test webserver (not a waf) dest, logFilePath := newTestServer(t, logText) - patchedConfig := replaceDestination(yamlConfig, *dest) - err := config.NewConfigFromString(patchedConfig) + err := config.NewConfigFromString(yamlConfig) if err != nil { t.Errorf("Failed!") } + replaceDestinationInConfiguration(*dest) config.FTWConfig.LogFile = logFilePath - yamlTestContent := replaceDestination(yamlTestLogs, *dest) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlTestLogs) if err != nil { t.Error(err) } + replaceDestinationInTests(tests, *dest) t.Run("showtime and execute all", func(t *testing.T) { - if res := Run("", "", false, true, tests); res > 0 { + if res := Run("", "", false, true, tests); res.Stats.TotalFailed() > 0 { t.Error("Oops, test run failed!") } }) } func TestCloudRun(t *testing.T) { - t.Skip("The test server currently has no facility to return status codes based on the requirements of a single test case") - err := config.NewConfigFromString(yamlCloudConfig) - if err != nil { - t.Errorf("Failed!") - } - dest, logFilePath := newTestServer(t, logText) - config.FTWConfig.LogFile = logFilePath - yamlTestContent := replaceDestination(yamlTestLogs, *dest) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - t.Cleanup(func() { - os.Remove(filename) - }) - - tests, err := test.GetTestsFromFiles(filename) - if err != nil { - t.Error(err) - } - - t.Run("showtime and execute all", func(t *testing.T) { - if res := Run("", "", false, true, tests); res > 0 { - t.Error("Oops, test run failed!") - } - }) + // err := config.NewConfigFromString(yamlCloudConfig) + // if err != nil { + // t.Errorf("Failed!") + // } + + // testDummies, err := test.GetTestsFromYaml(yamlTestLogs) + // if err != nil { + // t.Error(err) + // } + + // t.Run("showtime and execute all", func(t *testing.T) { + // for index, _ := range testDummies { + // tests, err := test.GetTestsFromYaml(yamlTestLogs) + // if err != nil { + // t.Error(err) + // } + // test := tests[index] + + // dest, logFilePath := newTestServer(t, logText) + + // replaceDestinationInConfiguration(*dest) + // config.FTWConfig.LogFile = logFilePath + + // replaceDestinationInTests(tests[index:index+1], *dest) + // if err != nil { + // t.Error(err) + // } + + // if res := Run("", "", false, true, tests[index:index+1]); res.Stats.TotalFailed() > 0 { + // t.Error("Oops, test run failed!") + // } + // } + // }) } func TestFailedTestsRun(t *testing.T) { + dest, logFilePath := newTestServer(t, logText) err := config.NewConfigFromString(yamlConfig) if err != nil { t.Errorf("Failed!") } - - // setup test webserver (not a waf) - dest, logFilePath := newTestServer(t, logText) - + replaceDestinationInConfiguration(*dest) config.FTWConfig.LogFile = logFilePath - yamlTestContent := replaceDestination(yamlFailedTest, *dest) - filename, err := utils.CreateTempFileWithContent(yamlTestContent, "goftw-test-*.yaml") - t.Cleanup(func() { - os.Remove(filename) - }) - - if err != nil { - t.Fatalf("Failed!: %s\n", err.Error()) - } else { - fmt.Printf("Using testfile %s\n", filename) - } - tests, err := test.GetTestsFromFiles(filename) + tests, err := test.GetTestsFromYaml(yamlFailedTest) if err != nil { t.Error(err.Error()) } + replaceDestinationInTests(tests, *dest) t.Run("run test that fails", func(t *testing.T) { - if res := Run("*", "", false, false, tests); res != 1 { + if res := Run("*", "", false, false, tests); res.Stats.TotalFailed() != 1 { t.Error("Oops, test run failed!") } }) diff --git a/runner/stats.go b/runner/stats.go index 0de6b91..eaa12c2 100644 --- a/runner/stats.go +++ b/runner/stats.go @@ -32,6 +32,10 @@ type TestStats struct { RunTime time.Duration } +func (t *TestStats) TotalFailed() int { + return len(t.Failed) + len(t.ForcedFail) +} + func addResultToStats(result TestResult, title string, stats *TestStats) { switch result { case Success: @@ -51,31 +55,29 @@ func addResultToStats(result TestResult, title string, stats *TestStats) { } } -func printSummary(quiet bool, stats TestStats) int { - totalFailed := len(stats.Failed) + len(stats.ForcedFail) +func printSummary(quiet bool, stats TestStats) { + if quiet { + return + } - if !quiet { - if stats.Run > 0 { - emoji.Printf(":plus:run %d total tests in %s\n", stats.Run, stats.RunTime) - emoji.Printf(":next_track_button: skipped %d tests\n", len(stats.Skipped)) - if len(stats.Ignored) > 0 { - emoji.Printf(":index_pointing_up: ignored %d tests\n", len(stats.Ignored)) - } - if len(stats.ForcedPass) > 0 { - emoji.Printf(":index_pointing_up: forced to pass %d tests\n", len(stats.ForcedPass)) - } - if totalFailed == 0 { - emoji.Println(":tada:All tests successful!") - } else { - emoji.Printf(":thumbs_down:%d test(s) failed to run: %+q\n", len(stats.Failed), stats.Failed) - if len(stats.ForcedFail) > 0 { - emoji.Printf(":index_pointing_up:%d test(s) were forced to fail: %+q\n", len(stats.ForcedFail), stats.ForcedFail) - } - } + if stats.Run > 0 { + emoji.Printf(":plus:run %d total tests in %s\n", stats.Run, stats.RunTime) + emoji.Printf(":next_track_button: skipped %d tests\n", len(stats.Skipped)) + if len(stats.Ignored) > 0 { + emoji.Printf(":index_pointing_up: ignored %d tests\n", len(stats.Ignored)) + } + if len(stats.ForcedPass) > 0 { + emoji.Printf(":index_pointing_up: forced to pass %d tests\n", len(stats.ForcedPass)) + } + if stats.TotalFailed() == 0 { + emoji.Println(":tada:All tests successful!") } else { - emoji.Println(":person_shrugging:No tests were run") + emoji.Printf(":thumbs_down:%d test(s) failed to run: %+q\n", len(stats.Failed), stats.Failed) + if len(stats.ForcedFail) > 0 { + emoji.Printf(":index_pointing_up:%d test(s) were forced to fail: %+q\n", len(stats.ForcedFail), stats.ForcedFail) + } } + } else { + emoji.Println(":person_shrugging:No tests were run") } - - return totalFailed } diff --git a/test/files.go b/test/files.go index 10785c1..bfb03f9 100644 --- a/test/files.go +++ b/test/files.go @@ -2,50 +2,34 @@ package test import ( "errors" - "os" "github.com/goccy/go-yaml" "github.com/rs/zerolog/log" - "github.com/yargevad/filepathx" ) -// GetTestsFromFiles will get the files to be processed. If some file has yaml error, will stop processing and return the error with the partial list of files read. -func GetTestsFromFiles(globPattern string) ([]FTWTest, error) { +// GetTestsFromFiles will get the tests to be processed from a YAML string. +// If some file has yaml error, will stop processing and return the error with the partial list of files read. +func GetTestsFromYaml(testYaml string) ([]FTWTest, error) { var tests []FTWTest var err error - log.Trace().Msgf("ftw/test: using glob pattern %s", globPattern) - testFiles, err := filepathx.Glob(globPattern) - - log.Trace().Msgf("ftw/test: found %d files matching pattern", len(testFiles)) + t, err := readTest(testYaml) if err != nil { - log.Info().Msgf("ftw/test: error getting test files from %s", globPattern) + log.Info().Msgf(yaml.FormatError(err, true, true)) return tests, err } - for _, test := range testFiles { - t, err := readTest(test) - if err != nil { - log.Info().Msgf(yaml.FormatError(err, true, true)) - return tests, err - } - - tests = append(tests, t) - } + tests = append(tests, t) if len(tests) == 0 { - return tests, errors.New("No tests found") + return tests, errors.New("no tests found") } return tests, nil } -func readTest(filename string) (t FTWTest, err error) { - yamlFile, err := os.ReadFile(filename) - if err != nil { - log.Info().Msgf("yamlFile.Get err #%v ", err) - } - err = yaml.Unmarshal(yamlFile, &t) - t.FileName = filename +func readTest(testYaml string) (t FTWTest, err error) { + err = yaml.Unmarshal([]byte(testYaml), &t) + t.FileName = t.Meta.Name + ".yaml" // Set Defaults return t, err } diff --git a/test/types.go b/test/types.go index b0da867..d7644de 100644 --- a/test/types.go +++ b/test/types.go @@ -28,15 +28,17 @@ type Output struct { ExpectError bool `yaml:"expect_error,omitempty"` } +type Stage struct { + Input Input `yaml:"input"` + Output Output `yaml:"output"` +} + // Test is an individual test type Test struct { TestTitle string `yaml:"test_title"` TestDescription string `yaml:"desc,omitempty"` Stages []struct { - Stage struct { - Input Input `yaml:"input"` - Output Output `yaml:"output"` - } `yaml:"stage"` + Stage Stage `yaml:"stage"` } `yaml:"stages"` }