From 44f1632217c169b0a6e4f37158b4249ea592da53 Mon Sep 17 00:00:00 2001 From: Samuel Manzer Date: Wed, 22 Nov 2023 10:09:29 -0800 Subject: [PATCH] Document the `tests` Feature and Demo Integration Test Runner (#448) Summary: * Add documentation for the `tests` feature * Allow `ttpforge test` to accept multiple arguments * Create a demo integration test runner that works in fbcode via buck2 custom_unittest - will rebase on D51518892 and build on that integration test script Reviewed By: cedowens Differential Revision: D51520434 --- cmd/test.go | 141 ++++++++++-------- docs/foundations/README.md | 1 + docs/foundations/tests.md | 49 ++++++ .../introduction/dotfile-backdoor-demo.yaml | 3 +- run_all_ttp_tests.sh | 5 + 5 files changed, 135 insertions(+), 64 deletions(-) create mode 100644 docs/foundations/tests.md create mode 100755 run_all_ttp_tests.sh diff --git a/cmd/test.go b/cmd/test.go index 2ee76c0e..733a4d86 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -40,84 +40,35 @@ type testCase struct { DryRun bool `yaml:"dry_run"` } -type testSection struct { +type ttpFields struct { + Name string `yaml:"name"` Cases []testCase `yaml:"tests"` } +// Note - this command cannot be unit tested +// because it calls os.Executable() and actually re-executes +// the same binary ("itself", though with a different command) +// as a subprocess func buildTestCommand(cfg *Config) *cobra.Command { var timeoutSeconds int runCmd := &cobra.Command{ Use: "test [repo_name//path/to/ttp]", Short: "Test the TTP found in the specified YAML file.", - Args: cobra.ExactArgs(1), + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { // don't want confusing usage display for errors past this point cmd.SilenceUsage = true - // find the TTP file - ttpRef := args[0] - _, ttpAbsPath, err := cfg.repoCollection.ResolveTTPRef(ttpRef) - if err != nil { - return fmt.Errorf("failed to resolve TTP reference %v: %w", ttpRef, err) - } - - // preprocess to separate out the `tests:` section from the `steps:` - // section and avoid YAML parsing errors associated with template syntax - contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath) - if err != nil { - return fmt.Errorf("failed to read TTP file %v: %w", ttpAbsPath, err) - } - preprocessResult, err := preprocess.Parse(contents) - if err != nil { - return err - } - - // load the test cases - var ts testSection - err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ts) - if err != nil { - return fmt.Errorf("failed to parse `test:` section of TTP file %v: %w", ttpAbsPath, err) - } - - // look up the path of this binary (ttpforge) - selfPath, err := os.Executable() - if err != nil { - return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err) - } - - if len(ts.Cases) == 0 { - logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath) - return nil - } - - // run all cases - logging.DividerThick() - logging.L().Infof("EXECUTING %v TEST CASE(S)", len(ts.Cases)) - for tcIdx, tc := range ts.Cases { - logging.DividerThin() - logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name) - logging.DividerThin() - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, selfPath) - cmd.Args = append(cmd.Args, "run", ttpAbsPath) - for argName, argVal := range tc.Args { - cmd.Args = append(cmd.Args, "--arg") - cmd.Args = append(cmd.Args, argName+"="+argVal) - } - if tc.DryRun { - cmd.Args = append(cmd.Args, "--dry-run") - } - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Run() + for _, ttpRef := range args { + // find the TTP file + _, ttpAbsPath, err := cfg.repoCollection.ResolveTTPRef(ttpRef) if err != nil { - return fmt.Errorf("test case %q failed: %w", tc.Name, err) + return fmt.Errorf("failed to resolve TTP reference %v: %w", ttpRef, err) + } + if err := runTestsForTTP(ttpAbsPath, timeoutSeconds); err != nil { + return fmt.Errorf("test(s) for TTP %v failed: %w", ttpRef, err) } } - logging.DividerThick() - logging.L().Info("ALL TESTS COMPLETED SUCCESSFULLY!") return nil }, } @@ -125,3 +76,67 @@ func buildTestCommand(cfg *Config) *cobra.Command { return runCmd } + +func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error { + // preprocess to separate out the `tests:` section from the `steps:` + // section and avoid YAML parsing errors associated with template syntax + contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath) + if err != nil { + return fmt.Errorf("failed to read TTP file %v: %w", ttpAbsPath, err) + } + preprocessResult, err := preprocess.Parse(contents) + if err != nil { + return err + } + + // load the test cases - we don't want to load the entire ttp + // because that's one of the parts of the code that this + // command is trying to test + var ttpf ttpFields + err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ttpf) + if err != nil { + return fmt.Errorf("failed to parse `test:` section of TTP file %v: %w", ttpAbsPath, err) + } + + // look up the path of this binary (ttpforge) + selfPath, err := os.Executable() + if err != nil { + return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err) + } + + if len(ttpf.Cases) == 0 { + logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath) + return nil + } + + // run all cases + logging.DividerThick() + logging.L().Infof("TTP: %q", ttpf.Name) + logging.L().Infof("EXECUTING %v TEST CASE(S)", len(ttpf.Cases)) + for tcIdx, tc := range ttpf.Cases { + logging.DividerThin() + logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name) + logging.DividerThin() + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, selfPath) + cmd.Args = append(cmd.Args, "run", ttpAbsPath) + for argName, argVal := range tc.Args { + cmd.Args = append(cmd.Args, "--arg") + cmd.Args = append(cmd.Args, argName+"="+argVal) + } + if tc.DryRun { + cmd.Args = append(cmd.Args, "--dry-run") + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Run() + if err != nil { + return fmt.Errorf("test case %q failed: %w", tc.Name, err) + } + } + logging.DividerThin() + logging.L().Info("ALL TESTS COMPLETED SUCCESSFULLY!") + return nil +} diff --git a/docs/foundations/README.md b/docs/foundations/README.md index 40e83541..6a2e8bf2 100644 --- a/docs/foundations/README.md +++ b/docs/foundations/README.md @@ -5,5 +5,6 @@ Learn about the key features of TTPForge, including: - [Automating Attacker Actions with TTPForge](actions.md) - [Customizing TTPs with Command-Line Arguments](args.md) - [Ensuring Reliable TTP Cleanup](cleanup.md) +- [Writing Tests for TTPs](tests.md) More sections coming soon! diff --git a/docs/foundations/tests.md b/docs/foundations/tests.md new file mode 100644 index 00000000..f064de8f --- /dev/null +++ b/docs/foundations/tests.md @@ -0,0 +1,49 @@ +# Tests for TTPs + +You can write tests for your TTPs using the `tests:` section of a TTPForge YAML +file. These tests serve two purposes: + +- They act as continuously-validated documentation for how users should run your + TTP. +- They help ensure that the TTPForge engine will remain compatible with your TTP + and provide warning if this compatibility is broken for any reason. + +## Basic Test Cases + +The simplest-possible test case that you can define for a TTP is shown below: + +https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/minimal_test_case.yaml#L1-L14 + +When you run the test cases for this TTP via the command +`ttpforge test examples//tests/minimal_test_case.yaml`, TTPForge will call +`ttpforge run` and pass the absolute path to your TTP file as an argument. In +this instance, the `tests` syntax may seem superfluous, but even in this simple +case it plays a very important role: **by declaring a test case, you are telling +TTPForge that your TTP is safe to run as an automated test.** + +## Test Cases with Arguments + +The `tests` feature really starts to show its value when used for TTPs that +expect command-line arguments. An example of such a TTP, with two associated +test cases, is shown below: + +https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/with_args.yaml#L1-L42 + +When you test this TTP via `ttpforge test examples//tests/with_args.yaml`, both +of the test cases in the above file will be run sequentially. TTPForge will +parse the provided `args` list, encode each entry in the string format +`--arg foo=bar`, and then append each resulting string to a dynamically +generated `ttpforge run` command. The subsequent execution of that command +verifies that the TTP functions correctly for that test case. + +## Dry-Run Test Cases + +Some TTPs can only be executed except under very specific conditions - for +example, Active Directory exploits that target domain controllers. It may not be +feasible to test execution of such a TTP in an automated setting; however, it is +still possible to verify that the TTP parses its arguments correctly and that +all TTPForge validation phases _prior to actual execution_ complete +successfully. To perform "validation without execution" in this manner, add +`dry_run: true` to your test case, as shown below: + +https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/dry_run.yaml#L1-L30 diff --git a/example-ttps/introduction/dotfile-backdoor-demo.yaml b/example-ttps/introduction/dotfile-backdoor-demo.yaml index 5900eed2..50afe2f7 100644 --- a/example-ttps/introduction/dotfile-backdoor-demo.yaml +++ b/example-ttps/introduction/dotfile-backdoor-demo.yaml @@ -8,7 +8,8 @@ description: | - Checking Conditions at Runtime to Avoid Errors tests: - name: default - description: execute with the default setting + description: dry run with the default settings + dry_run: true args: - name: target_file_path type: path diff --git a/run_all_ttp_tests.sh b/run_all_ttp_tests.sh new file mode 100755 index 00000000..a509a5e4 --- /dev/null +++ b/run_all_ttp_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")" +"$1" test example-ttps/**/*.yaml