Document the tests Feature and Demo Integration Test Runner (#448)

Summary:
Pull Request resolved: #448

* Add documentation for the `tests` feature
* Allow `ttpforge test` to accept multiple arguments
* Create a demo integration test runner that works in fbcode via buck2 custom_unittest - will rebase on D51518892 and build on that integration test script

Reviewed By: cedowens

Differential Revision: D51520434

fbshipit-source-id: 48e23ae63bf35fc75da519e15e3a1c2971dbdd30

cmd/test.go

@@ -40,88 +40,103 @@ type testCase struct {
 	DryRun bool              `yaml:"dry_run"`
 }
 
-type testSection struct {
+type ttpFields struct {
+	Name  string     `yaml:"name"`
 	Cases []testCase `yaml:"tests"`
 }
 
+// Note - this command cannot be unit tested
+// because it calls os.Executable() and actually re-executes
+// the same binary ("itself", though with a different command)
+// as a subprocess
 func buildTestCommand(cfg *Config) *cobra.Command {
 	var timeoutSeconds int
 	runCmd := &cobra.Command{
 		Use:   "test [repo_name//path/to/ttp]",
 		Short: "Test the TTP found in the specified YAML file.",
-		Args:  cobra.ExactArgs(1),
+		Args:  cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			// don't want confusing usage display for errors past this point
 			cmd.SilenceUsage = true
 
-			// find the TTP file
-			ttpRef := args[0]
-			_, ttpAbsPath, err := cfg.repoCollection.ResolveTTPRef(ttpRef)
-			if err != nil {
-				return fmt.Errorf("failed to resolve TTP reference %v: %w", ttpRef, err)
-			}
-
-			// preprocess to separate out the `tests:` section from the `steps:`
-			// section and avoid YAML parsing errors associated with template syntax
-			contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath)
-			if err != nil {
-				return fmt.Errorf("failed to read TTP file %v: %w", ttpAbsPath, err)
-			}
-			preprocessResult, err := preprocess.Parse(contents)
-			if err != nil {
-				return err
-			}
-
-			// load the test cases
-			var ts testSection
-			err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ts)
-			if err != nil {
-				return fmt.Errorf("failed to parse `test:` section of TTP file %v: %w", ttpAbsPath, err)
-			}
-
-			// look up the path of this binary (ttpforge)
-			selfPath, err := os.Executable()
-			if err != nil {
-				return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err)
-			}
-
-			if len(ts.Cases) == 0 {
-				logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath)
-				return nil
-			}
-
-			// run all cases
-			logging.DividerThick()
-			logging.L().Infof("EXECUTING %v TEST CASE(S)", len(ts.Cases))
-			for tcIdx, tc := range ts.Cases {
-				logging.DividerThin()
-				logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name)
-				logging.DividerThin()
-				ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second)
-				defer cancel()
-				cmd := exec.CommandContext(ctx, selfPath)
-				cmd.Args = append(cmd.Args, "run", ttpAbsPath)
-				for argName, argVal := range tc.Args {
-					cmd.Args = append(cmd.Args, "--arg")
-					cmd.Args = append(cmd.Args, argName+"="+argVal)
-				}
-				if tc.DryRun {
-					cmd.Args = append(cmd.Args, "--dry-run")
-				}
-				cmd.Stdout = os.Stdout
-				cmd.Stderr = os.Stderr
-
-				err = cmd.Run()
+			for _, ttpRef := range args {
+				// find the TTP file
+				_, ttpAbsPath, err := cfg.repoCollection.ResolveTTPRef(ttpRef)
 				if err != nil {
-					return fmt.Errorf("test case %q failed: %w", tc.Name, err)
+					return fmt.Errorf("failed to resolve TTP reference %v: %w", ttpRef, err)
+				}
+				if err := runTestsForTTP(ttpAbsPath, timeoutSeconds); err != nil {
+					return fmt.Errorf("test(s) for TTP %v failed: %w", ttpRef, err)
 				}
 			}
-			logging.DividerThick()
-			logging.L().Info("ALL TESTS COMPLETED SUCCESSFULLY!")
 			return nil
 		},
 	}
 	runCmd.PersistentFlags().IntVar(&timeoutSeconds, "time-out-seconds", 10, "Timeout allowed for each test case")
 
 	return runCmd
 }
+
+func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error {
+	// preprocess to separate out the `tests:` section from the `steps:`
+	// section and avoid YAML parsing errors associated with template syntax
+	contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath)
+	if err != nil {
+		return fmt.Errorf("failed to read TTP file %v: %w", ttpAbsPath, err)
+	}
+	preprocessResult, err := preprocess.Parse(contents)
+	if err != nil {
+		return err
+	}
+
+	// load the test cases - we don't want to load the entire ttp
+	// because that's one of the parts of the code that this
+	// command is trying to test
+	var ttpf ttpFields
+	err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ttpf)
+	if err != nil {
+		return fmt.Errorf("failed to parse `test:` section of TTP file %v: %w", ttpAbsPath, err)
+	}
+
+	// look up the path of this binary (ttpforge)
+	selfPath, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err)
+	}
+
+	if len(ttpf.Cases) == 0 {
+		logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath)
+		return nil
+	}
+
+	// run all cases
+	logging.DividerThick()
+	logging.L().Infof("TTP: %q", ttpf.Name)
+	logging.L().Infof("EXECUTING %v TEST CASE(S)", len(ttpf.Cases))
+	for tcIdx, tc := range ttpf.Cases {
+		logging.DividerThin()
+		logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name)
+		logging.DividerThin()
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second)
+		defer cancel()
+		cmd := exec.CommandContext(ctx, selfPath)
+		cmd.Args = append(cmd.Args, "run", ttpAbsPath)
+		for argName, argVal := range tc.Args {
+			cmd.Args = append(cmd.Args, "--arg")
+			cmd.Args = append(cmd.Args, argName+"="+argVal)
+		}
+		if tc.DryRun {
+			cmd.Args = append(cmd.Args, "--dry-run")
+		}
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("test case %q failed: %w", tc.Name, err)
+		}
+	}
+	logging.DividerThin()
+	logging.L().Info("ALL TESTS COMPLETED SUCCESSFULLY!")
+	return nil
+}

docs/foundations/README.md

@@ -5,5 +5,6 @@ Learn about the key features of TTPForge, including:
 - [Automating Attacker Actions with TTPForge](actions.md)
 - [Customizing TTPs with Command-Line Arguments](args.md)
 - [Ensuring Reliable TTP Cleanup](cleanup.md)
+- [Writing Tests for TTPs](tests.md)
 
 More sections coming soon!

docs/foundations/tests.md

@@ -0,0 +1,49 @@
+# Tests for TTPs
+
+You can write tests for your TTPs using the `tests:` section of a TTPForge YAML
+file. These tests serve two purposes:
+
+- They act as continuously-validated documentation for how users should run your
+  TTP.
+- They help ensure that the TTPForge engine will remain compatible with your TTP
+  and provide warning if this compatibility is broken for any reason.
+
+## Basic Test Cases
+
+The simplest-possible test case that you can define for a TTP is shown below:
+
+https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/minimal_test_case.yaml#L1-L14
+
+When you run the test cases for this TTP via the command
+`ttpforge test examples//tests/minimal_test_case.yaml`, TTPForge will call
+`ttpforge run` and pass the absolute path to your TTP file as an argument. In
+this instance, the `tests` syntax may seem superfluous, but even in this simple
+case it plays a very important role: **by declaring a test case, you are telling
+TTPForge that your TTP is safe to run as an automated test.**
+
+## Test Cases with Arguments
+
+The `tests` feature really starts to show its value when used for TTPs that
+expect command-line arguments. An example of such a TTP, with two associated
+test cases, is shown below:
+
+https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/with_args.yaml#L1-L42
+
+When you test this TTP via `ttpforge test examples//tests/with_args.yaml`, both
+of the test cases in the above file will be run sequentially. TTPForge will
+parse the provided `args` list, encode each entry in the string format
+`--arg foo=bar`, and then append each resulting string to a dynamically
+generated `ttpforge run` command. The subsequent execution of that command
+verifies that the TTP functions correctly for that test case.
+
+## Dry-Run Test Cases
+
+Some TTPs can only be executed except under very specific conditions - for
+example, Active Directory exploits that target domain controllers. It may not be
+feasible to test execution of such a TTP in an automated setting; however, it is
+still possible to verify that the TTP parses its arguments correctly and that
+all TTPForge validation phases _prior to actual execution_ complete
+successfully. To perform "validation without execution" in this manner, add
+`dry_run: true` to your test case, as shown below:
+
+https://github.com/facebookincubator/TTPForge/blob/bf2fbb3312a227323d1930ba500b76f041329ca2/example-ttps/tests/dry_run.yaml#L1-L30

example-ttps/introduction/dotfile-backdoor-demo.yaml

@@ -8,7 +8,8 @@ description: |
     - Checking Conditions at Runtime to Avoid Errors
 tests:
   - name: default
-    description: execute with the default setting
+    description: dry run with the default settings
+    dry_run: true
 args:
   - name: target_file_path
     type: path

run_all_ttp_tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")"
+"$1" test example-ttps/**/*.yaml