add manual tests exclusion

README.md

@@ -22,6 +22,7 @@ Usage:
 Flags:
   -c, --ciphers string          List of colon-separated TLS cipher names
       --dryrun                  Display only the title of test cases
+  -x, --exclude strings         Disable specific tests
       --help                    Display this help and exit
   -h, --host string             Target host (default "127.0.0.1")
   -k, --insecure                Don't verify server's certificate
@@ -86,6 +87,12 @@ When *Strict Mode* is enabled, h2spec will run the test cases related to the con
 $ h2spec --strict
 ```
 
+### Disable Certain Tests
+If you use `h2spec` as a part of your CI/CD pipeline, you may find it useful to be able to manually exclude specific tests by their *Spec ID*s before the corresponding issue gets resolved:
+```
+$ h2spec --exclude hpack/2 --exclude hpack/4/1
+```
+
 ## Screenshot
 
 ![Sceenshot](https://cloud.githubusercontent.com/assets/230145/22183160/9e9fbb4c-e0fa-11e6-9383-e2cc1ed6750a.png)

cmd/h2spec/h2spec.go

@@ -41,6 +41,7 @@ func main() {
 	flags.BoolP("verbose", "v", false, "Output verbose log")
 	flags.Bool("version", false, "Display version information and exit")
 	flags.Bool("help", false, "Display this help and exit")
+	flags.StringSliceP("exclude", "x", []string{}, "Disable specific tests")
 
 	err := cmd.Execute()
 	if err != nil {
@@ -122,6 +123,11 @@ func run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	exclude, err := flags.GetStringSlice("exclude")
+	if err != nil {
+		return err
+	}
+
 	if port == 0 {
 		if tls {
 			port = 443
@@ -144,6 +150,7 @@ func run(cmd *cobra.Command, args []string) error {
 		Insecure:     insecure,
 		Verbose:      verbose,
 		Sections:     args,
+		Excluded:     exclude,
 	}
 
 	success, err := h2spec.Run(c)

config/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	CertKeyFile  string
 	Exec         string
 	FromPort     int
+	Excluded     []string
 }
 
 // Addr returns the string concatenated with hostname and port number.

spec/spec.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 	"time"
+	"regexp"
 
 	"github.com/summerwind/h2spec/config"
 	"github.com/summerwind/h2spec/log"
@@ -154,6 +155,28 @@ type TestCase struct {
 
 // Test runs itself as a test case.
 func (tc *TestCase) Test(c *config.Config, seq int) error {
+	fullTestID := fmt.Sprintf("%v/%v", tc.Parent.ID(), seq)
+	// TODO: As for now our default target tests system is Ubuntu 20.04,
+	// `go` version there is 1.13.8, which doesn't have `slices.Contains` yet.
+	// Let's check it out later.
+	for _, exc := range c.Excluded {
+		matched, _ := regexp.MatchString(exc, fullTestID)
+		if matched {
+			if c.Verbose {
+				message := fmt.Sprintf("Test %#v is manually disabled by --exclude/-x %#v, skipping.",
+				                       fullTestID, exc)
+				log.Println(message)
+			}
+			tc.Result = &TestResult{
+				TestCase: tc,
+				Sequence: seq,
+				Error: ErrSkipped,
+				Skipped: true,
+			}
+			return nil
+		}
+	}
+
 	if tc.Strict && !c.Strict {
 		return nil
 	}