✨ Detect fuzzing in Haskell by the presence of property tests. (#2843)

* Add Haskell as a language. Signed-off-by: Yoo Chung <[email protected]> * Detect fuzzing in Haskell using presence of property-based testing. Signed-off-by: Yoo Chung <[email protected]> * Mention fuzzing detection for Haskell in documentation. Signed-off-by: Yoo Chung <[email protected]> * Fix pattern and test. Add test case. Signed-off-by: Yoo Chung <[email protected]> --------- Signed-off-by: Yoo Chung <[email protected]>
ossf · Apr 12, 2023 · 71eda75 · 71eda75
1 parent 358de6b
commit 71eda75
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 12 deletions.
diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go
@@ -28,10 +28,11 @@ import (
 )
 
 const (
-	fuzzerOSSFuzz         = "OSSFuzz"
-	fuzzerClusterFuzzLite = "ClusterFuzzLite"
-	oneFuzz               = "OneFuzz"
-	fuzzerBuiltInGo       = "GoBuiltInFuzzer"
+	fuzzerOSSFuzz              = "OSSFuzz"
+	fuzzerClusterFuzzLite      = "ClusterFuzzLite"
+	oneFuzz                    = "OneFuzz"
+	fuzzerBuiltInGo            = "GoBuiltInFuzzer"
+	fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting"
 	// TODO: add more fuzzing check supports.
 )
 
@@ -42,8 +43,12 @@ type filesWithPatternStr struct {
 
 // Configurations for language-specified fuzzers.
 type languageFuzzConfig struct {
-	URL, Desc                      *string
-	filePattern, funcPattern, Name string
+	URL, Desc *string
+
+	// Pattern is according to path.Match.
+	filePattern string
+
+	funcPattern, Name string
 	// TODO: add more language fuzzing-related fields.
 }
 
@@ -59,6 +64,29 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
 		Desc: asPointer(
 			"Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."),
 	},
+	// Fuzz patterns for Haskell based on property-based testing.
+	//
+	// Based on the import of one of these packages:
+	// * https://hackage.haskell.org/package/QuickCheck
+	// * https://hedgehog.qa/
+	// * https://github.com/NorfairKing/validity
+	// * https://hackage.haskell.org/package/smallcheck
+	//
+	// They can also be imported indirectly through these test frameworks:
+	// * https://hspec.github.io/
+	// * https://hackage.haskell.org/package/tasty
+	//
+	// This is not an exhaustive list.
+	clients.Haskell: {
+		filePattern: "*.hs",
+		// Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck,
+		// or their indirect imports through the higher-level Hspec or Tasty testing frameworks.
+		funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`,
+		Name:        fuzzerPropertyBasedHaskell,
+		Desc: asPointer(
+			"Property-based testing in Haskell generates test instances randomly or exhaustively " +
+				"and test that specific properties are satisfied."),
+	},
 	// TODO: add more language-specific fuzz patterns & configs.
 }
 

diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go
@@ -316,6 +316,103 @@ func Test_checkFuzzFunc(t *testing.T) {
 			},
 			fileContent: "func TestFoo (t *testing.T)",
 		},
+		{
+			name:     "Haskell QuickCheck",
+			want:     true,
+			fileName: []string{"ModuleSpec.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.QuickCheck",
+		},
+		{
+			name:     "Haskell Hedgehog",
+			want:     true,
+			fileName: []string{"TestSpec.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.Hedgehog",
+		},
+		{
+			name:     "Haskell Validity",
+			want:     true,
+			fileName: []string{"validity_test.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.Validity",
+		},
+		{
+			name:     "Haskell SmallCheck",
+			want:     true,
+			fileName: []string{"SmallSpec.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.SmallCheck",
+		},
+		{
+			name:     "Haskell QuickCheck with qualified import",
+			want:     true,
+			fileName: []string{"QualifiedSpec.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import qualified Test.QuickCheck",
+		},
+		{
+			name:     "Haskell QuickCheck through Hspec",
+			want:     true,
+			fileName: []string{"ArrowSpec.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.Hspec.QuickCheck",
+		},
+		{
+			name:     "Haskell QuickCheck through Tasty",
+			want:     true,
+			fileName: []string{"test.hs"},
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.Tasty.QuickCheck",
+		},
+		{
+			name:     "Haskell with no property-based testing",
+			want:     false,
+			fileName: []string{"PropertySpec.hs"},
+			wantErr:  true,
+			langs: []clients.Language{
+				{
+					Name:     clients.Haskell,
+					NumLines: 50,
+				},
+			},
+			fileContent: "import Test.Hspec",
+		},
 	}
 	for _, tt := range tests {
 		tt := tt
@@ -325,12 +422,12 @@ func Test_checkFuzzFunc(t *testing.T) {
 			defer ctrl.Finish()
 			mockClient := mockrepo.NewMockRepoClient(ctrl)
 			mockClient.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes()
-			mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) {
+			mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) ([]byte, error) {
 				if tt.wantErr {
 					//nolint
-					return "", errors.New("error")
+					return nil, errors.New("error")
 				}
-				return tt.fileContent, nil
+				return []byte(tt.fileContent), nil
 			}).AnyTimes()
 			req := checker.CheckRequest{
 				RepoClient: mockClient,

diff --git a/checks/write.md b/checks/write.md
@@ -75,7 +75,7 @@ The steps to writing a check are as follows:
 8.  Create e2e tests in `e2e/mycheck_test.go`. Use a dedicated repo that will
     not change over time, so that it's reliable for the tests.
 
-9.  Update the `checks/checks.yaml` with a description of your check.
+9.  Update the `docs/checks/internal/checks.yaml` with a description of your check.
 
 10. Generate `docs/check.md` using `make generate-docs`. This will validate and
     generate `docs/check.md`.

diff --git a/clients/languages.go b/clients/languages.go
@@ -71,6 +71,9 @@ const (
 	// Dockerfile: https://docs.docker.com/engine/reference/builder/
 	Dockerfile LanguageName = "dockerfile"
 
+	// Haskell: https://www.haskell.org/
+	Haskell LanguageName = "haskell"
+
 	// Other indicates other languages not listed by the GitHub API.
 	Other LanguageName = "other"
 

diff --git a/docs/checks.md b/docs/checks.md
@@ -336,7 +336,8 @@ This check tries to determine if the project uses
 [fuzzing](https://owasp.org/www-community/Fuzzing) by checking:
 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list;
 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository;
-3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository.
+3. if there are user-defined language-specified fuzzing functions in the repository.
+   - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell.
 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz);
 
 Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data

diff --git a/docs/checks/internal/checks.yaml b/docs/checks/internal/checks.yaml
@@ -396,7 +396,8 @@ checks:
       [fuzzing](https://owasp.org/www-community/Fuzzing) by checking:
       1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list;
       2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository;
-      3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository.
+      3. if there are user-defined language-specified fuzzing functions in the repository.
+         - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell.
       4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz);
 
       Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data