Skip to content

Commit

Permalink
✨ Add additional fuzzing probes (#3473)
Browse files Browse the repository at this point in the history 
* Extend with additional fuzzing probes

Signed-off-by: David Korczynski <[email protected]>

* fix formatting

Signed-off-by: David Korczynski <[email protected]>

* cleanup formatting

Signed-off-by: David Korczynski <[email protected]>

* make skip testing optional

Signed-off-by: David Korczynski <[email protected]>

* address reviews

Signed-off-by: David Korczynski <[email protected]>

* add todo

Signed-off-by: David Korczynski <[email protected]>

* nit

Signed-off-by: David Korczynski <[email protected]>

* nit

Signed-off-by: David Korczynski <[email protected]>

* add swift fuzzing probe

Signed-off-by: David Korczynski <[email protected]>

* avoid changing OnMatchingFileContentDo

Signed-off-by: David Korczynski <[email protected]>

* nit

Signed-off-by: David Korczynski <[email protected]>

* undo matching file content extension

Signed-off-by: David Korczynski <[email protected]>

* nit: fix constant

Signed-off-by: David Korczynski <[email protected]>

* test all fileMatchPatterns per client

Signed-off-by: David Korczynski <[email protected]>

* fix test logging counts

Signed-off-by: David Korczynski <[email protected]>

* nit

Signed-off-by: David Korczynski <[email protected]>

---------

Signed-off-by: David Korczynski <[email protected]>
  • Loading branch information
@DavidKorczynski
DavidKorczynski authored Oct 9, 2023
1 parent 034e6b2 commit bd640f7
Show file tree
Hide file tree
Showing 25 changed files with 1,457 additions and 37 deletions.
14 changes: 14 additions & 0 deletions checks/evaluation/fuzzing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,19 @@ import (
"github.com/ossf/scorecard/v4/checker"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
"github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
"github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris"
"github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer"
)

// Fuzzing applies the score policy for the Fuzzing check.
Expand All @@ -35,12 +41,20 @@ func Fuzzing(name string,
expectedProbes := []string{
fuzzedWithClusterFuzzLite.Probe,
fuzzedWithGoNative.Probe,
fuzzedWithPythonAtheris.Probe,
fuzzedWithCLibFuzzer.Probe,
fuzzedWithCppLibFuzzer.Probe,
fuzzedWithRustCargofuzz.Probe,
fuzzedWithSwiftLibFuzzer.Probe,
fuzzedWithJavaJazzerFuzzer.Probe,
fuzzedWithOneFuzz.Probe,
fuzzedWithOSSFuzz.Probe,
fuzzedWithPropertyBasedHaskell.Probe,
fuzzedWithPropertyBasedJavascript.Probe,
fuzzedWithPropertyBasedTypescript.Probe,
}
// TODO: other packages to consider:
// - github.com/google/fuzztest

if !finding.UniqueProbesEqual(findings, expectedProbes) {
e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
Expand Down
50 changes: 49 additions & 1 deletion checks/evaluation/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,30 @@ func TestFuzzing(t *testing.T) {
Probe: "fuzzedWithGoNative",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPythonAtheris",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithCLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithCppLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithRustCargofuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithSwiftLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithJavaJazzerFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
Expand All @@ -63,7 +87,7 @@ func TestFuzzing(t *testing.T) {
},
result: scut.TestReturn{
Score: checker.MinResultScore,
NumberOfWarn: 7,
NumberOfWarn: 13,
},
},
{
Expand All @@ -77,6 +101,30 @@ func TestFuzzing(t *testing.T) {
Probe: "fuzzedWithGoNative",
Outcome: finding.OutcomePositive,
},
{
Probe: "fuzzedWithPythonAtheris",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithCLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithCppLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithRustCargofuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithSwiftLibFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithJavaJazzerFuzzer",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
Expand Down
6 changes: 3 additions & 3 deletions checks/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func TestFuzzing(t *testing.T) {
wantErr: false,
expected: scut.TestReturn{
Error: nil,
NumberOfWarn: 7,
NumberOfWarn: 13,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
Expand Down Expand Up @@ -110,7 +110,7 @@ func TestFuzzing(t *testing.T) {
wantFuzzErr: false,
expected: scut.TestReturn{
Error: nil,
NumberOfWarn: 7,
NumberOfWarn: 13,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
Expand All @@ -121,7 +121,7 @@ func TestFuzzing(t *testing.T) {
wantFuzzErr: true,
expected: scut.TestReturn{
Error: nil,
NumberOfWarn: 7,
NumberOfWarn: 13,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
Expand Down
95 changes: 74 additions & 21 deletions checks/raw/fuzzing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ const (
fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting"
fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting"
fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting"
fuzzerPythonAtheris = "PythonAtherisFuzzer"
fuzzerCLibFuzzer = "CLibFuzzer"
fuzzerCppLibFuzzer = "CppLibFuzzer"
fuzzerSwiftLibFuzzer = "SwiftLibFuzzer"
fuzzerRustCargoFuzz = "RustCargoFuzzer"
fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer"
// TODO: add more fuzzing check supports.
)

Expand All @@ -47,8 +53,8 @@ type filesWithPatternStr struct {
type languageFuzzConfig struct {

Check failure on line 53 in checks/raw/fuzzing.go

View workflow job for this annotation

GitHub Actions / check-linter 

fieldalignment: struct with 64 pointer bytes could be 56 (govet)
URL, Desc *string

// Pattern is according to path.Match.
filePattern string
// Patterns are according to path.Match.
filePatterns []string

funcPattern, Name string
// TODO: add more language fuzzing-related fields.
Expand All @@ -59,10 +65,10 @@ type languageFuzzConfig struct {
var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
// Default fuzz patterns for Go.
clients.Go: {
filePattern: "*_test.go",
funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`,
Name: fuzzerBuiltInGo,
URL: asPointer("https://go.dev/doc/fuzz/"),
filePatterns: []string{"*_test.go"},
funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`,
Name: fuzzerBuiltInGo,
URL: asPointer("https://go.dev/doc/fuzz/"),
Desc: asPointer(
"Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."),
},
Expand All @@ -80,7 +86,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
//
// This is not an exhaustive list.
clients.Haskell: {
filePattern: "*.hs",
filePatterns: []string{"*.hs", "*.lhs"},
// Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck,
// or their indirect imports through the higher-level Hspec or Tasty testing frameworks.
funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`,
Expand All @@ -96,7 +102,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
//
// This is not an exhaustive list.
clients.JavaScript: {
filePattern: "*.js",
filePatterns: []string{"*.js"},
// Look for direct imports of fast-check.
funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`,
Name: fuzzerPropertyBasedJavaScript,
Expand All @@ -105,14 +111,56 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
"and test that specific properties are satisfied."),
},
clients.TypeScript: {
filePattern: "*.ts",
filePatterns: []string{"*.ts"},
// Look for direct imports of fast-check.
funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`,
Name: fuzzerPropertyBasedTypeScript,
Desc: asPointer(
"Property-based testing in TypeScript generates test instances randomly or exhaustively " +
"and test that specific properties are satisfied."),
},
clients.Python: {
filePatterns: []string{"*.py"},
funcPattern: `import atheris`,
Name: fuzzerPythonAtheris,
Desc: asPointer(
"Python fuzzing by way of Atheris"),
},
clients.C: {
filePatterns: []string{"*.c"},
funcPattern: `LLVMFuzzerTestOneInput`,
Name: fuzzerCLibFuzzer,
Desc: asPointer(
"Fuzzed with C LibFuzzer"),
},
clients.Cpp: {
filePatterns: []string{"*.cc", "*.cpp"},
funcPattern: `LLVMFuzzerTestOneInput`,
Name: fuzzerCppLibFuzzer,
Desc: asPointer(
"Fuzzed with cpp LibFuzzer"),
},
clients.Rust: {
filePatterns: []string{"*.rs"},
funcPattern: `libfuzzer_sys`,
Name: fuzzerRustCargoFuzz,
Desc: asPointer(
"Fuzzed with Cargo-fuzz"),
},
clients.Java: {
filePatterns: []string{"*.java"},
funcPattern: `com.code_intelligence.jazzer.api.FuzzedDataProvider;`,
Name: fuzzerJavaJazzerFuzzer,
Desc: asPointer(
"Fuzzed with Jazzer fuzzer"),
},
clients.Swift: {
filePatterns: []string{"*.swift"},
funcPattern: `LLVMFuzzerTestOneInput`,
Name: fuzzerSwiftLibFuzzer,
Desc: asPointer(
"Fuzzed with Swift LibFuzzer"),
},
// TODO: add more language-specific fuzz patterns & configs.
}

Expand Down Expand Up @@ -254,22 +302,26 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, []
// Get patterns for file and func.
// We use the file pattern in the matcher to match the test files,
// and put the func pattern in var data to match file contents (func names).
filePattern, funcPattern := pattern.filePattern, pattern.funcPattern
matcher := fileparser.PathMatcher{
Pattern: filePattern,
CaseSensitive: false,
}
data.pattern = funcPattern
err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data)
if err != nil {
return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err)
filePatterns, funcPattern := pattern.filePatterns, pattern.funcPattern
var dataFiles []checker.File
for _, filePattern := range filePatterns {
matcher := fileparser.PathMatcher{
Pattern: filePattern,
CaseSensitive: false,
}
data.pattern = funcPattern
err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data)
if err != nil {
return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err)
}
dataFiles = append(dataFiles, data.files...)
}

if len(data.files) == 0 {
if len(dataFiles) == 0 {
// This means no fuzz funcs matched for this language.
return false, nil, nil
}
return true, data.files, nil
return true, dataFiles, nil
}

// This is the callback func for interface OnMatchingFileContentDo
Expand Down Expand Up @@ -320,9 +372,10 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName {
// This var can stay as an int, no need for a precise float value.
avgLoC := totalLoC / numLangs
// Languages that have lines of code above average will be considered prominent.
prominentThreshold := avgLoC / 4.0
ret := []clients.LanguageName{}
for lName, loC := range langMap {
if loC >= avgLoC {
if loC >= prominentThreshold {
lang := clients.LanguageName(strings.ToLower(string(lName)))
ret = append(ret, lang)
}
Expand Down
23 changes: 13 additions & 10 deletions checks/raw/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,8 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) {
expectedFileMatch: false,
expectedFuncMatch: false,
lang: clients.LanguageName("not_a_supported_one"),
fileName: "a_fuzz_test.py",
fileContent: `def NotSupported (foo)`,
fileName: "a_fuzz_test.php",
fileContent: `function function-not-supported (foo)`,
wantErr: true,
},
}
Expand All @@ -274,16 +274,19 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) {
if !ok && !tt.wantErr {
t.Errorf("retrieve supported language error")
}
fileMatchPattern := langSpecs.filePattern
fileMatch, err := path.Match(fileMatchPattern, tt.fileName)
if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr {
t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name)
var found bool
for _, fileMatchPattern := range langSpecs.filePatterns {
fileMatch, err := path.Match(fileMatchPattern, tt.fileName)
if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr {
t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name)
}
funcRegexPattern := langSpecs.funcPattern
r := regexp.MustCompile(funcRegexPattern)
found = found || r.MatchString(tt.fileContent)
}
funcRegexPattern := langSpecs.funcPattern
r := regexp.MustCompile(funcRegexPattern)
found := r.MatchString(tt.fileContent)

if (found != tt.expectedFuncMatch) && !tt.wantErr {
t.Errorf("funcMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name)
t.Errorf("found = %v, want %v for %v", found, tt.expectedFileMatch, tt.name)
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions e2e/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
NumberOfInfo: 12,
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expand Down Expand Up @@ -192,7 +192,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
expected := scut.TestReturn{
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 7,
NumberOfWarn: 13,
NumberOfInfo: 0,
NumberOfDebug: 0,
}
Expand Down
12 changes: 12 additions & 0 deletions probes/entries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,19 @@ package probes
import (
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
"github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
"github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris"
"github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer"
"github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks"
"github.com/ossf/scorecard/v4/probes/securityPolicyContainsText"
"github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure"
Expand Down Expand Up @@ -60,6 +66,12 @@ var (
fuzzedWithOSSFuzz.Run,
fuzzedWithOneFuzz.Run,
fuzzedWithGoNative.Run,
fuzzedWithPythonAtheris.Run,
fuzzedWithCLibFuzzer.Run,
fuzzedWithCppLibFuzzer.Run,
fuzzedWithSwiftLibFuzzer.Run,
fuzzedWithRustCargofuzz.Run,
fuzzedWithJavaJazzerFuzzer.Run,
fuzzedWithClusterFuzzLite.Run,
fuzzedWithPropertyBasedHaskell.Run,
fuzzedWithPropertyBasedTypescript.Run,
Expand Down
Loading

0 comments on commit bd640f7

Please sign in to comment.