diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index f058c2e6fe0..132dda90df3 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -18,13 +18,19 @@ import ( "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" + "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer" ) // Fuzzing applies the score policy for the Fuzzing check. @@ -35,12 +41,20 @@ func Fuzzing(name string, expectedProbes := []string{ fuzzedWithClusterFuzzLite.Probe, fuzzedWithGoNative.Probe, + fuzzedWithPythonAtheris.Probe, + fuzzedWithCLibFuzzer.Probe, + fuzzedWithCppLibFuzzer.Probe, + fuzzedWithRustCargofuzz.Probe, + fuzzedWithSwiftLibFuzzer.Probe, + fuzzedWithJavaJazzerFuzzer.Probe, fuzzedWithOneFuzz.Probe, fuzzedWithOSSFuzz.Probe, fuzzedWithPropertyBasedHaskell.Probe, fuzzedWithPropertyBasedJavascript.Probe, fuzzedWithPropertyBasedTypescript.Probe, } + // TODO: other packages to consider: + // - github.com/google/fuzztest if !finding.UniqueProbesEqual(findings, expectedProbes) { e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index a1b54847c45..85018eadb25 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -40,6 +40,30 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomeNegative, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithSwiftLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, @@ -63,7 +87,7 @@ func TestFuzzing(t *testing.T) { }, result: scut.TestReturn{ Score: checker.MinResultScore, - NumberOfWarn: 7, + NumberOfWarn: 13, }, }, { @@ -77,6 +101,30 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomePositive, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithSwiftLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go index 191eedfd698..dd2f3a17994 100644 --- a/checks/fuzzing_test.go +++ b/checks/fuzzing_test.go @@ -53,7 +53,7 @@ func TestFuzzing(t *testing.T) { wantErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -110,7 +110,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -121,7 +121,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: true, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index a8a161d9847..9e92767d5a7 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -35,6 +35,12 @@ const ( fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting" fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting" + fuzzerPythonAtheris = "PythonAtherisFuzzer" + fuzzerCLibFuzzer = "CLibFuzzer" + fuzzerCppLibFuzzer = "CppLibFuzzer" + fuzzerSwiftLibFuzzer = "SwiftLibFuzzer" + fuzzerRustCargoFuzz = "RustCargoFuzzer" + fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer" // TODO: add more fuzzing check supports. ) @@ -47,8 +53,8 @@ type filesWithPatternStr struct { type languageFuzzConfig struct { URL, Desc *string - // Pattern is according to path.Match. - filePattern string + // Patterns are according to path.Match. + filePatterns []string funcPattern, Name string // TODO: add more language fuzzing-related fields. @@ -59,10 +65,10 @@ type languageFuzzConfig struct { var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // Default fuzz patterns for Go. clients.Go: { - filePattern: "*_test.go", - funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`, - Name: fuzzerBuiltInGo, - URL: asPointer("https://go.dev/doc/fuzz/"), + filePatterns: []string{"*_test.go"}, + funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`, + Name: fuzzerBuiltInGo, + URL: asPointer("https://go.dev/doc/fuzz/"), Desc: asPointer( "Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."), }, @@ -80,7 +86,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.Haskell: { - filePattern: "*.hs", + filePatterns: []string{"*.hs", "*.lhs"}, // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, @@ -96,7 +102,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.JavaScript: { - filePattern: "*.js", + filePatterns: []string{"*.js"}, // Look for direct imports of fast-check. funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, Name: fuzzerPropertyBasedJavaScript, @@ -105,7 +111,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "and test that specific properties are satisfied."), }, clients.TypeScript: { - filePattern: "*.ts", + filePatterns: []string{"*.ts"}, // Look for direct imports of fast-check. funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, Name: fuzzerPropertyBasedTypeScript, @@ -113,6 +119,48 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "Property-based testing in TypeScript generates test instances randomly or exhaustively " + "and test that specific properties are satisfied."), }, + clients.Python: { + filePatterns: []string{"*.py"}, + funcPattern: `import atheris`, + Name: fuzzerPythonAtheris, + Desc: asPointer( + "Python fuzzing by way of Atheris"), + }, + clients.C: { + filePatterns: []string{"*.c"}, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCLibFuzzer, + Desc: asPointer( + "Fuzzed with C LibFuzzer"), + }, + clients.Cpp: { + filePatterns: []string{"*.cc", "*.cpp"}, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCppLibFuzzer, + Desc: asPointer( + "Fuzzed with cpp LibFuzzer"), + }, + clients.Rust: { + filePatterns: []string{"*.rs"}, + funcPattern: `libfuzzer_sys`, + Name: fuzzerRustCargoFuzz, + Desc: asPointer( + "Fuzzed with Cargo-fuzz"), + }, + clients.Java: { + filePatterns: []string{"*.java"}, + funcPattern: `com.code_intelligence.jazzer.api.FuzzedDataProvider;`, + Name: fuzzerJavaJazzerFuzzer, + Desc: asPointer( + "Fuzzed with Jazzer fuzzer"), + }, + clients.Swift: { + filePatterns: []string{"*.swift"}, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerSwiftLibFuzzer, + Desc: asPointer( + "Fuzzed with Swift LibFuzzer"), + }, // TODO: add more language-specific fuzz patterns & configs. } @@ -254,22 +302,26 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] // Get patterns for file and func. // We use the file pattern in the matcher to match the test files, // and put the func pattern in var data to match file contents (func names). - filePattern, funcPattern := pattern.filePattern, pattern.funcPattern - matcher := fileparser.PathMatcher{ - Pattern: filePattern, - CaseSensitive: false, - } - data.pattern = funcPattern - err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) - if err != nil { - return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) + filePatterns, funcPattern := pattern.filePatterns, pattern.funcPattern + var dataFiles []checker.File + for _, filePattern := range filePatterns { + matcher := fileparser.PathMatcher{ + Pattern: filePattern, + CaseSensitive: false, + } + data.pattern = funcPattern + err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) + if err != nil { + return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) + } + dataFiles = append(dataFiles, data.files...) } - if len(data.files) == 0 { + if len(dataFiles) == 0 { // This means no fuzz funcs matched for this language. return false, nil, nil } - return true, data.files, nil + return true, dataFiles, nil } // This is the callback func for interface OnMatchingFileContentDo @@ -320,9 +372,10 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName { // This var can stay as an int, no need for a precise float value. avgLoC := totalLoC / numLangs // Languages that have lines of code above average will be considered prominent. + prominentThreshold := avgLoC / 4.0 ret := []clients.LanguageName{} for lName, loC := range langMap { - if loC >= avgLoC { + if loC >= prominentThreshold { lang := clients.LanguageName(strings.ToLower(string(lName))) ret = append(ret, lang) } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index c0a06edfe72..976ce65f4a4 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -261,8 +261,8 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { expectedFileMatch: false, expectedFuncMatch: false, lang: clients.LanguageName("not_a_supported_one"), - fileName: "a_fuzz_test.py", - fileContent: `def NotSupported (foo)`, + fileName: "a_fuzz_test.php", + fileContent: `function function-not-supported (foo)`, wantErr: true, }, } @@ -274,16 +274,19 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { if !ok && !tt.wantErr { t.Errorf("retrieve supported language error") } - fileMatchPattern := langSpecs.filePattern - fileMatch, err := path.Match(fileMatchPattern, tt.fileName) - if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr { - t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + var found bool + for _, fileMatchPattern := range langSpecs.filePatterns { + fileMatch, err := path.Match(fileMatchPattern, tt.fileName) + if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr { + t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + } + funcRegexPattern := langSpecs.funcPattern + r := regexp.MustCompile(funcRegexPattern) + found = found || r.MatchString(tt.fileContent) } - funcRegexPattern := langSpecs.funcPattern - r := regexp.MustCompile(funcRegexPattern) - found := r.MatchString(tt.fileContent) + if (found != tt.expectedFuncMatch) && !tt.wantErr { - t.Errorf("funcMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + t.Errorf("found = %v, want %v for %v", found, tt.expectedFileMatch, tt.name) } }) } diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index cf94f297688..83d6e720123 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, - NumberOfInfo: 1, + NumberOfInfo: 12, NumberOfDebug: 0, } result := checks.Fuzzing(&req) @@ -192,7 +192,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { expected := scut.TestReturn{ Error: nil, Score: checker.MinResultScore, - NumberOfWarn: 7, + NumberOfWarn: 13, NumberOfInfo: 0, NumberOfDebug: 0, } diff --git a/probes/entries.go b/probes/entries.go index 7e7dd72944a..4d96d051f65 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -17,13 +17,19 @@ package probes import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" + "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" @@ -60,6 +66,12 @@ var ( fuzzedWithOSSFuzz.Run, fuzzedWithOneFuzz.Run, fuzzedWithGoNative.Run, + fuzzedWithPythonAtheris.Run, + fuzzedWithCLibFuzzer.Run, + fuzzedWithCppLibFuzzer.Run, + fuzzedWithSwiftLibFuzzer.Run, + fuzzedWithRustCargofuzz.Run, + fuzzedWithJavaJazzerFuzzer.Run, fuzzedWithClusterFuzzLite.Run, fuzzedWithPropertyBasedHaskell.Run, fuzzedWithPropertyBasedTypescript.Run, diff --git a/probes/fuzzedWithCLibFuzzer/def.yml b/probes/fuzzedWithCLibFuzzer/def.yml new file mode 100644 index 00000000000..132f11af466 --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithCLibFuzzer +short: Check that the project is fuzzed using LibFuzzer +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of functions with the signature 'LLVMFuzzerTestOneInput' in .c files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://llvm.org/docs/LibFuzzer.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithCLibFuzzer/impl.go b/probes/fuzzedWithCLibFuzzer/impl.go new file mode 100644 index 00000000000..b2d198c033c --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithCLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "CLibFuzzer") +} diff --git a/probes/fuzzedWithCLibFuzzer/impl_test.go b/probes/fuzzedWithCLibFuzzer/impl_test.go new file mode 100644 index 00000000000..e4fe0dce5bf --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + { + Name: "CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + { + Name: "not-CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithCppLibFuzzer/def.yml b/probes/fuzzedWithCppLibFuzzer/def.yml new file mode 100644 index 00000000000..bb35db9f4ca --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithCppLibFuzzer +short: Check that the project is fuzzed using LibFuzzer +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of functions with the signature 'LLVMFuzzerTestOneInput' in .cpp or .cc files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://llvm.org/docs/LibFuzzer.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithCppLibFuzzer/impl.go b/probes/fuzzedWithCppLibFuzzer/impl.go new file mode 100644 index 00000000000..cbbb3146243 --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCppLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithCppLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "CppLibFuzzer") +} diff --git a/probes/fuzzedWithCppLibFuzzer/impl_test.go b/probes/fuzzedWithCppLibFuzzer/impl_test.go new file mode 100644 index 00000000000..48ea6bae6d9 --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCppLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + { + Name: "CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + { + Name: "not-CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithJavaJazzerFuzzer/def.yml b/probes/fuzzedWithJavaJazzerFuzzer/def.yml new file mode 100644 index 00000000000..6b71fd132b1 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithJavaJazzerFuzzer +short: Check that the project is fuzzed using the Jazzer Java fuzzing framework +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of an import of 'com.code_intelligence.jazzer.api.FuzzedDataProvider' in .java files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://github.com/CodeIntelligenceTesting/jazzer to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://github.com/CodeIntelligenceTesting/jazzer](https://github.com/CodeIntelligenceTesting/jazzer) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithJavaJazzerFuzzer/impl.go b/probes/fuzzedWithJavaJazzerFuzzer/impl.go new file mode 100644 index 00000000000..4fb46f0b113 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithJavaJazzerFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithJavaJazzerFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "JavaJazzerFuzzer") +} diff --git a/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go b/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go new file mode 100644 index 00000000000..3dcb5bb3549 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithJavaJazzerFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + { + Name: "JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + { + Name: "not-JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithPythonAtheris/def.yml b/probes/fuzzedWithPythonAtheris/def.yml new file mode 100644 index 00000000000..73eb086897d --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithPythonAtheris +short: Check that the project is fuzzed using Python Atheris fuzzing framework +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of 'import atheris' in .py files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://github.com/google/atheris to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://github.com/google/atheris](https://github.com/google/atheris) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithPythonAtheris/impl.go b/probes/fuzzedWithPythonAtheris/impl.go new file mode 100644 index 00000000000..eb26c2981a0 --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithPythonAtheris + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithPythonAtheris" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "PythonAtherisFuzzer") +} diff --git a/probes/fuzzedWithPythonAtheris/impl_test.go b/probes/fuzzedWithPythonAtheris/impl_test.go new file mode 100644 index 00000000000..34c918bb102 --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithPythonAtheris + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + { + Name: "PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + { + Name: "not-PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithRustCargofuzz/def.yml b/probes/fuzzedWithRustCargofuzz/def.yml new file mode 100644 index 00000000000..cf7550584d1 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithRustCargofuzz +short: Check that the project is fuzzed using Rust Cargo-fuzz framework. +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of 'libfuzzer_sys' in .rs files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://rust-fuzz.github.io/book/cargo-fuzz.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://rust-fuzz.github.io/book/cargo-fuzz.html](https://rust-fuzz.github.io/book/cargo-fuzz.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithRustCargofuzz/impl.go b/probes/fuzzedWithRustCargofuzz/impl.go new file mode 100644 index 00000000000..df531b4af25 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithRustCargofuzz + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithRustCargofuzz" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "RustCargoFuzzer") +} diff --git a/probes/fuzzedWithRustCargofuzz/impl_test.go b/probes/fuzzedWithRustCargofuzz/impl_test.go new file mode 100644 index 00000000000..bfcfb052692 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithRustCargofuzz + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + { + Name: "RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + { + Name: "not-RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithSwiftLibFuzzer/def.yml b/probes/fuzzedWithSwiftLibFuzzer/def.yml new file mode 100644 index 00000000000..288e5ee91f1 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithSwiftLibFuzzer +short: Check that the project is fuzzed using the LibFuzzer for Swift code. +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of an import of '@_cdecl("LLVMFuzzerTestOneInput")' in .swift files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/ to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/](https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithSwiftLibFuzzer/impl.go b/probes/fuzzedWithSwiftLibFuzzer/impl.go new file mode 100644 index 00000000000..7236d18e870 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithSwiftLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithSwiftLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "SwiftLibFuzzer") +} diff --git a/probes/fuzzedWithSwiftLibFuzzer/impl_test.go b/probes/fuzzedWithSwiftLibFuzzer/impl_test.go new file mode 100644 index 00000000000..5f3be6a4b65 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithSwiftLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + { + Name: "SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + { + Name: "not-SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}