✨ [Check split]: Binary-Artifacts (#1244)

* split binary artifact check

* fix

* missing file

* comments

* linter

* fix

* comments

* linter

checks/binary_artifact.go

@@ -15,14 +15,9 @@
 package checks
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/h2non/filetype"
-	"github.com/h2non/filetype/types"
-
 	"github.com/ossf/scorecard/v3/checker"
+	"github.com/ossf/scorecard/v3/checks/evaluation"
+	"github.com/ossf/scorecard/v3/checks/raw"
 	sce "github.com/ossf/scorecard/v3/errors"
 )
 
@@ -34,86 +29,13 @@ func init() {
 	registerCheck(CheckBinaryArtifacts, BinaryArtifacts)
 }
 
-// BinaryArtifacts  will check the repository if it contains binary artifacts.
+// BinaryArtifacts  will check the repository contains binary artifacts.
 func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult {
-	var binFound bool
-	err := CheckFilesContent("*", false, c, checkBinaryFileContent, &binFound)
+	rawData, err := raw.BinaryArtifacts(c)
 	if err != nil {
 		e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
 		return checker.CreateRuntimeErrorResult(CheckBinaryArtifacts, e)
 	}
-	if binFound {
-		return checker.CreateMinScoreResult(CheckBinaryArtifacts, "binaries present in source code")
-	}
-
-	return checker.CreateMaxScoreResult(CheckBinaryArtifacts, "no binaries found in the repo")
-}
-
-func checkBinaryFileContent(path string, content []byte,
-	dl checker.DetailLogger, data FileCbData) (bool, error) {
-	pfound := FileGetCbDataAsBoolPointer(data)
-	binaryFileTypes := map[string]bool{
-		"crx":     true,
-		"deb":     true,
-		"dex":     true,
-		"dey":     true,
-		"elf":     true,
-		"bin":     true,
-		"o":       true,
-		"so":      true,
-		"iso":     true,
-		"class":   true,
-		"jar":     true,
-		"bundle":  true,
-		"dylib":   true,
-		"lib":     true,
-		"msi":     true,
-		"acm":     true,
-		"ax":      true,
-		"cpl":     true,
-		"dll":     true,
-		"drv":     true,
-		"efi":     true,
-		"exe":     true,
-		"mui":     true,
-		"ocx":     true,
-		"scr":     true,
-		"sys":     true,
-		"tsp":     true,
-		"pyc":     true,
-		"pyo":     true,
-		"par":     true,
-		"rpm":     true,
-		"swf":     true,
-		"torrent": true,
-		"cab":     true,
-		"whl":     true,
-	}
-	var t types.Type
-	var err error
-	if len(content) == 0 {
-		return true, nil
-	}
-	if t, err = filetype.Get(content); err != nil {
-		return false, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("filetype.Get:%v", err))
-	}
-
-	if _, ok := binaryFileTypes[t.Extension]; ok {
-		dl.Warn3(&checker.LogMessage{
-			Path: path, Type: checker.FileTypeBinary,
-			Text: "binary detected",
-		})
-		*pfound = true
-		return true, nil
-	} else if _, ok := binaryFileTypes[strings.ReplaceAll(filepath.Ext(path), ".", "")]; ok {
-		// Falling back to file based extension.
-		dl.Warn3(&checker.LogMessage{
-			Path: path, Type: checker.FileTypeBinary,
-			Text: "binary detected",
-		})
-		*pfound = true
-		return true, nil
-	}
 
-	return true, nil
+	return evaluation.BinaryArtifacts(CheckBinaryArtifacts, c.Dlogger, &rawData)
 }

checks/dangerous_workflow.go

@@ -46,14 +46,14 @@ func DangerousWorkflow(c *checker.CheckRequest) checker.CheckResult {
 	data := patternCbData{
 		workflowPattern: make(map[string]bool),
 	}
-	err := CheckFilesContent(".github/workflows/*", false,
+	err := fileparser.CheckFilesContent(".github/workflows/*", false,
 		c, validateGitHubActionWorkflowPatterns, &data)
 	return createResultForDangerousWorkflowPatterns(data, err)
 }
 
 // Check file content.
 func validateGitHubActionWorkflowPatterns(path string, content []byte, dl checker.DetailLogger,
-	data FileCbData) (bool, error) {
+	data fileparser.FileCbData) (bool, error) {
 	if !fileparser.IsWorkflowFile(path) {
 		return true, nil
 	}
@@ -65,7 +65,7 @@ func validateGitHubActionWorkflowPatterns(path string, content []byte, dl checke
 		panic("invalid type")
 	}
 
-	if !CheckFileContainsCommands(content, "#") {
+	if !fileparser.CheckFileContainsCommands(content, "#") {
 		return true, nil
 	}
 

checks/dependency_update_tool.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 
 	"github.com/ossf/scorecard/v3/checker"
+	"github.com/ossf/scorecard/v3/checks/fileparser"
 	sce "github.com/ossf/scorecard/v3/errors"
 )
 
@@ -32,7 +33,7 @@ func init() {
 // UsesDependencyUpdateTool will check the repository uses a dependency update tool.
 func UsesDependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult {
 	var r bool
-	err := CheckIfFileExists(CheckDependencyUpdateTool, c, fileExists, &r)
+	err := fileparser.CheckIfFileExists(CheckDependencyUpdateTool, c, fileExists, &r)
 	if err != nil {
 		e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
 		return checker.CreateRuntimeErrorResult(CheckDependencyUpdateTool, e)
@@ -54,8 +55,8 @@ func UsesDependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult {
 }
 
 // fileExists will validate the if frozen dependencies file name exists.
-func fileExists(name string, dl checker.DetailLogger, data FileCbData) (bool, error) {
-	pdata := FileGetCbDataAsBoolPointer(data)
+func fileExists(name string, dl checker.DetailLogger, data fileparser.FileCbData) (bool, error) {
+	pdata := fileparser.FileGetCbDataAsBoolPointer(data)
 
 	switch strings.ToLower(name) {
 	case ".github/dependabot.yml":

checks/evaluation/binary_artifacts.go

@@ -0,0 +1,44 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package evaluation
+
+import (
+	"github.com/ossf/scorecard/v3/checker"
+	"github.com/ossf/scorecard/v3/checks/raw"
+	sce "github.com/ossf/scorecard/v3/errors"
+)
+
+// BinaryArtifacts applies the score policy for the Binary-Artiacts check.
+func BinaryArtifacts(name string, dl checker.DetailLogger,
+	r *raw.BinaryArtifactData) checker.CheckResult {
+	if r == nil {
+		e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
+		return checker.CreateRuntimeErrorResult(name, e)
+	}
+
+	// Apply the policy evaluation.
+	if r.Files == nil || len(r.Files) == 0 {
+		return checker.CreateMaxScoreResult(name, "no binaries found in the repo")
+	}
+
+	for _, f := range r.Files {
+		dl.Warn3(&checker.LogMessage{
+			Path: f.Path, Type: checker.FileTypeBinary,
+			Text: "binary detected",
+		})
+	}
+
+	return checker.CreateMinScoreResult(name, "binaries present in source code")
+}

checks/fileparser/errors.go

@@ -18,4 +18,7 @@ import (
 	"errors"
 )
 
-var errInvalidGitHubWorkflow = errors.New("invalid GitHub workflow")
+var (
+	errInvalidGitHubWorkflow = errors.New("invalid GitHub workflow")
+	errInternalFilenameMatch = errors.New("filename match error")
+)

checks/file_utils.go → checks/fileparser/listing.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package checks
+package fileparser
 
 import (
 	"bufio"
@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/ossf/scorecard/v3/checker"
+	"github.com/ossf/scorecard/v3/clients"
 	sce "github.com/ossf/scorecard/v3/errors"
 )
 
@@ -115,6 +116,58 @@ func CheckFilesContent(shellPathFnPattern string,
 	return nil
 }
 
+// FileContentCbV6 is the callback.
+// The bool returned indicates whether the CheckFilesContent2
+// should continue iterating over files or not.
+type FileContentCbV6 func(path string, content []byte, data FileCbData) (bool, error)
+
+// CheckFilesContentV6 is the same as CheckFilesContent
+// but for use with separated check/policy code.
+func CheckFilesContentV6(shellPathFnPattern string,
+	caseSensitive bool,
+	repoClient clients.RepoClient,
+	onFileContent FileContentCbV6,
+	data FileCbData,
+) error {
+	predicate := func(filepath string) (bool, error) {
+		// Filter out test files.
+		if isTestdataFile(filepath) {
+			return false, nil
+		}
+		// Filter out files based on path/names using the pattern.
+		b, err := isMatchingPath(shellPathFnPattern, filepath, caseSensitive)
+		if err != nil {
+			return false, err
+		}
+		return b, nil
+	}
+
+	matchedFiles, err := repoClient.ListFiles(predicate)
+	if err != nil {
+		// nolint: wrapcheck
+		return err
+	}
+
+	for _, file := range matchedFiles {
+		content, err := repoClient.GetFileContent(file)
+		if err != nil {
+			//nolint
+			return err
+		}
+
+		continueIter, err := onFileContent(file, content, data)
+		if err != nil {
+			return err
+		}
+
+		if !continueIter {
+			break
+		}
+	}
+
+	return nil
+}
+
 // FileCb represents a callback fn.
 type FileCb func(path string,
 	dl checker.DetailLogger, data FileCbData) (bool, error)

checks/fuzzing.go

@@ -21,6 +21,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/ossf/scorecard/v3/checker"
+	"github.com/ossf/scorecard/v3/checks/fileparser"
 	"github.com/ossf/scorecard/v3/clients"
 	"github.com/ossf/scorecard/v3/clients/githubrepo"
 	sce "github.com/ossf/scorecard/v3/errors"
@@ -44,13 +45,13 @@ func init() {
 
 func checkCFLite(c *checker.CheckRequest) (bool, error) {
 	result := false
-	e := CheckFilesContent(".clusterfuzzlite/Dockerfile", true, c,
-		func(path string, content []byte, dl checker.DetailLogger, data FileCbData) (bool, error) {
-			result = CheckFileContainsCommands(content, "#")
+	e := fileparser.CheckFilesContent(".clusterfuzzlite/Dockerfile", true, c,
+		func(path string, content []byte, dl checker.DetailLogger, data fileparser.FileCbData) (bool, error) {
+			result = fileparser.CheckFileContainsCommands(content, "#")
 			return false, nil
 		}, nil)
 
-	return result, e
+	return result, fmt.Errorf("%w", e)
 }
 
 func checkOSSFuzz(c *checker.CheckRequest) (bool, error) {

checks/permissions.go

@@ -48,7 +48,7 @@ func TokenPermissions(c *checker.CheckRequest) checker.CheckResult {
 		topLevelWritePermissions: make(map[string]bool),
 		runLevelWritePermissions: make(map[string]bool),
 	}
-	err := CheckFilesContent(".github/workflows/*", false,
+	err := fileparser.CheckFilesContent(".github/workflows/*", false,
 		c, validateGitHubActionTokenPermissions, &data)
 	return createResultForLeastPrivilegeTokens(data, err)
 }
@@ -326,7 +326,7 @@ func testValidateGitHubActionTokenPermissions(pathfn string,
 
 // Check file content.
 func validateGitHubActionTokenPermissions(path string, content []byte,
-	dl checker.DetailLogger, data FileCbData) (bool, error) {
+	dl checker.DetailLogger, data fileparser.FileCbData) (bool, error) {
 	if !fileparser.IsWorkflowFile(path) {
 		return true, nil
 	}
@@ -337,7 +337,7 @@ func validateGitHubActionTokenPermissions(path string, content []byte,
 		panic("invalid type")
 	}
 
-	if !CheckFileContainsCommands(content, "#") {
+	if !fileparser.CheckFileContainsCommands(content, "#") {
 		return true, nil
 	}