✨ Use checks.yaml to store which repo types are supported by each check

clients/localdir/client.go

@@ -32,10 +32,7 @@ import (
 	clients "github.com/ossf/scorecard/v3/clients"
 )
 
-var (
-	errUnsupportedFeature = errors.New("unsupported feature")
-	errInputRepoType      = errors.New("input repo should be of type repoLocal")
-)
+var errInputRepoType = errors.New("input repo should be of type repoLocal")
 
 //nolint:govet
 type localDirClient struct {
@@ -66,7 +63,7 @@ func (client *localDirClient) URI() string {
 
 // IsArchived implements RepoClient.IsArchived.
 func (client *localDirClient) IsArchived() (bool, error) {
-	return false, fmt.Errorf("IsArchived: %w", errUnsupportedFeature)
+	return false, fmt.Errorf("IsArchived: %w", clients.ErrUnsupportedFeature)
 }
 
 func isDir(p string) (bool, error) {
@@ -142,51 +139,51 @@ func (client *localDirClient) GetFileContent(filename string) ([]byte, error) {
 
 // ListMergedPRs implements RepoClient.ListMergedPRs.
 func (client *localDirClient) ListMergedPRs() ([]clients.PullRequest, error) {
-	return nil, fmt.Errorf("ListMergedPRs: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListMergedPRs: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListBranches implements RepoClient.ListBranches.
 func (client *localDirClient) ListBranches() ([]*clients.BranchRef, error) {
-	return nil, fmt.Errorf("ListBranches: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListBranches: %w", clients.ErrUnsupportedFeature)
 }
 
 // GetDefaultBranch implements RepoClient.GetDefaultBranch.
 func (client *localDirClient) GetDefaultBranch() (*clients.BranchRef, error) {
-	return nil, fmt.Errorf("GetDefaultBranch: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("GetDefaultBranch: %w", clients.ErrUnsupportedFeature)
 }
 
 func (client *localDirClient) ListCommits() ([]clients.Commit, error) {
-	return nil, fmt.Errorf("ListCommits: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListCommits: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListReleases implements RepoClient.ListReleases.
 func (client *localDirClient) ListReleases() ([]clients.Release, error) {
-	return nil, fmt.Errorf("ListReleases: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListReleases: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListContributors implements RepoClient.ListContributors.
 func (client *localDirClient) ListContributors() ([]clients.Contributor, error) {
-	return nil, fmt.Errorf("ListContributors: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListContributors: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListSuccessfulWorkflowRuns implements RepoClient.WorkflowRunsByFilename.
 func (client *localDirClient) ListSuccessfulWorkflowRuns(filename string) ([]clients.WorkflowRun, error) {
-	return nil, fmt.Errorf("ListSuccessfulWorkflowRuns: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListSuccessfulWorkflowRuns: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListCheckRunsForRef implements RepoClient.ListCheckRunsForRef.
 func (client *localDirClient) ListCheckRunsForRef(ref string) ([]clients.CheckRun, error) {
-	return nil, fmt.Errorf("ListCheckRunsForRef: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListCheckRunsForRef: %w", clients.ErrUnsupportedFeature)
 }
 
 // ListStatuses implements RepoClient.ListStatuses.
 func (client *localDirClient) ListStatuses(ref string) ([]clients.Status, error) {
-	return nil, fmt.Errorf("ListStatuses: %w", errUnsupportedFeature)
+	return nil, fmt.Errorf("ListStatuses: %w", clients.ErrUnsupportedFeature)
 }
 
 // Search implements RepoClient.Search.
 func (client *localDirClient) Search(request clients.SearchRequest) (clients.SearchResponse, error) {
-	return clients.SearchResponse{}, fmt.Errorf("Search: %w", errUnsupportedFeature)
+	return clients.SearchResponse{}, fmt.Errorf("Search: %w", clients.ErrUnsupportedFeature)
 }
 
 func (client *localDirClient) Close() error {

clients/localdir/client_test.go

@@ -39,21 +39,21 @@ func TestClient_CreationAndCaching(t *testing.T) {
 		{
 			name:        "invalid fullpath",
 			outputFiles: []string{},
-			inputFolder: "/invalid/fullpath",
+			inputFolder: "file:///invalid/fullpath",
 			err:         os.ErrNotExist,
 		},
 		{
 			name:        "invalid relative path",
 			outputFiles: []string{},
-			inputFolder: "invalid/relative/path",
+			inputFolder: "file://invalid/relative/path",
 			err:         os.ErrNotExist,
 		},
 		{
 			name: "repo 0",
 			outputFiles: []string{
 				"file0", "dir1/file1", "dir1/dir2/file2",
 			},
-			inputFolder: "testdata/repo0",
+			inputFolder: "file://testdata/repo0",
 			err:         nil,
 		},
 	}

clients/localdir/repo.go

@@ -21,11 +21,17 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 
 	clients "github.com/ossf/scorecard/v3/clients"
 )
 
-var errNotDirectory = errors.New("not a directory")
+var (
+	errNotDirectory = errors.New("not a directory")
+	errInvalidURI   = errors.New("invalid URI")
+)
+
+var filePrefix = "file://"
 
 type repoLocal struct {
 	path     string
@@ -77,14 +83,16 @@ func (r *repoLocal) IsScorecardRepo() bool {
 
 // MakeLocalDirRepo returns an implementation of clients.Repo interface.
 func MakeLocalDirRepo(pathfn string) (clients.Repo, error) {
-	p := path.Clean(pathfn)
+	if !strings.HasPrefix(pathfn, filePrefix) {
+		return nil, fmt.Errorf("%w", errInvalidURI)
+	}
+	p := path.Clean(pathfn[len(filePrefix):])
 	repo := &repoLocal{
 		path: p,
 	}
 
 	if err := repo.IsValid(); err != nil {
 		return nil, fmt.Errorf("error in IsValid: %w", err)
 	}
-
 	return repo, nil
 }

clients/repo_client.go

@@ -15,6 +15,11 @@
 // Package clients defines the interface for RepoClient and related structs.
 package clients
 
+import "errors"
+
+// ErrUnsupportedFeature indicates an API that is not supported by the client.
+var ErrUnsupportedFeature = errors.New("unsupported feature")
+
 // RepoClient interface is used by Scorecard checks to access a repo.
 type RepoClient interface {
 	InitRepo(repo Repo) error

cmd/root.go

@@ -61,6 +61,14 @@ const (
 	formatDefault = "default"
 )
 
+// These strings must be the same as the ones used in
+// checks.yaml for the "repos" field.
+const (
+	repoTypeUnknown = "unknown"
+	repoTypeLocal   = "local"
+	repoTypeGitHub  = "GitHub"
+)
+
 const (
 	scorecardLong = "A program that shows security scorecard for an open source software."
 	scorecardUse  = `./scorecard --repo=<repo_url> [--checks=check1,...] [--show-details] [--policy=file]
@@ -96,27 +104,76 @@ func checksHavePolicies(sp *spol.ScorecardPolicy, enabledChecks checker.CheckNam
 	return true
 }
 
-func getEnabledChecks(sp *spol.ScorecardPolicy, argsChecks []string) (checker.CheckNameToFnMap, error) {
+func getSupportedChecks(r string, checkDocs docs.Doc) ([]string, error) {
+	allChecks := checks.AllChecks
+	supportedChecks := []string{}
+	for check := range allChecks {
+		c, e := checkDocs.GetCheck(check)
+		if e != nil {
+			return nil, fmt.Errorf("checkDocs.GetCheck: %w", e)
+		}
+		types := c.GetSupportedRepoTypes()
+		for _, t := range types {
+			if r == t {
+				supportedChecks = append(supportedChecks, c.GetName())
+			}
+		}
+	}
+	return supportedChecks, nil
+}
+
+func isSupportedCheck(names []string, name string) bool {
+	for _, n := range names {
+		if n == name {
+			return true
+		}
+	}
+	return false
+}
+
+func getEnabledChecks(sp *spol.ScorecardPolicy, argsChecks []string,
+	supportedChecks []string, repoType string) (checker.CheckNameToFnMap, error) {
 	enabledChecks := checker.CheckNameToFnMap{}
 
 	switch {
 	case len(argsChecks) != 0:
 		// Populate checks to run with the CLI arguments.
 		for _, checkName := range argsChecks {
+			if !isSupportedCheck(supportedChecks, checkName) {
+				return enabledChecks,
+					sce.WithMessage(sce.ErrScorecardInternal,
+						fmt.Sprintf("repo type %s: unsupported check: %s", repoType, checkName))
+			}
 			if !enableCheck(checkName, &enabledChecks) {
 				return enabledChecks,
-					sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("Invalid check: %s", checkName))
+					sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("invalid check: %s", checkName))
 			}
 		}
 	case sp != nil:
 		// Populate checks to run with policy file.
 		for checkName := range sp.GetPolicies() {
+			if !isSupportedCheck(supportedChecks, checkName) {
+				return enabledChecks,
+					sce.WithMessage(sce.ErrScorecardInternal,
+						fmt.Sprintf("repo type %s: unsupported check: %s", repoType, checkName))
+			}
+
 			if !enableCheck(checkName, &enabledChecks) {
 				return enabledChecks,
-					sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("Invalid check: %s", checkName))
+					sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("invalid check: %s", checkName))
 			}
 		}
 	default:
+		// Enable all checks that are supported.
+		for checkName := range checks.AllChecks {
+			if !isSupportedCheck(supportedChecks, checkName) {
+				continue
+			}
+			if !enableCheck(checkName, &enabledChecks) {
+				return enabledChecks,
+					sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("invalid check: %s", checkName))
+			}
+		}
 		enabledChecks = checks.AllChecks
 	}
 
@@ -138,16 +195,22 @@ func validateFormat(format string) bool {
 	}
 }
 
-func getRepoAccessors(ctx context.Context, uri string, logger *zap.Logger) (clients.Repo, clients.RepoClient, error) {
-	if repo, err := localdir.MakeLocalDirRepo(uri); err == nil {
+func getRepoAccessors(ctx context.Context, uri string, logger *zap.Logger) (clients.Repo,
+	clients.RepoClient, string, error) {
+	var repo clients.Repo
+	var errLocal error
+	var errGitHub error
+	if repo, errLocal = localdir.MakeLocalDirRepo(uri); errLocal == nil {
 		// Local directory.
-		return repo, localdir.CreateLocalDirClient(ctx, logger), nil
+		return repo, localdir.CreateLocalDirClient(ctx, logger), repoTypeLocal, nil
 	}
-	if repo, err := githubrepo.MakeGithubRepo(uri); err == nil {
+
+	if repo, errGitHub = githubrepo.MakeGithubRepo(uri); errGitHub == nil {
 		// GitHub URL.
-		return repo, githubrepo.CreateGithubRepoClient(ctx, logger), nil
+		return repo, githubrepo.CreateGithubRepoClient(ctx, logger), repoTypeGitHub, nil
 	}
-	return nil, nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %s", uri))
+	return nil, nil, repoTypeUnknown,
+		sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %s: [%v, %v]", uri, errLocal, errGitHub))
 }
 
 var rootCmd = &cobra.Command{
@@ -215,13 +278,24 @@ var rootCmd = &cobra.Command{
 		// nolint
 		defer logger.Sync() // Flushes buffer, if any.
 
-		repoURI, repoClient, err := getRepoAccessors(ctx, repo, logger)
+		repoURI, repoClient, repoType, err := getRepoAccessors(ctx, repo, logger)
 		if err != nil {
 			log.Fatal(err)
 		}
 		defer repoClient.Close()
 
-		enabledChecks, err := getEnabledChecks(policy, checksToRun)
+		// Read docs.
+		checkDocs, err := docs.Read()
+		if err != nil {
+			log.Fatalf("cannot read yaml file: %v", err)
+		}
+
+		supportedChecks, err := getSupportedChecks(repoType, checkDocs)
+		if err != nil {
+			log.Fatalf("cannot read supported checks: %v", err)
+		}
+
+		enabledChecks, err := getEnabledChecks(policy, checksToRun, supportedChecks, repoType)
 		if err != nil {
 			panic(err)
 		}
@@ -250,12 +324,6 @@ var rootCmd = &cobra.Command{
 			fmt.Println("\nRESULTS\n-------")
 		}
 
-		// TODO: move the doc inside Scorecard structure.
-		checkDocs, e := docs.Read()
-		if e != nil {
-			log.Fatalf("cannot read yaml file: %v", err)
-		}
-
 		switch format {
 		case formatDefault:
 			err = repoResult.AsString(showDetails, *logLevel, checkDocs, os.Stdout)

cron/format/mock_doc.go

@@ -22,7 +22,7 @@ import (
 
 type mockCheck struct {
 	name, risk, short, description, url string
-	tags, remediation                   []string
+	tags, remediation, repos            []string
 }
 
 func (c *mockCheck) GetName() string {
@@ -53,6 +53,14 @@ func (c *mockCheck) GetTags() []string {
 	return l
 }
 
+func (c *mockCheck) GetSupportedRepoTypes() []string {
+	l := make([]string, len(c.repos))
+	for i := range c.repos {
+		l[i] = strings.TrimSpace(c.repos[i])
+	}
+	return l
+}
+
 func (c *mockCheck) GetDocumentationURL(commitish string) string {
 	return c.url
 }

docs/checks/doc.go

@@ -30,5 +30,6 @@ type CheckDoc interface {
 	GetDescription() string
 	GetRemediation() []string
 	GetTags() []string
+	GetSupportedRepoTypes() []string
 	GetDocumentationURL(commitish string) string
 }

docs/checks/impl.go

@@ -108,6 +108,16 @@ func (c *CheckDocImpl) GetRemediation() []string {
 	return c.internalCheck.Remediation
 }
 
+// GetSupportedRepoTypes returns the list of repo
+// types the check supports.
+func (c *CheckDocImpl) GetSupportedRepoTypes() []string {
+	l := strings.Split(c.internalCheck.Repos, ",")
+	for i := range l {
+		l[i] = strings.TrimSpace(l[i])
+	}
+	return l
+}
+
 // GetTags returns the list of tags or the check.
 func (c *CheckDocImpl) GetTags() []string {
 	l := strings.Split(c.internalCheck.Tags, ",")