From bfaa9febc2a4bec3b985239ab23d420b8876d45f Mon Sep 17 00:00:00 2001 From: Raghav Kaul <8695110+raghavkaul@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:15:20 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20probe:=20releases=20with=20verified?= =?UTF-8?q?=20provenance=20(#4141)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add projectpackageversions to signed releases raw results Signed-off-by: Raghav Kaul * finding: add NewNot* helpers, fix error msg Signed-off-by: Raghav Kaul * probe: releasesHaveVerifiedProvenance Signed-off-by: Raghav Kaul * logging Signed-off-by: Raghav Kaul * fix tests and lint Signed-off-by: Raghav Kaul * address comments Signed-off-by: Raghav Kaul * remove unused Signed-off-by: Raghav Kaul * fix merge conflict Signed-off-by: Raghav Kaul --------- Signed-off-by: Raghav Kaul --- checks/evaluation/signed_releases.go | 9 ++ finding/finding.go | 12 ++ finding/probe.go | 2 +- options/flags.go | 1 + probes/entries.go | 2 + probes/releasesHaveVerifiedProvenance/def.yml | 34 +++++ probes/releasesHaveVerifiedProvenance/impl.go | 70 ++++++++++ .../impl_test.go | 121 ++++++++++++++++++ 8 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 probes/releasesHaveVerifiedProvenance/def.yml create mode 100644 probes/releasesHaveVerifiedProvenance/impl.go create mode 100644 probes/releasesHaveVerifiedProvenance/impl_test.go diff --git a/checks/evaluation/signed_releases.go b/checks/evaluation/signed_releases.go index f432556beb5..1fc9cfb0524 100644 --- a/checks/evaluation/signed_releases.go +++ b/checks/evaluation/signed_releases.go @@ -24,6 +24,7 @@ import ( "github.com/ossf/scorecard/v5/finding" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" ) var errNoReleaseFound = errors.New("no release found") @@ -55,6 +56,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + // Debug release name if f.Outcome == finding.OutcomeNotApplicable { // Generic summary. @@ -86,6 +91,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + releaseName := getReleaseName(f) if releaseName == "" { return checker.CreateRuntimeErrorResult(name, errNoReleaseFound) diff --git a/finding/finding.go b/finding/finding.go index ad1da29cc1f..6745f8a0ce1 100644 --- a/finding/finding.go +++ b/finding/finding.go @@ -144,12 +144,24 @@ func NewFalse(efs embed.FS, probeID, text string, loc *Location, return NewWith(efs, probeID, text, loc, OutcomeFalse) } +// NewNotApplicable create a finding with a NotApplicable outcome and the desired location. +func NewNotApplicable(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotApplicable) +} + // NewNotAvailable create a finding with a NotAvailable outcome and the desired location. func NewNotAvailable(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { return NewWith(efs, probeID, text, loc, OutcomeNotAvailable) } +// NewNotSupported create a finding with a NotSupported outcome and the desired location. +func NewNotSupported(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotSupported) +} + // NewTrue create a true finding with the desired location. func NewTrue(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { diff --git a/finding/probe.go b/finding/probe.go index ec6ef66c83b..0acf821e561 100644 --- a/finding/probe.go +++ b/finding/probe.go @@ -142,7 +142,7 @@ func validateID(actual, expected string) error { func validateRemediation(r yamlRemediation) error { if err := validateRemediationOutcomeTrigger(r.OnOutcome); err != nil { - return err + return fmt.Errorf("remediation: %w", err) } switch r.Effort { case RemediationEffortHigh, RemediationEffortMedium, RemediationEffortLow: diff --git a/options/flags.go b/options/flags.go index 07eaeae21c1..dba6fcc2611 100644 --- a/options/flags.go +++ b/options/flags.go @@ -193,6 +193,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { allowedFormats := []string{ FormatDefault, FormatJSON, + FormatProbe, } if o.isSarifEnabled() { diff --git a/probes/entries.go b/probes/entries.go index 527bb485898..4ec8941188c 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -48,6 +48,7 @@ import ( "github.com/ossf/scorecard/v5/probes/pinsDependencies" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" "github.com/ossf/scorecard/v5/probes/requiresApproversForPullRequests" "github.com/ossf/scorecard/v5/probes/requiresCodeOwnersReview" "github.com/ossf/scorecard/v5/probes/requiresLastPushApproval" @@ -168,6 +169,7 @@ var ( hasPermissiveLicense.Run, codeReviewOneReviewers.Run, hasBinaryArtifacts.Run, + releasesHaveVerifiedProvenance.Run, } // Probes which don't use pre-computed raw data but rather collect it themselves. diff --git a/probes/releasesHaveVerifiedProvenance/def.yml b/probes/releasesHaveVerifiedProvenance/def.yml new file mode 100644 index 00000000000..698c520bf3d --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/def.yml @@ -0,0 +1,34 @@ +# Copyright 2024 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesHaveVerifiedProvenance +short: Checks if the project releases with provenance attestations that have been verified +motivation: > + Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity. +implementation: > + This probe checks how many packages published by the repository are associated with verified SLSA provenance attestations. It uses data from a ProjectPackageClient, which associates a GitHub/GitLab project with a package in a package manager. Using the data from the package manager (whom we rely on to verify the provenance attestation), this probe returns a finding for each release. For now, only NPM is supported. +outcome: + - For each release, the probe returns OutcomeTrue or OutcomeFalse, depending on if the package has a verified provenance attestation. + - If we didn't find a package or didn't find releases, return OutcomeNotAvailable. +remediation: + onOutcome: False + effort: Low + text: + - For NPM, publish provenance alongside your package using the `--provenance` flag (See (Introducing npm package provenance)[https://github.blog/2023-04-19-introducing-npm-package-provenance/]) +ecosystem: + languages: + - javascript + clients: + - github + - gitlab diff --git a/probes/releasesHaveVerifiedProvenance/impl.go b/probes/releasesHaveVerifiedProvenance/impl.go new file mode 100644 index 00000000000..f73b20fcc56 --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl.go @@ -0,0 +1,70 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/internal/probes" +) + +func init() { + probes.MustRegister(Probe, Run, []probes.CheckName{probes.SignedReleases}) +} + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesHaveVerifiedProvenance" +) + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + var findings []finding.Finding + + if len(raw.SignedReleasesResults.Packages) == 0 { + f, err := finding.NewNotApplicable(fs, Probe, "no package manager releases found", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + return findings, Probe, nil + } + + for i := range raw.SignedReleasesResults.Packages { + p := raw.SignedReleasesResults.Packages[i] + + if !p.Provenance.IsVerified { + f, err := finding.NewFalse(fs, Probe, "release without verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + continue + } + + f, err := finding.NewTrue(fs, Probe, "release with verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + + return findings, Probe, nil +} diff --git a/probes/releasesHaveVerifiedProvenance/impl_test.go b/probes/releasesHaveVerifiedProvenance/impl_test.go new file mode 100644 index 00000000000..317638b5b8a --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl_test.go @@ -0,0 +1,121 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "errors" + "testing" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet + tests := []struct { + desc string + pkgs []checker.ProjectPackage + outcomes []finding.Outcome + err error + }{ + { + desc: "no packages found", + outcomes: []finding.Outcome{finding.OutcomeNotApplicable}, + }, + { + desc: "some releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeFalse}, + }, + { + desc: "all releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeTrue}, + }, + { + desc: "no verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeFalse, finding.OutcomeFalse}, + }, + } + + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + raw := checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Packages: tt.pkgs, + }, + } + + outcomes, _, err := Run(&raw) + + if !errors.Is(tt.err, err) { + t.Errorf("expected %+v got %+v", tt.err, err) + } + + if !cmpOutcomes(tt.outcomes, outcomes) { + t.Errorf("expected %+v got %+v", tt.outcomes, outcomes) + } + }) + } +} + +func cmpOutcomes(ex []finding.Outcome, act []finding.Finding) bool { + if len(ex) != len(act) { + return false + } + + for i := range ex { + if act[i].Outcome != ex[i] { + return false + } + } + + return true +}