From 8bbb4b0a1a0ab3a761301119849f1fec1aa13ed9 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Wed, 17 Apr 2024 19:43:13 +0000 Subject: [PATCH 1/8] add projectpackageversions to signed releases raw results Signed-off-by: Raghav Kaul --- clients/mockclients/repo.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/clients/mockclients/repo.go b/clients/mockclients/repo.go index 3f1640f6fee..ce1860771f1 100644 --- a/clients/mockclients/repo.go +++ b/clients/mockclients/repo.go @@ -64,6 +64,20 @@ func (mr *MockRepoMockRecorder) AppendMetadata(metadata ...interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendMetadata", reflect.TypeOf((*MockRepo)(nil).AppendMetadata), metadata...) } +// Path mocks base method. +func (m *MockRepo) Path() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Path") + ret0, _ := ret[0].(string) + return ret0 +} + +// Path indicates an expected call of Path. +func (mr *MockRepoMockRecorder) Path() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Path", reflect.TypeOf((*MockRepo)(nil).Path)) +} + // Host mocks base method. func (m *MockRepo) Host() string { m.ctrl.T.Helper() From b61b28c7e8ed17ba271ef2d30b0646e38403bcf4 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Thu, 16 May 2024 19:19:57 +0000 Subject: [PATCH 2/8] finding: add NewNot* helpers, fix error msg Signed-off-by: Raghav Kaul --- checks/raw/signed_releases.go | 18 ++++++++++++++++++ clients/mockclients/repo.go | 14 -------------- finding/finding.go | 12 ++++++++++++ finding/probe.go | 2 +- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/checks/raw/signed_releases.go b/checks/raw/signed_releases.go index 5183e9eb532..87201d2e07f 100644 --- a/checks/raw/signed_releases.go +++ b/checks/raw/signed_releases.go @@ -55,6 +55,24 @@ func SignedReleases(c *checker.CheckRequest) (checker.SignedReleasesData, error) }) } + for _, v := range versions.Versions { + prov := checker.PackageProvenance{} + + if len(v.SLSAProvenances) > 0 { + prov = checker.PackageProvenance{ + Commit: v.SLSAProvenances[0].Commit, + IsVerified: v.SLSAProvenances[0].Verified, + } + } + + pkgs = append(pkgs, checker.ProjectPackage{ + System: v.VersionKey.System, + Name: v.VersionKey.Name, + Version: v.VersionKey.Version, + Provenance: prov, + }) + } + return checker.SignedReleasesData{ Releases: releases, Packages: pkgs, diff --git a/clients/mockclients/repo.go b/clients/mockclients/repo.go index ce1860771f1..3f1640f6fee 100644 --- a/clients/mockclients/repo.go +++ b/clients/mockclients/repo.go @@ -64,20 +64,6 @@ func (mr *MockRepoMockRecorder) AppendMetadata(metadata ...interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendMetadata", reflect.TypeOf((*MockRepo)(nil).AppendMetadata), metadata...) } -// Path mocks base method. -func (m *MockRepo) Path() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Path") - ret0, _ := ret[0].(string) - return ret0 -} - -// Path indicates an expected call of Path. -func (mr *MockRepoMockRecorder) Path() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Path", reflect.TypeOf((*MockRepo)(nil).Path)) -} - // Host mocks base method. func (m *MockRepo) Host() string { m.ctrl.T.Helper() diff --git a/finding/finding.go b/finding/finding.go index ad1da29cc1f..6745f8a0ce1 100644 --- a/finding/finding.go +++ b/finding/finding.go @@ -144,12 +144,24 @@ func NewFalse(efs embed.FS, probeID, text string, loc *Location, return NewWith(efs, probeID, text, loc, OutcomeFalse) } +// NewNotApplicable create a finding with a NotApplicable outcome and the desired location. +func NewNotApplicable(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotApplicable) +} + // NewNotAvailable create a finding with a NotAvailable outcome and the desired location. func NewNotAvailable(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { return NewWith(efs, probeID, text, loc, OutcomeNotAvailable) } +// NewNotSupported create a finding with a NotSupported outcome and the desired location. +func NewNotSupported(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotSupported) +} + // NewTrue create a true finding with the desired location. func NewTrue(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { diff --git a/finding/probe.go b/finding/probe.go index ec6ef66c83b..0acf821e561 100644 --- a/finding/probe.go +++ b/finding/probe.go @@ -142,7 +142,7 @@ func validateID(actual, expected string) error { func validateRemediation(r yamlRemediation) error { if err := validateRemediationOutcomeTrigger(r.OnOutcome); err != nil { - return err + return fmt.Errorf("remediation: %w", err) } switch r.Effort { case RemediationEffortHigh, RemediationEffortMedium, RemediationEffortLow: From afbb35adbd50ff2d0d2b5cc7128a02c243478098 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Tue, 28 May 2024 20:58:51 +0000 Subject: [PATCH 3/8] probe: releasesHaveVerifiedProvenance Signed-off-by: Raghav Kaul --- probes/entries.go | 2 + probes/releasesHaveVerifiedProvenance/def.yml | 34 +++++ probes/releasesHaveVerifiedProvenance/impl.go | 70 +++++++++++ .../impl_test.go | 117 ++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 probes/releasesHaveVerifiedProvenance/def.yml create mode 100644 probes/releasesHaveVerifiedProvenance/impl.go create mode 100644 probes/releasesHaveVerifiedProvenance/impl_test.go diff --git a/probes/entries.go b/probes/entries.go index 527bb485898..b722fd9de41 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -48,6 +48,7 @@ import ( "github.com/ossf/scorecard/v5/probes/pinsDependencies" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" "github.com/ossf/scorecard/v5/probes/requiresApproversForPullRequests" "github.com/ossf/scorecard/v5/probes/requiresCodeOwnersReview" "github.com/ossf/scorecard/v5/probes/requiresLastPushApproval" @@ -139,6 +140,7 @@ var ( SignedReleases = []ProbeImpl{ releasesAreSigned.Run, releasesHaveProvenance.Run, + releasesHaveVerifiedProvenance.Run, } BranchProtection = []ProbeImpl{ blocksDeleteOnBranches.Run, diff --git a/probes/releasesHaveVerifiedProvenance/def.yml b/probes/releasesHaveVerifiedProvenance/def.yml new file mode 100644 index 00000000000..572ae781ff8 --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/def.yml @@ -0,0 +1,34 @@ +# Copyright 2024 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesHaveVerifiedProvenance +short: Checks if the project releases with provenance attestations +motivation: > + Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity. +implementation: > + This probe checks how many packages published by the repository are associated with verified SLSA provenance attestations. It uses data from a ProjectPackageClient, which associates a GitHub/GitLab project with a package in a package manager. Using the data from the package manager (whom we rely on to verify the provenance attestation), this probe returns a finding for each release. For now, only NPM is supported. +outcome: + - For each release, the probe returns OutcomeTrue or OutcomeFalse, depending on if the package has a verified provenance attestation. + - If we didn't find a package or didn't find releases, return OutcomeNotAvailable. +remediation: + onOutcome: False + effort: Low + text: + - For NPM, publish provenance alongside your package using the `--provenance` flag (See (Introducing npm package provenance)[https://github.blog/2023-04-19-introducing-npm-package-provenance/]) +ecosystem: + languages: + - javascript + clients: + - github + - gitlab diff --git a/probes/releasesHaveVerifiedProvenance/impl.go b/probes/releasesHaveVerifiedProvenance/impl.go new file mode 100644 index 00000000000..80bad527306 --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl.go @@ -0,0 +1,70 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/internal/probes" +) + +func init() { + probes.MustRegister(Probe, Run, []probes.CheckName{probes.SignedReleases}) +} + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesHaveVerifiedProvenance" +) + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + var findings []finding.Finding + + if len(raw.SignedReleasesResults.Packages) == 0 { + f, err := finding.NewNotAvailable(fs, Probe, "no package manager releases found", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + return findings, Probe, nil + } + + for i := range raw.SignedReleasesResults.Packages { + p := raw.SignedReleasesResults.Packages[i] + + if !p.Provenance.IsVerified { + f, err := finding.NewFalse(fs, Probe, "release without verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + continue + } + + f, err := finding.NewTrue(fs, Probe, "release with verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + + return findings, Probe, nil +} diff --git a/probes/releasesHaveVerifiedProvenance/impl_test.go b/probes/releasesHaveVerifiedProvenance/impl_test.go new file mode 100644 index 00000000000..d770a4d6c63 --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl_test.go @@ -0,0 +1,117 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "testing" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" +) + +func Test_Run(t *testing.T) { + tests := []struct { + desc string + pkgs []checker.ProjectPackage + outcomes []finding.Outcome + err error + }{ + { + desc: "no packages found", + outcomes: []finding.Outcome{finding.OutcomeNotAvailable}, + }, + { + desc: "some releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeFalse}, + }, + { + desc: "all releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeTrue}, + }, + { + desc: "no verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeFalse, finding.OutcomeFalse}, + }, + } + + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.desc, func(t *testing.T) { + raw := checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Packages: tt.pkgs, + }, + } + + outcomes, _, err := Run(&raw) + + if tt.err != err { + t.Errorf("expected %+v got %+v", tt.err, err) + } + + if !cmpOutcomes(tt.outcomes, outcomes) { + t.Errorf("expected %+v got %+v", tt.outcomes, outcomes) + } + }) + } +} + +func cmpOutcomes(ex []finding.Outcome, act []finding.Finding) bool { + if len(ex) != len(act) { + return false + } + + for i := range ex { + if act[i].Outcome != ex[i] { + return false + } + } + + return true +} From 779bbb13dbf7cd5ee87fb231c1a2f501209de108 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Tue, 4 Jun 2024 17:09:16 +0000 Subject: [PATCH 4/8] logging Signed-off-by: Raghav Kaul --- options/flags.go | 1 + 1 file changed, 1 insertion(+) diff --git a/options/flags.go b/options/flags.go index 07eaeae21c1..dba6fcc2611 100644 --- a/options/flags.go +++ b/options/flags.go @@ -193,6 +193,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { allowedFormats := []string{ FormatDefault, FormatJSON, + FormatProbe, } if o.isSarifEnabled() { From 7df77ec0db7fcaa02bd57b09598ced0cd853c61c Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Tue, 4 Jun 2024 21:06:00 +0000 Subject: [PATCH 5/8] fix tests and lint Signed-off-by: Raghav Kaul --- checks/evaluation/signed_releases.go | 10 ++++++++++ checks/evaluation/signed_releases_test.go | 15 +++++++++++++++ .../releasesHaveVerifiedProvenance/impl_test.go | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/checks/evaluation/signed_releases.go b/checks/evaluation/signed_releases.go index f432556beb5..39d64aaae41 100644 --- a/checks/evaluation/signed_releases.go +++ b/checks/evaluation/signed_releases.go @@ -24,6 +24,7 @@ import ( "github.com/ossf/scorecard/v5/finding" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" ) var errNoReleaseFound = errors.New("no release found") @@ -37,6 +38,7 @@ func SignedReleases(name string, expectedProbes := []string{ releasesAreSigned.Probe, releasesHaveProvenance.Probe, + releasesHaveVerifiedProvenance.Probe, } if !finding.UniqueProbesEqual(findings, expectedProbes) { @@ -55,6 +57,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + // Debug release name if f.Outcome == finding.OutcomeNotApplicable { // Generic summary. @@ -86,6 +92,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + releaseName := getReleaseName(f) if releaseName == "" { return checker.CreateRuntimeErrorResult(name, errNoReleaseFound) diff --git a/checks/evaluation/signed_releases_test.go b/checks/evaluation/signed_releases_test.go index a72b605033e..502451f77cc 100644 --- a/checks/evaluation/signed_releases_test.go +++ b/checks/evaluation/signed_releases_test.go @@ -23,6 +23,7 @@ import ( "github.com/ossf/scorecard/v5/finding" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" scut "github.com/ossf/scorecard/v5/utests" ) @@ -64,6 +65,13 @@ func provenanceProbe(release, asset int, outcome finding.Outcome) finding.Findin } } +func verifiedProvenanceProbe(outcome finding.Outcome) finding.Finding { + return finding.Finding{ + Probe: releasesHaveVerifiedProvenance.Probe, + Outcome: outcome, + } +} + func TestSignedReleases(t *testing.T) { t.Parallel() tests := []struct { @@ -76,6 +84,7 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeTrue), provenanceProbe(0, 0, finding.OutcomeFalse), + verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: 8, @@ -89,6 +98,7 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeTrue), provenanceProbe(0, 0, finding.OutcomeTrue), + verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: 10, @@ -101,6 +111,7 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeFalse), provenanceProbe(0, 0, finding.OutcomeTrue), + verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: checker.MaxResultScore, @@ -122,6 +133,7 @@ func TestSignedReleases(t *testing.T) { // Release 3 signedProbe(release2, asset0, finding.OutcomeFalse), provenanceProbe(release2, asset1, finding.OutcomeTrue), + verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 6, @@ -148,6 +160,7 @@ func TestSignedReleases(t *testing.T) { // Release 5, Asset 1: signedProbe(release4, asset0, finding.OutcomeFalse), provenanceProbe(release4, asset0, finding.OutcomeFalse), + verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 7, @@ -174,6 +187,7 @@ func TestSignedReleases(t *testing.T) { // Release 5: signedProbe(release4, asset0, finding.OutcomeTrue), provenanceProbe(release4, asset0, finding.OutcomeFalse), + verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 8, @@ -205,6 +219,7 @@ func TestSignedReleases(t *testing.T) { // Release 6, Asset 1: signedProbe(release5, asset0, finding.OutcomeTrue), provenanceProbe(release5, asset0, finding.OutcomeTrue), + verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: checker.InconclusiveResultScore, diff --git a/probes/releasesHaveVerifiedProvenance/impl_test.go b/probes/releasesHaveVerifiedProvenance/impl_test.go index d770a4d6c63..46cfe3d569b 100644 --- a/probes/releasesHaveVerifiedProvenance/impl_test.go +++ b/probes/releasesHaveVerifiedProvenance/impl_test.go @@ -16,6 +16,7 @@ package releasesHaveVerifiedProvenance import ( + "errors" "testing" "github.com/ossf/scorecard/v5/checker" @@ -23,6 +24,8 @@ import ( ) func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet tests := []struct { desc string pkgs []checker.ProjectPackage @@ -83,6 +86,7 @@ func Test_Run(t *testing.T) { for _, tt := range tests { tt := tt // Re-initializing variable so it is not changed while executing the closure below t.Run(tt.desc, func(t *testing.T) { + t.Parallel() raw := checker.RawResults{ SignedReleasesResults: checker.SignedReleasesData{ Packages: tt.pkgs, @@ -91,7 +95,7 @@ func Test_Run(t *testing.T) { outcomes, _, err := Run(&raw) - if tt.err != err { + if !errors.Is(tt.err, err) { t.Errorf("expected %+v got %+v", tt.err, err) } From 3e50703b789ce7a93f386ff0cc0ebaee292a674b Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Thu, 6 Jun 2024 18:21:03 +0000 Subject: [PATCH 6/8] address comments Signed-off-by: Raghav Kaul --- checks/evaluation/signed_releases.go | 1 - checks/evaluation/signed_releases_test.go | 7 ------- probes/entries.go | 2 +- probes/releasesHaveVerifiedProvenance/def.yml | 2 +- probes/releasesHaveVerifiedProvenance/impl.go | 2 +- probes/releasesHaveVerifiedProvenance/impl_test.go | 2 +- 6 files changed, 4 insertions(+), 12 deletions(-) diff --git a/checks/evaluation/signed_releases.go b/checks/evaluation/signed_releases.go index 39d64aaae41..1fc9cfb0524 100644 --- a/checks/evaluation/signed_releases.go +++ b/checks/evaluation/signed_releases.go @@ -38,7 +38,6 @@ func SignedReleases(name string, expectedProbes := []string{ releasesAreSigned.Probe, releasesHaveProvenance.Probe, - releasesHaveVerifiedProvenance.Probe, } if !finding.UniqueProbesEqual(findings, expectedProbes) { diff --git a/checks/evaluation/signed_releases_test.go b/checks/evaluation/signed_releases_test.go index 502451f77cc..e53096e98d4 100644 --- a/checks/evaluation/signed_releases_test.go +++ b/checks/evaluation/signed_releases_test.go @@ -84,7 +84,6 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeTrue), provenanceProbe(0, 0, finding.OutcomeFalse), - verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: 8, @@ -98,7 +97,6 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeTrue), provenanceProbe(0, 0, finding.OutcomeTrue), - verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: 10, @@ -111,7 +109,6 @@ func TestSignedReleases(t *testing.T) { findings: []finding.Finding{ signedProbe(0, 0, finding.OutcomeFalse), provenanceProbe(0, 0, finding.OutcomeTrue), - verifiedProvenanceProbe(finding.OutcomeTrue), }, result: scut.TestReturn{ Score: checker.MaxResultScore, @@ -133,7 +130,6 @@ func TestSignedReleases(t *testing.T) { // Release 3 signedProbe(release2, asset0, finding.OutcomeFalse), provenanceProbe(release2, asset1, finding.OutcomeTrue), - verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 6, @@ -160,7 +156,6 @@ func TestSignedReleases(t *testing.T) { // Release 5, Asset 1: signedProbe(release4, asset0, finding.OutcomeFalse), provenanceProbe(release4, asset0, finding.OutcomeFalse), - verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 7, @@ -187,7 +182,6 @@ func TestSignedReleases(t *testing.T) { // Release 5: signedProbe(release4, asset0, finding.OutcomeTrue), provenanceProbe(release4, asset0, finding.OutcomeFalse), - verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: 8, @@ -219,7 +213,6 @@ func TestSignedReleases(t *testing.T) { // Release 6, Asset 1: signedProbe(release5, asset0, finding.OutcomeTrue), provenanceProbe(release5, asset0, finding.OutcomeTrue), - verifiedProvenanceProbe(finding.OutcomeFalse), }, result: scut.TestReturn{ Score: checker.InconclusiveResultScore, diff --git a/probes/entries.go b/probes/entries.go index b722fd9de41..4ec8941188c 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -140,7 +140,6 @@ var ( SignedReleases = []ProbeImpl{ releasesAreSigned.Run, releasesHaveProvenance.Run, - releasesHaveVerifiedProvenance.Run, } BranchProtection = []ProbeImpl{ blocksDeleteOnBranches.Run, @@ -170,6 +169,7 @@ var ( hasPermissiveLicense.Run, codeReviewOneReviewers.Run, hasBinaryArtifacts.Run, + releasesHaveVerifiedProvenance.Run, } // Probes which don't use pre-computed raw data but rather collect it themselves. diff --git a/probes/releasesHaveVerifiedProvenance/def.yml b/probes/releasesHaveVerifiedProvenance/def.yml index 572ae781ff8..698c520bf3d 100644 --- a/probes/releasesHaveVerifiedProvenance/def.yml +++ b/probes/releasesHaveVerifiedProvenance/def.yml @@ -13,7 +13,7 @@ # limitations under the License. id: releasesHaveVerifiedProvenance -short: Checks if the project releases with provenance attestations +short: Checks if the project releases with provenance attestations that have been verified motivation: > Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity. implementation: > diff --git a/probes/releasesHaveVerifiedProvenance/impl.go b/probes/releasesHaveVerifiedProvenance/impl.go index 80bad527306..f73b20fcc56 100644 --- a/probes/releasesHaveVerifiedProvenance/impl.go +++ b/probes/releasesHaveVerifiedProvenance/impl.go @@ -39,7 +39,7 @@ func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { var findings []finding.Finding if len(raw.SignedReleasesResults.Packages) == 0 { - f, err := finding.NewNotAvailable(fs, Probe, "no package manager releases found", nil) + f, err := finding.NewNotApplicable(fs, Probe, "no package manager releases found", nil) if err != nil { return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) } diff --git a/probes/releasesHaveVerifiedProvenance/impl_test.go b/probes/releasesHaveVerifiedProvenance/impl_test.go index 46cfe3d569b..317638b5b8a 100644 --- a/probes/releasesHaveVerifiedProvenance/impl_test.go +++ b/probes/releasesHaveVerifiedProvenance/impl_test.go @@ -34,7 +34,7 @@ func Test_Run(t *testing.T) { }{ { desc: "no packages found", - outcomes: []finding.Outcome{finding.OutcomeNotAvailable}, + outcomes: []finding.Outcome{finding.OutcomeNotApplicable}, }, { desc: "some releases with verified provenance", From 67eed67ba2357af4178e0ba251203df3bc2b5885 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Thu, 6 Jun 2024 18:36:26 +0000 Subject: [PATCH 7/8] remove unused Signed-off-by: Raghav Kaul --- checks/evaluation/signed_releases_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/checks/evaluation/signed_releases_test.go b/checks/evaluation/signed_releases_test.go index e53096e98d4..a72b605033e 100644 --- a/checks/evaluation/signed_releases_test.go +++ b/checks/evaluation/signed_releases_test.go @@ -23,7 +23,6 @@ import ( "github.com/ossf/scorecard/v5/finding" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" - "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" scut "github.com/ossf/scorecard/v5/utests" ) @@ -65,13 +64,6 @@ func provenanceProbe(release, asset int, outcome finding.Outcome) finding.Findin } } -func verifiedProvenanceProbe(outcome finding.Outcome) finding.Finding { - return finding.Finding{ - Probe: releasesHaveVerifiedProvenance.Probe, - Outcome: outcome, - } -} - func TestSignedReleases(t *testing.T) { t.Parallel() tests := []struct { From 61309702355b22583733559be2ed2e0c70c76277 Mon Sep 17 00:00:00 2001 From: Raghav Kaul Date: Thu, 6 Jun 2024 18:41:05 +0000 Subject: [PATCH 8/8] fix merge conflict Signed-off-by: Raghav Kaul --- checks/raw/signed_releases.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/checks/raw/signed_releases.go b/checks/raw/signed_releases.go index 87201d2e07f..5183e9eb532 100644 --- a/checks/raw/signed_releases.go +++ b/checks/raw/signed_releases.go @@ -55,24 +55,6 @@ func SignedReleases(c *checker.CheckRequest) (checker.SignedReleasesData, error) }) } - for _, v := range versions.Versions { - prov := checker.PackageProvenance{} - - if len(v.SLSAProvenances) > 0 { - prov = checker.PackageProvenance{ - Commit: v.SLSAProvenances[0].Commit, - IsVerified: v.SLSAProvenances[0].Verified, - } - } - - pkgs = append(pkgs, checker.ProjectPackage{ - System: v.VersionKey.System, - Name: v.VersionKey.Name, - Version: v.VersionKey.Version, - Provenance: prov, - }) - } - return checker.SignedReleasesData{ Releases: releases, Packages: pkgs,