✨ checks: add GitHub Webhook check (#1675)

* checks: add GitHub Webhook check Signed-off-by: Carlos Panato <[email protected]> * update per feedback Signed-off-by: cpanato <[email protected]> * add evaluation code Signed-off-by: cpanato <[email protected]> * add feature gate check Signed-off-by: cpanato <[email protected]> * fix lint Signed-off-by: cpanato <[email protected]>
ossf · Mar 31, 2022 · 7dcb3cb · 7dcb3cb
1 parent 93889a8
commit 7dcb3cb
Show file tree

Hide file tree

Showing 15 changed files with 786 additions and 5 deletions.
diff --git a/checker/raw_result.go b/checker/raw_result.go
@@ -25,6 +25,7 @@ type RawResults struct {
 	DependencyUpdateToolResults DependencyUpdateToolData
 	BranchProtectionResults     BranchProtectionsData
 	CodeReviewResults           CodeReviewData
+	WebhookResults              WebhooksData
 	MaintainedResults           MaintainedData
 }
 
@@ -70,6 +71,20 @@ type DependencyUpdateToolData struct {
 	Tools []Tool
 }
 
+// WebhooksData contains the raw results
+// for the Webhook check.
+type WebhooksData struct {
+	Webhook []WebhookData
+}
+
+// WebhookData contains the raw results
+// for webhook check.
+type WebhookData struct {
+	Path           string
+	ID             int64
+	UsesAuthSecret bool
+}
+
 // BranchProtectionsData contains the raw results
 // for the Branch-Protection check.
 type BranchProtectionsData struct {

diff --git a/checks/evaluation/webhooks.go b/checks/evaluation/webhooks.go
@@ -0,0 +1,60 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package evaluation
+
+import (
+	"fmt"
+
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+)
+
+// Webhooks applies the score policy for the Webhooks check.
+func Webhooks(name string, dl checker.DetailLogger,
+	r *checker.WebhooksData,
+) checker.CheckResult {
+	if r == nil {
+		e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
+		return checker.CreateRuntimeErrorResult(name, e)
+	}
+
+	if len(r.Webhook) < 1 {
+		return checker.CreateMaxScoreResult(name, "no webhooks defined")
+	}
+
+	hasNoSecretCount := 0
+	for _, hook := range r.Webhook {
+		if !hook.UsesAuthSecret {
+			dl.Warn(&checker.LogMessage{
+				Path: hook.Path,
+				Type: checker.FileTypeURL,
+				Text: "Webhook with no secret configured",
+			})
+			hasNoSecretCount++
+		}
+	}
+
+	if hasNoSecretCount == 0 {
+		return checker.CreateMaxScoreResult(name, fmt.Sprintf("all %d hook(s) have a secret configured", len(r.Webhook)))
+	}
+
+	if len(r.Webhook) == hasNoSecretCount {
+		return checker.CreateMinScoreResult(name, fmt.Sprintf("%d hook(s) do not have a secret configured", len(r.Webhook)))
+	}
+
+	return checker.CreateProportionalScoreResult(name,
+		fmt.Sprintf("%d/%d hook(s) with no secrets configured detected",
+			hasNoSecretCount, len(r.Webhook)), hasNoSecretCount, len(r.Webhook))
+}
diff --git a/checks/evaluation/webhooks_test.go b/checks/evaluation/webhooks_test.go
@@ -0,0 +1,152 @@
+// Copyright 2022 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package evaluation
+
+import (
+	"testing"
+
+	"github.com/ossf/scorecard/v4/checker"
+	scut "github.com/ossf/scorecard/v4/utests"
+)
+
+// TestWebhooks tests the webhooks check.
+func TestWebhooks(t *testing.T) {
+	t.Parallel()
+	//nolint
+	type args struct {
+		name string
+		dl   checker.DetailLogger
+		r    *checker.WebhooksData
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    checker.CheckResult
+		wantErr bool
+	}{
+		{
+			name: "r nil",
+			args: args{
+				name: "test_webhook_check_pass",
+				dl:   &scut.TestDetailLogger{},
+			},
+			wantErr: true,
+		},
+		{
+			name: "no webhooks",
+			args: args{
+				name: "no webhooks",
+				dl:   &scut.TestDetailLogger{},
+				r:    &checker.WebhooksData{},
+			},
+			want: checker.CheckResult{
+				Score: checker.MaxResultScore,
+			},
+		},
+		{
+			name: "1 webhook with secret",
+			args: args{
+				name: "1 webhook with secret",
+				dl:   &scut.TestDetailLogger{},
+				r: &checker.WebhooksData{
+					Webhook: []checker.WebhookData{
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/1234",
+							ID:             1234,
+							UsesAuthSecret: true,
+						},
+					},
+				},
+			},
+			want: checker.CheckResult{
+				Score: 10,
+			},
+		},
+		{
+			name: "1 webhook with no secret",
+			args: args{
+				name: "1 webhook with no secret",
+				dl:   &scut.TestDetailLogger{},
+				r: &checker.WebhooksData{
+					Webhook: []checker.WebhookData{
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/1234",
+							ID:             1234,
+							UsesAuthSecret: false,
+						},
+					},
+				},
+			},
+			want: checker.CheckResult{
+				Score: 0,
+			},
+		},
+		{
+			name: "many webhooks with no secret and with secret",
+			args: args{
+				name: "many webhooks with no secret and with secret",
+				dl:   &scut.TestDetailLogger{},
+				r: &checker.WebhooksData{
+					Webhook: []checker.WebhookData{
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/1234",
+							ID:             1234,
+							UsesAuthSecret: false,
+						},
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/1111",
+							ID:             1111,
+							UsesAuthSecret: true,
+						},
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/4444",
+							ID:             4444,
+							UsesAuthSecret: true,
+						},
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/3333",
+							ID:             3333,
+							UsesAuthSecret: false,
+						},
+						{
+							Path:           "https://github.com/owner/repo/settings/hooks/2222",
+							ID:             2222,
+							UsesAuthSecret: false,
+						},
+					},
+				},
+			},
+			want: checker.CheckResult{
+				Score: 6,
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got := Webhooks(tt.args.name, tt.args.dl, tt.args.r)
+			if tt.wantErr {
+				if got.Error2 == nil {
+					t.Errorf("Webhooks() error = %v, wantErr %v", got.Error2, tt.wantErr)
+				}
+			} else {
+				if got.Score != tt.want.Score {
+					t.Errorf("Webhooks() = %v, want %v", got.Score, tt.want.Score)
+				}
+			}
+		})
+	}
+}
diff --git a/checks/raw/webhook.go b/checks/raw/webhook.go
@@ -0,0 +1,48 @@
+// Copyright 2022 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package raw
+
+import (
+	"fmt"
+
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+)
+
+// WebHook retrieves the raw data for the WebHooks check.
+func WebHook(c *checker.CheckRequest) (checker.WebhooksData, error) {
+	hooksResp, err := c.RepoClient.ListWebhooks()
+	if err != nil {
+		return checker.WebhooksData{},
+			sce.WithMessage(sce.ErrScorecardInternal, "Client.Repositories.ListWebhooks")
+	}
+
+	if len(hooksResp) < 1 {
+		return checker.WebhooksData{}, nil
+	}
+
+	hooks := []checker.WebhookData{}
+	for _, hook := range hooksResp {
+		v := checker.WebhookData{
+			ID:             hook.ID,
+			UsesAuthSecret: hook.UsesAuthSecret,
+			Path:           fmt.Sprintf("https://%s/settings/hooks/%d", c.RepoClient.URI(), hook.ID),
+			// Note: add fields if needed.
+		}
+		hooks = append(hooks, v)
+	}
+
+	return checker.WebhooksData{Webhook: hooks}, nil
+}