✨ Use blob-based CII client

clients/cii_blob_client.go

@@ -0,0 +1,55 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clients
+
+import (
+	"context"
+	"fmt"
+
+	"gocloud.dev/blob"
+
+	// Needed to link GCP drivers.
+	_ "gocloud.dev/blob/gcsblob"
+)
+
+// blobClientCIIBestPractices implements the CIIBestPracticesClient interface.
+// A gocloud blob client is used to communicate with the CII Best Practices data.
+type blobClientCIIBestPractices struct {
+	bucketURL string
+}
+
+// GetBadgeLevel implements CIIBestPracticesClient.GetBadgeLevel.
+func (client *blobClientCIIBestPractices) GetBadgeLevel(ctx context.Context, uri string) (BadgeLevel, error) {
+	bucket, err := blob.OpenBucket(ctx, client.bucketURL)
+	if err != nil {
+		return Unknown, fmt.Errorf("error during blob.OpenBucket: %w", err)
+	}
+	defer bucket.Close()
+
+	objectName := fmt.Sprintf("%s/result.json", uri)
+	jsonData, err := bucket.ReadAll(ctx, objectName)
+	if err != nil {
+		return Unknown, fmt.Errorf("error during bucket.ReadAll: %w", err)
+	}
+
+	parsedResponse, err := ParseBadgeResponseFromJSON(jsonData)
+	if err != nil {
+		return Unknown, fmt.Errorf("error parsing data: %w", err)
+	}
+	if len(parsedResponse) < 1 {
+		return NotFound, nil
+	}
+	return parsedResponse[0].getBadgeLevel()
+}

clients/cii_client.go

@@ -18,10 +18,6 @@ import (
 	"context"
 )
 
-// BadgeLevel corresponds to CII-Best-Practices badge levels.
-// https://bestpractices.coreinfrastructure.org/en
-type BadgeLevel uint
-
 const (
 	// Unknown or non-parsable CII Best Practices badge.
 	Unknown BadgeLevel = iota
@@ -37,12 +33,23 @@ const (
 	Gold
 )
 
+// BadgeLevel corresponds to CII-Best-Practices badge levels.
+// https://bestpractices.coreinfrastructure.org/en
+type BadgeLevel uint
+
 // CIIBestPracticesClient interface returns the BadgeLevel for a repo URL.
 type CIIBestPracticesClient interface {
 	GetBadgeLevel(ctx context.Context, uri string) (BadgeLevel, error)
 }
 
-// DefaultCIIBestPracticesClient returns HTTPClientCIIBestPractices implementation of the interface.
+// DefaultCIIBestPracticesClient returns http-based implementation of the interface.
 func DefaultCIIBestPracticesClient() CIIBestPracticesClient {
-	return &HTTPClientCIIBestPractices{}
+	return &httpClientCIIBestPractices{}
+}
+
+// BlobCIIBestPracticesClient returns a blob-based implementation of the interface.
+func BlobCIIBestPracticesClient(bucketURL string) CIIBestPracticesClient {
+	return &blobClientCIIBestPractices{
+		bucketURL: bucketURL,
+	}
 }

clients/cii_http_client.go

@@ -16,35 +16,19 @@ package clients
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"math"
 	"net/http"
-	"strings"
 	"time"
 )
 
-const (
-	inProgressResp = "in_progress"
-	passingResp    = "passing"
-	silverResp     = "silver"
-	goldResp       = "gold"
-)
-
-var (
-	errTooManyRequests  = errors.New("failed after exponential backoff")
-	errUnsupportedBadge = errors.New("unsupported badge")
-)
+var errTooManyRequests = errors.New("failed after exponential backoff")
 
-// HTTPClientCIIBestPractices implements the CIIBestPracticesClient interface.
+// httpClientCIIBestPractices implements the CIIBestPracticesClient interface.
 // A HTTP client with exponential backoff is used to communicate with the CII Best Practices servers.
-type HTTPClientCIIBestPractices struct{}
-
-type response struct {
-	BadgeLevel string `json:"badge_level"`
-}
+type httpClientCIIBestPractices struct{}
 
 type expBackoffTransport struct {
 	numRetries uint8
@@ -63,7 +47,7 @@ func (transport *expBackoffTransport) RoundTrip(req *http.Request) (*http.Respon
 }
 
 // GetBadgeLevel implements CIIBestPracticesClient.GetBadgeLevel.
-func (client *HTTPClientCIIBestPractices) GetBadgeLevel(ctx context.Context, uri string) (BadgeLevel, error) {
+func (client *httpClientCIIBestPractices) GetBadgeLevel(ctx context.Context, uri string) (BadgeLevel, error) {
 	repoURI := fmt.Sprintf("https://%s", uri)
 	url := fmt.Sprintf("https://bestpractices.coreinfrastructure.org/projects.json?url=%s", repoURI)
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
@@ -82,31 +66,17 @@ func (client *HTTPClientCIIBestPractices) GetBadgeLevel(ctx context.Context, uri
 	}
 	defer resp.Body.Close()
 
-	b, err := io.ReadAll(resp.Body)
+	jsonData, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return Unknown, fmt.Errorf("error during io.ReadAll: %w", err)
 	}
 
-	parsedResponse := []response{}
-	if err := json.Unmarshal(b, &parsedResponse); err != nil {
-		return Unknown, fmt.Errorf("error during json.Unmarshal: %w", err)
+	parsedResponse, err := ParseBadgeResponseFromJSON(jsonData)
+	if err != nil {
+		return Unknown, fmt.Errorf("error during json parsing: %w", err)
 	}
-
 	if len(parsedResponse) < 1 {
 		return NotFound, nil
 	}
-	badgeLevel := parsedResponse[0].BadgeLevel
-	if strings.Contains(badgeLevel, inProgressResp) {
-		return InProgress, nil
-	}
-	if strings.Contains(badgeLevel, passingResp) {
-		return Passing, nil
-	}
-	if strings.Contains(badgeLevel, silverResp) {
-		return Silver, nil
-	}
-	if strings.Contains(badgeLevel, goldResp) {
-		return Gold, nil
-	}
-	return Unknown, fmt.Errorf("%w: %s", errUnsupportedBadge, badgeLevel)
+	return parsedResponse[0].getBadgeLevel()
 }

clients/cii_response.go

@@ -0,0 +1,72 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clients
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+const (
+	inProgressResp = "in_progress"
+	passingResp    = "passing"
+	silverResp     = "silver"
+	goldResp       = "gold"
+)
+
+var errUnsupportedBadge = errors.New("unsupported badge")
+
+// BadgeResponse struct is used to read/write CII Best Practices badge data.
+type BadgeResponse struct {
+	BadgeLevel string `json:"badge_level"`
+}
+
+// getBadgeLevel parses a string badge value into BadgeLevel enum.
+func (resp BadgeResponse) getBadgeLevel() (BadgeLevel, error) {
+	if strings.Contains(resp.BadgeLevel, inProgressResp) {
+		return InProgress, nil
+	}
+	if strings.Contains(resp.BadgeLevel, passingResp) {
+		return Passing, nil
+	}
+	if strings.Contains(resp.BadgeLevel, silverResp) {
+		return Silver, nil
+	}
+	if strings.Contains(resp.BadgeLevel, goldResp) {
+		return Gold, nil
+	}
+	return Unknown, fmt.Errorf("%w: %s", errUnsupportedBadge, resp.BadgeLevel)
+}
+
+// AsJSON outputs BadgeResponse struct in JSON format.
+func (resp BadgeResponse) AsJSON() ([]byte, error) {
+	ret := []BadgeResponse{resp}
+	jsonData, err := json.Marshal(ret)
+	if err != nil {
+		return nil, fmt.Errorf("error during json.Marshal: %w", err)
+	}
+	return jsonData, nil
+}
+
+// ParseBadgeResponseFromJSON parses input []byte value into []BadgeResponse.
+func ParseBadgeResponseFromJSON(data []byte) ([]BadgeResponse, error) {
+	parsedResponse := []BadgeResponse{}
+	if err := json.Unmarshal(data, &parsedResponse); err != nil {
+		return nil, fmt.Errorf("error during json.Unmarshal: %w", err)
+	}
+	return parsedResponse, nil
+}

cron/cii/main.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/ossf/scorecard/v3/clients"
 	"github.com/ossf/scorecard/v3/cron/config"
 	"github.com/ossf/scorecard/v3/cron/data"
 )
@@ -38,9 +39,15 @@ func writeToCIIDataBucket(ctx context.Context, pageResp []ciiPageResp, bucketURL
 	for _, project := range pageResp {
 		projectURL := strings.TrimPrefix(project.RepoURL, "https://")
 		projectURL = strings.TrimPrefix(projectURL, "http://")
+		jsonData, err := clients.BadgeResponse{
+			BadgeLevel: project.BadgeLevel,
+		}.AsJSON()
+		if err != nil {
+			return fmt.Errorf("error during AsJSON: %w", err)
+		}
 		fmt.Printf("Writing result for: %s\n", projectURL)
 		if err := data.WriteToBlobStore(ctx, bucketURL,
-			fmt.Sprintf("%s/result.json", projectURL), []byte(project.BadgeLevel)); err != nil {
+			fmt.Sprintf("%s/result.json", projectURL), jsonData); err != nil {
 			return fmt.Errorf("error during data.WriteToBlobStore: %w", err)
 		}
 	}

cron/worker/main.go

@@ -173,12 +173,17 @@ func main() {
 		panic(err)
 	}
 
+	ciiDataBucketURL, err := config.GetCIIDataBucketURL()
+	if err != nil {
+		panic(err)
+	}
+
 	logger, err := githubrepo.NewLogger(zap.InfoLevel)
 	if err != nil {
 		panic(err)
 	}
 	repoClient := githubrepo.CreateGithubRepoClient(ctx, logger)
-	ciiClient := clients.DefaultCIIBestPracticesClient()
+	ciiClient := clients.BlobCIIBestPracticesClient(ciiDataBucketURL)
 	ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(ctx, logger)
 	if err != nil {
 		panic(err)