Fetch verification targets by TUF custom metadata (sigstore#1423)

* Add TUF client method for fetching by metadata

Signed-off-by: Hayden Blauzvern <[email protected]>

* Fetch verification targets by TUF custom metadata

This uses GetTargetsByMeta to read the targets
using the custom metadata, or fallback to the old
targets by filename.

Signed-off-by: Hayden Blauzvern <[email protected]>

* Resolve PR comments, linter, and update tests

Signed-off-by: Hayden Blauzvern <[email protected]>

cmd/cosign/cli/fulcio/fulcioroots/fulcioroots.go

@@ -63,26 +63,25 @@ func initRoots() (*x509.CertPool, error) {
 			return nil, errors.New("error creating root cert pool")
 		}
 	} else {
-		tuf, err := tuf.NewFromEnv(context.Background())
+		tufClient, err := tuf.NewFromEnv(context.Background())
 		if err != nil {
 			return nil, errors.Wrap(err, "initializing tuf")
 		}
-		defer tuf.Close()
+		defer tufClient.Close()
 		// Retrieve from the embedded or cached TUF root. If expired, a network
 		// call is made to update the root.
-		rootFound := false
-		for _, fulcioTarget := range []string{fulcioTargetStr, fulcioV1TargetStr} {
-			b, err := tuf.GetTarget(fulcioTarget)
-			if err == nil {
-				rootFound = true
-				if !cp.AppendCertsFromPEM(b) {
-					return nil, errors.New("error creating root cert pool")
-				}
-			}
+		targets, err := tufClient.GetTargetsByMeta(tuf.Fulcio, []string{fulcioTargetStr, fulcioV1TargetStr})
+		if err != nil {
+			return nil, errors.New("error getting targets")
 		}
-		if !rootFound {
+		if len(targets) == 0 {
 			return nil, errors.New("none of the Fulcio roots have been found")
 		}
+		for _, t := range targets {
+			if !cp.AppendCertsFromPEM(t.Target) {
+				return nil, errors.New("error creating root cert pool")
+			}
+		}
 	}
 	return cp, nil
 }

cmd/cosign/cli/fulcio/fulcioroots/fulcioroots_test.go

@@ -0,0 +1,24 @@
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fulcioroots
+
+import "testing"
+
+func TestGetFulcioRoots(t *testing.T) {
+	certPool := Get()
+	if len(certPool.Subjects()) == 0 {
+		t.Errorf("expected 1 or more certificates, got 0")
+	}
+}

cmd/cosign/cli/fulcio/fulcioverifier/fulcioverifier.go

@@ -51,26 +51,27 @@ const altCTLogPublicKeyLocation = "SIGSTORE_CT_LOG_PUBLIC_KEY_FILE"
 // The SCT is a `Signed Certificate Timestamp`, which promises that
 // the certificate issued by Fulcio was also added to the public CT log within
 // some defined time period
-func verifySCT(certPEM, rawSCT []byte) error {
+func verifySCT(ctx context.Context, certPEM, rawSCT []byte) error {
 	var pubKeys []crypto.PublicKey
 	rootEnv := os.Getenv(altCTLogPublicKeyLocation)
 	if rootEnv == "" {
-		ctx := context.TODO()
-		tuf, err := tuf.NewFromEnv(ctx)
+		tufClient, err := tuf.NewFromEnv(ctx)
 		if err != nil {
 			return err
 		}
-		defer tuf.Close()
-		ctPub, err := tuf.GetTarget(ctPublicKeyStr)
+		defer tufClient.Close()
+
+		targets, err := tufClient.GetTargetsByMeta(tuf.CTFE, []string{ctPublicKeyStr})
 		if err != nil {
 			return err
 		}
-		// Is there a reason why this must be ECDSA key?
-		pubKey, err := cosign.PemToECDSAKey(ctPub)
-		if err != nil {
-			return errors.Wrap(err, "converting Public CT to ECDSAKey")
+		for _, t := range targets {
+			ctPub, err := cosign.PemToECDSAKey(t.Target)
+			if err != nil {
+				return errors.Wrap(err, "converting Public CT to ECDSAKey")
+			}
+			pubKeys = append(pubKeys, ctPub)
 		}
-		pubKeys = append(pubKeys, pubKey)
 	} else {
 		fmt.Fprintf(os.Stderr, "**Warning** Using a non-standard public key for verifying SCT: %s\n", rootEnv)
 		raw, err := os.ReadFile(rootEnv)
@@ -83,6 +84,9 @@ func verifySCT(certPEM, rawSCT []byte) error {
 		}
 		pubKeys = append(pubKeys, pubKey)
 	}
+	if len(pubKeys) == 0 {
+		return errors.New("none of the CTFE keys have been found")
+	}
 	cert, err := x509util.CertificateFromPEM(certPEM)
 	if err != nil {
 		return err
@@ -110,7 +114,7 @@ func NewSigner(ctx context.Context, idToken, oidcIssuer, oidcClientID, oidcClien
 	}
 
 	// verify the sct
-	if err := verifySCT(fs.Cert, fs.SCT); err != nil {
+	if err := verifySCT(ctx, fs.Cert, fs.SCT); err != nil {
 		return nil, errors.Wrap(err, "verifying SCT")
 	}
 	fmt.Fprintln(os.Stderr, "Successfully verified SCT...")

pkg/cosign/tlog.go

@@ -46,20 +46,27 @@ var rekorTargetStr = `rekor.pub`
 // GetRekorPubs retrieves trusted Rekor public keys from the embedded or cached
 // TUF root. If expired, makes a network call to retrieve the updated targets.
 func GetRekorPubs(ctx context.Context) ([]*ecdsa.PublicKey, error) {
-	tuf, err := tuf.NewFromEnv(ctx)
+	tufClient, err := tuf.NewFromEnv(ctx)
 	if err != nil {
 		return nil, err
 	}
-	defer tuf.Close()
-	b, err := tuf.GetTarget(rekorTargetStr)
+	defer tufClient.Close()
+	targets, err := tufClient.GetTargetsByMeta(tuf.Rekor, []string{rekorTargetStr})
 	if err != nil {
 		return nil, err
 	}
-	rekorPubKey, err := PemToECDSAKey(b)
-	if err != nil {
-		return nil, errors.Wrap(err, "pem to ecdsa")
+	publicKeys := make([]*ecdsa.PublicKey, 0, len(targets))
+	for _, t := range targets {
+		rekorPubKey, err := PemToECDSAKey(t.Target)
+		if err != nil {
+			return nil, errors.Wrap(err, "pem to ecdsa")
+		}
+		publicKeys = append(publicKeys, rekorPubKey)
+	}
+	if len(publicKeys) == 0 {
+		return nil, errors.New("none of the Rekor public keys have been found")
 	}
-	return []*ecdsa.PublicKey{rekorPubKey}, nil
+	return publicKeys, nil
 }
 
 // TLogUpload will upload the signature, public key and payload to the transparency log.

pkg/cosign/tuf/client.go

@@ -61,6 +61,20 @@ type RootStatus struct {
 	Targets    []string          `json:"targets"`
 }
 
+type TargetFile struct {
+	Target []byte
+	Status StatusKind
+}
+
+type customMetadata struct {
+	Usage  UsageKind  `json:"usage"`
+	Status StatusKind `json:"status"`
+}
+
+type sigstoreCustomMetadata struct {
+	Sigstore customMetadata `json:"sigstore"`
+}
+
 // RemoteCache contains information to cache on the location of the remote
 // repository.
 type remoteCache struct {
@@ -247,6 +261,45 @@ func (t *TUF) GetTarget(name string) ([]byte, error) {
 	return targetBytes, nil
 }
 
+// Get target files by a custom usage metadata tag. If there are no files found,
+// use the fallback target names to fetch the targets by name.
+func (t *TUF) GetTargetsByMeta(usage UsageKind, fallbacks []string) ([]TargetFile, error) {
+	targets, err := t.client.Targets()
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting targets")
+	}
+	var matchedTargets []TargetFile
+	for name, targetMeta := range targets {
+		// Skip any targets that do not include custom metadata.
+		if targetMeta.Custom == nil {
+			continue
+		}
+		var scm sigstoreCustomMetadata
+		err := json.Unmarshal(*targetMeta.Custom, &scm)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "**Warning** Custom metadata not configured properly for target %s, skipping target\n", name)
+			continue
+		}
+		if scm.Sigstore.Usage == usage {
+			target, err := t.GetTarget(name)
+			if err != nil {
+				return nil, errors.Wrap(err, "error getting target")
+			}
+			matchedTargets = append(matchedTargets, TargetFile{Target: target, Status: scm.Sigstore.Status})
+		}
+	}
+	if len(matchedTargets) == 0 {
+		for _, fallback := range fallbacks {
+			target, err := t.GetTarget(fallback)
+			if err != nil {
+				return nil, errors.Wrap(err, "error getting target")
+			}
+			matchedTargets = append(matchedTargets, TargetFile{Target: target, Status: Active})
+		}
+	}
+	return matchedTargets, nil
+}
+
 func localStore(cacheRoot string) (client.LocalStore, error) {
 	local, err := tuf_leveldbstore.FileLocalStore(cacheRoot)
 	if err != nil {

pkg/cosign/tuf/client_test.go

@@ -24,6 +24,9 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -206,6 +209,93 @@ func TestCustomRoot(t *testing.T) {
 	tufObj.Close()
 }
 
+func TestGetTargetsByMeta(t *testing.T) {
+	ctx := context.Background()
+	// Create a remote repository.
+	td := t.TempDir()
+	remote, _ := newTufCustomRepo(t, td, "foo")
+
+	// Serve remote repository.
+	s := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(td, "repository"))))
+	defer s.Close()
+
+	// Initialize with custom root.
+	tufRoot := t.TempDir()
+	t.Setenv("TUF_ROOT", tufRoot)
+	meta, err := remote.GetMeta()
+	if err != nil {
+		t.Error(err)
+	}
+	rootBytes, ok := meta["root.json"]
+	if !ok {
+		t.Error(err)
+	}
+	if err := Initialize(ctx, s.URL, rootBytes); err != nil {
+		t.Error(err)
+	}
+	if l := dirLen(t, tufRoot); l == 0 {
+		t.Errorf("expected filesystem writes, got %d entries", l)
+	}
+
+	tufObj, err := NewFromEnv(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tufObj.Close()
+	// Fetch a target with no custom metadata.
+	targets, err := tufObj.GetTargetsByMeta(UnknownUsage, []string{"fooNoCustom.txt"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(targets) != 1 {
+		t.Fatalf("expected one target without custom metadata, got %d targets", len(targets))
+	}
+	if !bytes.Equal(targets[0].Target, []byte("foo")) {
+		t.Fatalf("target metadata mismatched, expected: %s, got: %s", "foo", string(targets[0].Target))
+	}
+	if targets[0].Status != Active {
+		t.Fatalf("target without custom metadata not active, got: %v", targets[0].Status)
+	}
+	// Fetch multiple targets with no custom metadata.
+	targets, err = tufObj.GetTargetsByMeta(UnknownUsage, []string{"fooNoCustom.txt", "fooNoCustomOther.txt"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(targets) != 2 {
+		t.Fatalf("expected two targets without custom metadata, got %d targets", len(targets))
+	}
+	if targets[0].Status != Active || targets[1].Status != Active {
+		t.Fatalf("target without custom metadata not active, got: %v and %v", targets[0].Status, targets[1].Status)
+	}
+	// Fetch targets with custom metadata.
+	targets, err = tufObj.GetTargetsByMeta(Fulcio, []string{"fooNoCustom.txt"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(targets) != 2 {
+		t.Fatalf("expected two targets without custom metadata, got %d targets", len(targets))
+	}
+	targetBytes := []string{string(targets[0].Target), string(targets[1].Target)}
+	expectedTB := []string{"foo", "foo"}
+	if !reflect.DeepEqual(targetBytes, expectedTB) {
+		t.Fatalf("target metadata mismatched, expected: %v, got: %v", expectedTB, targetBytes)
+	}
+	targetStatuses := []StatusKind{targets[0].Status, targets[1].Status}
+	sort.Slice(targetStatuses, func(i, j int) bool {
+		return targetStatuses[i] < targetStatuses[j]
+	})
+	expectedTS := []StatusKind{Active, Expired}
+	if !reflect.DeepEqual(targetStatuses, expectedTS) {
+		t.Fatalf("unexpected target status with custom metadata, expected %v, got: %v", expectedTS, targetStatuses)
+	}
+	// Error when fetching target that does not exist.
+	_, err = tufObj.GetTargetsByMeta(UsageKind(UnknownStatus), []string{"unknown.txt"})
+	expectedErr := "file not found: unknown.txt"
+	if !strings.Contains(err.Error(), "file not found: unknown.txt") {
+		t.Fatalf("unexpected error fetching missing metadata, expected: %s, got: %s", expectedErr, err.Error())
+	}
+}
+
 func checkTargetsAndMeta(t *testing.T, tuf *TUF) {
 	// Check the targets
 	t.Helper()
@@ -268,6 +358,58 @@ func forceExpirationVersion(t *testing.T, version int) {
 	})
 }
 
+// newTufCustomRepo initializes a TUF repository with root, targets, snapshot, and timestamp roles
+// 4 targets are created to exercise various code paths, including two targets with no custom metadata,
+// one target with custom metadata marked as active, and another with custom metadata marked as expired.
+func newTufCustomRepo(t *testing.T, td string, targetData string) (tuf.LocalStore, *tuf.Repo) {
+	scmActive, err := json.Marshal(&sigstoreCustomMetadata{Sigstore: customMetadata{Usage: Fulcio, Status: Active}})
+	if err != nil {
+		t.Error(err)
+	}
+	scmExpired, err := json.Marshal(&sigstoreCustomMetadata{Sigstore: customMetadata{Usage: Fulcio, Status: Expired}})
+	if err != nil {
+		t.Error(err)
+	}
+
+	remote := tuf.FileSystemStore(td, nil)
+	r, err := tuf.NewRepo(remote)
+	if err != nil {
+		t.Error(err)
+	}
+	if err := r.Init(false); err != nil {
+		t.Error(err)
+	}
+	for _, role := range []string{"root", "targets", "snapshot", "timestamp"} {
+		if _, err := r.GenKey(role); err != nil {
+			t.Error(err)
+		}
+	}
+	for name, scm := range map[string]json.RawMessage{
+		"fooNoCustom.txt": nil, "fooNoCustomOther.txt": nil,
+		"fooActive.txt": scmActive, "fooExpired.txt": scmExpired} {
+		targetPath := filepath.Join(td, "staged", "targets", name)
+		if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
+			t.Error(err)
+		}
+		if err := ioutil.WriteFile(targetPath, []byte(targetData), 0600); err != nil {
+			t.Error(err)
+		}
+		if err := r.AddTarget(name, scm); err != nil {
+			t.Error(err)
+		}
+	}
+	if err := r.Snapshot(); err != nil {
+		t.Error(err)
+	}
+	if err := r.Timestamp(); err != nil {
+		t.Error(err)
+	}
+	if err := r.Commit(); err != nil {
+		t.Error(err)
+	}
+	return remote, r
+}
+
 func newTufRepo(t *testing.T, td string, targetData string) (tuf.LocalStore, *tuf.Repo) {
 	remote := tuf.FileSystemStore(td, nil)
 	r, err := tuf.NewRepo(remote)