Start wiring tests together. Not complete.

Signed-off-by: Ville Aikas <[email protected]>

.github/workflows/kind-cluster-image-policy.yaml

@@ -16,9 +16,7 @@ name: Test cosigned with ClusterImagePolicy
 
 on:
   pull_request:
-    # TODO(vaikas): DO NOT SUBMIT
-    #branches: [ main ]
-    branches: [ '*' ]
+    branches: [ 'main', 'release-*' ]
 
 defaults:
   run:
@@ -31,14 +29,19 @@ jobs:
     name: ClusterImagePolicy e2e tests
     runs-on: ubuntu-latest
 
-    matrix:
-      k8s-version:
-      - v1.21.x
+    strategy:
+      matrix:
+        k8s-version:
+        - v1.21.x
 
     env:
       KNATIVE_VERSION: "1.1.0"
       KO_DOCKER_REPO: "registry.local:5000/knative"
       SCAFFOLDING_RELEASE_VERSION: "v0.2.2"
+      GO111MODULE: on
+      GOFLAGS: -ldflags=-s -ldflags=-w
+      KOCACHE: ~/ko
+      COSIGN_EXPERIMENTAL: true
 
     steps:
     - name: Configure DockerHub mirror
@@ -48,6 +51,14 @@ jobs:
         sudo mv "$tmp" /etc/docker/daemon.json
         sudo service docker restart
 
+    - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # v2.4.0
+    - uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2.2.0
+      with:
+        go-version: '1.17.x'
+
+    # will use the latest release available for ko
+    - uses: imjasonh/setup-ko@2c3450ca27f6e6f2b02e72a40f2163c281a1f675 # v0.4
+
     - name: Setup Cluster
       run: |
         curl -Lo ./setup-kind.sh https://github.com/sigstore/scaffolding/releases/download/${{ env.SCAFFOLDING_RELEASE_VERSION }}/setup-kind.sh
@@ -82,89 +93,108 @@ jobs:
 
         kubectl wait --for=condition=Complete --timeout=180s job/sign-job job/checktree job/verify-job
 
+    - name: Install cosigned
+      env:
+        GIT_HASH: $GITHUB_SHA
+        GIT_VERSION: ci
+        LDFLAGS: ""
+      run: |
+        ko apply -Bf config/
+
+        # Wait for the webhook to come up and become Ready
+        kubectl rollout status --timeout 5m --namespace cosign-system deployments/webhook
+
+    - name: Create sample image - demoimage
+      run: |
+        pushd $(mktemp -d)
+        go mod init example.com/demo
+        cat <<EOF > main.go
+        package main
+        import "fmt"
+        func main() {
+          fmt.Println("hello world")
+        }
+        EOF
+        demoimage=`ko publish -B example.com/demo`
+        echo "demoimage=$demoimage" >> $GITHUB_ENV
+        echo Created image $demoimage
+        popd
+
+    - name: Create sample image2 - demoimage2
+      run: |
+        pushd $(mktemp -d)
+        go mod init example.com/demo2
+        cat <<EOF > main.go
+        package main
+        import "fmt"
+        func main() {
+          fmt.Println("hello world 2")
+        }
+        EOF
+        demoimage2=`ko publish -B example.com/demo2`
+        echo "demoimage2=$demoimage2" >> $GITHUB_ENV
+        echo Created image $demoimage2
+        popd
+
+    # TODO(vaikas): There should be a fake issuer on the cluster. This one
+    # fetches a k8s auth token from the kind cluster that we spin up above. We
+    # do not want to use the Github OIDC token, but do want PRs to run this
+    # flow.
+    - name: Install a Knative service for fetch tokens off the cluster
+      run: |
+        ko apply -f ./test/config/gettoken
+        sleep 2
+        kubectl wait --for=condition=Ready --timeout=15s ksvc gettoken
+
+    # These set up the env variables so that
     - name: Set the endpoints on the cluster and grab secrets
       run: |
         REKOR_URL=`kubectl -n rekor-system get --no-headers ksvc rekor | cut -d ' ' -f 4`
         echo "REKOR_URL=$REKOR_URL" >> $GITHUB_ENV
+        curl -s $REKOR_URL/api/v1/log/publicKey > ./rekor-public.pem
+
         FULCIO_URL=`kubectl -n fulcio-system get --no-headers ksvc fulcio | cut -d ' ' -f 4`
         echo "FULCIO_URL=$FULCIO_URL" >> $GITHUB_ENV
         CTLOG_URL=`kubectl -n ctlog-system get --no-headers ksvc ctlog | cut -d ' ' -f 4`
         echo "CTLOG_URL=$CTLOG_URL" >> $GITHUB_ENV
 
+        ISSUER_URL=`kubectl get --no-headers ksvc gettoken | cut -d ' ' -f 4`
+        echo "ISSUER_URL=$ISSUER_URL" >> $GITHUB_ENV
+        OIDC_TOKEN=`curl -s $ISSUER_URL`
+        echo "OIDC_TOKEN=$OIDC_TOKEN" >> $GITHUB_ENV
+
         kubectl -n ctlog-system get secrets ctlog-public-key -o=jsonpath='{.data.public}' | base64 -d > ./ctlog-public.pem
         echo "SIGSTORE_CT_LOG_PUBLIC_KEY_FILE=./ctlog-public.pem" >> $GITHUB_ENV
 
         kubectl -n fulcio-system get secrets fulcio-secret -ojsonpath='{.data.cert}' | base64 -d > ./fulcio-root.pem
         echo "SIGSTORE_ROOT_FILE=./fulcio-root.pem" >> $GITHUB_ENV
 
-      - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # v2.4.0
-      - uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2.2.0
-        with:
-          go-version: '1.17.x'
-
-      # will use the latest release available for ko
-      - uses: imjasonh/setup-ko@2c3450ca27f6e6f2b02e72a40f2163c281a1f675 # v0.4
-
-      - name: build cosign
-        run: |
-          make cosign
-
-      - name: Create sample image
-        run: |
-          pushd $(mktemp -d)
-          go mod init example.com/demo
-          cat <<EOF > main.go
-          package main
-          import "fmt"
-          func main() {
-            fmt.Println("hello world")
-          }
-          EOF
-          demoimage=`ko publish -B example.com/demo`
-          echo "demoimage=$demoimage" >> $GITHUB_ENV
-          echo Created image $demoimage
-          popd
-
-
-      - name: Sign with cosign
-        run: |
-        COSIGN_EXPERIMENTAL=1 ./cosign sign --rekor-url ${{ env.REKOR_URL }} --fulcio-url ${{ env.FULCIO_URL }} --force --allow-insecure-registry ${{ env.demoimage }}
-
-      - name: Verify with cosign
-        run: |
-        SIGSTORE_TRUST_REKOR_API_PUBLIC_KEY=1 COSIGN_EXPERIMENTAL=1 ./cosign verify --rekor-url ${{ env.REKOR_URL }} --allow-insecure-registry ${{ env.demoimage }}
+    - name: Deploy ClusterImagePolicy
+      run: |
+        kubectl apply -f ./test/testdata/cosigned/e2e/cip.yaml
 
-    - name: Collect node diagnostics
-      if: ${{ failure() }}
+    - name: build cosign
       run: |
-        for x in $(kubectl get nodes -oname); do
-          echo "::group:: describe $x"
-          kubectl describe $x
-          echo '::endgroup::'
-        done
+        make cosign
 
-    - name: Collect pod diagnostics
-      if: ${{ failure() }}
+    - name: Sign demoimage with cosign
       run: |
-        for ns in default fulcio-system rekor-system trillian-system ctlog-system; do
-          kubectl get pods -n${ns}
+        ./cosign sign --rekor-url ${{ env.REKOR_URL }} --fulcio-url ${{ env.FULCIO_URL }} --force --allow-insecure-registry ${{ env.demoimage }} --identity-token ${{ env.OIDC_TOKEN }}
 
-          for x in $(kubectl get pods -n${ns} -oname); do
-            echo "::group:: describe $x"
-            kubectl describe -n${ns} $x
-            echo '::endgroup::'
-          done
-        done
+    - name: Verify with cosign
+      run: |
+        SIGSTORE_TRUST_REKOR_API_PUBLIC_KEY=1 COSIGN_EXPERIMENTAL=1 ./cosign verify --rekor-url ${{ env.REKOR_URL }} --allow-insecure-registry ${{ env.demoimage }}
 
-    - name: Collect logs
-      if: ${{ failure() }}
+    - name: Deploy jobs and verify signed works, unsigned fails
       run: |
-        mkdir -p /tmp/logs
-        kind export logs /tmp/logs --name sigstore
+        kubectl create namespace demo
+        kubectl label namespace demo cosigned.sigstore.dev/include=true
+        # We signed this above, this should work
+        kubectl create -n demo job demo --image=${{ env.demoimage }}
 
-    - name: Upload artifacts
+        # We did not sign this, should fail
+        kubectl create -n demo job demo2 --image=${{ env.demoimage2 }}
+
+    - name: Collect diagnostics
       if: ${{ failure() }}
-      uses: actions/upload-artifact@v2
-      with:
-        name: logs
-        path: /tmp/logs
+      uses: chainguard-dev/actions/kind-diag@84c993eaf02da1c325854fb272a4df9184bd80fc # main

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.3
 	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87
 	github.com/in-toto/in-toto-golang v0.3.4-0.20211211042327-af1f9fb822bf
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/manifoldco/promptui v0.9.0
 	github.com/miekg/pkcs11 v1.1.1
 	github.com/mitchellh/go-homedir v1.1.0
@@ -218,7 +219,6 @@ require (
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect

pkg/cosign/kubernetes/webhook/validator.go

@@ -148,14 +148,36 @@ func (v *Validator) validatePodSpec(ctx context.Context, ps *corev1.PodSpec, opt
 
 			containerKeys := keys
 			config := config.FromContext(ctx)
+
+			// During the migration from the secret only validation into policy
+			// based ones. If there's a policy (or rather authorities) that
+			// successfully validated the image, keep tally of it and if Policy
+			// validated, skip the traditional one since they are not
+			// necessarily going to play nicely together.
+			passedAuthorityCheck := false
 			if config != nil {
-				fieldErrors := validateAuthorities(ctx, ref, kc, config)
+				av, fieldErrors := validateAuthorities(ctx, ref, kc, config)
 				if fieldErrors != nil && len(fieldErrors) > 0 {
 					// TODO:(dennyhoang) Enforce currently non-breaking errors https://github.com/sigstore/cosign/issues/1642
-					logging.FromContext(ctx).Warnf("Failed to validate authorities for %s : %v", ref.Name(), fieldErrors)
+					logging.FromContext(ctx).Warnf("Failed to validate 	authorities for %s : %v", ref.Name(), fieldErrors)
+					for _, fe := range fieldErrors {
+						errorField := apis.ErrGeneric(fe.Error(), "image").ViaFieldIndex(field, i)
+						errorField.Details = c.Image
+						errs = errs.Also(errorField)
+					}
 				} else {
 					logging.FromContext(ctx).Warnf("Validated authorities for %s", ref.Name())
+					// Only say we passed (aka, we skip the traditidional check
+					// below) if more than one authority was validated, which
+					// means that there was a matching ClusterImagePolicy.
+					if av > 0 {
+						passedAuthorityCheck = true
+					}
 				}
+			}
+
+			if passedAuthorityCheck {
+				logging.FromContext(ctx).Debugf("Found at least one matching policy and it was validated for %s", ref.Name())
 				continue
 			}
 
@@ -174,19 +196,26 @@ func (v *Validator) validatePodSpec(ctx context.Context, ps *corev1.PodSpec, opt
 	return errs
 }
 
-//
-func validateAuthorities(ctx context.Context, ref name.Reference, defaultKC authn.Keychain, config *config.Config, opts ...ociremote.Option) []error {
+// validateAuthorities will go through all the matching authorities for a given
+// image. Returns the number of Authority that it was successfully validated
+// against as well as any errors. Note that if an image does not match
+// any policies, it's perfectly reasonable that the return value is 0, nil since
+// there were no errors, but the image was not validated against any matching
+// authority.
+func validateAuthorities(ctx context.Context, ref name.Reference, defaultKC authn.Keychain, config *config.Config, opts ...ociremote.Option) (int, []error) {
 	ret := []error{}
 	authorities, err := config.ImagePolicyConfig.GetAuthorities(ref.Name())
 	if err != nil {
-		return append(ret, errors.Wrap(err, "failed to GetAuthorities"))
+		return 0, append(ret, errors.Wrap(err, "failed to GetAuthorities"))
 	}
 
+	authoritiesValidated := 0
 	for _, authority := range authorities {
-		logging.FromContext(ctx).Infof("Checking Authority: %+v", authority)
+		logging.FromContext(ctx).Debugf("Checking Authority: %+v", authority)
 		// TODO(vaikas): We currently only use the defaultKC, we have to look
 		// at authority.Sources to determine additional information for the
 		// WithRemoteOptions below, at least the 'TargetRepository'
+		// https://github.com/sigstore/cosign/issues/1651
 		opts := ociremote.WithRemoteOptions(remote.WithAuthFromKeychain(defaultKC))
 
 		if authority.Key != nil {
@@ -198,42 +227,50 @@ func validateAuthorities(ctx context.Context, ref name.Reference, defaultKC auth
 				// says it has to match all the policies, and I think we may
 				// have to ensure _all_ the keys match, but valid (well, and
 				// things it calls) succeed if any key matches.
+				// https://github.com/sigstore/cosign/issues/1652
 				if err := valid(ctx, ref, authorityKeys, opts); err != nil {
 					ret = append(ret, errors.Wrap(err, "failed to validate keys"))
 					continue
 				}
+				// Since there can be only one Key per authority, if we found
+				// multiple, then add them all since that many authorities
+				// matched.
+				authoritiesValidated += len(authorityKeys)
 			}
 		}
 		if authority.Keyless != nil && authority.Keyless.URL != nil {
-			logging.FromContext(ctx).Infof("Fetching FulcioRoot for %s : From: %s ", ref.Name(), authority.Keyless.URL)
+			logging.FromContext(ctx).Debugf("Fetching FulcioRoot for %s : From: %s ", ref.Name(), authority.Keyless.URL)
 			fulcioroot, err := getFulcioCert(authority.Keyless.URL)
 			if err != nil {
 				ret = append(ret, errors.Wrap(err, "failed to fetch FulcioRoot"))
 				continue
 			}
 			var rekorClient *client.Rekor
 			if authority.CTLog != nil && authority.CTLog.URL != nil {
+				logging.FromContext(ctx).Debugf("Using CTLog %s for %s", authority.CTLog.URL, ref.Name())
 				rekorClient, err = rekor.GetRekorClient(authority.CTLog.URL.String())
 				if err != nil {
-					logging.FromContext(ctx).Errorf("vaikas failed creating rekor client: +v", err)
-					ret = append(ret, errors.Wrap(err, "failed creating Rekor client"))
+					logging.FromContext(ctx).Errorf("failed creating rekor client: +v", err)
+					ret = append(ret, errors.Wrap(err, "creating Rekor client"))
 					continue
 				}
 			}
 			sps, err := validSignaturesWithFulcio(ctx, ref, fulcioroot, rekorClient, opts)
 			if err != nil {
-				logging.FromContext(ctx).Errorf("vaikas FAILED validSignatures: %v", err)
-				ret = append(ret, errors.Wrap(err, "failed to validate signatures"))
+				logging.FromContext(ctx).Errorf("failed validSignatures for %s: %v", ref.Name(), err)
+				ret = append(ret, errors.Wrap(err, "validate signatures"))
 			} else {
 				if len(sps) > 0 {
-					logging.FromContext(ctx).Infof("vaikas zomg validated signature, got %d signatures", len(sps))
+					logging.FromContext(ctx).Debugf("validated signature for %s, got %d signatures", len(sps))
+					authoritiesValidated++
 				} else {
-					logging.FromContext(ctx).Errorf("vaikas No validSignatures found ")
+					logging.FromContext(ctx).Errorf("no validSignatures found for %s", ref.Name())
+					ret = append(ret, fmt.Errorf("No valid signatures found for %s", ref.Name()))
 				}
 			}
 		}
 	}
-	return ret
+	return authoritiesValidated, ret
 }
 
 // ResolvePodSpecable implements duckv1.PodSpecValidator
@@ -327,6 +364,15 @@ func (v *Validator) resolvePodSpec(ctx context.Context, ps *corev1.PodSpec, opt
 }
 
 func getFulcioCert(u *apis.URL) (*x509.CertPool, error) {
+	// TODO(vaikas): Check the URL and if it's not containing the fully path
+	// to rootCert, aka something like:
+	// http://fulcio.fulcio-system.svc/api/v1/rootCert
+	// but is instead just the
+	// http://fulcio.fulcio-system.svc
+	// or
+	// http://fulcio.fulcio-system.svc/
+	// Construct the full URL above.
+	// Right now it must be the first, as in fully specified.
 	resp, err := http.Get(u.String())
 	if err != nil {
 		return nil, errors.Wrap(err, "doing http get")