Write the repository into secret, test air-gap mode.

.github/workflows/fulcio-rekor-kind.yaml

@@ -156,6 +156,8 @@ jobs:
     - name: Initialize cosign with our custom tuf root and make root copy
       run: |
         kubectl -n tuf-system get secrets tuf-root -ojsonpath='{.data.root}' | base64 -d > ./root.json
+        # Also grab the compressed repository for airgap testing.
+        kubectl -n tuf-system get secrets tuf-root -ojsonpath='{.data.repository}'  | base64 -d > ./repository.tar.gz
         TUF_MIRROR=$(kubectl -n tuf-system get ksvc tuf -ojsonpath='{.status.url}')
         echo "TUF_MIRROR=$TUF_MIRROR" >> $GITHUB_ENV
         # Then initialize cosign
@@ -203,6 +205,30 @@ jobs:
       run: |
         cosign verify --rekor-url ${{ env.REKOR_URL }} --allow-insecure-registry ${{ env.demoimage }}
 
+    # Test with cosign in 'airgapped mode'
+    # Uncomment these once modified cosign goes in.
+    #- name: Checkout modified cosign for testing.
+    #  uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
+    #  with:
+    #    repository: vaikas/cosign
+    #    ref: air-gap
+    #    path: ./src/github.com/sigstore/cosign
+    #- name: Build cosign
+    #  working-directory: ./src/github.com/sigstore/cosign
+    #  run: |
+    #    go build -o ./cosign ./cmd/cosign/main.go
+    #- name: Untar the repository from the fetched secret, initialize and verify with it
+    #  working-directory: ./src/github.com/sigstore/cosign
+    #  run: |
+    #    # Also grab the compressed repository for airgap testing.
+    #    kubectl -n tuf-system get secrets tuf-root -ojsonpath='{.data.repository}'  | base64 -d > ./repository.tar.gz
+    #    tar -zxvf ./repository.tar.gz
+    #    PWD=$(pwd)
+    #    ROOT=${PWD}/repository/1.root.json
+    #    REPOSITORY=${PWD}/repository
+    #    ./cosign initialize --root ${ROOT} --mirror file://${REPOSITORY}
+    #    ./cosign verify --rekor-url ${{ env.REKOR_URL }} --allow-insecure-registry ${{ env.demoimage }}
+
     - name: Collect diagnostics
       if: ${{ failure() }}
       uses: chainguard-dev/actions/kind-diag@main

cmd/tuf/server/main.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"net/http"
@@ -33,7 +34,9 @@ import (
 
 var (
 	dir = flag.String("file-dir", "/var/run/tuf-secrets", "Directory where all the files that need to be added to TUF root live. File names are used to as targets.")
-	// Name of the "secret" initial 1.root.json.
+	// Name of the "secret" where we create two entries, one for:
+	// root = Which holds 1.root.json
+	// repository - Compressed repo, which has been tar/gzipped.
 	secretName = flag.String("rootsecret", "tuf-root", "Name of the secret to create for the initial root file")
 )
 
@@ -100,6 +103,15 @@ func main() {
 	data := make(map[string][]byte)
 	data["root"] = rootJSON
 
+	// Then compress the root directory and put it into a secret
+	// Secrets have 1MiB and the repository as tested goes to about ~3k, so no
+	// worries here.
+	var compressed bytes.Buffer
+	if err := repo.CompressFS(os.DirFS(dir), &compressed, map[string]bool{"keys": true, "staged": true}); err != nil {
+		logging.FromContext(ctx).Fatalf("Failed to compress the repo: %v", err)
+	}
+	data["repository"] = compressed.Bytes()
+
 	nsSecret := clientset.CoreV1().Secrets(ns)
 	if err := secret.ReconcileSecret(ctx, *secretName, ns, data, nsSecret); err != nil {
 		logging.FromContext(ctx).Panicf("Failed to reconcile secret %s/%s: %v", ns, *secretName, err)

pkg/repo/repo.go

@@ -15,11 +15,16 @@
 package repo
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"context"
 	"fmt"
+	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/theupdateframework/go-tuf"
@@ -102,3 +107,117 @@ func writeStagedTarget(dir, path string, data []byte) error {
 	}
 	return nil
 }
+
+// CompressFS archives a TUF repository so that it can be written to Secret
+// for later use.
+func CompressFS(fsys fs.FS, buf io.Writer, skipDirs map[string]bool) error {
+	// tar > gzip > buf
+	zr := gzip.NewWriter(buf)
+	tw := tar.NewWriter(zr)
+
+	err := fs.WalkDir(fsys, "repository", func(file string, d fs.DirEntry, err error) error {
+		// Skip the 'keys' and 'staged' directory
+		if d.IsDir() && skipDirs[d.Name()] {
+			return filepath.SkipDir
+		}
+
+		// Stat the file to get the details of it.
+		fi, err := fs.Stat(fsys, file)
+		if err != nil {
+			return fmt.Errorf("fs.Stat %s: %w", file, err)
+		}
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return fmt.Errorf("FileInfoHeader %s: %w", file, err)
+		}
+		header.Name = filepath.ToSlash(file)
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		// For files, write the contents.
+		if !d.IsDir() {
+			data, err := fsys.Open(file)
+			if err != nil {
+				return fmt.Errorf("opening %s: %w", file, err)
+			}
+			if _, err := io.Copy(tw, data); err != nil {
+				return fmt.Errorf("copying %s: %w", file, err)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		tw.Close()
+		zr.Close()
+		return fmt.Errorf("WalkDir: %w", err)
+	}
+
+	if err := tw.Close(); err != nil {
+		zr.Close()
+		return fmt.Errorf("tar.NewWriter Close(): %w", err)
+	}
+	return zr.Close()
+}
+
+// check for path traversal and correct forward slashes
+func validRelPath(p string) bool {
+	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+		return false
+	}
+	return true
+}
+
+// Uncompress takes a TUF repository that's been compressed with Compress and
+// writes to dst directory.
+func Uncompress(src io.Reader, dst string) error {
+	zr, err := gzip.NewReader(src)
+	if err != nil {
+		return err
+	}
+	tr := tar.NewReader(zr)
+
+	// uncompress each element
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break // End of archive
+		}
+		if err != nil {
+			return err
+		}
+		target := header.Name
+
+		// validate name against path traversal
+		if !validRelPath(header.Name) {
+			return fmt.Errorf("tar contained invalid name error %q\n", target)
+		}
+
+		// add dst + re-format slashes according to system
+		target = filepath.Join(dst, header.Name)
+		// check the type
+		switch header.Typeflag {
+		// Create directories
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, os.ModePerm); err != nil {
+					return err
+				}
+			}
+		// Write out files
+		case tar.TypeReg:
+			fileToWrite, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			// copy over contents
+			if _, err := io.Copy(fileToWrite, tr); err != nil {
+				return err
+			}
+			if err := fileToWrite.Close(); err != nil {
+				return fmt.Errorf("failed to close file %s: %w", target, err)
+			}
+		}
+	}
+	return nil
+}

pkg/repo/repo_test.go

@@ -15,8 +15,10 @@
 package repo
 
 import (
+	"bytes"
 	"context"
 	"os"
+	"path/filepath"
 	"testing"
 )
 
@@ -73,3 +75,52 @@ func TestCreateRepo(t *testing.T) {
 	}
 	t.Logf("Got repo meta as: %+v", meta)
 }
+
+func TestCompressUncompressFS(t *testing.T) {
+	files := map[string][]byte{
+		"fulcio_v1.crt.pem": []byte(fulcioRootCert),
+		"ctfe.pub":          []byte(ctlogPublicKey),
+		"rekor.pub":         []byte(rekorPublicKey),
+	}
+	repo, dir, err := CreateRepo(context.Background(), files)
+	if err != nil {
+		t.Fatalf("Failed to CreateRepo: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	var buf bytes.Buffer
+	fsys := os.DirFS(dir)
+	if err = CompressFS(fsys, &buf, map[string]bool{"keys": true, "staged": true}); err != nil {
+		t.Fatalf("Failed to compress: %v", err)
+	}
+	os.WriteFile("/tmp/newcompressed", buf.Bytes(), os.ModePerm)
+	dstDir := t.TempDir()
+	if err = Uncompress(&buf, dstDir); err != nil {
+		t.Fatalf("Failed to uncompress: %v", err)
+	}
+	// Then check that files have been uncompressed there.
+	meta, err := repo.GetMeta()
+	if err != nil {
+		t.Errorf("Failed to GetMeta: %s", err)
+	}
+	root := meta["root.json"]
+
+	// This should have roundtripped to the new directory.
+	rtRoot, err := os.ReadFile(filepath.Join(dstDir, "repository", "root.json"))
+	if err != nil {
+		t.Errorf("Failed to read the roundtripped root %v", err)
+	}
+	if bytes.Compare(root, rtRoot) != 0 {
+		t.Errorf("Roundtripped root differs:\n%s\n%s", string(root), string(rtRoot))
+	}
+
+	// As well as, say rekor.pub under targets dir
+	rtRekor, err := os.ReadFile(filepath.Join(dstDir, "repository", "targets", "rekor.pub"))
+	if err != nil {
+		t.Errorf("Failed to read the roundtripped rekor %v", err)
+	}
+	if bytes.Compare(files["rekor.pub"], rtRekor) != 0 {
+		t.Errorf("Roundtripped rekor differs:\n%s\n%s", rekorPublicKey, string(rtRekor))
+	}
+
+}