feat: implement gc command (#1811)

* feat: implement prune flag

* address changes

* revert

* boilerplate

* rename

* boilerplate

* Update .gitattributes

---------

Co-authored-by: Jason Hall <[email protected]>

.gitattributes

@@ -5,3 +5,4 @@
 **/zz_deepcopy_generated.go linguist-generated=true
 cmd/crane/doc/crane*.md linguist-generated=true
 go.sum linguist-generated=true
+**/testdata/** ignore-lint=true

cmd/crane/cmd/gc.go

@@ -0,0 +1,66 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/spf13/cobra"
+)
+
+func NewCmdLayout() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "layout",
+	}
+	cmd.AddCommand(newCmdGc())
+	return cmd
+}
+
+// NewCmdGc creates a new cobra.Command for the pull subcommand.
+func newCmdGc() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "gc OCI-LAYOUT",
+		Short:  "Garbage collect unreferenced blobs in a local oci-layout",
+		Args:   cobra.ExactArgs(1),
+		Hidden: true, // TODO: promote to public once theres some milage
+		RunE: func(_ *cobra.Command, args []string) error {
+			path := args[0]
+
+			p, err := layout.FromPath(path)
+
+			if err != nil {
+				return err
+			}
+
+			blobs, err := p.GarbageCollect()
+			if err != nil {
+				return err
+			}
+
+			for _, blob := range blobs {
+				if err := p.RemoveBlob(blob); err != nil {
+					return err
+				}
+				fmt.Fprintf(os.Stderr, "garbage collecting: %s\n", blob.String())
+			}
+
+			return nil
+		},
+	}
+
+	return cmd
+}

cmd/crane/cmd/root.go

@@ -129,7 +129,8 @@ func New(use, short string, options []crane.Option) *cobra.Command {
 		NewCmdTag(&options),
 		NewCmdValidate(&options),
 		NewCmdVersion(),
-		newCmdRegistry(),
+		NewCmdRegistry(),
+		NewCmdLayout(),
 	)
 
 	root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable debug logs")

cmd/crane/cmd/serve.go

@@ -28,7 +28,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/registry"
 )
 
-func newCmdRegistry() *cobra.Command {
+func NewCmdRegistry() *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "registry",
 	}

pkg/v1/layout/gc.go

@@ -0,0 +1,137 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice.
+package layout
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+)
+
+// GarbageCollect removes unreferenced blobs from the oci-layout
+//
+//	This is an experimental api, and not subject to any stability guarantees
+//	We may abandon it at any time, without prior notice.
+//	Deprecated: Use it at your own risk!
+func (l Path) GarbageCollect() ([]v1.Hash, error) {
+	idx, err := l.ImageIndex()
+	if err != nil {
+		return nil, err
+	}
+	blobsToKeep := map[string]bool{}
+	if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
+		return nil, err
+	}
+	blobsDir := l.path("blobs")
+	removedBlobs := []v1.Hash{}
+
+	err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		rel, err := filepath.Rel(blobsDir, path)
+		if err != nil {
+			return err
+		}
+		hashString := strings.Replace(rel, "/", ":", 1)
+		if present := blobsToKeep[hashString]; !present {
+			h, err := v1.NewHash(hashString)
+			if err != nil {
+				return err
+			}
+			removedBlobs = append(removedBlobs, h)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return removedBlobs, nil
+}
+
+func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error {
+	idxm, err := index.IndexManifest()
+	if err != nil {
+		return err
+	}
+
+	h, err := index.Digest()
+	if err != nil {
+		return err
+	}
+
+	blobsToKeep[h.String()] = true
+
+	for _, descriptor := range idxm.Manifests {
+		if descriptor.MediaType.IsImage() {
+			img, err := index.Image(descriptor.Digest)
+			if err != nil {
+				return err
+			}
+			if err := l.garbageCollectImage(img, blobsToKeep); err != nil {
+				return err
+			}
+		} else if descriptor.MediaType.IsIndex() {
+			idx, err := index.ImageIndex(descriptor.Digest)
+			if err != nil {
+				return err
+			}
+			if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
+				return err
+			}
+		} else {
+			return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType)
+		}
+	}
+	return nil
+}
+
+func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error {
+	h, err := image.Digest()
+	if err != nil {
+		return err
+	}
+	blobsToKeep[h.String()] = true
+
+	h, err = image.ConfigName()
+	if err != nil {
+		return err
+	}
+	blobsToKeep[h.String()] = true
+
+	ls, err := image.Layers()
+	if err != nil {
+		return err
+	}
+	for _, l := range ls {
+		h, err := l.Digest()
+		if err != nil {
+			return err
+		}
+		blobsToKeep[h.String()] = true
+	}
+	return nil
+}

pkg/v1/layout/gc_test.go

@@ -0,0 +1,96 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package layout
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+var (
+	gcIndexPath              = filepath.Join("testdata", "test_gc_index")
+	gcIndexBlobHash          = "sha256:492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0"
+	gcUnknownMediaTypePath   = filepath.Join("testdata", "test_gc_image_unknown_mediatype")
+	gcUnknownMediaTypeErr    = "gc: unknown media type: application/vnd.oci.descriptor.v1+json"
+	gcTestOneImagePath       = filepath.Join("testdata", "test_index_one_image")
+	gcTestIndexMediaTypePath = filepath.Join("testdata", "test_index_media_type")
+)
+
+func TestGcIndex(t *testing.T) {
+	lp, err := FromPath(gcIndexPath)
+	if err != nil {
+		t.Fatalf("FromPath() = %v", err)
+	}
+
+	removed, err := lp.GarbageCollect()
+	if err != nil {
+		t.Fatalf("GarbageCollect() = %v", err)
+	}
+
+	if len(removed) != 1 {
+		t.Fatalf("expected to have only one gc-able blob")
+	}
+	if removed[0].String() != gcIndexBlobHash {
+		t.Fatalf("wrong blob is gc-ed: expected '%s', got '%s'", gcIndexBlobHash, removed[0].String())
+	}
+}
+
+func TestGcOneImage(t *testing.T) {
+	lp, err := FromPath(gcTestOneImagePath)
+	if err != nil {
+		t.Fatalf("FromPath() = %v", err)
+	}
+
+	removed, err := lp.GarbageCollect()
+	if err != nil {
+		t.Fatalf("GarbageCollect() = %v", err)
+	}
+
+	if len(removed) != 0 {
+		t.Fatalf("expected to have to gc-able blobs")
+	}
+}
+
+func TestGcIndexMediaType(t *testing.T) {
+	lp, err := FromPath(gcTestIndexMediaTypePath)
+	if err != nil {
+		t.Fatalf("FromPath() = %v", err)
+	}
+
+	removed, err := lp.GarbageCollect()
+	if err != nil {
+		t.Fatalf("GarbageCollect() = %v", err)
+	}
+
+	if len(removed) != 0 {
+		t.Fatalf("expected to have to gc-able blobs")
+	}
+}
+
+func TestGcUnknownMediaType(t *testing.T) {
+	lp, err := FromPath(gcUnknownMediaTypePath)
+	if err != nil {
+		t.Fatalf("FromPath() = %v", err)
+	}
+
+	_, err = lp.GarbageCollect()
+	if err == nil {
+		t.Fatalf("expected GarbageCollect to return err but did not")
+	}
+
+	if err.Error() != gcUnknownMediaTypeErr {
+		t.Fatalf("expected error '%s', got '%s'", gcUnknownMediaTypeErr, err.Error())
+	}
+}

pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.descriptor.v1+json",
+      "size": 423,
+      "digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720"
+    }
+  ]
+}

pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/oci-layout

@@ -0,0 +1,3 @@
+{
+    "imageLayoutVersion": "1.0.0"
+}

pkg/v1/layout/testdata/test_gc_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5

@@ -0,0 +1,13 @@
+{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 423,
+      "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650",
+      "annotations": {
+        "org.opencontainers.image.ref.name": "1"
+      }
+    }
+  ]
+}

pkg/v1/layout/testdata/test_gc_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb

@@ -0,0 +1,13 @@
+{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 423,
+      "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650",
+      "annotations": {
+        "org.opencontainers.image.ref.name": "4"
+      }
+    }
+  ]
+}

pkg/v1/layout/testdata/test_gc_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e

@@ -0,0 +1 @@
+{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}}

pkg/v1/layout/testdata/test_gc_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]}