fix(sync): fixed skipping docker images when they already synced

before syncing an image we first check if it's already present in our storage to do that we get the manifest from remote and compare it with the local one but in the case of syncing docker images, because the conversion to OCI format is done while syncing, we get a docker manifest before conversion, so sync detects that local manifest and remote one are different, so it starts syncing again. to overcome this, convert remote docker manifests to OCI manifests and then compare. Signed-off-by: Petu Eusebiu <[email protected]>
project-zot · Jun 21, 2023 · 20d6240 · 20d6240
1 parent d5487d5
commit 20d6240
Show file tree

Hide file tree

Showing 5 changed files with 689 additions and 1 deletion.
diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go
@@ -9,8 +9,10 @@ import (
 
 	"github.com/containers/image/v5/docker"
 	dockerReference "github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/types"
 	"github.com/opencontainers/go-digest"
+	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"zotregistry.io/zot/pkg/api/constants"
 	"zotregistry.io/zot/pkg/common"
@@ -109,7 +111,21 @@ func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageRef
 		return []byte{}, "", "", err
 	}
 
-	return manifestBuf, mediaType, digest.FromBytes(manifestBuf), nil
+	// if mediatype is docker then convert to OCI
+	switch mediaType {
+	case manifest.DockerV2Schema2MediaType:
+		manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf)
+		if err != nil {
+			return []byte{}, "", "", err
+		}
+	case manifest.DockerV2ListMediaType:
+		manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf)
+		if err != nil {
+			return []byte{}, "", "", err
+		}
+	}
+
+	return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil
 }
 
 func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) {

diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go
@@ -9,8 +9,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path"
 	"testing"
 
+	dockerManifest "github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	godigest "github.com/opencontainers/go-digest"
@@ -426,3 +428,197 @@ func TestLocalRegistry(t *testing.T) {
 		})
 	})
 }
+
+func TestConvertDockerToOCI(t *testing.T) {
+	Convey("test converting docker to oci functions", t, func() {
+		dir := t.TempDir()
+
+		test.CopyTestFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
+
+		imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1")
+		So(err, ShouldBeNil)
+
+		imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
+		So(err, ShouldBeNil)
+
+		defer imageSource.Close()
+
+		Convey("trigger Unmarshal manifest error", func() {
+			_, err = convertDockerManifestToOCI(imageSource, []byte{})
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("trigger getImageConfigContent() error", func() {
+			manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
+			So(err, ShouldBeNil)
+
+			var manifest ispec.Manifest
+
+			err = json.Unmarshal(manifestBuf, &manifest)
+			So(err, ShouldBeNil)
+
+			err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000)
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("trigger Unmarshal config error", func() {
+			manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
+			So(err, ShouldBeNil)
+
+			var manifest ispec.Manifest
+
+			err = json.Unmarshal(manifestBuf, &manifest)
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()),
+				[]byte{}, storageConstants.DefaultFilePerms)
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("trigger convertDockerLayersToOCI error", func() {
+			manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
+			So(err, ShouldBeNil)
+
+			var manifest ispec.Manifest
+
+			err = json.Unmarshal(manifestBuf, &manifest)
+			So(err, ShouldBeNil)
+
+			manifestDigest := godigest.FromBytes(manifestBuf)
+
+			manifest.Layers[0].MediaType = "unknown"
+
+			newManifest, err := json.Marshal(manifest)
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()),
+				newManifest, storageConstants.DefaultFilePerms)
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("trigger convertDockerIndexToOCI error", func() {
+			manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerIndexToOCI(imageSource, manifestBuf)
+			So(err, ShouldNotBeNil)
+
+			// make zot-test image an index image
+
+			var manifest ispec.Manifest
+
+			err = json.Unmarshal(manifestBuf, &manifest)
+			So(err, ShouldBeNil)
+
+			dockerNewManifest := ispec.Manifest{
+				MediaType: dockerManifest.DockerV2Schema2MediaType,
+				Config:    manifest.Config,
+				Layers:    manifest.Layers,
+			}
+
+			dockerNewManifestBuf, err := json.Marshal(dockerNewManifest)
+			So(err, ShouldBeNil)
+
+			dockerManifestDigest := godigest.FromBytes(manifestBuf)
+
+			err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()),
+				dockerNewManifestBuf, storageConstants.DefaultFilePerms)
+			So(err, ShouldBeNil)
+
+			var index ispec.Index
+
+			index.Manifests = append(index.Manifests, ispec.Descriptor{
+				Digest:    dockerManifestDigest,
+				Size:      int64(len(dockerNewManifestBuf)),
+				MediaType: dockerManifest.DockerV2Schema2MediaType,
+			})
+
+			index.MediaType = dockerManifest.DockerV2ListMediaType
+
+			dockerIndexBuf, err := json.Marshal(index)
+			So(err, ShouldBeNil)
+
+			dockerIndexDigest := godigest.FromBytes(dockerIndexBuf)
+
+			err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()),
+				dockerIndexBuf, storageConstants.DefaultFilePerms)
+			So(err, ShouldBeNil)
+
+			// write index.json
+
+			var indexJSON ispec.Index
+
+			indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json"))
+			So(err, ShouldBeNil)
+
+			err = json.Unmarshal(indexJSONBuf, &indexJSON)
+			So(err, ShouldBeNil)
+
+			indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{
+				Digest:    dockerIndexDigest,
+				Size:      int64(len(dockerIndexBuf)),
+				MediaType: ispec.MediaTypeImageIndex,
+				Annotations: map[string]string{
+					ispec.AnnotationRefName: "0.0.2",
+				},
+			})
+
+			indexJSONBuf, err = json.Marshal(indexJSON)
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms)
+			So(err, ShouldBeNil)
+
+			imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2")
+			So(err, ShouldBeNil)
+
+			imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
+			So(err, ShouldNotBeNil)
+
+			err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000)
+			So(err, ShouldBeNil)
+
+			_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
+			So(err, ShouldNotBeNil)
+		})
+	})
+}
+
+func TestConvertDockerLayersToOCI(t *testing.T) {
+	Convey("test converting docker to oci functions", t, func() {
+		dockerLayers := []ispec.Descriptor{
+			{
+				MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType,
+			},
+			{
+				MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip,
+			},
+			{
+				MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed,
+			},
+			{
+				MediaType: dockerManifest.DockerV2Schema2LayerMediaType,
+			},
+		}
+
+		err := convertDockerLayersToOCI(dockerLayers)
+		So(err, ShouldBeNil)
+
+		So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable)     //nolint: staticcheck
+		So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck
+		So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer)
+		So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip)
+	})
+}