From 20d6240eb3dbc4c0df54b567cc6829a0b7ea0cd8 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Wed, 14 Jun 2023 10:39:03 +0300 Subject: [PATCH] fix(sync): fixed skipping docker images when they already synced before syncing an image we first check if it's already present in our storage to do that we get the manifest from remote and compare it with the local one but in the case of syncing docker images, because the conversion to OCI format is done while syncing, we get a docker manifest before conversion, so sync detects that local manifest and remote one are different, so it starts syncing again. to overcome this, convert remote docker manifests to OCI manifests and then compare. Signed-off-by: Petu Eusebiu --- pkg/extensions/sync/remote.go | 18 +- pkg/extensions/sync/sync_internal_test.go | 196 +++++++++++++ pkg/extensions/sync/sync_test.go | 328 ++++++++++++++++++++++ pkg/extensions/sync/utils.go | 130 +++++++++ test/blackbox/sync_docker.bats | 18 ++ 5 files changed, 689 insertions(+), 1 deletion(-) diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go index 0bb4a5bc22..0694e22b9c 100644 --- a/pkg/extensions/sync/remote.go +++ b/pkg/extensions/sync/remote.go @@ -9,8 +9,10 @@ import ( "github.com/containers/image/v5/docker" dockerReference "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.io/zot/pkg/api/constants" "zotregistry.io/zot/pkg/common" @@ -109,7 +111,21 @@ func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageRef return []byte{}, "", "", err } - return manifestBuf, mediaType, digest.FromBytes(manifestBuf), nil + // if mediatype is docker then convert to OCI + switch mediaType { + case manifest.DockerV2Schema2MediaType: + manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf) + if err != nil { + return []byte{}, "", "", err + } + case manifest.DockerV2ListMediaType: + manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf) + if err != nil { + return []byte{}, "", "", err + } + } + + return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil } func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) { diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index cca2c52709..8ef0fafd35 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -9,8 +9,10 @@ import ( "encoding/json" "fmt" "os" + "path" "testing" + dockerManifest "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" godigest "github.com/opencontainers/go-digest" @@ -426,3 +428,197 @@ func TestLocalRegistry(t *testing.T) { }) }) } + +func TestConvertDockerToOCI(t *testing.T) { + Convey("test converting docker to oci functions", t, func() { + dir := t.TempDir() + + test.CopyTestFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) + + imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1") + So(err, ShouldBeNil) + + imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) + So(err, ShouldBeNil) + + defer imageSource.Close() + + Convey("trigger Unmarshal manifest error", func() { + _, err = convertDockerManifestToOCI(imageSource, []byte{}) + So(err, ShouldNotBeNil) + }) + + Convey("trigger getImageConfigContent() error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger Unmarshal config error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), + []byte{}, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger convertDockerLayersToOCI error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + + manifest.Layers[0].MediaType = "unknown" + + newManifest, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()), + newManifest, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger convertDockerIndexToOCI error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + + // make zot-test image an index image + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + dockerNewManifest := ispec.Manifest{ + MediaType: dockerManifest.DockerV2Schema2MediaType, + Config: manifest.Config, + Layers: manifest.Layers, + } + + dockerNewManifestBuf, err := json.Marshal(dockerNewManifest) + So(err, ShouldBeNil) + + dockerManifestDigest := godigest.FromBytes(manifestBuf) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), + dockerNewManifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + var index ispec.Index + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: dockerManifestDigest, + Size: int64(len(dockerNewManifestBuf)), + MediaType: dockerManifest.DockerV2Schema2MediaType, + }) + + index.MediaType = dockerManifest.DockerV2ListMediaType + + dockerIndexBuf, err := json.Marshal(index) + So(err, ShouldBeNil) + + dockerIndexDigest := godigest.FromBytes(dockerIndexBuf) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()), + dockerIndexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + // write index.json + + var indexJSON ispec.Index + + indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + + err = json.Unmarshal(indexJSONBuf, &indexJSON) + So(err, ShouldBeNil) + + indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{ + Digest: dockerIndexDigest, + Size: int64(len(dockerIndexBuf)), + MediaType: ispec.MediaTypeImageIndex, + Annotations: map[string]string{ + ispec.AnnotationRefName: "0.0.2", + }, + }) + + indexJSONBuf, err = json.Marshal(indexJSON) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2") + So(err, ShouldBeNil) + + imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestConvertDockerLayersToOCI(t *testing.T) { + Convey("test converting docker to oci functions", t, func() { + dockerLayers := []ispec.Descriptor{ + { + MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType, + }, + { + MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip, + }, + { + MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed, + }, + { + MediaType: dockerManifest.DockerV2Schema2LayerMediaType, + }, + } + + err := convertDockerLayersToOCI(dockerLayers) + So(err, ShouldBeNil) + + So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck + So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck + So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer) + So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip) + }) +} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index a2889eb467..9b2829e2f0 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + dockerManifest "github.com/containers/image/v5/manifest" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -1101,6 +1102,333 @@ func TestSyncWithNonDistributableBlob(t *testing.T) { }) } +func TestDockerImagesAreSkipped(t *testing.T) { + Convey("Verify docker images are skipped when they are already synced", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false) + + scm := test.NewControllerManager(sctlr) + scm.StartAndWait(sctlr.Config.HTTP.Port) + defer scm.StopServer() + + var tlsVerify bool + + maxRetries := 1 + delay := 1 * time.Second + + indexRepoName := "index" + + syncRegistryConfig := syncconf.RegistryConfig{ + Content: []syncconf.Content{ + { + Prefix: testImage, + }, + { + Prefix: indexRepoName, + }, + }, + URLs: []string{srcBaseURL}, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + MaxRetries: &maxRetries, + OnDemand: true, + RetryDelay: &delay, + } + + defaultVal := true + syncConfig := &syncconf.Config{ + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, destDir, _ := makeDownstreamServer(t, false, syncConfig) + + Convey("skipping already synced docker image", func() { + // because we can not store images in docker format, modify the test image so that it has docker mediatype + indexContent, err := os.ReadFile(path.Join(srcDir, testImage, "index.json")) + So(err, ShouldBeNil) + So(indexContent, ShouldNotBeNil) + + var index ispec.Index + err = json.Unmarshal(indexContent, &index) + So(err, ShouldBeNil) + + var configBlobDigest godigest.Digest + + for idx, manifestDesc := range index.Manifests { + manifestContent, err := os.ReadFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestContent, &manifest) + So(err, ShouldBeNil) + + configBlobDigest = manifest.Config.Digest + + manifest.MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType + index.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + + for idx := range manifest.Layers { + manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + } + + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + index.Manifests[idx].Digest = manifestDigest + + // write modified manifest, remove old one + err = os.WriteFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDigest.Encoded()), + manifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = os.Remove(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + } + + indexBuf, err := json.Marshal(index) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, testImage, "index.json"), indexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // now it should be skipped + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, + "skipping image because it's already synced", 20*time.Second) + if err != nil { + panic(err) + } + + if !found { + data, err := os.ReadFile(dctlr.Config.Log.Output) + So(err, ShouldBeNil) + + t.Logf("downstream log: %s", string(data)) + } + + So(found, ShouldBeTrue) + + Convey("trigger config blob upstream error", func() { + // remove synced image + err := os.RemoveAll(path.Join(destDir, testImage)) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", configBlobDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) + }) + + Convey("skipping already synced multiarch docker image", func() { + // create an image index on upstream + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + // upload multiple manifests + for i := 0; i < 4; i++ { + config, layers, manifest, err := test.GetImageComponents(1000 + i) + So(err, ShouldBeNil) + + manifestContent, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestContent) + + err = test.UploadImage( + test.Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Reference: manifestDigest.String(), + }, + srcBaseURL, + "index") + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: manifestDigest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(manifestContent)), + }) + } + + content, err := json.Marshal(index) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(srcBaseURL + "/v2/index/manifests/latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(srcBaseURL + "/v2/index/manifests/latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // 'convert' oci multi arch image to docker multi arch + + indexContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "index.json")) + So(err, ShouldBeNil) + So(indexContent, ShouldNotBeNil) + + var newIndex ispec.Index + err = json.Unmarshal(indexContent, &newIndex) + So(err, ShouldBeNil) + + /* first find multiarch manifest in index.json + so that we can update both multiarch manifest and index.json at the same time*/ + var indexManifest ispec.Index + indexManifest.Manifests = make([]ispec.Descriptor, 4) + + var indexManifestIdx int + for idx, manifestDesc := range newIndex.Manifests { + if manifestDesc.MediaType == ispec.MediaTypeImageIndex { + indexManifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + err = json.Unmarshal(indexManifestContent, &indexManifest) + So(err, ShouldBeNil) + indexManifestIdx = idx + } + } + + var configBlobDigest godigest.Digest + var indexManifestContent []byte + for idx, manifestDesc := range newIndex.Manifests { + if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + manifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestContent, &manifest) + So(err, ShouldBeNil) + + configBlobDigest = manifest.Config.Digest + + manifest.MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType + newIndex.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + indexManifest.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + + for idx := range manifest.Layers { + manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + } + + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + newIndex.Manifests[idx].Digest = manifestDigest + indexManifest.Manifests[idx].Digest = manifestDigest + + // write modified manifest, remove old one + err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDigest.Encoded()), + manifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = os.Remove(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + } + + indexManifest.MediaType = dockerManifest.DockerV2ListMediaType + // write converted multi arch manifest + indexManifestContent, err = json.Marshal(indexManifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + godigest.FromBytes(indexManifestContent).Encoded()), indexManifestContent, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + } + + newIndex.Manifests[indexManifestIdx].MediaType = dockerManifest.DockerV2ListMediaType + newIndex.Manifests[indexManifestIdx].Digest = godigest.FromBytes(indexManifestContent) + + indexBuf, err := json.Marshal(newIndex) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, indexRepoName, "index.json"), indexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + // sync + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // sync again, should skip + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, + "skipping image because it's already synced", 20*time.Second) + if err != nil { + panic(err) + } + + if !found { + data, err := os.ReadFile(dctlr.Config.Log.Output) + So(err, ShouldBeNil) + + t.Logf("downstream log: %s", string(data)) + } + + So(found, ShouldBeTrue) + + Convey("trigger config blob upstream error", func() { + // remove synced image + err := os.RemoveAll(path.Join(destDir, indexRepoName)) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(srcDir, indexRepoName, "blobs/sha256", configBlobDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) + }) + }) +} + func TestPeriodically(t *testing.T) { Convey("Verify sync feature", t, func() { updateDuration, _ := time.ParseDuration("30m") diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 04e69d4cce..5eb210065c 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -4,6 +4,7 @@ package sync import ( + "bytes" "context" "encoding/json" "fmt" @@ -14,6 +15,7 @@ import ( "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" "github.com/docker/distribution/reference" @@ -170,3 +172,131 @@ func isSupportedMediaType(mediaType string) bool { return false } + +// given an imageSource and a docker manifest, convert it to OCI. +func convertDockerManifestToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { + var ociManifest ispec.Manifest + + // unmarshal docker manifest into OCI manifest + err := json.Unmarshal(dockerManifestBuf, &ociManifest) + if err != nil { + return []byte{}, err + } + + configContent, err := getImageConfigContent(imageSource, ociManifest.Config.Digest) + if err != nil { + return []byte{}, err + } + + // marshal config blob into OCI config, will remove keys specific to docker + var ociConfig ispec.Image + + err = json.Unmarshal(configContent, &ociConfig) + if err != nil { + return []byte{}, err + } + + ociConfigContent, err := json.Marshal(ociConfig) + if err != nil { + return []byte{}, err + } + + // convert layers + err = convertDockerLayersToOCI(ociManifest.Layers) + if err != nil { + return []byte{}, err + } + + // convert config and manifest mediatype + ociManifest.Config.Size = int64(len(ociConfigContent)) + ociManifest.Config.Digest = digest.FromBytes(ociConfigContent) + ociManifest.Config.MediaType = ispec.MediaTypeImageConfig + ociManifest.MediaType = ispec.MediaTypeImageManifest + + return json.Marshal(ociManifest) +} + +// convert docker layers mediatypes to OCI mediatypes. +func convertDockerLayersToOCI(dockerLayers []ispec.Descriptor) error { + for idx, layer := range dockerLayers { + switch layer.MediaType { + case manifest.DockerV2Schema2ForeignLayerMediaType: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck + case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributableGzip //nolint: staticcheck + case manifest.DockerV2SchemaLayerMediaTypeUncompressed: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayer + case manifest.DockerV2Schema2LayerMediaType: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerGzip + default: + return zerr.ErrMediaTypeNotSupported + } + } + + return nil +} + +// given an imageSource and a docker index manifest, convert it to OCI. +func convertDockerIndexToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { + // get docker index + originalIndex, err := manifest.ListFromBlob(dockerManifestBuf, manifest.DockerV2ListMediaType) + if err != nil { + return []byte{}, err + } + + // get manifests digests + manifestsDigests := originalIndex.Instances() + + manifestsUpdates := make([]manifest.ListUpdate, 0, len(manifestsDigests)) + + // convert each manifests in index from docker to OCI + for _, manifestDigest := range manifestsDigests { + digestCopy := manifestDigest + + indexManifestBuf, _, err := imageSource.GetManifest(context.Background(), &digestCopy) + if err != nil { + return []byte{}, err + } + + convertedIndexManifest, err := convertDockerManifestToOCI(imageSource, indexManifestBuf) + if err != nil { + return []byte{}, err + } + + manifestsUpdates = append(manifestsUpdates, manifest.ListUpdate{ + Digest: digest.FromBytes(convertedIndexManifest), + Size: int64(len(convertedIndexManifest)), + MediaType: ispec.MediaTypeImageManifest, + }) + } + + // update all manifests in index + if err := originalIndex.UpdateInstances(manifestsUpdates); err != nil { + return []byte{}, err + } + + // convert index to OCI + convertedList, err := originalIndex.ConvertToMIMEType(ispec.MediaTypeImageIndex) + if err != nil { + return []byte{}, err + } + + return convertedList.Serialize() +} + +// given an image source and a config blob digest, get blob config content. +func getImageConfigContent(imageSource types.ImageSource, configDigest digest.Digest, +) ([]byte, error) { + configBlob, _, err := imageSource.GetBlob(context.Background(), types.BlobInfo{ + Digest: configDigest, + }, none.NoCache) + if err != nil { + return nil, err + } + + configBuf := new(bytes.Buffer) + + _, err = configBuf.ReadFrom(configBlob) + + return configBuf.Bytes(), err +} diff --git a/test/blackbox/sync_docker.bats b/test/blackbox/sync_docker.bats index 47bda01e6d..625c33f626 100644 --- a/test/blackbox/sync_docker.bats +++ b/test/blackbox/sync_docker.bats @@ -129,6 +129,15 @@ function teardown_file() { run curl http://127.0.0.1:8090/v2/registry/tags/list [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] + + # make sure image is skipped when synced again + run skopeo --insecure-policy copy --multi-arch=all --src-tls-verify=false \ + docker://127.0.0.1:8090/registry \ + oci:${TEST_DATA_DIR} + [ "$status" -eq 0 ] + + run bash -c "cat /tmp/blackbox.log | grep -q registry:latest.*.skipping.image.because.it's.already.synced" + [ "$status" -eq 0 ] } @test "sync docker image on demand" { @@ -143,6 +152,15 @@ function teardown_file() { run curl http://127.0.0.1:8090/v2/archlinux/tags/list [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] + + # make sure image is skipped when synced again + run skopeo --insecure-policy copy --src-tls-verify=false \ + docker://127.0.0.1:8090/archlinux \ + oci:${TEST_DATA_DIR} + [ "$status" -eq 0 ] + + run bash -c "cat /tmp/blackbox.log | grep -q archlinux:latest.*.skipping.image.because.it's.already.synced" + [ "$status" -eq 0 ] } @test "sync k8s image list on demand" {