From 11dc0a3bc7ec1b248810f87632c981b33552afcb Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 22 Aug 2022 15:51:42 +0300 Subject: [PATCH 1/3] Select layer by OCI media type Allow specifying the media type of the layer which should be extracted from the OCI artifact. Signed-off-by: Stefan Prodan --- api/v1beta2/ocirepository_types.go | 22 +++++++ api/v1beta2/zz_generated.deepcopy.go | 20 ++++++ ...rce.toolkit.fluxcd.io_ocirepositories.yaml | 10 +++ controllers/ocirepository_controller.go | 36 ++++++++++- controllers/ocirepository_controller_test.go | 14 ++-- docs/api/source.md | 64 +++++++++++++++++++ 6 files changed, 160 insertions(+), 6 deletions(-) diff --git a/api/v1beta2/ocirepository_types.go b/api/v1beta2/ocirepository_types.go index 83ff7f3ff..24ea674c4 100644 --- a/api/v1beta2/ocirepository_types.go +++ b/api/v1beta2/ocirepository_types.go @@ -60,6 +60,11 @@ type OCIRepositorySpec struct { // +optional Reference *OCIRepositoryRef `json:"ref,omitempty"` + // LayerSelector specifies which layer should be extracted from the OCI artifact. + // When not specified, the first layer found in the artifact is selected. + // +optional + LayerSelector *OCILayerSelector `json:"layerSelector,omitempty"` + // The provider used for authentication, can be 'aws', 'azure', 'gcp' or 'generic'. // When not specified, defaults to 'generic'. // +kubebuilder:validation:Enum=generic;aws;azure;gcp @@ -130,6 +135,14 @@ type OCIRepositoryRef struct { Tag string `json:"tag,omitempty"` } +// OCILayerSelector specifies which layer should be extracted from an OCI Artifact +type OCILayerSelector struct { + // MediaType specifies the OCI media type of the layer + // which should be extracted from the OCI Artifact. + // +optional + MediaType string `json:"mediaType,omitempty"` +} + // OCIRepositoryVerification verifies the authenticity of an OCI Artifact type OCIRepositoryVerification struct { // Provider specifies the technology used to sign the OCI Artifact. @@ -192,6 +205,15 @@ func (in *OCIRepository) GetArtifact() *Artifact { return in.Status.Artifact } +// GetLayerMediaType returns the media type layer selector if found in spec. +func (in *OCIRepository) GetLayerMediaType() string { + if in.Spec.LayerSelector == nil { + return "" + } + + return in.Spec.LayerSelector.MediaType +} + // +genclient // +genclient:Namespaced // +kubebuilder:storageversion diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index fc186d4df..25652de71 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -622,6 +622,21 @@ func (in *LocalHelmChartSourceReference) DeepCopy() *LocalHelmChartSourceReferen return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OCILayerSelector) DeepCopyInto(out *OCILayerSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCILayerSelector. +func (in *OCILayerSelector) DeepCopy() *OCILayerSelector { + if in == nil { + return nil + } + out := new(OCILayerSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OCIRepository) DeepCopyInto(out *OCIRepository) { *out = *in @@ -704,6 +719,11 @@ func (in *OCIRepositorySpec) DeepCopyInto(out *OCIRepositorySpec) { *out = new(OCIRepositoryRef) **out = **in } + if in.LayerSelector != nil { + in, out := &in.LayerSelector, &out.LayerSelector + *out = new(OCILayerSelector) + **out = **in + } if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef *out = new(meta.LocalObjectReference) diff --git a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml index 5e214ccd8..39c7fbd2e 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml @@ -75,6 +75,16 @@ spec: interval: description: The interval at which to check for image updates. type: string + layerSelector: + description: LayerSelector specifies which layer should be extracted + from the OCI artifact. When not specified, the first layer found + in the artifact is selected. + properties: + mediaType: + description: MediaType specifies the OCI media type of the layer + which should be extracted from the OCI Artifact. + type: string + type: object provider: default: generic description: The provider used for authentication, can be 'aws', 'azure', diff --git a/controllers/ocirepository_controller.go b/controllers/ocirepository_controller.go index 2a4993bbb..f9965842d 100644 --- a/controllers/ocirepository_controller.go +++ b/controllers/ocirepository_controller.go @@ -33,6 +33,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn/k8schain" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/name" + gcrv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -433,7 +434,40 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour return sreconcile.ResultEmpty, e } - blob, err := layers[0].Compressed() + var layer gcrv1.Layer + + switch { + case obj.GetLayerMediaType() != "": + var found bool + for i, l := range layers { + md, err := l.MediaType() + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + if string(md) == obj.GetLayerMediaType() { + layer = layers[i] + found = true + break + } + } + if !found { + e := serror.NewGeneric( + fmt.Errorf("failed to find layer with media type '%s' in artifact: %w", obj.GetLayerMediaType(), err), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + default: + layer = layers[0] + } + + blob, err := layer.Compressed() if err != nil { e := serror.NewGeneric( fmt.Errorf("failed to extract the first layer from artifact: %w", err), diff --git a/controllers/ocirepository_controller_test.go b/controllers/ocirepository_controller_test.go index b72413b1f..b138224df 100644 --- a/controllers/ocirepository_controller_test.go +++ b/controllers/ocirepository_controller_test.go @@ -80,13 +80,15 @@ func TestOCIRepository_Reconcile(t *testing.T) { tag string semver string digest string + mediaType string assertArtifact []artifactFixture }{ { - name: "public tag", - url: podinfoVersions["6.1.6"].url, - tag: podinfoVersions["6.1.6"].tag, - digest: podinfoVersions["6.1.6"].digest.Hex, + name: "public tag", + url: podinfoVersions["6.1.6"].url, + tag: podinfoVersions["6.1.6"].tag, + digest: podinfoVersions["6.1.6"].digest.Hex, + mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", assertArtifact: []artifactFixture{ { expectedPath: "kustomize/deployment.yaml", @@ -142,7 +144,9 @@ func TestOCIRepository_Reconcile(t *testing.T) { if tt.semver != "" { obj.Spec.Reference.SemVer = tt.semver } - + if tt.mediaType != "" { + obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: tt.mediaType} + } g.Expect(testEnv.Create(ctx, obj)).To(Succeed()) key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace} diff --git a/docs/api/source.md b/docs/api/source.md index 09f072743..b497c2688 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -968,6 +968,21 @@ defaults to the latest tag.

+layerSelector
+ + +OCILayerSelector + + + + +(Optional) +

LayerSelector specifies which layer should be extracted from the OCI artifact. +When not specified, the first layer found in the artifact is selected.

+ + + + provider
string @@ -2529,6 +2544,40 @@ string +

OCILayerSelector +

+

+(Appears on: +OCIRepositorySpec) +

+

OCILayerSelector specifies which layer should be extracted from an OCI Artifact

+
+
+ + + + + + + + + + + + + +
FieldDescription
+mediaType
+ +string + +		 +(Optional) +

MediaType specifies the OCI media type of the layer +which should be extracted from the OCI Artifact.

+
+
+

OCIRepositoryRef

@@ -2634,6 +2683,21 @@ defaults to the latest tag.

+layerSelector
+ + +OCILayerSelector + + + + +(Optional) +

LayerSelector specifies which layer should be extracted from the OCI artifact. +When not specified, the first layer found in the artifact is selected.

+ + + + provider
string From 49dc30922dd6356ba454583033a6a5b176eed799 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 24 Aug 2022 12:27:30 +0300 Subject: [PATCH 2/3] Add tests for OCI layer selector Signed-off-by: Stefan Prodan --- controllers/ocirepository_controller.go | 2 +- controllers/ocirepository_controller_test.go | 103 +++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/controllers/ocirepository_controller.go b/controllers/ocirepository_controller.go index f9965842d..58646313f 100644 --- a/controllers/ocirepository_controller.go +++ b/controllers/ocirepository_controller.go @@ -457,7 +457,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour } if !found { e := serror.NewGeneric( - fmt.Errorf("failed to find layer with media type '%s' in artifact: %w", obj.GetLayerMediaType(), err), + fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()), sourcev1.OCILayerOperationFailedReason, ) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) diff --git a/controllers/ocirepository_controller_test.go b/controllers/ocirepository_controller_test.go index b138224df..a0835100f 100644 --- a/controllers/ocirepository_controller_test.go +++ b/controllers/ocirepository_controller_test.go @@ -248,6 +248,109 @@ func TestOCIRepository_Reconcile(t *testing.T) { } } +func TestOCIRepository_Reconcile_MediaType(t *testing.T) { + g := NewWithT(t) + + // Registry server with public images + tmpDir := t.TempDir() + regServer, err := setupRegistryServer(ctx, tmpDir, registryOptions{}) + if err != nil { + g.Expect(err).ToNot(HaveOccurred()) + } + + podinfoVersions, err := pushMultiplePodinfoImages(regServer.registryHost, "6.1.4", "6.1.5", "6.1.6") + + tests := []struct { + name string + url string + tag string + mediaType string + wantErr bool + }{ + { + name: "Works with no media type", + url: podinfoVersions["6.1.4"].url, + tag: podinfoVersions["6.1.4"].tag, + }, + { + name: "Works with Flux CLI media type", + url: podinfoVersions["6.1.5"].url, + tag: podinfoVersions["6.1.5"].tag, + mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + }, + { + name: "Fails with unknown media type", + url: podinfoVersions["6.1.6"].url, + tag: podinfoVersions["6.1.6"].tag, + mediaType: "application/invalid.tar.gzip", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + g := NewWithT(t) + + ns, err := testEnv.CreateNamespace(ctx, "ocirepository-mediatype-test") + g.Expect(err).ToNot(HaveOccurred()) + defer func() { g.Expect(testEnv.Delete(ctx, ns)).To(Succeed()) }() + + obj := &sourcev1.OCIRepository{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ocirepository-reconcile", + Namespace: ns.Name, + }, + Spec: sourcev1.OCIRepositorySpec{ + URL: tt.url, + Interval: metav1.Duration{Duration: 60 * time.Minute}, + Reference: &sourcev1.OCIRepositoryRef{ + Tag: tt.tag, + }, + LayerSelector: &sourcev1.OCILayerSelector{ + MediaType: tt.mediaType, + }, + }, + } + + g.Expect(testEnv.Create(ctx, obj)).To(Succeed()) + + key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace} + + // Wait for the finalizer to be set + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return false + } + return len(obj.Finalizers) > 0 + }, timeout).Should(BeTrue()) + + // Wait for the object to be reconciled + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return false + } + readyCondition := conditions.Get(obj, meta.ReadyCondition) + return readyCondition != nil + }, timeout).Should(BeTrue()) + + g.Expect(conditions.IsReady(obj)).To(BeIdenticalTo(!tt.wantErr)) + if tt.wantErr { + g.Expect(conditions.Get(obj, meta.ReadyCondition).Message).Should(ContainSubstring("failed to find layer with media type")) + } + + // Wait for the object to be deleted + g.Expect(testEnv.Delete(ctx, obj)).To(Succeed()) + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return apierrors.IsNotFound(err) + } + return false + }, timeout).Should(BeTrue()) + }) + } +} + func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { type secretOptions struct { username string From e5cb32b0f248014f5512879dd93851a101c72e61 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 24 Aug 2022 12:46:04 +0300 Subject: [PATCH 3/3] Add OCI layer selector to API docs Signed-off-by: Stefan Prodan --- api/v1beta2/ocirepository_types.go | 3 ++- ...rce.toolkit.fluxcd.io_ocirepositories.yaml | 3 ++- docs/api/source.md | 3 ++- docs/spec/v1beta2/ocirepositories.md | 24 +++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/api/v1beta2/ocirepository_types.go b/api/v1beta2/ocirepository_types.go index 24ea674c4..5c89a4ac0 100644 --- a/api/v1beta2/ocirepository_types.go +++ b/api/v1beta2/ocirepository_types.go @@ -138,7 +138,8 @@ type OCIRepositoryRef struct { // OCILayerSelector specifies which layer should be extracted from an OCI Artifact type OCILayerSelector struct { // MediaType specifies the OCI media type of the layer - // which should be extracted from the OCI Artifact. + // which should be extracted from the OCI Artifact. The + // first layer matching this type is selected. // +optional MediaType string `json:"mediaType,omitempty"` } diff --git a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml index 39c7fbd2e..d5308a130 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml @@ -82,7 +82,8 @@ spec: properties: mediaType: description: MediaType specifies the OCI media type of the layer - which should be extracted from the OCI Artifact. + which should be extracted from the OCI Artifact. The first layer + matching this type is selected. type: string type: object provider: diff --git a/docs/api/source.md b/docs/api/source.md index b497c2688..ec0b1daf7 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -2571,7 +2571,8 @@ string (Optional)

MediaType specifies the OCI media type of the layer -which should be extracted from the OCI Artifact.

+which should be extracted from the OCI Artifact. The +first layer matching this type is selected.

diff --git a/docs/spec/v1beta2/ocirepositories.md b/docs/spec/v1beta2/ocirepositories.md index d540d8131..6bb67650b 100644 --- a/docs/spec/v1beta2/ocirepositories.md +++ b/docs/spec/v1beta2/ocirepositories.md @@ -368,6 +368,30 @@ spec: This field takes precedence over all other fields. +### Layer selector + +`spec.layerSelector` is an optional field to specify which layer should be extracted from the OCI Artifact. +If not specified, the controller will extract the first layer found in the artifact. + +To extract a layer matching a specific +[OCI media type](https://github.com/opencontainers/image-spec/blob/v1.0.2/media-types.md): + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: +spec: + layerSelector: + mediaType: "application/deployment.content.v1.tar+gzip" +``` + +If the layer selector matches more than one layer, the first layer matching the specified media type will be used. +Note that the selected OCI layer must be +[compressed](https://github.com/opencontainers/image-spec/blob/v1.0.2/layer.md#gzip-media-types) +in the `tar+gzip` format. + ### Ignore `.spec.ignore` is an optional field to specify rules in [the `.gitignore`