From f310777af25b2766c933b89d959ed9e34c8a1462 Mon Sep 17 00:00:00 2001 From: Piotr Halama Date: Tue, 18 Jun 2024 09:56:32 +0200 Subject: [PATCH] Add Azure storage support (#49) * Add Azure storage support * move cr-azure to examples * apply suggestions * pass Azure secrets as secrets * remove json info from azure secrets * rename getSecret function * remove example --- .../api/v1alpha1/dockerregistry_types.go | 19 +++++- .../api/v1alpha1/zz_generated.deepcopy.go | 57 +++++++++++++++- components/operator/internal/chart/flags.go | 18 +++++ .../operator/internal/registry/secret.go | 14 ++++ .../state/controller_configuration.go | 17 +++++ .../state/controller_configuration_test.go | 49 +++++++++++++- .../operator/internal/state/registry.go | 27 ++++++++ .../operator/internal/state/registry_test.go | 67 ++++++++++++++++++- components/operator/internal/state/state.go | 4 +- config/docker-registry/values.yaml | 2 - ...ator.kyma-project.io_dockerregistries.yaml | 12 ++++ 11 files changed, 279 insertions(+), 7 deletions(-) diff --git a/components/operator/api/v1alpha1/dockerregistry_types.go b/components/operator/api/v1alpha1/dockerregistry_types.go index 04fbd74b..c68fb7fd 100644 --- a/components/operator/api/v1alpha1/dockerregistry_types.go +++ b/components/operator/api/v1alpha1/dockerregistry_types.go @@ -28,7 +28,22 @@ type Endpoint struct { // DockerRegistrySpec defines the desired state of DockerRegistry type DockerRegistrySpec struct { // Sets the timeout for the Function health check. The default value in seconds is `10` - HealthzLivenessTimeout string `json:"healthzLivenessTimeout,omitempty"` //TODO: probably it was only used by serverless so it could be removed + HealthzLivenessTimeout string `json:"healthzLivenessTimeout,omitempty"` //TODO: probably it was only used by serverless so it could be removed + Storage *Storage `json:"storage,omitempty"` +} + +type Storage struct { + Azure *StorageAzure `json:"azure,omitempty"` +} + +type StorageAzure struct { + SecretName string `json:"secretName"` +} + +type StorageAzureSecrets struct { + AccountName string + AccountKey string + Container string } type State string @@ -75,6 +90,8 @@ const ( type DockerRegistryStatus struct { SecretName string `json:"secretName,omitempty"` + Storage string `json:"storage,omitempty"` + HealthzLivenessTimeout string `json:"healthzLivenessTimeout,omitempty"` // State signifies current state of DockerRegistry. diff --git a/components/operator/api/v1alpha1/zz_generated.deepcopy.go b/components/operator/api/v1alpha1/zz_generated.deepcopy.go index 74d48ab7..49222b1e 100644 --- a/components/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/components/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -30,7 +30,7 @@ func (in *DockerRegistry) DeepCopyInto(out *DockerRegistry) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -87,6 +87,11 @@ func (in *DockerRegistryList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DockerRegistrySpec) DeepCopyInto(out *DockerRegistrySpec) { *out = *in + if in.Storage != nil { + in, out := &in.Storage, &out.Storage + *out = new(Storage) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerRegistrySpec. @@ -135,3 +140,53 @@ func (in *Endpoint) DeepCopy() *Endpoint { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Storage) DeepCopyInto(out *Storage) { + *out = *in + if in.Azure != nil { + in, out := &in.Azure, &out.Azure + *out = new(StorageAzure) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Storage. +func (in *Storage) DeepCopy() *Storage { + if in == nil { + return nil + } + out := new(Storage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageAzure) DeepCopyInto(out *StorageAzure) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageAzure. +func (in *StorageAzure) DeepCopy() *StorageAzure { + if in == nil { + return nil + } + out := new(StorageAzure) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageAzureSecrets) DeepCopyInto(out *StorageAzureSecrets) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageAzureSecrets. +func (in *StorageAzureSecrets) DeepCopy() *StorageAzureSecrets { + if in == nil { + return nil + } + out := new(StorageAzureSecrets) + in.DeepCopyInto(out) + return out +} diff --git a/components/operator/internal/chart/flags.go b/components/operator/internal/chart/flags.go index 85e029f5..59079684 100644 --- a/components/operator/internal/chart/flags.go +++ b/components/operator/internal/chart/flags.go @@ -3,6 +3,8 @@ package chart import ( "fmt" "strings" + + "github.com/kyma-project/docker-registry/components/operator/api/v1alpha1" ) type FlagsBuilder interface { @@ -11,6 +13,8 @@ type FlagsBuilder interface { WithRegistryCredentials(username string, password string) *flagsBuilder WithRegistryHttpSecret(httpSecret string) *flagsBuilder WithNodePort(nodePort int64) *flagsBuilder + WithAzure(secret *v1alpha1.StorageAzureSecrets) *flagsBuilder + WithFilesystem() *flagsBuilder } type flagsBuilder struct { @@ -92,3 +96,17 @@ func (fb *flagsBuilder) WithNodePort(nodePort int64) *flagsBuilder { fb.flags["registryNodePort"] = nodePort return fb } + +func (fb *flagsBuilder) WithAzure(secret *v1alpha1.StorageAzureSecrets) *flagsBuilder { + fb.flags["storage"] = "azure" + fb.flags["secrets.azure.accountName"] = secret.AccountName + fb.flags["secrets.azure.accountKey"] = secret.AccountKey + fb.flags["secrets.azure.container"] = secret.Container + return fb +} + +func (fb *flagsBuilder) WithFilesystem() *flagsBuilder { + fb.flags["storage"] = "filesystem" + fb.flags["configData.storage.filesystem.rootdirectory"] = "/var/lib/registry" + return fb +} diff --git a/components/operator/internal/registry/secret.go b/components/operator/internal/registry/secret.go index 7a5cc2a6..37f829d2 100644 --- a/components/operator/internal/registry/secret.go +++ b/components/operator/internal/registry/secret.go @@ -59,3 +59,17 @@ func GetRegistryHTTPSecretEnvValue(ctx context.Context, c client.Client, namespa return "", nil } + +func GetSecret(ctx context.Context, c client.Client, name, namespace string) (*corev1.Secret, error) { + secret := corev1.Secret{} + key := client.ObjectKey{ + Namespace: namespace, + Name: name, + } + err := c.Get(ctx, key, &secret) + if err != nil { + return nil, err + } + + return &secret, nil +} diff --git a/components/operator/internal/state/controller_configuration.go b/components/operator/internal/state/controller_configuration.go index b6170ede..4e9307e3 100644 --- a/components/operator/internal/state/controller_configuration.go +++ b/components/operator/internal/state/controller_configuration.go @@ -8,6 +8,11 @@ import ( controllerruntime "sigs.k8s.io/controller-runtime" ) +const ( + AzureStorageName = "azure" + FilesystemStorageName = "filesystem" +) + func sFnControllerConfiguration(_ context.Context, r *reconciler, s *systemState) (stateFn, *controllerruntime.Result, error) { err := updateControllerConfigurationStatus(r, &s.instance) if err != nil { @@ -28,15 +33,27 @@ func sFnControllerConfiguration(_ context.Context, r *reconciler, s *systemState func updateControllerConfigurationStatus(r *reconciler, instance *v1alpha1.DockerRegistry) error { spec := instance.Spec + storageField := getStorageField(spec.Storage, instance) fields := fieldsToUpdate{ {spec.HealthzLivenessTimeout, &instance.Status.HealthzLivenessTimeout, "Duration of health check", ""}, {registry.SecretName, &instance.Status.SecretName, "Name of secret with registry access data", ""}, + storageField, } updateStatusFields(r.k8s, instance, fields) return nil } +func getStorageField(storage *v1alpha1.Storage, instance *v1alpha1.DockerRegistry) fieldToUpdate { + storageName := FilesystemStorageName + if storage != nil { + if storage.Azure != nil { + storageName = AzureStorageName + } + } + return fieldToUpdate{storageName, &instance.Status.Storage, "Storage type", ""} +} + func configureControllerConfigurationFlags(s *systemState) { s.flagsBuilder. WithControllerConfiguration( diff --git a/components/operator/internal/state/controller_configuration_test.go b/components/operator/internal/state/controller_configuration_test.go index 431231b2..f03bd848 100644 --- a/components/operator/internal/state/controller_configuration_test.go +++ b/components/operator/internal/state/controller_configuration_test.go @@ -2,9 +2,10 @@ package state import ( "context" - "github.com/kyma-project/docker-registry/components/operator/internal/registry" "testing" + "github.com/kyma-project/docker-registry/components/operator/internal/registry" + "github.com/kyma-project/docker-registry/components/operator/api/v1alpha1" "github.com/kyma-project/docker-registry/components/operator/internal/chart" "github.com/stretchr/testify/require" @@ -43,6 +44,52 @@ func Test_sFnControllerConfiguration(t *testing.T) { status := s.instance.Status require.Equal(t, healthzLivenessTimeoutTest, status.HealthzLivenessTimeout) require.Equal(t, registry.SecretName, status.SecretName) + require.Equal(t, FilesystemStorageName, status.Storage) + + require.Equal(t, v1alpha1.StateProcessing, status.State) + requireContainsCondition(t, status, + v1alpha1.ConditionTypeConfigured, + metav1.ConditionTrue, + v1alpha1.ConditionReasonConfigured, + configurationReadyMsg, + ) + + expectedEvents := []string{ + "Normal Configuration Duration of health check set from '' to 'test-healthz-liveness-timeout'", + } + + for _, expectedEvent := range expectedEvents { + require.Equal(t, expectedEvent, <-eventRecorder.Events) + } + }) + + t.Run("update status additional configuration overrides", func(t *testing.T) { + s := &systemState{ + instance: v1alpha1.DockerRegistry{ + Spec: v1alpha1.DockerRegistrySpec{ + HealthzLivenessTimeout: healthzLivenessTimeoutTest, + Storage: &v1alpha1.Storage{ + Azure: &v1alpha1.StorageAzure{ + SecretName: "azureSecret", + }, + }, + }, + }, + flagsBuilder: chart.NewFlagsBuilder(), + } + + c := fake.NewClientBuilder().Build() + eventRecorder := record.NewFakeRecorder(10) + r := &reconciler{log: zap.NewNop().Sugar(), k8s: k8s{client: c, EventRecorder: eventRecorder}} + next, result, err := sFnControllerConfiguration(context.TODO(), r, s) + require.Nil(t, err) + require.Nil(t, result) + requireEqualFunc(t, sFnApplyResources, next) + + status := s.instance.Status + require.Equal(t, healthzLivenessTimeoutTest, status.HealthzLivenessTimeout) + require.Equal(t, registry.SecretName, status.SecretName) + require.Equal(t, AzureStorageName, status.Storage) require.Equal(t, v1alpha1.StateProcessing, status.State) requireContainsCondition(t, status, diff --git a/components/operator/internal/state/registry.go b/components/operator/internal/state/registry.go index c39d56c6..fe26ed2b 100644 --- a/components/operator/internal/state/registry.go +++ b/components/operator/internal/state/registry.go @@ -2,6 +2,8 @@ package state import ( "context" + "fmt" + "github.com/kyma-project/docker-registry/components/operator/api/v1alpha1" "github.com/kyma-project/docker-registry/components/operator/internal/registry" "github.com/pkg/errors" @@ -55,6 +57,11 @@ func setInternalRegistryConfig(ctx context.Context, r *reconciler, s *systemStat ) } + err = prepareStorage(ctx, r, s) //s.instance.Spec.Storage, s.flagsBuilder) + if err != nil { + return errors.Wrap(err, "while preparing storage") + } + resolver := registry.NewNodePortResolver(registry.RandomNodePort) nodePort, err := resolver.ResolveDockerRegistryNodePortFn(ctx, r.client, s.instance.Namespace) if err != nil { @@ -64,3 +71,23 @@ func setInternalRegistryConfig(ctx context.Context, r *reconciler, s *systemStat s.flagsBuilder.WithNodePort(int64(nodePort)) return nil } + +func prepareStorage(ctx context.Context, r *reconciler, s *systemState) error { //storage *v1alpha1.Storage, flagsBuilder chart.FlagsBuilder, s *systemState) { + if s.instance.Spec.Storage != nil { + if s.instance.Spec.Storage.Azure != nil { + azureSecret, err := registry.GetSecret(ctx, r.client, s.instance.Spec.Storage.Azure.SecretName, s.instance.Namespace) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("while fetching azure storage secret from %s", s.instance.Namespace)) + } + storageAzureSecret := &v1alpha1.StorageAzureSecrets{ + AccountName: string(azureSecret.Data["accountName"]), + AccountKey: string(azureSecret.Data["accountKey"]), + Container: string(azureSecret.Data["container"]), + } + s.flagsBuilder.WithAzure(storageAzureSecret) + return nil + } + } + s.flagsBuilder.WithFilesystem() + return nil +} diff --git a/components/operator/internal/state/registry_test.go b/components/operator/internal/state/registry_test.go index 49bb4724..7898dff7 100644 --- a/components/operator/internal/state/registry_test.go +++ b/components/operator/internal/state/registry_test.go @@ -8,11 +8,13 @@ import ( "github.com/kyma-project/docker-registry/components/operator/internal/chart" "github.com/stretchr/testify/require" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func Test_sFnRegistryConfiguration(t *testing.T) { - t.Run("internal registry and update", func(t *testing.T) { + t.Run("internal registry using default storage", func(t *testing.T) { s := &systemState{ instance: v1alpha1.DockerRegistry{}, statusSnapshot: v1alpha1.DockerRegistryStatus{}, @@ -23,6 +25,69 @@ func Test_sFnRegistryConfiguration(t *testing.T) { log: zap.NewNop().Sugar(), } expectedFlags := map[string]interface{}{ + "configData": map[string]interface{}{ + "storage": map[string]interface{}{ + "filesystem": map[string]interface{}{ + "rootdirectory": "/var/lib/registry", + }, + }, + }, + "storage": "filesystem", + "registryNodePort": int64(32_137), + } + + next, result, err := sFnRegistryConfiguration(context.Background(), r, s) + require.NoError(t, err) + require.Nil(t, result) + requireEqualFunc(t, sFnControllerConfiguration, next) + + require.EqualValues(t, expectedFlags, s.flagsBuilder.Build()) + require.Equal(t, v1alpha1.StateProcessing, s.instance.Status.State) + }) + + t.Run("internal registry using azure storage", func(t *testing.T) { + s := &systemState{ + instance: v1alpha1.DockerRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kyma-system", + }, + Spec: v1alpha1.DockerRegistrySpec{ + Storage: &v1alpha1.Storage{ + Azure: &v1alpha1.StorageAzure{ + SecretName: "azureSecret", + }, + }, + }, + }, + statusSnapshot: v1alpha1.DockerRegistryStatus{}, + flagsBuilder: chart.NewFlagsBuilder(), + } + r := &reconciler{ + k8s: k8s{client: fake.NewClientBuilder().Build()}, + log: zap.NewNop().Sugar(), + } + azureSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azureSecret", + Namespace: "kyma-system", + }, + Data: map[string][]byte{ + "accountName": []byte("accountName"), + "accountKey": []byte("accountKey"), + "container": []byte("container"), + }, + } + require.NoError(t, r.k8s.client.Create(context.Background(), azureSecret)) + + expectedFlags := map[string]interface{}{ + "storage": "azure", + "secrets": map[string]interface{}{ + "azure": map[string]interface{}{ + "accountName": "accountName", + "accountKey": "accountKey", + "container": "container", + }, + }, "registryNodePort": int64(32_137), } diff --git a/components/operator/internal/state/state.go b/components/operator/internal/state/state.go index fdbf8a99..f2d4e41d 100644 --- a/components/operator/internal/state/state.go +++ b/components/operator/internal/state/state.go @@ -34,7 +34,9 @@ func requeueAfter(duration time.Duration) (stateFn, *ctrl.Result, error) { }, nil } -type fieldsToUpdate []struct { +type fieldsToUpdate []fieldToUpdate + +type fieldToUpdate struct { specField string statusField *string fieldName string diff --git a/config/docker-registry/values.yaml b/config/docker-registry/values.yaml index 129b1690..a122ec95 100644 --- a/config/docker-registry/values.yaml +++ b/config/docker-registry/values.yaml @@ -121,8 +121,6 @@ configData: # example: https://github.com/docker/distribution/blob/master/cmd/re storage: cache: blobdescriptor: inmemory - filesystem: - rootdirectory: /var/lib/registry http: addr: :5000 # same as .Values.service.port headers: diff --git a/config/operator/base/crd/bases/operator.kyma-project.io_dockerregistries.yaml b/config/operator/base/crd/bases/operator.kyma-project.io_dockerregistries.yaml index 088e8e70..2a80aec5 100644 --- a/config/operator/base/crd/bases/operator.kyma-project.io_dockerregistries.yaml +++ b/config/operator/base/crd/bases/operator.kyma-project.io_dockerregistries.yaml @@ -59,6 +59,16 @@ spec: description: Sets the timeout for the Function health check. The default value in seconds is `10` type: string + storage: + properties: + azure: + properties: + secretName: + type: string + required: + - secretName + type: object + type: object type: object status: properties: @@ -155,6 +165,8 @@ spec: - Error - Warning type: string + storage: + type: string required: - served type: object