From bebde0e31b810db5973d2304c697e06659375a2d Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Sat, 25 Nov 2023 14:12:14 +0100 Subject: [PATCH 1/8] Restructure ML Storage CR --- docs/api/ArangoMLStorage.V1Alpha1.md | 67 ++++++---- pkg/apis/ml/v1alpha1/storage_s3_spec.go | 58 -------- pkg/apis/ml/v1alpha1/storage_spec.go | 47 +++---- pkg/apis/ml/v1alpha1/storage_spec_backend.go | 42 ++++++ .../ml/v1alpha1/storage_spec_backend_s3.go | 116 ++++++++++++++++ pkg/apis/ml/v1alpha1/storage_spec_mode.go | 37 ++++++ .../ml/v1alpha1/storage_spec_mode_sidecar.go | 75 +++++++++++ pkg/apis/ml/v1alpha1/storage_spec_test.go | 46 +++++-- pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go | 124 ++++++++++++++++-- 9 files changed, 470 insertions(+), 142 deletions(-) delete mode 100644 pkg/apis/ml/v1alpha1/storage_s3_spec.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_backend.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_mode.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index a15311fc2..a3f99b02d 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -2,71 +2,84 @@ ## Spec -### .spec.listenPort +### .spec.backend.s3.allowInsecure -Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec.go#L32) +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L39) -ListenPort defines on which port the sidecar container will be listening for connections +AllowInsecure if set to true, the Endpoint certificates won't be checked -Default Value: `9201` +Default Value: `false` *** -### .spec.resources +### .spec.backend.s3.bucketName -Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec.go#L37) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L33) -Resources holds resource requests & limits for container running the S3 proxy +BucketName specifies the name of the bucket +Required -Links: -* [Documentation of core.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core) +*** + +### .spec.backend.s3.caSecretName + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L45) + +CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint +The specified `Secret`, must contain the following data fields: +- `ca.crt` PEM encoded public key of the CA certificate +- `ca.key` PEM encoded private key of the CA certificate + +Default Value: `""` *** -### .spec.s3.bucketName +### .spec.backend.s3.credentialsSecretName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L39) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L36) -BucketName specifies the name of the bucket +CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization Required *** -### .spec.s3.credentialsSecret +### .spec.backend.s3.endpoint -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L42) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L30) -CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization +Endpoint specifies the S3 API-compatible endpoint which implements storage Required *** -### .spec.s3.disableSSL +### .spec.backend.s3.region -Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L33) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L48) -DisableSSL if set to true, no certificate checks will be performed for Endpoint +Region defines the availability zone name. If empty, defaults to 'us-east-1' -Default Value: `false` +Default Value: `""` *** -### .spec.s3.endpoint +### .spec.mode.sidecar.listenPort -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L30) +Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L33) -Endpoint specifies the S3 API-compatible endpoint which implements storage -Required +ListenPort defines on which port the sidecar container will be listening for connections + +Default Value: `9201` *** -### .spec.s3.region +### .spec.mode.sidecar.resources -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L36) +Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L38) -Region defines the availability zone name. If empty, defaults to 'us-east-1' +Resources holds resource requests & limits for container running the S3 proxy -Default Value: `""` +Links: +* [Documentation of core.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core) ## Status diff --git a/pkg/apis/ml/v1alpha1/storage_s3_spec.go b/pkg/apis/ml/v1alpha1/storage_s3_spec.go deleted file mode 100644 index 33340f048..000000000 --- a/pkg/apis/ml/v1alpha1/storage_s3_spec.go +++ /dev/null @@ -1,58 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2023 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// - -package v1alpha1 - -import ( - "github.com/pkg/errors" -) - -type ArangoMLStorageS3Spec struct { - // Endpoint specifies the S3 API-compatible endpoint which implements storage - // Required - Endpoint string `json:"endpoint"` - // DisableSSL if set to true, no certificate checks will be performed for Endpoint - // +doc/default: false - DisableSSL bool `json:"disableSSL,omitempty"` - // Region defines the availability zone name. If empty, defaults to 'us-east-1' - // +doc/default: "" - Region string `json:"region,omitempty"` - // BucketName specifies the name of the bucket - // Required - BucketName string `json:"bucketName"` - // CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization - // Required - CredentialsSecretName string `json:"credentialsSecret"` -} - -func (s *ArangoMLStorageS3Spec) Validate() error { - if s.BucketName == "" { - return errors.New("S3 BucketName must be not empty") - } - - if s.Endpoint == "" { - return errors.New("S3 Endpoint must be not empty") - } - - if s.CredentialsSecretName == "" { - return errors.New("S3 CredentialsSecretName must be not empty") - } - return nil -} diff --git a/pkg/apis/ml/v1alpha1/storage_spec.go b/pkg/apis/ml/v1alpha1/storage_spec.go index 3e250cd70..0aed5c4ec 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec.go +++ b/pkg/apis/ml/v1alpha1/storage_spec.go @@ -21,47 +21,30 @@ package v1alpha1 import ( - "github.com/pkg/errors" - core "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" + "github.com/arangodb/kube-arangodb/pkg/apis/shared" ) type ArangoMLStorageSpec struct { - // ListenPort defines on which port the sidecar container will be listening for connections - // +doc/default: 9201 - ListenPort *uint16 `json:"listenPort,omitempty"` - - // Resources holds resource requests & limits for container running the S3 proxy - // +doc/type: core.ResourceRequirements - // +doc/link: Documentation of core.ResourceRequirements|https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core - Resources core.ResourceRequirements `json:"resources,omitempty"` - - S3 *ArangoMLStorageS3Spec `json:"s3,omitempty"` + Mode *ArangoMLStorageSpecMode `json:"mode,omitempty"` + Backend *ArangoMLStorageSpecBackend `json:"backend,omitempty"` } func (s *ArangoMLStorageSpec) Validate() error { - if s.S3 == nil { - return errors.New("Currently only s3 storage type is supported") - } - - return s.S3.Validate() -} - -// SetDefaults fills in missing defaults -func (s *ArangoMLStorageSpec) SetDefaults() { if s == nil { - return + s = &ArangoMLStorageSpec{} } - resources := s.Resources - if len(resources.Requests) == 0 { - resources.Requests = make(core.ResourceList) - resources.Requests[core.ResourceCPU] = resource.MustParse("100m") - resources.Requests[core.ResourceMemory] = resource.MustParse("100m") + if err := shared.WithErrors( + shared.PrefixResourceError("backend", s.Backend.Validate()), + ); err != nil { + return err } - if len(resources.Limits) == 0 { - resources.Limits = make(core.ResourceList) - resources.Limits[core.ResourceCPU] = resource.MustParse("250m") - resources.Limits[core.ResourceMemory] = resource.MustParse("250m") + + if err := shared.WithErrors(shared.PrefixResourceErrors("spec", + shared.PrefixResourceError("mode", s.Mode.Validate()), + )); err != nil { + return err } + + return nil } diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend.go b/pkg/apis/ml/v1alpha1/storage_spec_backend.go new file mode 100644 index 000000000..555d28235 --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend.go @@ -0,0 +1,42 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecBackend struct { + S3 *ArangoMLStorageSpecBackendS3 `json:"s3,omitempty"` +} + +func (s *ArangoMLStorageSpecBackend) Validate() error { + if s == nil { + return errors.Newf("Backend is not specified") + } + + if s.S3 == nil { + return errors.Newf("At least one backend needs to be defined") + } + + return shared.WithErrors(shared.PrefixResourceError("s3", s.S3.Validate())) +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go new file mode 100644 index 000000000..d84a2338f --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go @@ -0,0 +1,116 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "net/url" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecBackendS3 struct { + // Endpoint specifies the S3 API-compatible endpoint which implements storage + // Required + Endpoint *string `json:"endpoint"` + // BucketName specifies the name of the bucket + // Required + BucketName *string `json:"bucketName"` + // CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization + // Required + CredentialsSecretName *string `json:"credentialsSecretName"` + // AllowInsecure if set to true, the Endpoint certificates won't be checked + // +doc/default: false + AllowInsecure *bool `json:"allowInsecure,omitempty"` + // CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint + // The specified `Secret`, must contain the following data fields: + // - `ca.crt` PEM encoded public key of the CA certificate + // - `ca.key` PEM encoded private key of the CA certificate + // +doc/default: "" + CASecretName *string `json:"caSecretName,omitempty"` + // Region defines the availability zone name. If empty, defaults to 'us-east-1' + // +doc/default: "" + Region *string `json:"region,omitempty"` +} + +func (s *ArangoMLStorageSpecBackendS3) Validate() error { + if s == nil { + s = &ArangoMLStorageSpecBackendS3{} + } + + if s.GetBucketName() == "" { + return errors.New("bucketName must be not empty") + } + + if s.GetEndpoint() == "" { + return errors.New("endpoint must be not empty") + } + + if _, err := url.Parse(s.GetEndpoint()); err != nil { + return errors.Newf("invalid endpoint URL was provided: %s", err.Error()) + } + + if s.GetCredentialsSecretName() == "" { + return errors.New("credentialsSecretName must be not empty") + } + return nil +} + +func (s *ArangoMLStorageSpecBackendS3) GetEndpoint() string { + if s.Endpoint == nil { + return "" + } + return *s.Endpoint +} + +func (s *ArangoMLStorageSpecBackendS3) GetBucketName() string { + if s.BucketName == nil { + return "" + } + return *s.BucketName +} + +func (s *ArangoMLStorageSpecBackendS3) GetCredentialsSecretName() string { + if s.CredentialsSecretName == nil { + return "" + } + return *s.CredentialsSecretName +} + +func (s *ArangoMLStorageSpecBackendS3) GetAllowInsecure() bool { + if s.AllowInsecure == nil { + return false + } + return *s.AllowInsecure +} + +func (s *ArangoMLStorageSpecBackendS3) GetCASecretName() string { + if s.CASecretName == nil { + return "" + } + return *s.CASecretName +} + +func (s *ArangoMLStorageSpecBackendS3) GetRegion() string { + if s.Region == nil { + return "" + } + return *s.Region +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode.go b/pkg/apis/ml/v1alpha1/storage_spec_mode.go new file mode 100644 index 000000000..4a697c216 --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecMode struct { + Sidecar *ArangoMLStorageSpecModeSidecar `json:"sidecar,omitempty"` +} + +func (s *ArangoMLStorageSpecMode) Validate() error { + if s == nil { + return errors.Newf("Mode is not defined") + } + return shared.WithErrors(shared.PrefixResourceError("sidecar", s.Sidecar.Validate())) +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go new file mode 100644 index 000000000..a6185bffa --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go @@ -0,0 +1,75 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecModeSidecar struct { + // ListenPort defines on which port the sidecar container will be listening for connections + // +doc/default: 9201 + ListenPort *uint16 `json:"listenPort,omitempty"` + + // Resources holds resource requests & limits for container running the S3 proxy + // +doc/type: core.ResourceRequirements + // +doc/link: Documentation of core.ResourceRequirements|https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core + Resources *core.ResourceRequirements `json:"resources,omitempty"` +} + +func (s *ArangoMLStorageSpecModeSidecar) Validate() error { + if s == nil { + s = &ArangoMLStorageSpecModeSidecar{} + } + if s.GetListenPort() < 1 { + return errors.Newf("invalid listenPort value: must be positive") + } + return nil +} + +func (s *ArangoMLStorageSpecModeSidecar) GetListenPort() uint16 { + if s.ListenPort == nil { + return 9201 + } + return *s.ListenPort +} + +func (s *ArangoMLStorageSpecModeSidecar) GetResources() core.ResourceRequirements { + var resources core.ResourceRequirements + if s.Resources != nil { + resources = *s.Resources + } + + if len(resources.Requests) == 0 { + resources.Requests = make(core.ResourceList) + resources.Requests[core.ResourceCPU] = resource.MustParse("100m") + resources.Requests[core.ResourceMemory] = resource.MustParse("100Mi") + } + if len(resources.Limits) == 0 { + resources.Limits = make(core.ResourceList) + resources.Limits[core.ResourceCPU] = resource.MustParse("200m") + resources.Limits[core.ResourceMemory] = resource.MustParse("200Mi") + } + return resources +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_test.go b/pkg/apis/ml/v1alpha1/storage_spec_test.go index 02e5d1f78..2cdb4bce2 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_test.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_test.go @@ -25,23 +25,45 @@ import ( "github.com/stretchr/testify/require" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/arangodb/kube-arangodb/pkg/util" ) func Test_ArangoMLStorageSpec(t *testing.T) { - s := ArangoMLStorageSpec{ - ListenPort: nil, - Resources: core.ResourceRequirements{}, - S3: nil, - } - s.SetDefaults() + s := ArangoMLStorageSpec{} + require.Error(t, s.Validate()) + + s.Mode = &ArangoMLStorageSpecMode{} + s.Backend = &ArangoMLStorageSpecBackend{} require.Error(t, s.Validate()) - s.S3 = &ArangoMLStorageS3Spec{ - Endpoint: "some-endpoint", - DisableSSL: false, - Region: "", - BucketName: "test-bucket", - CredentialsSecretName: "some-secret", + s.Mode.Sidecar = &ArangoMLStorageSpecModeSidecar{} + s.Backend.S3 = &ArangoMLStorageSpecBackendS3{ + Endpoint: util.NewType("http://test.s3.example.com"), + BucketName: util.NewType("bucket"), + CredentialsSecretName: util.NewType("a-secret"), } require.NoError(t, s.Validate()) + + t.Run("default requests and limits assigned", func(t *testing.T) { + assignedRequirements := core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceCPU: resource.MustParse("200m"), + core.ResourceMemory: resource.MustParse("200Mi"), + }, + } + s.Mode.Sidecar.Resources = &assignedRequirements + + expectedRequirements := core.ResourceRequirements{ + Requests: assignedRequirements.Requests, + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("200m"), + core.ResourceMemory: resource.MustParse("200Mi"), + }, + } + + actualRequirements := s.Mode.Sidecar.GetResources() + require.Equal(t, expectedRequirements, actualRequirements) + }) } diff --git a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go index 70e817fdf..0f8f36ae6 100644 --- a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,7 @@ package v1alpha1 import ( v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -392,44 +393,141 @@ func (in *ArangoMLStorageList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ArangoMLStorageS3Spec) DeepCopyInto(out *ArangoMLStorageS3Spec) { +func (in *ArangoMLStorageSpec) DeepCopyInto(out *ArangoMLStorageSpec) { *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(ArangoMLStorageSpecMode) + (*in).DeepCopyInto(*out) + } + if in.Backend != nil { + in, out := &in.Backend, &out.Backend + *out = new(ArangoMLStorageSpecBackend) + (*in).DeepCopyInto(*out) + } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageS3Spec. -func (in *ArangoMLStorageS3Spec) DeepCopy() *ArangoMLStorageS3Spec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpec. +func (in *ArangoMLStorageSpec) DeepCopy() *ArangoMLStorageSpec { if in == nil { return nil } - out := new(ArangoMLStorageS3Spec) + out := new(ArangoMLStorageSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ArangoMLStorageSpec) DeepCopyInto(out *ArangoMLStorageSpec) { +func (in *ArangoMLStorageSpecBackend) DeepCopyInto(out *ArangoMLStorageSpecBackend) { + *out = *in + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(ArangoMLStorageSpecBackendS3) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecBackend. +func (in *ArangoMLStorageSpecBackend) DeepCopy() *ArangoMLStorageSpecBackend { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecBackendS3) DeepCopyInto(out *ArangoMLStorageSpecBackendS3) { + *out = *in + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint + *out = new(string) + **out = **in + } + if in.BucketName != nil { + in, out := &in.BucketName, &out.BucketName + *out = new(string) + **out = **in + } + if in.CredentialsSecretName != nil { + in, out := &in.CredentialsSecretName, &out.CredentialsSecretName + *out = new(string) + **out = **in + } + if in.AllowInsecure != nil { + in, out := &in.AllowInsecure, &out.AllowInsecure + *out = new(bool) + **out = **in + } + if in.CASecretName != nil { + in, out := &in.CASecretName, &out.CASecretName + *out = new(string) + **out = **in + } + if in.Region != nil { + in, out := &in.Region, &out.Region + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecBackendS3. +func (in *ArangoMLStorageSpecBackendS3) DeepCopy() *ArangoMLStorageSpecBackendS3 { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecBackendS3) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecMode) DeepCopyInto(out *ArangoMLStorageSpecMode) { + *out = *in + if in.Sidecar != nil { + in, out := &in.Sidecar, &out.Sidecar + *out = new(ArangoMLStorageSpecModeSidecar) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecMode. +func (in *ArangoMLStorageSpecMode) DeepCopy() *ArangoMLStorageSpecMode { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecModeSidecar) DeepCopyInto(out *ArangoMLStorageSpecModeSidecar) { *out = *in if in.ListenPort != nil { in, out := &in.ListenPort, &out.ListenPort *out = new(uint16) **out = **in } - in.Resources.DeepCopyInto(&out.Resources) - if in.S3 != nil { - in, out := &in.S3, &out.S3 - *out = new(ArangoMLStorageS3Spec) - **out = **in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpec. -func (in *ArangoMLStorageSpec) DeepCopy() *ArangoMLStorageSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecModeSidecar. +func (in *ArangoMLStorageSpecModeSidecar) DeepCopy() *ArangoMLStorageSpecModeSidecar { if in == nil { return nil } - out := new(ArangoMLStorageSpec) + out := new(ArangoMLStorageSpecModeSidecar) in.DeepCopyInto(out) return out } From 391840c813752a4775cb7baf746a6412ad9fa10e Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Sun, 26 Nov 2023 09:46:48 +0100 Subject: [PATCH 2/8] Restructure ML Storage CR --- docs/api/ArangoMLStorage.V1Alpha1.md | 12 +-- pkg/crd/crds/ml-storage.schema.generated.yaml | 86 +++++++++++-------- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index a3f99b02d..7422f8110 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -4,7 +4,7 @@ ### .spec.backend.s3.allowInsecure -Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L39) +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L41) AllowInsecure if set to true, the Endpoint certificates won't be checked @@ -14,7 +14,7 @@ Default Value: `false` ### .spec.backend.s3.bucketName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L33) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L35) BucketName specifies the name of the bucket Required @@ -23,7 +23,7 @@ Required ### .spec.backend.s3.caSecretName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L45) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L47) CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint The specified `Secret`, must contain the following data fields: @@ -36,7 +36,7 @@ Default Value: `""` ### .spec.backend.s3.credentialsSecretName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L36) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L38) CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization Required @@ -45,7 +45,7 @@ Required ### .spec.backend.s3.endpoint -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L30) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L32) Endpoint specifies the S3 API-compatible endpoint which implements storage Required @@ -54,7 +54,7 @@ Required ### .spec.backend.s3.region -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L48) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L50) Region defines the availability zone name. If empty, defaults to 'us-east-1' diff --git a/pkg/crd/crds/ml-storage.schema.generated.yaml b/pkg/crd/crds/ml-storage.schema.generated.yaml index e18e9bbba..7564d8d4a 100644 --- a/pkg/crd/crds/ml-storage.schema.generated.yaml +++ b/pkg/crd/crds/ml-storage.schema.generated.yaml @@ -3,45 +3,61 @@ v1alpha1: properties: spec: properties: - listenPort: - description: ListenPort defines on which port the sidecar container will be listening for connections - format: int32 - type: integer - resources: - description: Resources holds resource requests & limits for container running the S3 proxy + backend: properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string + s3: + properties: + allowInsecure: + description: AllowInsecure if set to true, the Endpoint certificates won't be checked + type: boolean + bucketName: + description: |- + BucketName specifies the name of the bucket + Required + type: string + caSecretName: + description: |- + CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint + The specified `Secret`, must contain the following data fields: + - `ca.crt` PEM encoded public key of the CA certificate + - `ca.key` PEM encoded private key of the CA certificate + type: string + credentialsSecretName: + description: |- + CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization + Required + type: string + endpoint: + description: |- + Endpoint specifies the S3 API-compatible endpoint which implements storage + Required + type: string + region: + description: Region defines the availability zone name. If empty, defaults to 'us-east-1' + type: string type: object type: object - s3: + mode: properties: - bucketName: - description: |- - BucketName specifies the name of the bucket - Required - type: string - credentialsSecret: - description: |- - CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization - Required - type: string - disableSSL: - description: DisableSSL if set to true, no certificate checks will be performed for Endpoint - type: boolean - endpoint: - description: |- - Endpoint specifies the S3 API-compatible endpoint which implements storage - Required - type: string - region: - description: Region defines the availability zone name. If empty, defaults to 'us-east-1' - type: string + sidecar: + properties: + listenPort: + description: ListenPort defines on which port the sidecar container will be listening for connections + format: int32 + type: integer + resources: + description: Resources holds resource requests & limits for container running the S3 proxy + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + type: object type: object type: object type: object From 923d4e2460898acf1c38b5346bdc363391d7383e Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Mon, 27 Nov 2023 11:18:48 +0100 Subject: [PATCH 3/8] Restructure ML Storage CR --- pkg/apis/ml/v1alpha1/storage_spec.go | 25 +++++++++++++------ pkg/apis/ml/v1alpha1/storage_spec_backend.go | 8 ++++++ .../ml/v1alpha1/storage_spec_backend_s3.go | 25 +++++++++++-------- pkg/apis/ml/v1alpha1/storage_spec_mode.go | 8 ++++++ .../ml/v1alpha1/storage_spec_mode_sidecar.go | 19 +++++++++----- pkg/apis/ml/v1alpha1/storage_spec_test.go | 9 +++++++ 6 files changed, 70 insertions(+), 24 deletions(-) diff --git a/pkg/apis/ml/v1alpha1/storage_spec.go b/pkg/apis/ml/v1alpha1/storage_spec.go index 0aed5c4ec..4d70176db 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec.go +++ b/pkg/apis/ml/v1alpha1/storage_spec.go @@ -25,22 +25,33 @@ import ( ) type ArangoMLStorageSpec struct { - Mode *ArangoMLStorageSpecMode `json:"mode,omitempty"` + // Mode defines how storage implementation should be deployed + Mode *ArangoMLStorageSpecMode `json:"mode,omitempty"` + // Backed defines how storage is implemented Backend *ArangoMLStorageSpecBackend `json:"backend,omitempty"` } +func (s *ArangoMLStorageSpec) GetMode() *ArangoMLStorageSpecMode { + if s == nil || s.Mode == nil { + return &ArangoMLStorageSpecMode{} + } + return s.Mode +} + +func (s *ArangoMLStorageSpec) GetBackend() *ArangoMLStorageSpecBackend { + if s == nil || s.Backend == nil { + return &ArangoMLStorageSpecBackend{} + } + return s.Backend +} + func (s *ArangoMLStorageSpec) Validate() error { if s == nil { s = &ArangoMLStorageSpec{} } - if err := shared.WithErrors( - shared.PrefixResourceError("backend", s.Backend.Validate()), - ); err != nil { - return err - } - if err := shared.WithErrors(shared.PrefixResourceErrors("spec", + shared.PrefixResourceError("backend", s.Backend.Validate()), shared.PrefixResourceError("mode", s.Mode.Validate()), )); err != nil { return err diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend.go b/pkg/apis/ml/v1alpha1/storage_spec_backend.go index 555d28235..eece830fd 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_backend.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend.go @@ -26,9 +26,17 @@ import ( ) type ArangoMLStorageSpecBackend struct { + // S3 backend implements storage as a proxy to the provided S3 API endpoint S3 *ArangoMLStorageSpecBackendS3 `json:"s3,omitempty"` } +func (s *ArangoMLStorageSpecBackend) GetS3() *ArangoMLStorageSpecBackendS3 { + if s == nil || s.S3 == nil { + return &ArangoMLStorageSpecBackendS3{} + } + return s.S3 +} + func (s *ArangoMLStorageSpecBackend) Validate() error { if s == nil { return errors.Newf("Backend is not specified") diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go index d84a2338f..c46513455 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go @@ -23,6 +23,7 @@ package v1alpha1 import ( "net/url" + "github.com/arangodb/kube-arangodb/pkg/apis/shared" "github.com/arangodb/kube-arangodb/pkg/util/errors" ) @@ -55,61 +56,63 @@ func (s *ArangoMLStorageSpecBackendS3) Validate() error { s = &ArangoMLStorageSpecBackendS3{} } + var errs []error + if s.GetBucketName() == "" { - return errors.New("bucketName must be not empty") + errs = append(errs, errors.New("bucketName must be not empty")) } if s.GetEndpoint() == "" { - return errors.New("endpoint must be not empty") + errs = append(errs, errors.New("endpoint must be not empty")) } if _, err := url.Parse(s.GetEndpoint()); err != nil { - return errors.Newf("invalid endpoint URL was provided: %s", err.Error()) + errs = append(errs, errors.Newf("invalid endpoint URL was provided: %s", err.Error())) } if s.GetCredentialsSecretName() == "" { - return errors.New("credentialsSecretName must be not empty") + errs = append(errs, errors.New("credentialsSecretName must be not empty")) } - return nil + return shared.WithErrors(errs...) } func (s *ArangoMLStorageSpecBackendS3) GetEndpoint() string { - if s.Endpoint == nil { + if s == nil || s.Endpoint == nil { return "" } return *s.Endpoint } func (s *ArangoMLStorageSpecBackendS3) GetBucketName() string { - if s.BucketName == nil { + if s == nil || s.BucketName == nil { return "" } return *s.BucketName } func (s *ArangoMLStorageSpecBackendS3) GetCredentialsSecretName() string { - if s.CredentialsSecretName == nil { + if s == nil || s.CredentialsSecretName == nil { return "" } return *s.CredentialsSecretName } func (s *ArangoMLStorageSpecBackendS3) GetAllowInsecure() bool { - if s.AllowInsecure == nil { + if s == nil || s.AllowInsecure == nil { return false } return *s.AllowInsecure } func (s *ArangoMLStorageSpecBackendS3) GetCASecretName() string { - if s.CASecretName == nil { + if s == nil || s.CASecretName == nil { return "" } return *s.CASecretName } func (s *ArangoMLStorageSpecBackendS3) GetRegion() string { - if s.Region == nil { + if s == nil || s.Region == nil { return "" } return *s.Region diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode.go b/pkg/apis/ml/v1alpha1/storage_spec_mode.go index 4a697c216..0dda1cdeb 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_mode.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode.go @@ -26,9 +26,17 @@ import ( ) type ArangoMLStorageSpecMode struct { + // Sidecar mode runs the storage implementation as a sidecar Sidecar *ArangoMLStorageSpecModeSidecar `json:"sidecar,omitempty"` } +func (s *ArangoMLStorageSpecMode) GetSidecar() *ArangoMLStorageSpecModeSidecar { + if s == nil || s.Sidecar == nil { + return &ArangoMLStorageSpecModeSidecar{} + } + return s.Sidecar +} + func (s *ArangoMLStorageSpecMode) Validate() error { if s == nil { return errors.Newf("Mode is not defined") diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go index a6185bffa..3bbb55ec0 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go @@ -27,6 +27,13 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" ) +var ( + defaultRequestsCPU = resource.MustParse("100m") + defaultRequestsMemory = resource.MustParse("100Mi") + defaultLimitsCPU = resource.MustParse("200m") + defaultLimitsMemory = resource.MustParse("200Mi") +) + type ArangoMLStorageSpecModeSidecar struct { // ListenPort defines on which port the sidecar container will be listening for connections // +doc/default: 9201 @@ -49,7 +56,7 @@ func (s *ArangoMLStorageSpecModeSidecar) Validate() error { } func (s *ArangoMLStorageSpecModeSidecar) GetListenPort() uint16 { - if s.ListenPort == nil { + if s == nil || s.ListenPort == nil { return 9201 } return *s.ListenPort @@ -57,19 +64,19 @@ func (s *ArangoMLStorageSpecModeSidecar) GetListenPort() uint16 { func (s *ArangoMLStorageSpecModeSidecar) GetResources() core.ResourceRequirements { var resources core.ResourceRequirements - if s.Resources != nil { + if s != nil && s.Resources != nil { resources = *s.Resources } if len(resources.Requests) == 0 { resources.Requests = make(core.ResourceList) - resources.Requests[core.ResourceCPU] = resource.MustParse("100m") - resources.Requests[core.ResourceMemory] = resource.MustParse("100Mi") + resources.Requests[core.ResourceCPU] = defaultRequestsCPU + resources.Requests[core.ResourceMemory] = defaultRequestsMemory } if len(resources.Limits) == 0 { resources.Limits = make(core.ResourceList) - resources.Limits[core.ResourceCPU] = resource.MustParse("200m") - resources.Limits[core.ResourceMemory] = resource.MustParse("200Mi") + resources.Limits[core.ResourceCPU] = defaultLimitsCPU + resources.Limits[core.ResourceMemory] = defaultLimitsMemory } return resources } diff --git a/pkg/apis/ml/v1alpha1/storage_spec_test.go b/pkg/apis/ml/v1alpha1/storage_spec_test.go index 2cdb4bce2..41fc10bcd 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_test.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_test.go @@ -33,12 +33,21 @@ import ( func Test_ArangoMLStorageSpec(t *testing.T) { s := ArangoMLStorageSpec{} require.Error(t, s.Validate()) + require.NotNil(t, s.GetMode()) + require.NotNil(t, s.GetBackend()) + require.NotNil(t, s.Mode.GetSidecar()) s.Mode = &ArangoMLStorageSpecMode{} + + require.NotNil(t, s.Backend.GetS3()) s.Backend = &ArangoMLStorageSpecBackend{} require.Error(t, s.Validate()) + require.NotNil(t, s.Mode.Sidecar.GetListenPort()) + require.NotNil(t, s.Mode.Sidecar.GetResources()) s.Mode.Sidecar = &ArangoMLStorageSpecModeSidecar{} + + require.Error(t, s.Backend.S3.Validate()) s.Backend.S3 = &ArangoMLStorageSpecBackendS3{ Endpoint: util.NewType("http://test.s3.example.com"), BucketName: util.NewType("bucket"), From 458ea8040d83cf7994d15ba59d24fd9b0b963bb5 Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Mon, 27 Nov 2023 11:19:29 +0100 Subject: [PATCH 4/8] Restructure ML Storage CR --- docs/api/ArangoMLStorage.V1Alpha1.md | 16 ++++++++-------- pkg/apis/ml/v1alpha1/storage_spec.go | 2 +- pkg/crd/crds/ml-storage.schema.generated.yaml | 4 ++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index 7422f8110..edf0b8c59 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -4,7 +4,7 @@ ### .spec.backend.s3.allowInsecure -Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L41) +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L42) AllowInsecure if set to true, the Endpoint certificates won't be checked @@ -14,7 +14,7 @@ Default Value: `false` ### .spec.backend.s3.bucketName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L35) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L36) BucketName specifies the name of the bucket Required @@ -23,7 +23,7 @@ Required ### .spec.backend.s3.caSecretName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L47) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L48) CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint The specified `Secret`, must contain the following data fields: @@ -36,7 +36,7 @@ Default Value: `""` ### .spec.backend.s3.credentialsSecretName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L38) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L39) CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization Required @@ -45,7 +45,7 @@ Required ### .spec.backend.s3.endpoint -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L32) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L33) Endpoint specifies the S3 API-compatible endpoint which implements storage Required @@ -54,7 +54,7 @@ Required ### .spec.backend.s3.region -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L50) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L51) Region defines the availability zone name. If empty, defaults to 'us-east-1' @@ -64,7 +64,7 @@ Default Value: `""` ### .spec.mode.sidecar.listenPort -Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L33) +Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L40) ListenPort defines on which port the sidecar container will be listening for connections @@ -74,7 +74,7 @@ Default Value: `9201` ### .spec.mode.sidecar.resources -Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L38) +Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L45) Resources holds resource requests & limits for container running the S3 proxy diff --git a/pkg/apis/ml/v1alpha1/storage_spec.go b/pkg/apis/ml/v1alpha1/storage_spec.go index 4d70176db..37e86e2ba 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec.go +++ b/pkg/apis/ml/v1alpha1/storage_spec.go @@ -27,7 +27,7 @@ import ( type ArangoMLStorageSpec struct { // Mode defines how storage implementation should be deployed Mode *ArangoMLStorageSpecMode `json:"mode,omitempty"` - // Backed defines how storage is implemented + // Backend defines how storage is implemented Backend *ArangoMLStorageSpecBackend `json:"backend,omitempty"` } diff --git a/pkg/crd/crds/ml-storage.schema.generated.yaml b/pkg/crd/crds/ml-storage.schema.generated.yaml index 7564d8d4a..f97a4e26b 100644 --- a/pkg/crd/crds/ml-storage.schema.generated.yaml +++ b/pkg/crd/crds/ml-storage.schema.generated.yaml @@ -4,8 +4,10 @@ v1alpha1: spec: properties: backend: + description: Backend defines how storage is implemented properties: s3: + description: S3 backend implements storage as a proxy to the provided S3 API endpoint properties: allowInsecure: description: AllowInsecure if set to true, the Endpoint certificates won't be checked @@ -38,8 +40,10 @@ v1alpha1: type: object type: object mode: + description: Mode defines how storage implementation should be deployed properties: sidecar: + description: Sidecar mode runs the storage implementation as a sidecar properties: listenPort: description: ListenPort defines on which port the sidecar container will be listening for connections From 6cab6859d584234be8268f094880bb2b95a6ac3d Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Mon, 27 Nov 2023 13:18:54 +0100 Subject: [PATCH 5/8] Introduce shared.Object --- docs/api/ArangoMLStorage.V1Alpha1.md | 46 +++++++----- internal/docs_test.go | 15 +++- .../ml/v1alpha1/storage_spec_backend_s3.go | 44 ++++++----- .../ml/v1alpha1/storage_spec_mode_sidecar.go | 3 +- pkg/apis/ml/v1alpha1/storage_spec_test.go | 10 ++- pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go | 17 +++-- pkg/apis/shared/v1/object.go | 75 +++++++++++++++++++ pkg/apis/shared/v1/zz_generated.deepcopy.go | 21 ++++++ pkg/crd/crds/ml-storage.schema.generated.yaml | 26 +++++-- 9 files changed, 197 insertions(+), 60 deletions(-) create mode 100644 pkg/apis/shared/v1/object.go diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index edf0b8c59..50d24130e 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -4,7 +4,7 @@ ### .spec.backend.s3.allowInsecure -Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L42) +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L43) AllowInsecure if set to true, the Endpoint certificates won't be checked @@ -14,38 +14,48 @@ Default Value: `false` ### .spec.backend.s3.bucketName -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L36) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L37) BucketName specifies the name of the bucket Required *** -### .spec.backend.s3.caSecretName +### .spec.backend.s3.caSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L48) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L23) -CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint -The specified `Secret`, must contain the following data fields: -- `ca.crt` PEM encoded public key of the CA certificate -- `ca.key` PEM encoded private key of the CA certificate +Name of the object -Default Value: `""` +*** + +### .spec.backend.s3.caSecret.namespace + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L6) + +Namespace of the object. Should default to the namespace of the parent object *** -### .spec.backend.s3.credentialsSecretName +### .spec.backend.s3.credentialsSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L39) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L23) -CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization -Required +Name of the object + +*** + +### .spec.backend.s3.credentialsSecret.namespace + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L6) + +Namespace of the object. Should default to the namespace of the parent object *** ### .spec.backend.s3.endpoint -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L33) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L34) Endpoint specifies the S3 API-compatible endpoint which implements storage Required @@ -54,9 +64,9 @@ Required ### .spec.backend.s3.region -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L51) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L52) -Region defines the availability zone name. If empty, defaults to 'us-east-1' +Region defines the availability zone name. Default Value: `""` @@ -64,7 +74,7 @@ Default Value: `""` ### .spec.mode.sidecar.listenPort -Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L40) +Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L41) ListenPort defines on which port the sidecar container will be listening for connections @@ -74,7 +84,7 @@ Default Value: `9201` ### .spec.mode.sidecar.resources -Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L45) +Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L46) Resources holds resource requests & limits for container running the S3 proxy diff --git a/internal/docs_test.go b/internal/docs_test.go index 8ca30cf55..06579a642 100644 --- a/internal/docs_test.go +++ b/internal/docs_test.go @@ -128,6 +128,8 @@ func Test_GenerateAPIDocs(t *testing.T) { root := os.Getenv("ROOT") require.NotEmpty(t, root) + sharedFields, sharedFilesSet := parseSourceFiles(t, fmt.Sprintf("%s/pkg/apis/shared/v1", root)) + // package path -> result doc file name -> name of the top-level field to be described -> field instance for reflection input := map[string]map[string]map[string]interface{}{ fmt.Sprintf("%s/pkg/apis/deployment/v1", root): { @@ -180,8 +182,17 @@ func Test_GenerateAPIDocs(t *testing.T) { resultPaths := make(map[string]string) for apiDir, docs := range input { - fields, fileSets := parseSourceFiles(t, apiDir) - util.CopyMap(resultPaths, generateDocs(t, docs, fields, fileSets)) + fields, fileSet := parseSourceFiles(t, apiDir) + + for n, f := range sharedFields { + fields[n] = f + } + sharedFilesSet.Iterate(func(file *token.File) bool { + fileSet.AddFile(file.Name(), fileSet.Base()+file.Base(), file.Size()) + return true + }) + + util.CopyMap(resultPaths, generateDocs(t, docs, fields, fileSet)) } generateIndex(t, resultPaths) } diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go index c46513455..11fbacc5e 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/arangodb/kube-arangodb/pkg/apis/shared" + sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" "github.com/arangodb/kube-arangodb/pkg/util/errors" ) @@ -34,19 +35,19 @@ type ArangoMLStorageSpecBackendS3 struct { // BucketName specifies the name of the bucket // Required BucketName *string `json:"bucketName"` - // CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization + // CredentialsSecret specifies the Kubernetes Secret containing AccessKey and SecretKey for S3 API authorization // Required - CredentialsSecretName *string `json:"credentialsSecretName"` + CredentialsSecret *sharedApi.Object `json:"credentialsSecret"` // AllowInsecure if set to true, the Endpoint certificates won't be checked // +doc/default: false AllowInsecure *bool `json:"allowInsecure,omitempty"` - // CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint - // The specified `Secret`, must contain the following data fields: + // CASecret if not empty, the given Kubernetes Secret will be used to check the authenticity of Endpoint + // The specified Secret, must contain the following data fields: // - `ca.crt` PEM encoded public key of the CA certificate // - `ca.key` PEM encoded private key of the CA certificate - // +doc/default: "" - CASecretName *string `json:"caSecretName,omitempty"` - // Region defines the availability zone name. If empty, defaults to 'us-east-1' + // +doc/default: nil + CASecret *sharedApi.Object `json:"caSecret,omitempty"` + // Region defines the availability zone name. // +doc/default: "" Region *string `json:"region,omitempty"` } @@ -59,20 +60,23 @@ func (s *ArangoMLStorageSpecBackendS3) Validate() error { var errs []error if s.GetBucketName() == "" { - errs = append(errs, errors.New("bucketName must be not empty")) + errs = append(errs, shared.PrefixResourceErrors("bucketName", errors.New("must be not empty"))) } if s.GetEndpoint() == "" { - errs = append(errs, errors.New("endpoint must be not empty")) + errs = append(errs, shared.PrefixResourceErrors("endpoint", errors.New("must be not empty"))) } if _, err := url.Parse(s.GetEndpoint()); err != nil { - errs = append(errs, errors.Newf("invalid endpoint URL was provided: %s", err.Error())) + errs = append(errs, shared.PrefixResourceErrors("endpoint", errors.Newf("invalid URL: %s", err.Error()))) } - if s.GetCredentialsSecretName() == "" { - errs = append(errs, errors.New("credentialsSecretName must be not empty")) + errs = append(errs, shared.PrefixResourceErrors("credentialsSecret", s.GetCredentialsSecret().Validate())) + + if caSecret := s.GetCASecret(); !caSecret.IsEmpty() { + errs = append(errs, shared.PrefixResourceErrors("caSecret", caSecret.Validate())) } + return shared.WithErrors(errs...) } @@ -90,11 +94,11 @@ func (s *ArangoMLStorageSpecBackendS3) GetBucketName() string { return *s.BucketName } -func (s *ArangoMLStorageSpecBackendS3) GetCredentialsSecretName() string { - if s == nil || s.CredentialsSecretName == nil { - return "" +func (s *ArangoMLStorageSpecBackendS3) GetCredentialsSecret() *sharedApi.Object { + if s == nil || s.CredentialsSecret == nil { + return &sharedApi.Object{} } - return *s.CredentialsSecretName + return s.CredentialsSecret } func (s *ArangoMLStorageSpecBackendS3) GetAllowInsecure() bool { @@ -104,11 +108,11 @@ func (s *ArangoMLStorageSpecBackendS3) GetAllowInsecure() bool { return *s.AllowInsecure } -func (s *ArangoMLStorageSpecBackendS3) GetCASecretName() string { - if s == nil || s.CASecretName == nil { - return "" +func (s *ArangoMLStorageSpecBackendS3) GetCASecret() *sharedApi.Object { + if s == nil || s.CASecret == nil { + return &sharedApi.Object{} } - return *s.CASecretName + return s.CASecret } func (s *ArangoMLStorageSpecBackendS3) GetRegion() string { diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go index 3bbb55ec0..4116a06e4 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go @@ -24,6 +24,7 @@ import ( core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "github.com/arangodb/kube-arangodb/pkg/apis/shared" "github.com/arangodb/kube-arangodb/pkg/util/errors" ) @@ -50,7 +51,7 @@ func (s *ArangoMLStorageSpecModeSidecar) Validate() error { s = &ArangoMLStorageSpecModeSidecar{} } if s.GetListenPort() < 1 { - return errors.Newf("invalid listenPort value: must be positive") + return shared.PrefixResourceErrors("database", errors.Newf("must be positive")) } return nil } diff --git a/pkg/apis/ml/v1alpha1/storage_spec_test.go b/pkg/apis/ml/v1alpha1/storage_spec_test.go index 41fc10bcd..159cdfedc 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_test.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_test.go @@ -27,6 +27,7 @@ import ( core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" "github.com/arangodb/kube-arangodb/pkg/util" ) @@ -49,9 +50,12 @@ func Test_ArangoMLStorageSpec(t *testing.T) { require.Error(t, s.Backend.S3.Validate()) s.Backend.S3 = &ArangoMLStorageSpecBackendS3{ - Endpoint: util.NewType("http://test.s3.example.com"), - BucketName: util.NewType("bucket"), - CredentialsSecretName: util.NewType("a-secret"), + Endpoint: util.NewType("http://test.s3.example.com"), + BucketName: util.NewType("bucket"), + CredentialsSecret: &sharedApi.Object{ + Name: "a-secret", + Namespace: nil, + }, } require.NoError(t, s.Validate()) diff --git a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go index 0f8f36ae6..250fb2645 100644 --- a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,7 @@ package v1alpha1 import ( v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + sharedv1 "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -452,20 +453,20 @@ func (in *ArangoMLStorageSpecBackendS3) DeepCopyInto(out *ArangoMLStorageSpecBac *out = new(string) **out = **in } - if in.CredentialsSecretName != nil { - in, out := &in.CredentialsSecretName, &out.CredentialsSecretName - *out = new(string) - **out = **in + if in.CredentialsSecret != nil { + in, out := &in.CredentialsSecret, &out.CredentialsSecret + *out = new(sharedv1.Object) + (*in).DeepCopyInto(*out) } if in.AllowInsecure != nil { in, out := &in.AllowInsecure, &out.AllowInsecure *out = new(bool) **out = **in } - if in.CASecretName != nil { - in, out := &in.CASecretName, &out.CASecretName - *out = new(string) - **out = **in + if in.CASecret != nil { + in, out := &in.CASecret, &out.CASecret + *out = new(sharedv1.Object) + (*in).DeepCopyInto(*out) } if in.Region != nil { in, out := &in.Region, &out.Region diff --git a/pkg/apis/shared/v1/object.go b/pkg/apis/shared/v1/object.go new file mode 100644 index 000000000..3a25caef6 --- /dev/null +++ b/pkg/apis/shared/v1/object.go @@ -0,0 +1,75 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1 + +import ( + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type Object struct { + // Name of the object + Name string `json:"name"` + + // Namespace of the object. Should default to the namespace of the parent object + Namespace *string `json:"namespace,omitempty"` +} + +func (o *Object) IsEmpty() bool { + return o == nil || + (o.Name == "" && o.Namespace != nil) +} + +func (o *Object) GetName() string { + if o == nil { + return "" + } + + return o.Name +} + +func (o *Object) GetNamespace(obj meta.Object) string { + if o != nil { + if n := o.Namespace; n != nil { + return *n + } + } + + return obj.GetNamespace() +} + +func (o *Object) Validate() error { + if o == nil { + o = &Object{} + } + + var errs []error + if o.Name == "" { + errs = append(errs, shared.PrefixResourceErrors("name", errors.New("must be not empty"))) + } + if o.Namespace != nil && *o.Namespace == "" { + errs = append(errs, shared.PrefixResourceErrors("namespace", errors.New("must be nil or non-empty string"))) + } + + return shared.WithErrors(errs...) +} diff --git a/pkg/apis/shared/v1/zz_generated.deepcopy.go b/pkg/apis/shared/v1/zz_generated.deepcopy.go index c356004dc..c996b978f 100644 --- a/pkg/apis/shared/v1/zz_generated.deepcopy.go +++ b/pkg/apis/shared/v1/zz_generated.deepcopy.go @@ -44,3 +44,24 @@ func (in HashList) DeepCopy() HashList { in.DeepCopyInto(out) return *out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Object) DeepCopyInto(out *Object) { + *out = *in + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Object. +func (in *Object) DeepCopy() *Object { + if in == nil { + return nil + } + out := new(Object) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/crd/crds/ml-storage.schema.generated.yaml b/pkg/crd/crds/ml-storage.schema.generated.yaml index f97a4e26b..b762f5c2f 100644 --- a/pkg/crd/crds/ml-storage.schema.generated.yaml +++ b/pkg/crd/crds/ml-storage.schema.generated.yaml @@ -17,25 +17,35 @@ v1alpha1: BucketName specifies the name of the bucket Required type: string - caSecretName: + caSecret: description: |- - CASecretName if not empty, the given secret will be used to check the authenticity of Endpoint - The specified `Secret`, must contain the following data fields: + CASecret if not empty, the given Kubernetes Secret will be used to check the authenticity of Endpoint + The specified Secret, must contain the following data fields: - `ca.crt` PEM encoded public key of the CA certificate - `ca.key` PEM encoded private key of the CA certificate - type: string - credentialsSecretName: + properties: + name: + type: string + namespace: + type: string + type: object + credentialsSecret: description: |- - CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization + CredentialsSecret specifies the Kubernetes Secret containing AccessKey and SecretKey for S3 API authorization Required - type: string + properties: + name: + type: string + namespace: + type: string + type: object endpoint: description: |- Endpoint specifies the S3 API-compatible endpoint which implements storage Required type: string region: - description: Region defines the availability zone name. If empty, defaults to 'us-east-1' + description: Region defines the availability zone name. type: string type: object type: object From ef1bc12fb12a64bdcd9c7e636520b1bcdba18b4a Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Mon, 27 Nov 2023 17:59:01 +0100 Subject: [PATCH 6/8] Use KubernetesResourceName helper for validation --- pkg/apis/shared/v1/object.go | 9 ++---- pkg/apis/shared/v1/object_test.go | 52 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 pkg/apis/shared/v1/object_test.go diff --git a/pkg/apis/shared/v1/object.go b/pkg/apis/shared/v1/object.go index 3a25caef6..cf103c463 100644 --- a/pkg/apis/shared/v1/object.go +++ b/pkg/apis/shared/v1/object.go @@ -24,7 +24,6 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/kube-arangodb/pkg/apis/shared" - "github.com/arangodb/kube-arangodb/pkg/util/errors" ) type Object struct { @@ -64,11 +63,9 @@ func (o *Object) Validate() error { } var errs []error - if o.Name == "" { - errs = append(errs, shared.PrefixResourceErrors("name", errors.New("must be not empty"))) - } - if o.Namespace != nil && *o.Namespace == "" { - errs = append(errs, shared.PrefixResourceErrors("namespace", errors.New("must be nil or non-empty string"))) + errs = append(errs, shared.PrefixResourceErrors("name", AsKubernetesResourceName(&o.Name).Validate())) + if o.Namespace != nil { + errs = append(errs, shared.PrefixResourceErrors("namespace", AsKubernetesResourceName(o.Namespace).Validate())) } return shared.WithErrors(errs...) diff --git a/pkg/apis/shared/v1/object_test.go b/pkg/apis/shared/v1/object_test.go new file mode 100644 index 000000000..3abe136f1 --- /dev/null +++ b/pkg/apis/shared/v1/object_test.go @@ -0,0 +1,52 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/arangodb/kube-arangodb/pkg/util" +) + +func Test_Object_Validate(t *testing.T) { + var o *Object + require.Error(t, o.Validate()) + + o = &Object{} + require.Error(t, o.Validate()) + + o.Name = "#invalid" + require.Error(t, o.Validate()) + + o.Name = "valid" + require.NoError(t, o.Validate()) + + o.Namespace = util.NewType("") + require.Error(t, o.Validate()) + + o.Namespace = util.NewType("#invalid") + require.Error(t, o.Validate()) + + o.Namespace = util.NewType("valid") + require.NoError(t, o.Validate()) +} From a7d2bd260682550d9bae68319beea7fe3ad1c197 Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Mon, 27 Nov 2023 18:17:10 +0100 Subject: [PATCH 7/8] Use KubernetesResourceName helper for validation --- docs/api/ArangoMLStorage.V1Alpha1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index 50d24130e..a721fb72e 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -23,7 +23,7 @@ Required ### .spec.backend.s3.caSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L23) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L18) Name of the object @@ -31,7 +31,7 @@ Name of the object ### .spec.backend.s3.caSecret.namespace -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L6) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L4) Namespace of the object. Should default to the namespace of the parent object @@ -39,7 +39,7 @@ Namespace of the object. Should default to the namespace of the parent object ### .spec.backend.s3.credentialsSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L23) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L18) Name of the object @@ -47,7 +47,7 @@ Name of the object ### .spec.backend.s3.credentialsSecret.namespace -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L6) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L4) Namespace of the object. Should default to the namespace of the parent object From 17fd41bf6ad3e74c5645079ad8fc4c3723a1ef78 Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Tue, 28 Nov 2023 07:32:42 +0100 Subject: [PATCH 8/8] Use KubernetesResourceName helper for validation --- docs/api/ArangoMLStorage.V1Alpha1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index a721fb72e..8c325c73c 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -23,7 +23,7 @@ Required ### .spec.backend.s3.caSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L18) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12) Name of the object @@ -31,7 +31,7 @@ Name of the object ### .spec.backend.s3.caSecret.namespace -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L4) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13) Namespace of the object. Should default to the namespace of the parent object @@ -39,7 +39,7 @@ Namespace of the object. Should default to the namespace of the parent object ### .spec.backend.s3.credentialsSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L18) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12) Name of the object @@ -47,7 +47,7 @@ Name of the object ### .spec.backend.s3.credentialsSecret.namespace -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob.go#L4) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13) Namespace of the object. Should default to the namespace of the parent object