From b2986a01ec6d01143527594f90ca9b73c9bb625e Mon Sep 17 00:00:00 2001 From: Nikita Vaniasin Date: Tue, 28 Nov 2023 08:05:53 +0100 Subject: [PATCH] Restructure ML Storage CR (#1508) --- docs/api/ArangoMLStorage.V1Alpha1.md | 77 +++++++---- internal/docs_test.go | 15 ++- pkg/apis/ml/v1alpha1/storage_s3_spec.go | 58 -------- pkg/apis/ml/v1alpha1/storage_spec.go | 54 ++++---- pkg/apis/ml/v1alpha1/storage_spec_backend.go | 50 +++++++ .../ml/v1alpha1/storage_spec_backend_s3.go | 123 +++++++++++++++++ pkg/apis/ml/v1alpha1/storage_spec_mode.go | 45 +++++++ .../ml/v1alpha1/storage_spec_mode_sidecar.go | 83 ++++++++++++ pkg/apis/ml/v1alpha1/storage_spec_test.go | 59 +++++++-- pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go | 125 ++++++++++++++++-- pkg/apis/shared/v1/object.go | 72 ++++++++++ pkg/apis/shared/v1/object_test.go | 52 ++++++++ pkg/apis/shared/v1/zz_generated.deepcopy.go | 21 +++ pkg/crd/crds/ml-storage.schema.generated.yaml | 100 +++++++++----- 14 files changed, 757 insertions(+), 177 deletions(-) delete mode 100644 pkg/apis/ml/v1alpha1/storage_s3_spec.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_backend.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_mode.go create mode 100644 pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go create mode 100644 pkg/apis/shared/v1/object.go create mode 100644 pkg/apis/shared/v1/object_test.go diff --git a/docs/api/ArangoMLStorage.V1Alpha1.md b/docs/api/ArangoMLStorage.V1Alpha1.md index a15311fc2..8c325c73c 100644 --- a/docs/api/ArangoMLStorage.V1Alpha1.md +++ b/docs/api/ArangoMLStorage.V1Alpha1.md @@ -2,72 +2,95 @@ ## Spec -### .spec.listenPort +### .spec.backend.s3.allowInsecure -Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec.go#L32) +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L43) -ListenPort defines on which port the sidecar container will be listening for connections +AllowInsecure if set to true, the Endpoint certificates won't be checked -Default Value: `9201` +Default Value: `false` *** -### .spec.resources +### .spec.backend.s3.bucketName -Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec.go#L37) - -Resources holds resource requests & limits for container running the S3 proxy +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L37) -Links: -* [Documentation of core.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core) +BucketName specifies the name of the bucket +Required *** -### .spec.s3.bucketName +### .spec.backend.s3.caSecret.name -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L39) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12) -BucketName specifies the name of the bucket -Required +Name of the object *** -### .spec.s3.credentialsSecret +### .spec.backend.s3.caSecret.namespace -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L42) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13) -CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization -Required +Namespace of the object. Should default to the namespace of the parent object *** -### .spec.s3.disableSSL +### .spec.backend.s3.credentialsSecret.name -Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L33) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12) -DisableSSL if set to true, no certificate checks will be performed for Endpoint +Name of the object -Default Value: `false` +*** + +### .spec.backend.s3.credentialsSecret.namespace + +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13) + +Namespace of the object. Should default to the namespace of the parent object *** -### .spec.s3.endpoint +### .spec.backend.s3.endpoint -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L30) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L34) Endpoint specifies the S3 API-compatible endpoint which implements storage Required *** -### .spec.s3.region +### .spec.backend.s3.region -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_s3_spec.go#L36) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L52) -Region defines the availability zone name. If empty, defaults to 'us-east-1' +Region defines the availability zone name. Default Value: `""` +*** + +### .spec.mode.sidecar.listenPort + +Type: `integer` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L41) + +ListenPort defines on which port the sidecar container will be listening for connections + +Default Value: `9201` + +*** + +### .spec.mode.sidecar.resources + +Type: `core.ResourceRequirements` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go#L46) + +Resources holds resource requests & limits for container running the S3 proxy + +Links: +* [Documentation of core.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core) + ## Status ### .status.conditions diff --git a/internal/docs_test.go b/internal/docs_test.go index 8ca30cf55..06579a642 100644 --- a/internal/docs_test.go +++ b/internal/docs_test.go @@ -128,6 +128,8 @@ func Test_GenerateAPIDocs(t *testing.T) { root := os.Getenv("ROOT") require.NotEmpty(t, root) + sharedFields, sharedFilesSet := parseSourceFiles(t, fmt.Sprintf("%s/pkg/apis/shared/v1", root)) + // package path -> result doc file name -> name of the top-level field to be described -> field instance for reflection input := map[string]map[string]map[string]interface{}{ fmt.Sprintf("%s/pkg/apis/deployment/v1", root): { @@ -180,8 +182,17 @@ func Test_GenerateAPIDocs(t *testing.T) { resultPaths := make(map[string]string) for apiDir, docs := range input { - fields, fileSets := parseSourceFiles(t, apiDir) - util.CopyMap(resultPaths, generateDocs(t, docs, fields, fileSets)) + fields, fileSet := parseSourceFiles(t, apiDir) + + for n, f := range sharedFields { + fields[n] = f + } + sharedFilesSet.Iterate(func(file *token.File) bool { + fileSet.AddFile(file.Name(), fileSet.Base()+file.Base(), file.Size()) + return true + }) + + util.CopyMap(resultPaths, generateDocs(t, docs, fields, fileSet)) } generateIndex(t, resultPaths) } diff --git a/pkg/apis/ml/v1alpha1/storage_s3_spec.go b/pkg/apis/ml/v1alpha1/storage_s3_spec.go deleted file mode 100644 index 33340f048..000000000 --- a/pkg/apis/ml/v1alpha1/storage_s3_spec.go +++ /dev/null @@ -1,58 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2023 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// - -package v1alpha1 - -import ( - "github.com/pkg/errors" -) - -type ArangoMLStorageS3Spec struct { - // Endpoint specifies the S3 API-compatible endpoint which implements storage - // Required - Endpoint string `json:"endpoint"` - // DisableSSL if set to true, no certificate checks will be performed for Endpoint - // +doc/default: false - DisableSSL bool `json:"disableSSL,omitempty"` - // Region defines the availability zone name. If empty, defaults to 'us-east-1' - // +doc/default: "" - Region string `json:"region,omitempty"` - // BucketName specifies the name of the bucket - // Required - BucketName string `json:"bucketName"` - // CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization - // Required - CredentialsSecretName string `json:"credentialsSecret"` -} - -func (s *ArangoMLStorageS3Spec) Validate() error { - if s.BucketName == "" { - return errors.New("S3 BucketName must be not empty") - } - - if s.Endpoint == "" { - return errors.New("S3 Endpoint must be not empty") - } - - if s.CredentialsSecretName == "" { - return errors.New("S3 CredentialsSecretName must be not empty") - } - return nil -} diff --git a/pkg/apis/ml/v1alpha1/storage_spec.go b/pkg/apis/ml/v1alpha1/storage_spec.go index 3e250cd70..37e86e2ba 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec.go +++ b/pkg/apis/ml/v1alpha1/storage_spec.go @@ -21,47 +21,41 @@ package v1alpha1 import ( - "github.com/pkg/errors" - core "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" + "github.com/arangodb/kube-arangodb/pkg/apis/shared" ) type ArangoMLStorageSpec struct { - // ListenPort defines on which port the sidecar container will be listening for connections - // +doc/default: 9201 - ListenPort *uint16 `json:"listenPort,omitempty"` - - // Resources holds resource requests & limits for container running the S3 proxy - // +doc/type: core.ResourceRequirements - // +doc/link: Documentation of core.ResourceRequirements|https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core - Resources core.ResourceRequirements `json:"resources,omitempty"` - - S3 *ArangoMLStorageS3Spec `json:"s3,omitempty"` + // Mode defines how storage implementation should be deployed + Mode *ArangoMLStorageSpecMode `json:"mode,omitempty"` + // Backend defines how storage is implemented + Backend *ArangoMLStorageSpecBackend `json:"backend,omitempty"` } -func (s *ArangoMLStorageSpec) Validate() error { - if s.S3 == nil { - return errors.New("Currently only s3 storage type is supported") +func (s *ArangoMLStorageSpec) GetMode() *ArangoMLStorageSpecMode { + if s == nil || s.Mode == nil { + return &ArangoMLStorageSpecMode{} } + return s.Mode +} - return s.S3.Validate() +func (s *ArangoMLStorageSpec) GetBackend() *ArangoMLStorageSpecBackend { + if s == nil || s.Backend == nil { + return &ArangoMLStorageSpecBackend{} + } + return s.Backend } -// SetDefaults fills in missing defaults -func (s *ArangoMLStorageSpec) SetDefaults() { +func (s *ArangoMLStorageSpec) Validate() error { if s == nil { - return + s = &ArangoMLStorageSpec{} } - resources := s.Resources - if len(resources.Requests) == 0 { - resources.Requests = make(core.ResourceList) - resources.Requests[core.ResourceCPU] = resource.MustParse("100m") - resources.Requests[core.ResourceMemory] = resource.MustParse("100m") - } - if len(resources.Limits) == 0 { - resources.Limits = make(core.ResourceList) - resources.Limits[core.ResourceCPU] = resource.MustParse("250m") - resources.Limits[core.ResourceMemory] = resource.MustParse("250m") + if err := shared.WithErrors(shared.PrefixResourceErrors("spec", + shared.PrefixResourceError("backend", s.Backend.Validate()), + shared.PrefixResourceError("mode", s.Mode.Validate()), + )); err != nil { + return err } + + return nil } diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend.go b/pkg/apis/ml/v1alpha1/storage_spec_backend.go new file mode 100644 index 000000000..eece830fd --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecBackend struct { + // S3 backend implements storage as a proxy to the provided S3 API endpoint + S3 *ArangoMLStorageSpecBackendS3 `json:"s3,omitempty"` +} + +func (s *ArangoMLStorageSpecBackend) GetS3() *ArangoMLStorageSpecBackendS3 { + if s == nil || s.S3 == nil { + return &ArangoMLStorageSpecBackendS3{} + } + return s.S3 +} + +func (s *ArangoMLStorageSpecBackend) Validate() error { + if s == nil { + return errors.Newf("Backend is not specified") + } + + if s.S3 == nil { + return errors.Newf("At least one backend needs to be defined") + } + + return shared.WithErrors(shared.PrefixResourceError("s3", s.S3.Validate())) +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go new file mode 100644 index 000000000..11fbacc5e --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go @@ -0,0 +1,123 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "net/url" + + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecBackendS3 struct { + // Endpoint specifies the S3 API-compatible endpoint which implements storage + // Required + Endpoint *string `json:"endpoint"` + // BucketName specifies the name of the bucket + // Required + BucketName *string `json:"bucketName"` + // CredentialsSecret specifies the Kubernetes Secret containing AccessKey and SecretKey for S3 API authorization + // Required + CredentialsSecret *sharedApi.Object `json:"credentialsSecret"` + // AllowInsecure if set to true, the Endpoint certificates won't be checked + // +doc/default: false + AllowInsecure *bool `json:"allowInsecure,omitempty"` + // CASecret if not empty, the given Kubernetes Secret will be used to check the authenticity of Endpoint + // The specified Secret, must contain the following data fields: + // - `ca.crt` PEM encoded public key of the CA certificate + // - `ca.key` PEM encoded private key of the CA certificate + // +doc/default: nil + CASecret *sharedApi.Object `json:"caSecret,omitempty"` + // Region defines the availability zone name. + // +doc/default: "" + Region *string `json:"region,omitempty"` +} + +func (s *ArangoMLStorageSpecBackendS3) Validate() error { + if s == nil { + s = &ArangoMLStorageSpecBackendS3{} + } + + var errs []error + + if s.GetBucketName() == "" { + errs = append(errs, shared.PrefixResourceErrors("bucketName", errors.New("must be not empty"))) + } + + if s.GetEndpoint() == "" { + errs = append(errs, shared.PrefixResourceErrors("endpoint", errors.New("must be not empty"))) + } + + if _, err := url.Parse(s.GetEndpoint()); err != nil { + errs = append(errs, shared.PrefixResourceErrors("endpoint", errors.Newf("invalid URL: %s", err.Error()))) + } + + errs = append(errs, shared.PrefixResourceErrors("credentialsSecret", s.GetCredentialsSecret().Validate())) + + if caSecret := s.GetCASecret(); !caSecret.IsEmpty() { + errs = append(errs, shared.PrefixResourceErrors("caSecret", caSecret.Validate())) + } + + return shared.WithErrors(errs...) +} + +func (s *ArangoMLStorageSpecBackendS3) GetEndpoint() string { + if s == nil || s.Endpoint == nil { + return "" + } + return *s.Endpoint +} + +func (s *ArangoMLStorageSpecBackendS3) GetBucketName() string { + if s == nil || s.BucketName == nil { + return "" + } + return *s.BucketName +} + +func (s *ArangoMLStorageSpecBackendS3) GetCredentialsSecret() *sharedApi.Object { + if s == nil || s.CredentialsSecret == nil { + return &sharedApi.Object{} + } + return s.CredentialsSecret +} + +func (s *ArangoMLStorageSpecBackendS3) GetAllowInsecure() bool { + if s == nil || s.AllowInsecure == nil { + return false + } + return *s.AllowInsecure +} + +func (s *ArangoMLStorageSpecBackendS3) GetCASecret() *sharedApi.Object { + if s == nil || s.CASecret == nil { + return &sharedApi.Object{} + } + return s.CASecret +} + +func (s *ArangoMLStorageSpecBackendS3) GetRegion() string { + if s == nil || s.Region == nil { + return "" + } + return *s.Region +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode.go b/pkg/apis/ml/v1alpha1/storage_spec_mode.go new file mode 100644 index 000000000..0dda1cdeb --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode.go @@ -0,0 +1,45 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +type ArangoMLStorageSpecMode struct { + // Sidecar mode runs the storage implementation as a sidecar + Sidecar *ArangoMLStorageSpecModeSidecar `json:"sidecar,omitempty"` +} + +func (s *ArangoMLStorageSpecMode) GetSidecar() *ArangoMLStorageSpecModeSidecar { + if s == nil || s.Sidecar == nil { + return &ArangoMLStorageSpecModeSidecar{} + } + return s.Sidecar +} + +func (s *ArangoMLStorageSpecMode) Validate() error { + if s == nil { + return errors.Newf("Mode is not defined") + } + return shared.WithErrors(shared.PrefixResourceError("sidecar", s.Sidecar.Validate())) +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go new file mode 100644 index 000000000..4116a06e4 --- /dev/null +++ b/pkg/apis/ml/v1alpha1/storage_spec_mode_sidecar.go @@ -0,0 +1,83 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1alpha1 + +import ( + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/arangodb/kube-arangodb/pkg/apis/shared" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +var ( + defaultRequestsCPU = resource.MustParse("100m") + defaultRequestsMemory = resource.MustParse("100Mi") + defaultLimitsCPU = resource.MustParse("200m") + defaultLimitsMemory = resource.MustParse("200Mi") +) + +type ArangoMLStorageSpecModeSidecar struct { + // ListenPort defines on which port the sidecar container will be listening for connections + // +doc/default: 9201 + ListenPort *uint16 `json:"listenPort,omitempty"` + + // Resources holds resource requests & limits for container running the S3 proxy + // +doc/type: core.ResourceRequirements + // +doc/link: Documentation of core.ResourceRequirements|https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core + Resources *core.ResourceRequirements `json:"resources,omitempty"` +} + +func (s *ArangoMLStorageSpecModeSidecar) Validate() error { + if s == nil { + s = &ArangoMLStorageSpecModeSidecar{} + } + if s.GetListenPort() < 1 { + return shared.PrefixResourceErrors("database", errors.Newf("must be positive")) + } + return nil +} + +func (s *ArangoMLStorageSpecModeSidecar) GetListenPort() uint16 { + if s == nil || s.ListenPort == nil { + return 9201 + } + return *s.ListenPort +} + +func (s *ArangoMLStorageSpecModeSidecar) GetResources() core.ResourceRequirements { + var resources core.ResourceRequirements + if s != nil && s.Resources != nil { + resources = *s.Resources + } + + if len(resources.Requests) == 0 { + resources.Requests = make(core.ResourceList) + resources.Requests[core.ResourceCPU] = defaultRequestsCPU + resources.Requests[core.ResourceMemory] = defaultRequestsMemory + } + if len(resources.Limits) == 0 { + resources.Limits = make(core.ResourceList) + resources.Limits[core.ResourceCPU] = defaultLimitsCPU + resources.Limits[core.ResourceMemory] = defaultLimitsMemory + } + return resources +} diff --git a/pkg/apis/ml/v1alpha1/storage_spec_test.go b/pkg/apis/ml/v1alpha1/storage_spec_test.go index 02e5d1f78..159cdfedc 100644 --- a/pkg/apis/ml/v1alpha1/storage_spec_test.go +++ b/pkg/apis/ml/v1alpha1/storage_spec_test.go @@ -25,23 +25,58 @@ import ( "github.com/stretchr/testify/require" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + "github.com/arangodb/kube-arangodb/pkg/util" ) func Test_ArangoMLStorageSpec(t *testing.T) { - s := ArangoMLStorageSpec{ - ListenPort: nil, - Resources: core.ResourceRequirements{}, - S3: nil, - } - s.SetDefaults() + s := ArangoMLStorageSpec{} + require.Error(t, s.Validate()) + require.NotNil(t, s.GetMode()) + require.NotNil(t, s.GetBackend()) + + require.NotNil(t, s.Mode.GetSidecar()) + s.Mode = &ArangoMLStorageSpecMode{} + + require.NotNil(t, s.Backend.GetS3()) + s.Backend = &ArangoMLStorageSpecBackend{} require.Error(t, s.Validate()) - s.S3 = &ArangoMLStorageS3Spec{ - Endpoint: "some-endpoint", - DisableSSL: false, - Region: "", - BucketName: "test-bucket", - CredentialsSecretName: "some-secret", + require.NotNil(t, s.Mode.Sidecar.GetListenPort()) + require.NotNil(t, s.Mode.Sidecar.GetResources()) + s.Mode.Sidecar = &ArangoMLStorageSpecModeSidecar{} + + require.Error(t, s.Backend.S3.Validate()) + s.Backend.S3 = &ArangoMLStorageSpecBackendS3{ + Endpoint: util.NewType("http://test.s3.example.com"), + BucketName: util.NewType("bucket"), + CredentialsSecret: &sharedApi.Object{ + Name: "a-secret", + Namespace: nil, + }, } require.NoError(t, s.Validate()) + + t.Run("default requests and limits assigned", func(t *testing.T) { + assignedRequirements := core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceCPU: resource.MustParse("200m"), + core.ResourceMemory: resource.MustParse("200Mi"), + }, + } + s.Mode.Sidecar.Resources = &assignedRequirements + + expectedRequirements := core.ResourceRequirements{ + Requests: assignedRequirements.Requests, + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("200m"), + core.ResourceMemory: resource.MustParse("200Mi"), + }, + } + + actualRequirements := s.Mode.Sidecar.GetResources() + require.Equal(t, expectedRequirements, actualRequirements) + }) } diff --git a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go index 70e817fdf..250fb2645 100644 --- a/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,8 @@ package v1alpha1 import ( v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + sharedv1 "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -392,44 +394,141 @@ func (in *ArangoMLStorageList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ArangoMLStorageS3Spec) DeepCopyInto(out *ArangoMLStorageS3Spec) { +func (in *ArangoMLStorageSpec) DeepCopyInto(out *ArangoMLStorageSpec) { *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(ArangoMLStorageSpecMode) + (*in).DeepCopyInto(*out) + } + if in.Backend != nil { + in, out := &in.Backend, &out.Backend + *out = new(ArangoMLStorageSpecBackend) + (*in).DeepCopyInto(*out) + } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageS3Spec. -func (in *ArangoMLStorageS3Spec) DeepCopy() *ArangoMLStorageS3Spec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpec. +func (in *ArangoMLStorageSpec) DeepCopy() *ArangoMLStorageSpec { if in == nil { return nil } - out := new(ArangoMLStorageS3Spec) + out := new(ArangoMLStorageSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ArangoMLStorageSpec) DeepCopyInto(out *ArangoMLStorageSpec) { +func (in *ArangoMLStorageSpecBackend) DeepCopyInto(out *ArangoMLStorageSpecBackend) { + *out = *in + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(ArangoMLStorageSpecBackendS3) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecBackend. +func (in *ArangoMLStorageSpecBackend) DeepCopy() *ArangoMLStorageSpecBackend { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecBackendS3) DeepCopyInto(out *ArangoMLStorageSpecBackendS3) { + *out = *in + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint + *out = new(string) + **out = **in + } + if in.BucketName != nil { + in, out := &in.BucketName, &out.BucketName + *out = new(string) + **out = **in + } + if in.CredentialsSecret != nil { + in, out := &in.CredentialsSecret, &out.CredentialsSecret + *out = new(sharedv1.Object) + (*in).DeepCopyInto(*out) + } + if in.AllowInsecure != nil { + in, out := &in.AllowInsecure, &out.AllowInsecure + *out = new(bool) + **out = **in + } + if in.CASecret != nil { + in, out := &in.CASecret, &out.CASecret + *out = new(sharedv1.Object) + (*in).DeepCopyInto(*out) + } + if in.Region != nil { + in, out := &in.Region, &out.Region + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecBackendS3. +func (in *ArangoMLStorageSpecBackendS3) DeepCopy() *ArangoMLStorageSpecBackendS3 { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecBackendS3) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecMode) DeepCopyInto(out *ArangoMLStorageSpecMode) { + *out = *in + if in.Sidecar != nil { + in, out := &in.Sidecar, &out.Sidecar + *out = new(ArangoMLStorageSpecModeSidecar) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecMode. +func (in *ArangoMLStorageSpecMode) DeepCopy() *ArangoMLStorageSpecMode { + if in == nil { + return nil + } + out := new(ArangoMLStorageSpecMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMLStorageSpecModeSidecar) DeepCopyInto(out *ArangoMLStorageSpecModeSidecar) { *out = *in if in.ListenPort != nil { in, out := &in.ListenPort, &out.ListenPort *out = new(uint16) **out = **in } - in.Resources.DeepCopyInto(&out.Resources) - if in.S3 != nil { - in, out := &in.S3, &out.S3 - *out = new(ArangoMLStorageS3Spec) - **out = **in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpec. -func (in *ArangoMLStorageSpec) DeepCopy() *ArangoMLStorageSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLStorageSpecModeSidecar. +func (in *ArangoMLStorageSpecModeSidecar) DeepCopy() *ArangoMLStorageSpecModeSidecar { if in == nil { return nil } - out := new(ArangoMLStorageSpec) + out := new(ArangoMLStorageSpecModeSidecar) in.DeepCopyInto(out) return out } diff --git a/pkg/apis/shared/v1/object.go b/pkg/apis/shared/v1/object.go new file mode 100644 index 000000000..cf103c463 --- /dev/null +++ b/pkg/apis/shared/v1/object.go @@ -0,0 +1,72 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1 + +import ( + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/apis/shared" +) + +type Object struct { + // Name of the object + Name string `json:"name"` + + // Namespace of the object. Should default to the namespace of the parent object + Namespace *string `json:"namespace,omitempty"` +} + +func (o *Object) IsEmpty() bool { + return o == nil || + (o.Name == "" && o.Namespace != nil) +} + +func (o *Object) GetName() string { + if o == nil { + return "" + } + + return o.Name +} + +func (o *Object) GetNamespace(obj meta.Object) string { + if o != nil { + if n := o.Namespace; n != nil { + return *n + } + } + + return obj.GetNamespace() +} + +func (o *Object) Validate() error { + if o == nil { + o = &Object{} + } + + var errs []error + errs = append(errs, shared.PrefixResourceErrors("name", AsKubernetesResourceName(&o.Name).Validate())) + if o.Namespace != nil { + errs = append(errs, shared.PrefixResourceErrors("namespace", AsKubernetesResourceName(o.Namespace).Validate())) + } + + return shared.WithErrors(errs...) +} diff --git a/pkg/apis/shared/v1/object_test.go b/pkg/apis/shared/v1/object_test.go new file mode 100644 index 000000000..3abe136f1 --- /dev/null +++ b/pkg/apis/shared/v1/object_test.go @@ -0,0 +1,52 @@ +// +// DISCLAIMER +// +// Copyright 2023 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/arangodb/kube-arangodb/pkg/util" +) + +func Test_Object_Validate(t *testing.T) { + var o *Object + require.Error(t, o.Validate()) + + o = &Object{} + require.Error(t, o.Validate()) + + o.Name = "#invalid" + require.Error(t, o.Validate()) + + o.Name = "valid" + require.NoError(t, o.Validate()) + + o.Namespace = util.NewType("") + require.Error(t, o.Validate()) + + o.Namespace = util.NewType("#invalid") + require.Error(t, o.Validate()) + + o.Namespace = util.NewType("valid") + require.NoError(t, o.Validate()) +} diff --git a/pkg/apis/shared/v1/zz_generated.deepcopy.go b/pkg/apis/shared/v1/zz_generated.deepcopy.go index c356004dc..c996b978f 100644 --- a/pkg/apis/shared/v1/zz_generated.deepcopy.go +++ b/pkg/apis/shared/v1/zz_generated.deepcopy.go @@ -44,3 +44,24 @@ func (in HashList) DeepCopy() HashList { in.DeepCopyInto(out) return *out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Object) DeepCopyInto(out *Object) { + *out = *in + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Object. +func (in *Object) DeepCopy() *Object { + if in == nil { + return nil + } + out := new(Object) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/crd/crds/ml-storage.schema.generated.yaml b/pkg/crd/crds/ml-storage.schema.generated.yaml index e18e9bbba..b762f5c2f 100644 --- a/pkg/crd/crds/ml-storage.schema.generated.yaml +++ b/pkg/crd/crds/ml-storage.schema.generated.yaml @@ -3,45 +3,75 @@ v1alpha1: properties: spec: properties: - listenPort: - description: ListenPort defines on which port the sidecar container will be listening for connections - format: int32 - type: integer - resources: - description: Resources holds resource requests & limits for container running the S3 proxy + backend: + description: Backend defines how storage is implemented properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string + s3: + description: S3 backend implements storage as a proxy to the provided S3 API endpoint + properties: + allowInsecure: + description: AllowInsecure if set to true, the Endpoint certificates won't be checked + type: boolean + bucketName: + description: |- + BucketName specifies the name of the bucket + Required + type: string + caSecret: + description: |- + CASecret if not empty, the given Kubernetes Secret will be used to check the authenticity of Endpoint + The specified Secret, must contain the following data fields: + - `ca.crt` PEM encoded public key of the CA certificate + - `ca.key` PEM encoded private key of the CA certificate + properties: + name: + type: string + namespace: + type: string + type: object + credentialsSecret: + description: |- + CredentialsSecret specifies the Kubernetes Secret containing AccessKey and SecretKey for S3 API authorization + Required + properties: + name: + type: string + namespace: + type: string + type: object + endpoint: + description: |- + Endpoint specifies the S3 API-compatible endpoint which implements storage + Required + type: string + region: + description: Region defines the availability zone name. + type: string type: object type: object - s3: + mode: + description: Mode defines how storage implementation should be deployed properties: - bucketName: - description: |- - BucketName specifies the name of the bucket - Required - type: string - credentialsSecret: - description: |- - CredentialsSecretName specifies the name of the secret containing AccessKey and SecretKey for S3 API authorization - Required - type: string - disableSSL: - description: DisableSSL if set to true, no certificate checks will be performed for Endpoint - type: boolean - endpoint: - description: |- - Endpoint specifies the S3 API-compatible endpoint which implements storage - Required - type: string - region: - description: Region defines the availability zone name. If empty, defaults to 'us-east-1' - type: string + sidecar: + description: Sidecar mode runs the storage implementation as a sidecar + properties: + listenPort: + description: ListenPort defines on which port the sidecar container will be listening for connections + format: int32 + type: integer + resources: + description: Resources holds resource requests & limits for container running the S3 proxy + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + type: object type: object type: object type: object