(Feature) Add generator for CRD validation schemas (#1488)

* (Feature) Add generator for CRD validation schemas

* Improve docs / log output for crd installation

* Add ability to force-update CRDs to allow testing

CHANGELOG.md

@@ -10,6 +10,7 @@
 - (Maintenance) Bump Go to 1.20.11
 - (Feature) License ArangoDeployment Fetcher
 - (Feature) K8S Resources Compare Generic
+- (Feature) Add support for CRD validation schemas
 
 ## [1.2.35](https://github.com/arangodb/kube-arangodb/tree/1.2.35) (2023-11-06)
 - (Maintenance) Update go-driver to v1.6.0, update IsNotFound() checks

cmd/cmd.go

@@ -136,7 +136,8 @@ var (
 		timeout time.Duration
 	}
 	crdOptions struct {
-		install bool
+		install          bool
+		validationSchema []string
 	}
 	operatorKubernetesOptions struct {
 		maxBatchSize int64
@@ -224,6 +225,7 @@ func init() {
 	f.Float32Var(&operatorKubernetesOptions.qps, "kubernetes.qps", kclient.DefaultQPS, "Number of queries per second for k8s API")
 	f.IntVar(&operatorKubernetesOptions.burst, "kubernetes.burst", kclient.DefaultBurst, "Burst for the k8s API")
 	f.BoolVar(&crdOptions.install, "crd.install", true, "Install missing CRD if access is possible")
+	f.StringArrayVar(&crdOptions.validationSchema, "crd.validation-schema", defaultValidationSchemaEnabled, "Overrides default set of CRDs which should have validation schema enabled <crd-name>=<true/false>.")
 	f.IntVar(&operatorBackup.concurrentUploads, "backup-concurrent-uploads", globals.DefaultBackupConcurrentUploads, "Number of concurrent uploads per deployment")
 	f.Uint64Var(&memoryLimit.hardLimit, "memory-limit", 0, "Define memory limit for hard shutdown and the dump of goroutines. Used for testing")
 	f.StringArrayVar(&metricsOptions.excludedMetricPrefixes, "metrics.excluded-prefixes", nil, "List of the excluded metrics prefixes")
@@ -359,7 +361,12 @@ func executeMain(cmd *cobra.Command, args []string) {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 			defer cancel()
 
-			_ = crd.EnsureCRD(ctx, client, true)
+			crdOpts, err := prepareCRDOptions(crdOptions.validationSchema)
+			if err != nil {
+				logger.Fatal("Invalid --crd.validation-schema args: %s", err)
+			}
+
+			_ = crd.EnsureCRDWithOptions(ctx, client, crd.EnsureCRDOptions{IgnoreErrors: true, CRDOptions: crdOpts})
 		}
 
 		secrets := client.Kubernetes().CoreV1().Secrets(namespace)

cmd/crd.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,12 +22,17 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
 	"github.com/arangodb/kube-arangodb/pkg/crd"
+	"github.com/arangodb/kube-arangodb/pkg/crd/crds"
 	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
 )
 
@@ -44,23 +49,67 @@ var (
 	}
 )
 
+var (
+	crdInstallOptions struct {
+		validationSchema []string
+	}
+)
+
+var (
+	defaultValidationSchemaEnabled []string
+)
+
 func init() {
 	cmdMain.AddCommand(cmdCRD)
 	cmdOps.AddCommand(cmdCRD)
 
+	f := cmdCRDInstall.Flags()
+	f.StringArrayVar(&crdInstallOptions.validationSchema, "crd.validation-schema", defaultValidationSchemaEnabled, "Controls which CRD should have validation schema <crd-name>=<true/false>.")
 	cmdCRD.AddCommand(cmdCRDInstall)
 }
 
+func prepareCRDOptions(schemaEnabledArgs []string) (map[string]crds.CRDOptions, error) {
+	defaultOptions := crd.GetDefaultCRDOptions()
+	result := make(map[string]crds.CRDOptions)
+	var err error
+	for _, arg := range schemaEnabledArgs {
+		parts := strings.Split(arg, "=")
+
+		crdName := parts[0]
+		opts, ok := defaultOptions[crdName]
+		if !ok {
+			return nil, fmt.Errorf("unknown CRD %s", crdName)
+		}
+
+		if len(parts) == 2 {
+			opts.WithSchema, err = strconv.ParseBool(parts[1])
+			if err != nil {
+				return nil, errors.Wrapf(err, "not a bool value: %s", parts[1])
+			}
+		}
+
+		result[crdName] = opts
+	}
+	return result, nil
+}
+
 func cmdCRDInstallRun(cmd *cobra.Command, args []string) {
+	crdOpts, err := prepareCRDOptions(crdInstallOptions.validationSchema)
+	if err != nil {
+		logger.Fatal("Invalid --crd.validation-schema args: %s", err)
+		return
+	}
+
 	client, ok := kclient.GetDefaultFactory().Client()
 	if !ok {
 		logger.Fatal("Failed to get client")
+		return
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 
-	err := crd.EnsureCRD(ctx, client, false)
+	err = crd.EnsureCRDWithOptions(ctx, client, crd.EnsureCRDOptions{IgnoreErrors: false, CRDOptions: crdOpts})
 	if err != nil {
 		os.Exit(1)
 	}

docs/README.md

@@ -8,7 +8,7 @@
 - [Operator Metrics & Alerts](generated/metrics/README.md)
 - [Operator Actions](generated/actions.md)
 - [Authentication](authentication.md)
-- Custom resources overview:
+- [Custom resources overview](crds.md):
   - [ArangoDeployment](deployment-resource-reference.md)
   - [ArangoDeploymentReplication](deployment-replication-resource-reference.md)
   - [ArangoLocalStorage](storage-resource.md)

docs/crds.md

@@ -0,0 +1,38 @@
+# Custom resources overview
+
+Main CRDs:
+- [ArangoDeployment](deployment-resource-reference.md)
+- [ArangoDeploymentReplication](deployment-replication-resource-reference.md)
+- [ArangoLocalStorage](storage-resource.md)
+- [Backup](backup-resource.md)
+- [BackupPolicy](backuppolicy-resource.md)
+
+Operator manages the CustomResources based on CustomResourceDefinitions installed in your cluster.
+
+There are different options how CustomResourceDefinitions can be created.
+
+**Deprecated options:**
+- Install CRDs directly from `manifests` folder.
+- Install `kube-arangodb-crd` helm chart before installing `kube-arangodb` chart.
+- Install CRDs using kustomize `all` or `crd` manifests.
+
+**Recommended:**
+Use `kube-arangodb` Helm chart.
+If you've chosen not to install CRDs automatically (`--skip-crds`), the operator will try to install CRDs automatically.
+Make sure that ServiceAccount for operator has permissions to `create` CustomResourceDefinitions.
+
+To disable the automatic creation of CRDs, set `enableCRDManagement=false` template parameter, e.g.:
+```shell
+helm install --generate-name https://github.com/arangodb/kube-arangodb/releases/download/$VER/kube-arangodb-$VER.tgz --set "operator.enableCRDManagement=false"
+```
+
+## Schema validation
+
+Starting with v1.2.36, the [schema validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation) is supported for all CRDs.
+
+Schema validation can be enabled only on cluster with no CRDs installed or by upgrading your CR from one CRD version to another.
+
+To enable creation of CRD with validation schema, pass additional args to operator command line, e.g.:
+```
+--crd.validation-schema=arangobackuppolicies.backup.arangodb.com=true --crd.validation-schema=arangodeployments.database.arangodb.com=false
+```

docs/how-to/additional_configuration.md

@@ -3,15 +3,28 @@
 It is possible to additionally fine-tune operator behavior by
 providing arguments via `operator.args` chart template value.
 
-For example, you can specify burst size for k8s API requests or how long the operator
+The full list of available arguments can be retrieved using
+```
+export OPERATOR_IMAGE=arangodb/kube-arangodb:$VER
+kubectl run arango-operator-help --image=$OPERATOR_IMAGE -i --rm --restart=Never -- --help
+```
+
+
+### Example 1: kubernetes.burst 
+
+You can specify burst size for k8s API requests or how long the operator
 should wait for ArangoDeployment termination after receiving interruption signal:
 ```
 operator:
   args: ["--kubernetes.burst=40", --shutdown.timeout=2m"]
 ```
 
-The full list of available arguments can be retrieved using 
+### Example 2: CRD validation
+
+You can specify which of installed CRD should have a validation schema enabled:
 ```
-export OPERATOR_IMAGE=arangodb/kube-arangodb:1.2.9
-kubectl run arango-operator-help --image=$OPERATOR_IMAGE -i --rm --restart=Never -- --help
+operator:
+  args:
+    - --crd.validation-schema=arangobackuppolicies.backup.arangodb.com=true
+    - --crd.validation-schema=arangodeployments.database.arangodb.com=false
 ```

go.mod

@@ -61,6 +61,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.25.13
 	k8s.io/apimachinery v0.25.13
 	k8s.io/client-go v0.25.13
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1
 	sigs.k8s.io/yaml v1.2.0
 )
 
@@ -126,7 +127,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/klog/v2 v2.70.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect

pkg/crd/apply.go

@@ -31,94 +31,117 @@ import (
 
 	"github.com/arangodb/go-driver"
 
+	"github.com/arangodb/kube-arangodb/pkg/crd/crds"
 	"github.com/arangodb/kube-arangodb/pkg/logging"
 	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
 )
 
 var logger = logging.Global().RegisterAndGetLogger("crd", logging.Info)
 
+// Deprecated: use EnsureCRDWithOptions instead
 func EnsureCRD(ctx context.Context, client kclient.Client, ignoreErrors bool) error {
+	return EnsureCRDWithOptions(ctx, client, EnsureCRDOptions{
+		IgnoreErrors: ignoreErrors,
+	})
+}
+
+type EnsureCRDOptions struct {
+	// IgnoreErrors do not return errors if could not apply CRD
+	IgnoreErrors bool
+	// ForceUpdate if true, CRD will be updated even if definitions versions are the same
+	ForceUpdate bool
+	// CRDOptions defines options per each CRD
+	CRDOptions map[string]crds.CRDOptions
+}
+
+func EnsureCRDWithOptions(ctx context.Context, client kclient.Client, opts EnsureCRDOptions) error {
 	crdsLock.Lock()
 	defer crdsLock.Unlock()
 
-	for crd, spec := range registeredCRDs {
-		getAccess := verifyCRDAccess(ctx, client, crd, "get")
+	for crdName, crdReg := range registeredCRDs {
+		getAccess := verifyCRDAccess(ctx, client, crdName, "get")
 		if !getAccess.Allowed {
-			logger.Str("crd", crd).Info("Get Operations is not allowed. Continue")
+			logger.Str("crd", crdName).Info("Get Operations is not allowed. Continue")
 			continue
 		}
 
-		err := tryApplyCRD(ctx, client, crd, spec)
-		if !ignoreErrors && err != nil {
+		var opt *crds.CRDOptions
+		if o, ok := opts.CRDOptions[crdName]; ok {
+			opt = &o
+		}
+		def := crdReg.getter(opt)
+
+		err := tryApplyCRD(ctx, client, def, opts.ForceUpdate)
+		if !opts.IgnoreErrors && err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func tryApplyCRD(ctx context.Context, client kclient.Client, crd string, spec crd) error {
+func tryApplyCRD(ctx context.Context, client kclient.Client, def crds.Definition, forceUpdate bool) error {
 	crdDefinitions := client.KubernetesExtensions().ApiextensionsV1().CustomResourceDefinitions()
 
-	c, err := crdDefinitions.Get(ctx, crd, meta.GetOptions{})
+	crdName := def.CRD.Name
+
+	c, err := crdDefinitions.Get(ctx, crdName, meta.GetOptions{})
 	if err != nil {
 		if !errors.IsNotFound(err) {
-			logger.Err(err).Str("crd", crd).Warn("Get Operations is not allowed due to error")
+			logger.Err(err).Str("crd", crdName).Warn("Get Operations is not allowed due to error")
 			return err
 		}
 
-		createAccess := verifyCRDAccess(ctx, client, crd, "create")
+		createAccess := verifyCRDAccess(ctx, client, crdName, "create")
 
 		if !createAccess.Allowed {
-			logger.Str("crd", crd).Info("Create Operations is not allowed but CRD is missing. Continue")
+			logger.Str("crd", crdName).Info("Create Operations is not allowed but CRD is missing. Continue")
 			return nil
 		}
 
 		c = &apiextensions.CustomResourceDefinition{
 			ObjectMeta: meta.ObjectMeta{
-				Name: crd,
+				Name: crdName,
 				Labels: map[string]string{
-					Version: string(spec.version),
+					Version: string(def.Version),
 				},
 			},
-			Spec: spec.spec,
+			Spec: def.CRD.Spec,
 		}
 
 		if _, err := crdDefinitions.Create(ctx, c, meta.CreateOptions{}); err != nil {
-			logger.Err(err).Str("crd", crd).Warn("Create Operations is not allowed due to error")
+			logger.Err(err).Str("crd", crdName).Warn("Create Operations is not allowed due to error")
 			return err
 		}
 
-		logger.Str("crd", crd).Info("CRD Created")
+		logger.Str("crd", crdName).Info("CRD Created")
 		return nil
 	}
 
-	updateAccess := verifyCRDAccess(ctx, client, crd, "update")
+	updateAccess := verifyCRDAccess(ctx, client, crdName, "update")
 	if !updateAccess.Allowed {
-		logger.Str("crd", crd).Info("Update Operations is not allowed. Continue")
+		logger.Str("crd", crdName).Info("Update Operations is not allowed. Continue")
 		return nil
 	}
 
 	if c.ObjectMeta.Labels == nil {
 		c.ObjectMeta.Labels = map[string]string{}
 	}
 
-	if v, ok := c.ObjectMeta.Labels[Version]; ok {
-		if v != "" {
-			if !isUpdateRequired(spec.version, driver.Version(v)) {
-				logger.Str("crd", crd).Info("CRD Update not required")
-				return nil
-			}
+	if v, ok := c.ObjectMeta.Labels[Version]; ok && v != "" {
+		if !forceUpdate && !isUpdateRequired(def.Version, driver.Version(v)) {
+			logger.Str("crd", crdName).Info("CRD Update not required")
+			return nil
 		}
 	}
 
-	c.ObjectMeta.Labels[Version] = string(spec.version)
-	c.Spec = spec.spec
+	c.ObjectMeta.Labels[Version] = string(def.Version)
+	c.Spec = def.CRD.Spec
 
 	if _, err := crdDefinitions.Update(ctx, c, meta.UpdateOptions{}); err != nil {
-		logger.Err(err).Str("crd", crd).Warn("Create Operations is not allowed due to error")
+		logger.Err(err).Str("crd", crdName).Warn("Failed to update CRD definition")
 		return err
 	}
-	logger.Str("crd", crd).Info("CRD Updated")
+	logger.Str("crd", crdName).Info("CRD Updated")
 	return nil
 }