♻️ Set docker image location at the benchmark operator configmap and …

…use it when information is set Signed-off-by: vankichi <[email protected]>
vdaas · Jun 1, 2023 · dce512c · dce512c
1 parent 24ce258
commit dce512c
Show file tree

Hide file tree

Showing 14 changed files with 156 additions and 12 deletions.
diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml
@@ -81,6 +81,19 @@ spec:
                       type: string
                     tag:
                       type: string
+                job_image:
+                  type: object
+                  properties:
+                    pullPolicy:
+                      type: string
+                      enum:
+                        - Always
+                        - Never
+                        - IfNotPresent
+                    repository:
+                      type: string
+                    tag:
+                      type: string
                 logging:
                   type: object
                   properties:

diff --git a/charts/vald-benchmark-operator/templates/configmap.yaml b/charts/vald-benchmark-operator/templates/configmap.yaml
@@ -38,3 +38,6 @@ data:
     observability:
       {{- $observability := dict "Values" .Values.observability}}
       {{- include "vald.observability" $observability | nindent 6 }}
+    job_image:
+      image: "{{ .Values.job_image.repository }}:{{ .Values.job_image.tag }}"
+      pullPolicy: {{ .Values.job_image.pullPolicy }}
diff --git a/charts/vald-benchmark-operator/values.schema.json b/charts/vald-benchmark-operator/values.schema.json
@@ -16,8 +16,25 @@
           "description": "image pull policy",
           "enum": ["Always", "Never", "IfNotPresent"]
         },
-        "repository": { "type": "string", "description": "image repository" },
-        "tag": { "type": "string", "description": "image tag" }
+        "repository": {
+          "type": "string",
+          "description": "job image repository"
+        },
+        "tag": {
+          "type": "string",
+          "description": "image tag for job docker image"
+        }
+      }
+    },
+    "job_image": {
+      "type": "object",
+      "properties": {
+        "pullPolicy": {
+          "type": "string",
+          "enum": ["Always", "Never", "IfNotPresent"]
+        },
+        "repository": { "type": "string" },
+        "tag": { "type": "string" }
       }
     },
     "logging": {

diff --git a/charts/vald-benchmark-operator/values.yaml b/charts/vald-benchmark-operator/values.yaml
@@ -42,6 +42,18 @@ image:
   # image.pullPolicy -- image pull policy
   pullPolicy: Always
 
+# @schema {"name": "job_image", "type": "object"}
+job_image:
+  # @schema {"name": "job_image.repository", "type": "string"}
+  # image.repository -- job image repository
+  repository: vdaas/vald-benchmark-operator
+  # @schema {"name": "job_image.tag", "type": "string"}
+  # image.tag -- image tag for job docker image
+  tag: v1.7.5
+  # @schema {"name": "job_image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]}
+  # image.pullPolicy -- image pull policy
+  pullPolicy: Always
+
 # @schema {"name": "rbac", "type": "object"}
 rbac:
   # @schema {"name": "rbac.create", "type": "boolean"}

diff --git a/internal/config/benchmark.go b/internal/config/benchmark.go
@@ -229,3 +229,16 @@ func (b *BenchmarkJob) Bind() *BenchmarkJob {
 func (b *BenchmarkScenario) Bind() *BenchmarkScenario {
 	return b
 }
+
+// BenchmarkJobImageInfo represents the docker image information for benchmark job.
+type BenchmarkJobImageInfo struct {
+	Image      string `json:"image,omitempty"       yaml:"image"`
+	PullPolicy string `json:"pull_policy,omitempty" yaml:"pull_policy"`
+}
+
+// Bind binds the actual data from the BenchmarkJobImageInfo receiver fields.
+func (b *BenchmarkJobImageInfo) Bind() *BenchmarkJobImageInfo {
+	b.Image = GetActualValue(b.Image)
+	b.PullPolicy = GetActualValue(b.PullPolicy)
+	return b
+}
diff --git a/internal/k8s/vald/benchmark/job/job_template.go b/internal/k8s/vald/benchmark/job/job_template.go
@@ -22,11 +22,14 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
+var (
+	ContainerImage  string
+	ImagePullPolicy corev1.PullPolicy
+)
+
 const (
 	SvcAccountName = "vald-benchmark-operator"
 	ContainerName  = "vald-benchmark-job"
-	// TODO: Fix
-	ContainerImage = "vdaas/vald-benchmark-job:pr-2027"
 
 	RestartPolicyAlways    corev1.RestartPolicy = "Always"
 	RestartPolicyOnFailure corev1.RestartPolicy = "OnFailure"
@@ -35,6 +38,7 @@ const (
 
 // NewBenchmarkJobTemplate creates the job template for crating k8s job resource.
 func NewBenchmarkJobTemplate(opts ...BenchmarkJobOption) (jobs.Job, error) {
+	ContainerImage = "vdaas/vald-benchmark-job"
 	jobTmpl := new(jobs.Job)
 	for _, opt := range append(defaultBenchmarkJobOpts, opts...) {
 		err := opt(jobTmpl)

diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go
@@ -17,6 +17,7 @@
 package job
 
 import (
+	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/k8s"
 	jobs "github.com/vdaas/vald/internal/k8s/job"
 	corev1 "k8s.io/api/core/v1"
@@ -30,6 +31,36 @@ var defaultBenchmarkJobOpts = []BenchmarkJobOption{
 	WithRestartPolicy(RestartPolicyNever),
 }
 
+// WithImage sets the docker image path for benchmark job.
+func WithImage(name string) BenchmarkJobOption {
+	return func(_ *jobs.Job) error {
+		if len(name) > 0 {
+			ContainerImage = name
+		}
+		return nil
+	}
+}
+
+// WithImagePullPolicy sets the docker image pull policy for benchmark job.
+func WithImagePullPolicy(policy string) BenchmarkJobOption {
+	return func(_ *jobs.Job) error {
+		if len(policy) == 0 {
+			return nil
+		}
+		switch policy {
+		case string(corev1.PullAlways):
+			ImagePullPolicy = corev1.PullAlways
+		case string(corev1.PullIfNotPresent):
+			ImagePullPolicy = corev1.PullIfNotPresent
+		case string(corev1.PullNever):
+			ImagePullPolicy = corev1.PullNever
+		default:
+			return errors.NewErrInvalidOption("image pull policy", policy)
+		}
+		return nil
+	}
+}
+
 // WithSvcAccountName sets the service account name for benchmark job.
 func WithSvcAccountName(name string) BenchmarkJobOption {
 	return func(b *jobs.Job) error {

diff --git a/k8s/tools/benchmark/operator/configmap.yaml b/k8s/tools/benchmark/operator/configmap.yaml
@@ -138,3 +138,6 @@ data:
       trace:
         enabled: false
         sampling_rate: 1
+    job_image:
+      image: "local-registry:5000/vdaas/vald-benchmark-job"
+      pullPolicy: Always
diff --git a/k8s/tools/benchmark/operator/crds/valdbenchmarkoperatorrelease.yaml b/k8s/tools/benchmark/operator/crds/valdbenchmarkoperatorrelease.yaml
@@ -81,6 +81,19 @@ spec:
                       type: string
                     tag:
                       type: string
+                job_image:
+                  type: object
+                  properties:
+                    pullPolicy:
+                      type: string
+                      enum:
+                        - Always
+                        - Never
+                        - IfNotPresent
+                    repository:
+                      type: string
+                    tag:
+                      type: string
                 logging:
                   type: object
                   properties:

diff --git a/k8s/tools/benchmark/operator/deployment.yaml b/k8s/tools/benchmark/operator/deployment.yaml
@@ -43,7 +43,7 @@ spec:
       serviceAccountName: vald-benchmark-operator
       containers:
         - name: vald-benchmark-operator
-          image: "vdaas/vald-benchmark-operator:v1.7.5"
+          image: local-registry:5000/vdaas/vald-benchmark-operator
           imagePullPolicy: Always
           livenessProbe:
             failureThreshold: 2

diff --git a/pkg/tools/benchmark/operator/config/config.go b/pkg/tools/benchmark/operator/config/config.go
@@ -37,6 +37,9 @@ type Config struct {
 
 	// Scenario represents benchmark scenario configurations
 	Scenario *config.BenchmarkScenario `json:"scenario" yaml:"scenario"`
+
+	// JobImage represents the location of Docker image for benchmark job and its ImagePullPolicy
+	JobImage *config.BenchmarkJobImageInfo `json:"job_image" yaml:"job_image"`
 }
 
 // NewConfig represents the set config from the given setting file path.
@@ -58,6 +61,12 @@ func NewConfig(path string) (cfg *Config, err error) {
 		cfg.Observability = cfg.Observability.Bind()
 	}
 
+	if cfg.JobImage != nil {
+		cfg.JobImage = cfg.JobImage.Bind()
+	} else {
+		cfg.JobImage = new(config.BenchmarkJobImageInfo)
+	}
+
 	if cfg.Scenario != nil {
 		cfg.Scenario = cfg.Scenario.Bind()
 	} else {

diff --git a/pkg/tools/benchmark/operator/service/operator.go b/pkg/tools/benchmark/operator/service/operator.go
@@ -53,13 +53,15 @@ const (
 )
 
 type operator struct {
-	jobNamespace string
-	scenarios    atomic.Pointer[map[string]*scenario]
-	benchjobs    atomic.Pointer[map[string]*v1.ValdBenchmarkJob]
-	jobs         atomic.Pointer[map[string]string]
-	rcd          time.Duration // reconcile check duration
-	eg           errgroup.Group
-	ctrl         k8s.Controller
+	jobNamespace       string
+	jobImage           string
+	jobImagePullPolicy string
+	scenarios          atomic.Pointer[map[string]*scenario]
+	benchjobs          atomic.Pointer[map[string]*v1.ValdBenchmarkJob]
+	jobs               atomic.Pointer[map[string]string]
+	rcd                time.Duration // reconcile check duration
+	eg                 errgroup.Group
+	ctrl               k8s.Controller
 }
 
 // New creates the new scenario struct to handle vald benchmark job scenario.
@@ -462,6 +464,8 @@ func (o *operator) createJob(ctx context.Context, bjr v1.ValdBenchmarkJob) error
 			},
 		}),
 		benchjob.WithTTLSecondsAfterFinished(int32(bjr.Spec.TTLSecondsAfterFinished)),
+		benchjob.WithImage(o.jobImage),
+		benchjob.WithImagePullPolicy(o.jobImagePullPolicy),
 	)
 	if err != nil {
 		return err

diff --git a/pkg/tools/benchmark/operator/service/option.go b/pkg/tools/benchmark/operator/service/option.go
@@ -66,3 +66,23 @@ func WithJobNamespace(ns string) Option {
 		return nil
 	}
 }
+
+// WithJobImageInfo sets the benchmark job docker image info.
+func WithJobImage(image string) Option {
+	return func(o *operator) error {
+		if len(image) > 0 {
+			o.jobImage = image
+		}
+		return nil
+	}
+}
+
+// WithJobImagePullPolicy sets the benchmark job docker image pullPolicy.
+func WithJobImagePullPolicy(p string) Option {
+	return func(o *operator) error {
+		if len(p) > 0 {
+			o.jobImagePullPolicy = p
+		}
+		return nil
+	}
+}
diff --git a/pkg/tools/benchmark/operator/usecase/benchmarkd.go b/pkg/tools/benchmark/operator/usecase/benchmarkd.go
@@ -57,6 +57,8 @@ func New(cfg *config.Config) (r runner.Runner, err error) {
 
 	operator, err := service.New(
 		service.WithErrGroup(eg),
+		service.WithJobImage(cfg.JobImage.Image),
+		service.WithJobImagePullPolicy(cfg.JobImage.PullPolicy),
 	)
 	if err != nil {
 		return nil, err