Add liveness and readiness probes

Signed-off-by: Jason Parraga <[email protected]>

internal/controller/install/armadaserver_controller.go

@@ -449,6 +449,13 @@ func createArmadaServerDeployment(
 	volumeMounts := createVolumeMounts(GetConfigFilename(as.Name), as.Spec.AdditionalVolumeMounts)
 	volumeMounts = append(volumeMounts, createPulsarVolumeMounts(pulsarConfig)...)
 
+	asConfig, err := extractArmadaServerConfig(as.Spec.ApplicationConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(asConfig.Grpc.Tls))
+
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      as.Name,
@@ -483,6 +490,8 @@ func createArmadaServerDeployment(
 						Env:             env,
 						VolumeMounts:    volumeMounts,
 						SecurityContext: as.Spec.SecurityContext,
+						ReadinessProbe:  readinessProbe,
+						LivenessProbe:   livenessProbe,
 					}},
 					Volumes: volumes,
 				},
@@ -678,3 +687,21 @@ func createServerPrometheusRule(server *installv1alpha1.ArmadaServer) *monitorin
 		},
 	}
 }
+
+type ArmadaServerConfig struct {
+	Grpc GrpcConfig
+}
+
+// extractArmadaServerConfig will unmarshal the appconfig and return the AramadaServerConfig portion
+func extractArmadaServerConfig(config runtime.RawExtension) (ArmadaServerConfig, error) {
+	appConfig, err := builders.ConvertRawExtensionToYaml(config)
+	if err != nil {
+		return ArmadaServerConfig{}, err
+	}
+	var asConfig ArmadaServerConfig
+	err = yaml.Unmarshal([]byte(appConfig), &asConfig)
+	if err != nil {
+		return ArmadaServerConfig{}, err
+	}
+	return asConfig, err
+}

internal/controller/install/binoculars_controller.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"time"
 
+	"sigs.k8s.io/yaml"
+
 	"github.com/pkg/errors"
 
 	installv1alpha1 "github.com/armadaproject/armada-operator/api/install/v1alpha1"
@@ -227,6 +229,13 @@ func createBinocularsDeployment(
 	env := createEnv(binoculars.Spec.Environment)
 	volumes := createVolumes(binoculars.Name, binoculars.Spec.AdditionalVolumes)
 	volumeMounts := createVolumeMounts(GetConfigFilename(secret.Name), binoculars.Spec.AdditionalVolumeMounts)
+
+	binocularsConfig, err := extractBinocularsConfig(binoculars.Spec.ApplicationConfig)
+	if err != nil {
+		return nil, err
+	}
+	readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(binocularsConfig.Grpc.Tls))
+
 	containers := []corev1.Container{{
 		Name:            "binoculars",
 		ImagePullPolicy: corev1.PullIfNotPresent,
@@ -236,6 +245,8 @@ func createBinocularsDeployment(
 		Env:             env,
 		VolumeMounts:    volumeMounts,
 		SecurityContext: binoculars.Spec.SecurityContext,
+		ReadinessProbe:  readinessProbe,
+		LivenessProbe:   livenessProbe,
 	}}
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{Name: binoculars.Name, Namespace: binoculars.Namespace, Labels: AllLabels(binoculars.Name, binoculars.Labels)},
@@ -367,3 +378,21 @@ func (r *BinocularsReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&installv1alpha1.Binoculars{}).
 		Complete(r)
 }
+
+type BinocularsConfig struct {
+	Grpc GrpcConfig
+}
+
+// extractBinocularsConfig will unmarshal the appconfig and return the BinocularsConfig portion
+func extractBinocularsConfig(config runtime.RawExtension) (BinocularsConfig, error) {
+	appConfig, err := builders.ConvertRawExtensionToYaml(config)
+	if err != nil {
+		return BinocularsConfig{}, err
+	}
+	var binocularsConfig BinocularsConfig
+	err = yaml.Unmarshal([]byte(appConfig), &binocularsConfig)
+	if err != nil {
+		return BinocularsConfig{}, err
+	}
+	return binocularsConfig, err
+}

internal/controller/install/common_helpers.go

@@ -229,6 +229,14 @@ type PulsarConfig struct {
 	Cacert                string
 }
 
+type GrpcConfig struct {
+	Tls TlsConfig
+}
+
+type TlsConfig struct {
+	Enabled bool
+}
+
 // ArmadaInit used to initialize pulsar
 type ArmadaInit struct {
 	Enabled    bool
@@ -317,7 +325,16 @@ func ExtractPulsarConfig(config runtime.RawExtension) (PulsarConfig, error) {
 	if err != nil {
 		return PulsarConfig{}, err
 	}
-	return asConfig.Pulsar, nil
+	return asConfig.Pulsar, err
+}
+
+// GetServerScheme returns the URI scheme for the grpc server
+func GetServerScheme(tlsConfig TlsConfig) corev1.URIScheme {
+	if tlsConfig.Enabled {
+		return corev1.URISchemeHTTPS
+	}
+
+	return corev1.URISchemeHTTP
 }
 
 // waitForJob waits for the Job to reach a terminal state (complete or failed).
@@ -371,6 +388,36 @@ func createEnv(crdEnv []corev1.EnvVar) []corev1.EnvVar {
 	return envVars
 }
 
+func CreateProbesWithScheme(scheme corev1.URIScheme) (*corev1.Probe, *corev1.Probe) {
+	readinessProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/health",
+				Port:   intstr.FromString("http"),
+				Scheme: scheme,
+			},
+		},
+		InitialDelaySeconds: 5,
+		TimeoutSeconds:      5,
+		FailureThreshold:    2,
+	}
+
+	livenessProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/health",
+				Port:   intstr.FromString("http"),
+				Scheme: scheme,
+			},
+		},
+		InitialDelaySeconds: 10,
+		TimeoutSeconds:      10,
+		FailureThreshold:    3,
+	}
+
+	return readinessProbe, livenessProbe
+}
+
 // createVolumes creates the default appconfig Volume and appends the CRD AdditionalVolumes
 func createVolumes(configVolumeSecretName string, crdVolumes []corev1.Volume) []corev1.Volume {
 	volumes := []corev1.Volume{{
@@ -733,15 +780,6 @@ func newContainerPortsHTTPWithMetrics(config *builders.CommonApplicationConfig)
 	return ports
 }
 
-// newContainerPortsGRPCWithMetrics creates container ports for grpc and metrics server and optional port for profiling server.
-func newContainerPortsGRPCWithMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort {
-	ports := []corev1.ContainerPort{newContainerPortGRPC(config), newContainerPortMetrics(config)}
-	if config.Profiling.Port > 0 {
-		ports = append(ports, newContainerPortProfiling(config))
-	}
-	return ports
-}
-
 // newContainerPortsMetrics creates container ports for metrics server and optional port for profiling server.
 func newContainerPortsMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort {
 	ports := []corev1.ContainerPort{newContainerPortMetrics(config)}

internal/controller/install/executor_controller.go

@@ -250,6 +250,8 @@ func (r *ExecutorReconciler) createDeployment(
 	volumes := createVolumes(executor.Name, executor.Spec.AdditionalVolumes)
 	volumeMounts := createVolumeMounts(GetConfigFilename(executor.Name), executor.Spec.AdditionalVolumeMounts)
 
+	readinessProbe, livenessProbe := CreateProbesWithScheme(corev1.URISchemeHTTP)
+
 	env := []corev1.EnvVar{
 		{
 			Name: "SERVICE_ACCOUNT",
@@ -278,6 +280,8 @@ func (r *ExecutorReconciler) createDeployment(
 		Env:             env,
 		VolumeMounts:    volumeMounts,
 		SecurityContext: executor.Spec.SecurityContext,
+		ReadinessProbe:  readinessProbe,
+		LivenessProbe:   livenessProbe,
 	}}
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{Name: executor.Name, Namespace: executor.Namespace, Labels: AllLabels(executor.Name, executor.Labels)},

internal/controller/install/lookout_controller.go

@@ -139,6 +139,7 @@ func (r *LookoutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 
 type LookoutConfig struct {
 	Postgres PostgresConfig
+	Tls      TlsConfig
 }
 
 func generateLookoutInstallComponents(
@@ -271,6 +272,13 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam
 	volumes := createVolumes(lookout.Name, lookout.Spec.AdditionalVolumes)
 	volumeMounts := createVolumeMounts(GetConfigFilename(lookout.Name), lookout.Spec.AdditionalVolumeMounts)
 
+	lookoutConfig, err := extractLookoutConfig(lookout.Spec.ApplicationConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(lookoutConfig.Tls))
+
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{Name: lookout.Name, Namespace: lookout.Namespace, Labels: AllLabels(lookout.Name, lookout.Labels)},
 		Spec: appsv1.DeploymentSpec{
@@ -299,6 +307,8 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam
 						Env:             env,
 						VolumeMounts:    volumeMounts,
 						SecurityContext: lookout.Spec.SecurityContext,
+						ReadinessProbe:  readinessProbe,
+						LivenessProbe:   livenessProbe,
 					}},
 					Volumes: volumes,
 				},
@@ -458,12 +468,7 @@ func createLookoutCronJob(lookout *installv1alpha1.Lookout, serviceAccountName s
 		dbPruningSchedule = *lookout.Spec.DbPruningSchedule
 	}
 
-	appConfig, err := builders.ConvertRawExtensionToYaml(lookout.Spec.ApplicationConfig)
-	if err != nil {
-		return nil, err
-	}
-	var lookoutConfig LookoutConfig
-	err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig)
+	lookoutConfig, err := extractLookoutConfig(lookout.Spec.ApplicationConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -555,3 +560,17 @@ func (r *LookoutReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&installv1alpha1.Lookout{}).
 		Complete(r)
 }
+
+// extractLookoutConfig will unmarshal the appconfig and return the LookoutConfig portion
+func extractLookoutConfig(config runtime.RawExtension) (LookoutConfig, error) {
+	appConfig, err := builders.ConvertRawExtensionToYaml(config)
+	if err != nil {
+		return LookoutConfig{}, err
+	}
+	var lookoutConfig LookoutConfig
+	err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig)
+	if err != nil {
+		return LookoutConfig{}, err
+	}
+	return lookoutConfig, err
+}