WIP Initial support for TLS in image-upload

controllers/octavia_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"bufio"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"strings"
@@ -37,14 +38,15 @@ import (
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
 	oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
+	common_tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/octavia-operator/pkg/octavia"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/ptr"
 
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -1184,7 +1186,14 @@ func (r *OctaviaReconciler) reconcileAmphoraImages(
 		Log.Info("Image Upload Pod not ready")
 		return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, nil
 	}
-	endpoint, err := svc.GetAPIEndpoint(nil, nil, "")
+
+	// if TLS is enabled
+	var protocol *service.Protocol = nil
+	if instance.Spec.OctaviaAPI.TLS.API.Enabled(service.EndpointInternal) {
+		// set endpoint protocol to https
+		protocol = ptr.To(service.ProtocolHTTPS)
+	}
+	endpoint, err := svc.GetAPIEndpoint(nil, protocol, "")
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -1195,6 +1204,7 @@ func (r *OctaviaReconciler) reconcileAmphoraImages(
 		return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err
 	}
 
+	fmt.Printf(">>>>>>> %+v\n\n\n\n\n\n", urlMap)
 	ok, err := octavia.EnsureAmphoraImages(ctx, instance, &r.Log, helper, urlMap)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -1223,6 +1233,9 @@ func (r *OctaviaReconciler) getLocalImageURLs(
 	// Get the list of images and their hashes
 	listURL := fmt.Sprintf("%s/octavia-amphora-images.sha256sum", endpoint)
 
+	// TODO(gthiemonge) Remove when image-upload uses its own certificate
+	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
 	resp, err := http.Get(listURL)
 	if err != nil {
 		return nil, err
@@ -1264,9 +1277,9 @@ func (r *OctaviaReconciler) generateServiceConfigMaps(
 
 	cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(octavia.ServiceName), map[string]string{})
 
-	var tlsCfg *tls.Service
+	var tlsCfg *common_tls.Service
 	if instance.Spec.OctaviaAPI.TLS.Ca.CaBundleSecretName != "" {
-		tlsCfg = &tls.Service{}
+		tlsCfg = &common_tls.Service{}
 	}
 
 	// customData hold any customization for the service.
@@ -1304,6 +1317,12 @@ func (r *OctaviaReconciler) generateServiceConfigMaps(
 		),
 	}
 	templateParameters["ServiceUser"] = instance.Spec.ServiceUser
+	templateParameters["TLS"] = false // default TLS to false, and set it bellow to true if enabled
+	if instance.Spec.OctaviaAPI.TLS.API.Enabled(service.EndpointInternal) {
+		templateParameters["TLS"] = true
+		templateParameters["SSLCertificateFile"] = fmt.Sprintf("/var/lib/config-data/merged/%s.crt", string(service.EndpointInternal))
+		templateParameters["SSLCertificateKeyFile"] = fmt.Sprintf("/var/lib/config-data/merged/%s.key", string(service.EndpointInternal))
+	}
 
 	cms := []util.Template{
 		// ScriptsConfigMap

pkg/octavia/const.go

@@ -45,7 +45,7 @@ const (
 	OctaviaInternalPort int32 = 9876
 
 	// ApacheInternalPort -
-	ApacheInternalPort int32 = 80
+	ApacheInternalPort int32 = 443
 
 	// AdminTenantName
 	AdminTenant = "admin"

pkg/octavia/image_upload_deployment.go

@@ -18,6 +18,7 @@ package octavia
 import (
 	"fmt"
 
+	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -32,11 +33,12 @@ type ImageUploadDetails struct {
 
 const (
 	// ServiceCommand -
-	ServiceCommand = "cp -f /usr/local/apache2/conf/httpd.conf /etc/httpd/conf/httpd.conf && /usr/bin/run-httpd"
+	ServiceCommand = "/usr/local/bin/container-scripts/image_upload_run.sh"
 )
 
 func getVolumes(name string) []corev1.Volume {
-	var config0640AccessMode int32 = 0640
+	var scriptsVolumeDefaultMode int32 = 0755
+	var config0640AccessMode int32 = 0644
 
 	return []corev1.Volume{
 		{
@@ -46,22 +48,47 @@ func getVolumes(name string) []corev1.Volume {
 			},
 		},
 		{
-			Name: "httpd-config",
+			Name: "scripts",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					DefaultMode: &scriptsVolumeDefaultMode,
+					SecretName:  name + "-scripts",
+				},
+			},
+		},
+		{
+			Name: "config-data",
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					DefaultMode: &config0640AccessMode,
 					SecretName:  name + "-config-data",
 				},
 			},
 		},
+		{
+			Name: "config-data-merged",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""},
+			},
+		},
 	}
 }
 
 func getInitVolumeMounts() []corev1.VolumeMount {
 	return []corev1.VolumeMount{
 		{
 			Name:      "amphora-image",
-			MountPath: "/usr/local/apache2/htdocs",
+			MountPath: "/www",
+		},
+		{
+			Name:      "scripts",
+			MountPath: "/usr/local/bin/container-scripts",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "config-data-merged",
+			MountPath: "/var/lib/config-data/merged",
+			ReadOnly:  false,
 		},
 	}
 }
@@ -71,12 +98,22 @@ func getVolumeMounts() []corev1.VolumeMount {
 	return []corev1.VolumeMount{
 		{
 			Name:      "amphora-image",
-			MountPath: "/usr/local/apache2/htdocs",
+			MountPath: "/www",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "scripts",
+			MountPath: "/usr/local/bin/container-scripts",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "config-data",
+			MountPath: "/var/lib/config-data/default",
+			ReadOnly:  true,
 		},
 		{
-			Name:      "httpd-config",
-			MountPath: "/usr/local/apache2/conf/httpd.conf",
-			SubPath:   "httpd.conf",
+			Name:      "config-data-merged",
+			MountPath: "/var/lib/config-data/merged",
 			ReadOnly:  true,
 		},
 	}
@@ -87,12 +124,34 @@ func ImageUploadDeployment(
 	instance *octaviav1.Octavia,
 	labels map[string]string,
 ) *appsv1.Deployment {
-	initVolumeMounts := getInitVolumeMounts()
-
 	args := []string{"-c", ServiceCommand}
 
 	serviceName := fmt.Sprintf("%s-image-upload", ServiceName)
 
+	// create Volume and VolumeMounts
+	volumes := getVolumes(instance.Name)
+	volumeMounts := getVolumeMounts()
+	initVolumeMounts := getInitVolumeMounts()
+
+	// add CA cert if defined
+	if instance.Spec.OctaviaAPI.TLS.CaBundleSecretName != "" {
+		volumes = append(volumes, instance.Spec.OctaviaAPI.TLS.CreateVolume())
+		volumeMounts = append(volumeMounts, instance.Spec.OctaviaAPI.TLS.CreateVolumeMounts(nil)...)
+		initVolumeMounts = append(initVolumeMounts, instance.Spec.OctaviaAPI.TLS.CreateVolumeMounts(nil)...)
+	}
+
+	if instance.Spec.OctaviaAPI.TLS.API.Enabled(service.EndpointInternal) {
+		tlsEndptCfg := instance.Spec.OctaviaAPI.TLS.API.Internal
+
+		svc, err := tlsEndptCfg.ToService()
+		if err != nil {
+			return nil //, err
+		}
+		volumes = append(volumes, svc.CreateVolume(string(service.EndpointInternal)))
+		volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(string(service.EndpointInternal))...)
+		initVolumeMounts = append(initVolumeMounts, svc.CreateVolumeMounts(string(service.EndpointInternal))...)
+	}
+
 	depl := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      serviceName,
@@ -116,12 +175,12 @@ func ImageUploadDeployment(
 							},
 							Args:         args,
 							Image:        instance.Spec.ApacheContainerImage,
-							VolumeMounts: getVolumeMounts(),
+							VolumeMounts: volumeMounts,
 							Resources:    instance.Spec.Resources,
 							// TODO(gthiemonge) do we need probes?
 						},
 					},
-					Volumes: getVolumes(instance.Name),
+					Volumes: volumes,
 				},
 			},
 		},
@@ -131,29 +190,48 @@ func ImageUploadDeployment(
 		ContainerImage: instance.Spec.AmphoraImageContainerImage,
 		VolumeMounts:   initVolumeMounts,
 	}
-	depl.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails)
+	depl.Spec.Template.Spec.InitContainers = initContainer(instance, initContainerDetails)
 
 	return depl
 }
 
-func initContainer(init ImageUploadDetails) []corev1.Container {
+func initContainer(
+	instance *octaviav1.Octavia,
+	init ImageUploadDetails,
+) []corev1.Container {
 	runAsUser := int64(0)
 	envs := []corev1.EnvVar{
 		{
 			Name:  "DEST_DIR",
-			Value: "/usr/local/apache2/htdocs",
+			Value: "/www",
 		},
 	}
 
 	return []corev1.Container{
 		{
-			Name:  "init",
+			Name: "init",
+			Command: []string{
+				"/bin/bash",
+			},
+			Args: []string{
+				"-c",
+				"/usr/local/bin/container-scripts/image_upload_init.sh",
+			},
+			Image: instance.Spec.ApacheContainerImage,
+			SecurityContext: &corev1.SecurityContext{
+				RunAsUser: &runAsUser,
+			},
+			Env:          envs,
+			VolumeMounts: init.VolumeMounts,
+		},
+		{
+			Name:  "init-image",
 			Image: init.ContainerImage,
 			SecurityContext: &corev1.SecurityContext{
 				RunAsUser: &runAsUser,
 			},
 			Env:          envs,
-			VolumeMounts: getInitVolumeMounts(),
+			VolumeMounts: init.VolumeMounts,
 		},
 	}
 }

templates/octavia/bin/image_upload_init.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -xe
+
+TLS_SRC_DIR=/var/lib/config-data/tls
+TLS_DST_DIR=/var/lib/config-data/merged
+
+if [ -e $TLS_SRC_DIR/certs/internal.crt ]; then
+    cp $TLS_SRC_DIR/certs/internal.crt $TLS_DST_DIR/
+    chown default $TLS_DST_DIR/internal.crt
+    chmod 400 $TLS_DST_DIR/internal.crt
+fi
+
+if [ -e $TLS_SRC_DIR/private/internal.key ]; then
+    cp $TLS_SRC_DIR/private/internal.key $TLS_DST_DIR/
+    chown default $TLS_DST_DIR/internal.key
+    chmod 400 $TLS_DST_DIR/internal.key
+fi

templates/octavia/bin/image_upload_run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -xe
+
+cp -f /var/lib/config-data/default/httpd.conf /etc/httpd/conf/httpd.conf
+
+exec /usr/bin/run-httpd

templates/octavia/config/httpd.conf

@@ -116,8 +116,8 @@ ServerAdmin root@localhost
 # documents. By default, all requests are taken from this directory, but
 # symbolic links and aliases may be used to point to other locations.
 #
-DocumentRoot "/usr/local/apache2/htdocs"
-<Directory "/usr/local/apache2/htdocs">
+DocumentRoot "/www"
+<Directory "/www">
     #
     # Possible values for the Options directive are "None", "All",
     # or any combination of:
@@ -344,3 +344,12 @@ IncludeOptional conf.d/autoindex.conf
 IncludeOptional conf.d/mod_security.conf
 IncludeOptional conf.d/userdir.conf
 #IncludeOptional conf.d/welcome.conf
+
+{{ if .TLS }}
+SetEnvIf X-Forwarded-Proto https HTTPS=1
+
+## SSL directives
+SSLEngine on
+SSLCertificateFile      "{{ .SSLCertificateFile }}"
+SSLCertificateKeyFile   "{{ .SSLCertificateKeyFile }}"
+{{ end }}

templates/octavia/config/octavia-image-upload.json

@@ -0,0 +1,21 @@
+{
+    "command": "cp -f /usr/local/apache2/conf/httpd.conf /etc/httpd/conf/httpd.conf && /usr/bin/run-httpd",
+    "config_files": [
+          {
+            "source": "/var/lib/config-data/tls/certs/*",
+            "dest": "/etc/pki/tls/certs/",
+            "owner": "octavia",
+            "perm": "0440",
+            "optional": true,
+            "merge": true
+          },
+          {
+            "source": "/var/lib/config-data/tls/private/*",
+            "dest": "/etc/pki/tls/private/",
+            "owner": "octavia",
+            "perm": "0400",
+            "optional": true,
+            "merge": true
+          }
+    ]
+}