Run httpd using kolla

Instead of running the httpd -DFOREGROUND command as entrypoint for the -httpd sidecar container, this change moves the file copy and deployment logic to kolla. This is a requirement to not run the container as root user, because kolla helps to apply the right permissions to the config files (and pid) used by the process. The switch from root user to GlanceUID (already present as const) will be part of a different patch. Signed-off-by: Francesco Pantano <[email protected]>
openstack-k8s-operators · Sep 10, 2024 · 9e6cd1a · 9e6cd1a
1 parent 33fe3c0
commit 9e6cd1a
Show file tree

Hide file tree

Showing 9 changed files with 113 additions and 86 deletions.
diff --git a/pkg/glance/volumes.go b/pkg/glance/volumes.go
@@ -268,20 +268,13 @@ func GetHttpdVolumeMount() []corev1.VolumeMount {
 	return []corev1.VolumeMount{
 		{
 			Name:      "config-data",
-			MountPath: "/etc/httpd/conf/httpd.conf",
-			SubPath:   "httpd.conf",
-			ReadOnly:  true,
-		},
-		{
-			Name:      "config-data",
-			MountPath: "/etc/httpd/conf.d/10-glance.conf",
-			SubPath:   "10-glance-httpd.conf",
+			MountPath: "/var/lib/config-data/default",
 			ReadOnly:  true,
 		},
 		{
 			Name:      "config-data",
-			MountPath: "/etc/httpd/conf.d/ssl.conf",
-			SubPath:   "ssl.conf",
+			MountPath: "/var/lib/kolla/config_files/config.json",
+			SubPath:   "glance-httpd-config.json",
 			ReadOnly:  true,
 		},
 	}
@@ -339,3 +332,45 @@ func GetScriptVolumeMount() []corev1.VolumeMount {
 		},
 	}
 }
+
+// GetAPIVolumes -
+func GetAPIVolumes(name string) []corev1.Volume {
+	var config0644AccessMode int32 = 0644
+	apiVolumes := []corev1.Volume{
+		{
+			Name: "config-data-custom",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					DefaultMode: &config0644AccessMode,
+					SecretName:  name + "-config-data",
+				},
+			},
+		},
+	}
+	// Append LogVolume to the apiVolumes: this will be used to stream logging
+	apiVolumes = append(apiVolumes, GetLogVolume()...)
+	apiVolumes = append(apiVolumes, GetScriptVolume()...)
+	return apiVolumes
+}
+
+// GetAPIVolumeMount -
+func GetAPIVolumeMount(cacheSize string) []corev1.VolumeMount {
+	apiVolumeMounts := []corev1.VolumeMount{
+		{
+			Name:      "config-data",
+			MountPath: "/var/lib/kolla/config_files/config.json",
+			SubPath:   "glance-api-config.json",
+			ReadOnly:  true,
+		},
+	}
+	// Append LogVolume to apiVolumes: this will be used to stream logging
+	apiVolumeMounts = append(apiVolumeMounts, GetLogVolumeMount()...)
+	// Append ScriptsVolume to apiVolumes
+	apiVolumeMounts = append(apiVolumeMounts, GetScriptVolumeMount()...)
+	// If cache is provided, we expect the main glance_controller to request a
+	// PVC that should be used for that purpose (according to ImageCache.Size)
+	if len(cacheSize) > 0 {
+		apiVolumeMounts = append(apiVolumeMounts, GetCacheVolumeMount()...)
+	}
+	return apiVolumeMounts
+}
diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go
@@ -38,10 +38,8 @@ import (
 )
 
 const (
-	// GlanceAPIServiceCommand -
-	GlanceAPIServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start"
-	// GlanceAPIHttpdCommand -
-	GlanceAPIHttpdCommand = "/usr/sbin/httpd -DFOREGROUND"
+	// GlanceServiceCommand -
+	GlanceServiceCommand = "/usr/local/bin/kolla_start"
 )
 
 // StatefulSet func
@@ -54,8 +52,6 @@ func StatefulSet(
 ) (*appsv1.StatefulSet, error) {
 	runAsUser := int64(0)
 
-	var config0644AccessMode int32 = 0644
-
 	startupProbe := &corev1.Probe{
 		FailureThreshold: 6,
 		PeriodSeconds:    10,
@@ -111,41 +107,8 @@ func StatefulSet(
 	envVars["GLANCE_DOMAIN"] = env.SetValue(instance.Status.Domain)
 	envVars["URISCHEME"] = env.SetValue(string(glanceURIScheme))
 
-	apiVolumes := []corev1.Volume{
-		{
-			Name: "config-data-custom",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					DefaultMode: &config0644AccessMode,
-					SecretName:  instance.Name + "-config-data",
-				},
-			},
-		},
-	}
-	// Append LogVolume to the apiVolumes: this will be used to stream
-	// logging
-	apiVolumes = append(apiVolumes, glance.GetLogVolume()...)
-	apiVolumeMounts := []corev1.VolumeMount{
-		{
-			Name:      "config-data",
-			MountPath: "/var/lib/kolla/config_files/config.json",
-			SubPath:   "glance-api-config.json",
-			ReadOnly:  true,
-		},
-	}
-
-	// Append LogVolume to the apiVolumes: this will be used to stream logging
-	apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...)
-
-	// Append scripts
-	apiVolumes = append(apiVolumes, glance.GetScriptVolume()...)
-	apiVolumeMounts = append(apiVolumeMounts, glance.GetScriptVolumeMount()...)
-
-	// If cache is provided, we expect the main glance_controller to request a
-	// PVC that should be used for that purpose (according to ImageCacheSize)
-	if len(instance.Spec.ImageCache.Size) > 0 {
-		apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...)
-	}
+	apiVolumes := glance.GetAPIVolumes(instance.Name)
+	apiVolumeMounts := glance.GetAPIVolumeMount(instance.Spec.ImageCache.Size)
 
 	extraVolPropagation := append(glance.GlanceAPIPropagation,
 		storage.PropagationType(instance.APIName()))
@@ -255,7 +218,7 @@ func StatefulSet(
 								"--",
 								"/bin/bash",
 								"-c",
-								string(GlanceAPIHttpdCommand),
+								string(GlanceServiceCommand),
 							},
 							Image: instance.Spec.ContainerImage,
 							SecurityContext: &corev1.SecurityContext{
@@ -278,7 +241,7 @@ func StatefulSet(
 								"--",
 								"/bin/bash",
 								"-c",
-								string(GlanceAPIServiceCommand),
+								string(GlanceServiceCommand),
 							},
 							Image: instance.Spec.ContainerImage,
 							SecurityContext: &corev1.SecurityContext{

diff --git a/templates/glanceapi/config/glance-api-config.json b/templates/glanceapi/config/glance-api-config.json
@@ -56,22 +56,6 @@
         "owner": "root:root",
         "perm": "0755"
       },
-      {
-        "source": "/var/lib/config-data/tls/certs/*",
-        "dest": "/etc/pki/tls/certs/",
-        "owner": "root",
-        "perm": "0640",
-        "optional": true,
-        "merge": true
-      },
-      {
-        "source": "/var/lib/config-data/tls/private/*",
-        "dest": "/etc/pki/tls/private/",
-        "owner": "root",
-        "perm": "0600",
-        "optional": true,
-        "merge": true
-      },
       {
         "source": "/usr/local/bin/container-scripts/kolla_extend_start",
         "dest": "/usr/local/bin/kolla_extend_start",

diff --git a/templates/glanceapi/config/glance-httpd-config.json b/templates/glanceapi/config/glance-httpd-config.json
@@ -0,0 +1,49 @@
+{
+    "command": "/usr/sbin/httpd -DFOREGROUND",
+    "config_files": [
+      {
+        "source": "/var/lib/config-data/tls/certs/*",
+        "dest": "/etc/pki/tls/certs/",
+        "owner": "glance:glance",
+        "perm": "0640",
+        "optional": true,
+        "merge": true
+      },
+      {
+        "source": "/var/lib/config-data/tls/private/*",
+        "dest": "/etc/pki/tls/private/",
+        "owner": "glance:glance",
+        "perm": "0640",
+        "optional": true,
+        "merge": true
+      },
+      {
+        "source": "/var/lib/config-data/default/httpd.conf",
+        "dest": "/etc/httpd/conf/httpd.conf",
+        "owner": "glance:apache",
+        "optional": true,
+        "perm": "0644"
+      },
+      {
+        "source": "/var/lib/config-data/default/10-glance-httpd.conf",
+        "dest": "/etc/httpd/conf.d/10-glance.conf",
+        "owner": "glance:apache",
+        "optional": true,
+        "perm": "0644"
+      },
+      {
+        "source": "/var/lib/config-data/default/ssl.conf",
+        "dest": "/etc/httpd/conf.d/ssl.conf",
+        "owner": "glance:apache",
+        "optional": true,
+        "perm": "0644"
+      }
+    ],
+    "permissions": [
+        {
+            "path": "/etc/httpd/run",
+            "owner": "glance:apache",
+            "recurse": true
+        }
+    ]
+}
diff --git a/templates/glanceapi/config/httpd.conf b/templates/glanceapi/config/httpd.conf
@@ -19,5 +19,6 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A
 SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
 CustomLog /dev/stdout combined env=!forwarded
 CustomLog /dev/stdout proxy env=forwarded
+ErrorLog /dev/stdout
 
 Include conf.d/10-glance.conf
diff --git a/test/functional/glanceapi_controller_test.go b/test/functional/glanceapi_controller_test.go
@@ -217,7 +217,7 @@ var _ = Describe("Glanceapi controller", func() {
 
 			// Check the glance-httpd container
 			container = ss.Spec.Template.Spec.Containers[1]
-			Expect(container.VolumeMounts).To(HaveLen(3))
+			Expect(container.VolumeMounts).To(HaveLen(2))
 			Expect(container.Image).To(Equal(glanceTest.ContainerImage))
 
 			// Check the glance-log container

diff --git a/test/kuttl/tests/glance_single/01-assert.yaml b/test/kuttl/tests/glance_single/01-assert.yaml
@@ -68,7 +68,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/sbin/httpd -DFOREGROUND
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-httpd
@@ -77,7 +77,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-api

diff --git a/test/kuttl/tests/glance_single_tls/01-assert.yaml b/test/kuttl/tests/glance_single_tls/01-assert.yaml
@@ -66,20 +66,15 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/sbin/httpd -DFOREGROUND
+        - /usr/local/bin/kolla_start
         volumeMounts:
-         - mountPath: /etc/httpd/conf/httpd.conf
-           name: config-data
-           readOnly: true
-           subPath: httpd.conf
-         - mountPath: /etc/httpd/conf.d/10-glance.conf
+         - mountPath: /var/lib/config-data/default
            name: config-data
            readOnly: true
-           subPath: 10-glance-httpd.conf
-         - mountPath: /etc/httpd/conf.d/ssl.conf
+         - mountPath: /var/lib/kolla/config_files/config.json
            name: config-data
            readOnly: true
-           subPath: ssl.conf
+           subPath: glance-httpd-config.json
          - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
            name: combined-ca-bundle
            readOnly: true
@@ -106,7 +101,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start
+        - /usr/local/bin/kolla_start
         volumeMounts:
          - mountPath: /var/lib/config-data/default
            name: config-data

diff --git a/test/kuttl/tests/glance_split/01-assert.yaml b/test/kuttl/tests/glance_split/01-assert.yaml
@@ -81,7 +81,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/sbin/httpd -DFOREGROUND
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-httpd
@@ -90,7 +90,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-api
@@ -129,7 +129,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/sbin/httpd -DFOREGROUND
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-httpd
@@ -138,7 +138,7 @@ spec:
         - --
         - /bin/bash
         - -c
-        - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start
+        - /usr/local/bin/kolla_start
         command:
         - /usr/bin/dumb-init
         name: glance-api