From 727a14512838f17cafd66260a7a3108e36a13059 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 9 Sep 2024 22:29:52 +0200 Subject: [PATCH] Run httpd using kolla Instead of running the httpd -DFOREGROUND command as entrypoint for the -httpd sidecar container, this change moves the file copy and deployment logic to kolla. This is a requirement to not run the container as root user, because kolla helps to apply the right permissions to the config files (and pid) used by the process. The switch from root user to GlanceUID (already present as const) will be part of a different patch. Signed-off-by: Francesco Pantano --- pkg/glance/volumes.go | 55 +++++++++++++++---- pkg/glanceapi/statefulset.go | 49 ++--------------- .../glanceapi/config/glance-api-config.json | 16 ------ .../glanceapi/config/glance-httpd-config.json | 49 +++++++++++++++++ templates/glanceapi/config/httpd.conf | 1 + test/functional/glanceapi_controller_test.go | 2 +- test/kuttl/tests/glance_single/01-assert.yaml | 4 +- .../tests/glance_single_tls/01-assert.yaml | 4 +- test/kuttl/tests/glance_split/01-assert.yaml | 8 +-- 9 files changed, 110 insertions(+), 78 deletions(-) create mode 100644 templates/glanceapi/config/glance-httpd-config.json diff --git a/pkg/glance/volumes.go b/pkg/glance/volumes.go index 8953a198..16f6bce4 100644 --- a/pkg/glance/volumes.go +++ b/pkg/glance/volumes.go @@ -268,20 +268,13 @@ func GetHttpdVolumeMount() []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "config-data", - MountPath: "/etc/httpd/conf/httpd.conf", - SubPath: "httpd.conf", - ReadOnly: true, - }, - { - Name: "config-data", - MountPath: "/etc/httpd/conf.d/10-glance.conf", - SubPath: "10-glance-httpd.conf", + MountPath: "/var/lib/config-data/default", ReadOnly: true, }, { Name: "config-data", - MountPath: "/etc/httpd/conf.d/ssl.conf", - SubPath: "ssl.conf", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "glance-httpd-config.json", ReadOnly: true, }, } @@ -339,3 +332,45 @@ func GetScriptVolumeMount() []corev1.VolumeMount { }, } } + +// GetAPIVolumes - +func GetAPIVolumes(name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + apiVolumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + // Append LogVolume to the apiVolumes: this will be used to stream logging + apiVolumes = append(apiVolumes, GetLogVolume()...) + apiVolumes = append(apiVolumes, GetScriptVolume()...) + return apiVolumes +} + +// GetAPIVolumeMount - +func GetAPIVolumeMount(cacheSize string) []corev1.VolumeMount { + apiVolumeMounts := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "glance-api-config.json", + ReadOnly: true, + }, + } + // Append LogVolume to apiVolumes: this will be used to stream logging + apiVolumeMounts = append(apiVolumeMounts, GetLogVolumeMount()...) + // Append ScriptsVolume to apiVolumes + apiVolumeMounts = append(apiVolumeMounts, GetScriptVolumeMount()...) + // If cache is provided, we expect the main glance_controller to request a + // PVC that should be used for that purpose (according to ImageCache.Size) + if len(cacheSize) > 0 { + apiVolumeMounts = append(apiVolumeMounts, GetCacheVolumeMount()...) + } + return apiVolumeMounts +} diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go index 18908116..0d73dde4 100644 --- a/pkg/glanceapi/statefulset.go +++ b/pkg/glanceapi/statefulset.go @@ -38,10 +38,8 @@ import ( ) const ( - // GlanceAPIServiceCommand - - GlanceAPIServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" - // GlanceAPIHttpdCommand - - GlanceAPIHttpdCommand = "/usr/sbin/httpd -DFOREGROUND" + // GlanceServiceCommand - + GlanceServiceCommand = "/usr/local/bin/kolla_start" ) // StatefulSet func @@ -54,8 +52,6 @@ func StatefulSet( ) (*appsv1.StatefulSet, error) { runAsUser := int64(0) - var config0644AccessMode int32 = 0644 - startupProbe := &corev1.Probe{ FailureThreshold: 6, PeriodSeconds: 10, @@ -111,41 +107,8 @@ func StatefulSet( envVars["GLANCE_DOMAIN"] = env.SetValue(instance.Status.Domain) envVars["URISCHEME"] = env.SetValue(string(glanceURIScheme)) - apiVolumes := []corev1.Volume{ - { - Name: "config-data-custom", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - DefaultMode: &config0644AccessMode, - SecretName: instance.Name + "-config-data", - }, - }, - }, - } - // Append LogVolume to the apiVolumes: this will be used to stream - // logging - apiVolumes = append(apiVolumes, glance.GetLogVolume()...) - apiVolumeMounts := []corev1.VolumeMount{ - { - Name: "config-data", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: "glance-api-config.json", - ReadOnly: true, - }, - } - - // Append LogVolume to the apiVolumes: this will be used to stream logging - apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...) - - // Append scripts - apiVolumes = append(apiVolumes, glance.GetScriptVolume()...) - apiVolumeMounts = append(apiVolumeMounts, glance.GetScriptVolumeMount()...) - - // If cache is provided, we expect the main glance_controller to request a - // PVC that should be used for that purpose (according to ImageCacheSize) - if len(instance.Spec.ImageCache.Size) > 0 { - apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...) - } + apiVolumes := glance.GetAPIVolumes(instance.Name) + apiVolumeMounts := glance.GetAPIVolumeMount(instance.Spec.ImageCache.Size) extraVolPropagation := append(glance.GlanceAPIPropagation, storage.PropagationType(instance.APIName())) @@ -255,7 +218,7 @@ func StatefulSet( "--", "/bin/bash", "-c", - string(GlanceAPIHttpdCommand), + string(GlanceServiceCommand), }, Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ @@ -278,7 +241,7 @@ func StatefulSet( "--", "/bin/bash", "-c", - string(GlanceAPIServiceCommand), + string(GlanceServiceCommand), }, Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ diff --git a/templates/glanceapi/config/glance-api-config.json b/templates/glanceapi/config/glance-api-config.json index 9b6bd608..996d8b24 100644 --- a/templates/glanceapi/config/glance-api-config.json +++ b/templates/glanceapi/config/glance-api-config.json @@ -56,22 +56,6 @@ "owner": "root:root", "perm": "0755" }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "root", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "root", - "perm": "0600", - "optional": true, - "merge": true - }, { "source": "/usr/local/bin/container-scripts/kolla_extend_start", "dest": "/usr/local/bin/kolla_extend_start", diff --git a/templates/glanceapi/config/glance-httpd-config.json b/templates/glanceapi/config/glance-httpd-config.json new file mode 100644 index 00000000..109b5b52 --- /dev/null +++ b/templates/glanceapi/config/glance-httpd-config.json @@ -0,0 +1,49 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "glance:glance", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "glance:glance", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/default/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + }, + { + "source": "/var/lib/config-data/default/10-glance-httpd.conf", + "dest": "/etc/httpd/conf.d/10-glance.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + }, + { + "source": "/var/lib/config-data/default/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + } + ], + "permissions": [ + { + "path": "/etc/httpd/run", + "owner": "glance:apache", + "recurse": true + } + ] +} diff --git a/templates/glanceapi/config/httpd.conf b/templates/glanceapi/config/httpd.conf index 75c972c0..cd06bf92 100644 --- a/templates/glanceapi/config/httpd.conf +++ b/templates/glanceapi/config/httpd.conf @@ -19,5 +19,6 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout Include conf.d/10-glance.conf diff --git a/test/functional/glanceapi_controller_test.go b/test/functional/glanceapi_controller_test.go index 01c397e0..d504f7a3 100644 --- a/test/functional/glanceapi_controller_test.go +++ b/test/functional/glanceapi_controller_test.go @@ -217,7 +217,7 @@ var _ = Describe("Glanceapi controller", func() { // Check the glance-httpd container container = ss.Spec.Template.Spec.Containers[1] - Expect(container.VolumeMounts).To(HaveLen(3)) + Expect(container.VolumeMounts).To(HaveLen(2)) Expect(container.Image).To(Equal(glanceTest.ContainerImage)) // Check the glance-log container diff --git a/test/kuttl/tests/glance_single/01-assert.yaml b/test/kuttl/tests/glance_single/01-assert.yaml index d0a9cf98..1be4a233 100644 --- a/test/kuttl/tests/glance_single/01-assert.yaml +++ b/test/kuttl/tests/glance_single/01-assert.yaml @@ -68,7 +68,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -77,7 +77,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api diff --git a/test/kuttl/tests/glance_single_tls/01-assert.yaml b/test/kuttl/tests/glance_single_tls/01-assert.yaml index ab021b29..9103ad6f 100644 --- a/test/kuttl/tests/glance_single_tls/01-assert.yaml +++ b/test/kuttl/tests/glance_single_tls/01-assert.yaml @@ -66,7 +66,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start volumeMounts: - mountPath: /etc/httpd/conf/httpd.conf name: config-data @@ -106,7 +106,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start volumeMounts: - mountPath: /var/lib/config-data/default name: config-data diff --git a/test/kuttl/tests/glance_split/01-assert.yaml b/test/kuttl/tests/glance_split/01-assert.yaml index 676ac6ca..3faf4fd9 100644 --- a/test/kuttl/tests/glance_split/01-assert.yaml +++ b/test/kuttl/tests/glance_split/01-assert.yaml @@ -81,7 +81,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -90,7 +90,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api @@ -129,7 +129,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -138,7 +138,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api