diff --git a/Makefile b/Makefile index 1c229fff..f6969a1a 100644 --- a/Makefile +++ b/Makefile @@ -341,15 +341,15 @@ operator-lint: gowork ## Runs operator-lint go vet -vettool=$(LOCALBIN)/operator-lint ./... ./api/... # Used for webhook testing -# The configure_local_webhooks.sh script below will remove any OLM webhooks +# The configure_local_webhook.sh script below will remove any OLM webhooks # for the operator and also scale its deployment replicas down to 0 so that # the operator can run locally. -# Make sure to cleanup the webhook configuration for local testing by running -# ./hack/clean_local_webhook.sh before deplying with OLM again. +# We will attempt to catch SIGINT/SIGTERM and clean up the local webhooks, +# but it may be necessary to manually run ./hack/clean_local_webhook.sh +# before deploying with OLM again for other untrappable signals. SKIP_CERT ?=false .PHONY: run-with-webhook run-with-webhook: export METRICS_PORT?=8080 run-with-webhook: export HEALTH_PORT?=8081 run-with-webhook: manifests generate fmt vet ## Run a controller from your host. - /bin/bash hack/configure_local_webhook.sh - go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)" + /bin/bash hack/run_with_local_webhook.sh diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index e55ddef1..5c69b819 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -125,6 +125,7 @@ spec: type: string type: array createDefaultLbMgmtNetwork: + default: true description: CreateDefaultLbMgmtNetwork - when True, octavia-operator creates a Management Network for the default Availability Zone of the control plane. Can be set to false when deploying OpenStack @@ -137,6 +138,7 @@ spec: Attachment Definition type: string manageLbMgmtNetworks: + default: true description: ManageLbMgmtNetworks - when True, octavia-operator creates the Neutron resources needed for its Management Network type: boolean diff --git a/api/go.mod b/api/go.mod index 4375b299..678a6b71 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,10 +5,10 @@ go 1.21 require ( github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 - github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af - k8s.io/api v0.29.9 - k8s.io/apimachinery v0.29.9 - k8s.io/client-go v0.29.9 + github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333 + k8s.io/api v0.29.10 + k8s.io/apimachinery v0.29.10 + k8s.io/client-go v0.29.10 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -62,8 +62,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.9 // indirect - k8s.io/component-base v0.29.9 // indirect + k8s.io/apiextensions-apiserver v0.29.10 // indirect + k8s.io/component-base v0.29.10 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect diff --git a/api/go.sum b/api/go.sum index 607a956f..1f528578 100644 --- a/api/go.sum +++ b/api/go.sum @@ -71,8 +71,8 @@ github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af h1:MNNuYPfaXWf14lfsWGay3WjLjIIu9KN5+iHrLdCMQac= -github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af/go.mod h1:EKfyFMU2brJYqR4M2Hwfo5ZLBQxAJXAYfHjpdqX6N5g= +github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333 h1:yejekTWudX5af3mCJQ1MUPLEa0X6sIsklf07o9KilRk= +github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -175,16 +175,16 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.9 h1:FwdflpNsfMUYUOblMZNWJ4K/q0OSL5A4jGa0iOgcJco= -k8s.io/api v0.29.9/go.mod h1:fNhmzRfKaSEHCmczA/jRx6CiDKhYOnFLJBERMJAXEk8= -k8s.io/apiextensions-apiserver v0.29.9 h1:EB6RK06kFJjbzBwU1YiVznxrcgBE0hhDWt6EQQIcOy4= -k8s.io/apiextensions-apiserver v0.29.9/go.mod h1:jcaHG6R/bB1iU6XzC1DMhB1x2ktTJLt2KKpg6B65Z2c= -k8s.io/apimachinery v0.29.9 h1:YZ8HUid1TzQVz94cnNlsQjLdH0VoAhWSqz7t0q6B12A= -k8s.io/apimachinery v0.29.9/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= -k8s.io/client-go v0.29.9 h1:4f/Wz6li3rEyIPFj32XAQMtOGMM1tg7KQi1oeS6ibPg= -k8s.io/client-go v0.29.9/go.mod h1:2N1drQEZ5yiYrWVaE2Un8JiISUhl47D8pyZlYLszke4= -k8s.io/component-base v0.29.9 h1:lPENvp3CCwdeMEWGjiTfn5b287qQYuK7gX32OBOovmA= -k8s.io/component-base v0.29.9/go.mod h1:NGDa6Ih0EdcLA2G4K2ZYySoiB+2Tn+rmSqPyudCPgDY= +k8s.io/api v0.29.10 h1:Fao3HOxccbGRC1HZtXD+Y41xJhP0tEToVo5W7EEUBm0= +k8s.io/api v0.29.10/go.mod h1:rF0sRh64w1hMNAVGh4YYniSxODyHye3GLmymAbWBDvY= +k8s.io/apiextensions-apiserver v0.29.10 h1:2k2AHrWgs4S+07HrOjEdcyrQ/UqYlvBi6uon1fox1KE= +k8s.io/apiextensions-apiserver v0.29.10/go.mod h1:3rrJC52bW5frHf10N3JvaDpl++hgrzD9Gs6IZpKWMoc= +k8s.io/apimachinery v0.29.10 h1:57OLNqOJUgp5KlRRY3JOBFOTTa5Rt/LVkmKiiN2cvaQ= +k8s.io/apimachinery v0.29.10/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= +k8s.io/client-go v0.29.10 h1:hPmG1pmKslRhmCIzVd90sA58B0sJwNwduNgXFWsFqhI= +k8s.io/client-go v0.29.10/go.mod h1:gnMCQiRXGL9K0VtlW8gTkhzptGrHm2BJ4qBbujNemc4= +k8s.io/component-base v0.29.10 h1:YQrQ/bpzGPGqIPEPaBzxjH0/1DJOI+yZPZNbbz7ZCBY= +k8s.io/component-base v0.29.10/go.mod h1:IbwsBob2DnYiAONsSHIuYenchqcDycbHSLHrXshuLgM= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index e95e6a1e..01ce3f6e 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -231,10 +231,12 @@ type PasswordSelector struct { // OctaviaLbMgmtNetworks Settings for Octavia management networks type OctaviaLbMgmtNetworks struct { // +kubebuilder:validation:Optional + // +kubebuilder:default=true // ManageLbMgmtNetworks - when True, octavia-operator creates the Neutron resources needed for its Management Network ManageLbMgmtNetworks bool `json:"manageLbMgmtNetworks"` // +kubebuilder:validation:Optional + // +kubebuilder:default=true // CreateDefaultLbMgmtNetwork - when True, octavia-operator creates a // Management Network for the default Availability Zone of the control // plane. Can be set to false when deploying OpenStack in DCN mode. diff --git a/api/v1beta1/octaviaapi_types.go b/api/v1beta1/octaviaapi_types.go index 73c28179..a4137e8e 100644 --- a/api/v1beta1/octaviaapi_types.go +++ b/api/v1beta1/octaviaapi_types.go @@ -220,6 +220,6 @@ func init() { // IsReady - returns true if service is ready to server requests func (instance OctaviaAPI) IsReady() bool { - return instance.Status.Conditions.IsTrue(condition.ExposeServiceReadyCondition) && + return instance.Status.Conditions.IsTrue(condition.CreateServiceReadyCondition) && instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) } diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index e55ddef1..5c69b819 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -125,6 +125,7 @@ spec: type: string type: array createDefaultLbMgmtNetwork: + default: true description: CreateDefaultLbMgmtNetwork - when True, octavia-operator creates a Management Network for the default Availability Zone of the control plane. Can be set to false when deploying OpenStack @@ -137,6 +138,7 @@ spec: Attachment Definition type: string manageLbMgmtNetworks: + default: true description: ManageLbMgmtNetworks - when True, octavia-operator creates the Neutron resources needed for its Management Network type: boolean diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index cec149a0..5087ec2b 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -12,6 +12,7 @@ spec: - name: kube-rbac-proxy securityContext: allowPrivilegeEscalation: false + runAsNonRoot: true capabilities: drop: - "ALL" diff --git a/config/samples/network-attachment-definition/kustomization.yaml b/config/samples/network-attachment-definition/kustomization.yaml new file mode 100644 index 00000000..973feccb --- /dev/null +++ b/config/samples/network-attachment-definition/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- octavia-nad.yaml diff --git a/config/samples/network-attachment-definition/octavia-nad.yaml b/config/samples/network-attachment-definition/octavia-nad.yaml new file mode 100644 index 00000000..f9111f0a --- /dev/null +++ b/config/samples/network-attachment-definition/octavia-nad.yaml @@ -0,0 +1,24 @@ +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: octavia +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "octavia", + "type": "bridge", + "bridge": "octbr", + "ipam": { + "type": "whereabouts", + "range": "172.23.0.0/24", + "range_start": "172.23.0.30", + "range_end": "172.23.0.70", + "routes": [ + { + "dst": "172.24.0.0/16", + "gw" : "172.23.0.150" + } + ] + } + } diff --git a/config/samples/octavia_v1beta1_octavia.yaml b/config/samples/octavia_v1beta1_octavia.yaml index 93155e4c..4680e6eb 100644 --- a/config/samples/octavia_v1beta1_octavia.yaml +++ b/config/samples/octavia_v1beta1_octavia.yaml @@ -10,6 +10,9 @@ spec: rabbitMqClusterName: rabbitmq secret: osp-secret preserveJobs: false + lbMgmtNetwork: + availabilityZones: + - zone-1 customServiceConfig: | [DEFAULT] debug = true diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 6c3c94df..44af0964 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -187,7 +187,7 @@ func (r *OctaviaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAPIReadyCondition, condition.InitReason, octaviav1.OctaviaAPIReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), - condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAmphoraCertsReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraCertsReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaQuotasReadyCondition, condition.InitReason, octaviav1.OctaviaQuotasReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAmphoraSSHReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraSSHReadyInitMessage), @@ -1020,7 +1020,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav octaviav1.OctaviaAmphoraImagesReadyCondition, octaviav1.OctaviaAmphoraImagesReadyCompleteMessage) - instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) // create Deployment - end @@ -1213,10 +1213,10 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( ) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, + condition.CreateServiceReadyErrorMessage, err.Error())) return ctrl.Result{}, err @@ -1231,19 +1231,19 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( ctrlResult, err = svc.CreateOrPatch(ctx, helper) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, + condition.CreateServiceReadyErrorMessage, err.Error())) return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.RequestedReason, condition.SeverityInfo, - condition.ExposeServiceReadyRunningMessage)) + condition.CreateServiceReadyRunningMessage)) return ctrlResult, nil } Log.Info("Initializing amphora image upload deployment") diff --git a/controllers/octaviaapi_controller.go b/controllers/octaviaapi_controller.go index d76fe1b6..69cf38df 100644 --- a/controllers/octaviaapi_controller.go +++ b/controllers/octaviaapi_controller.go @@ -162,7 +162,7 @@ func (r *OctaviaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) // cl := condition.CreateList( condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), @@ -411,10 +411,10 @@ func (r *OctaviaAPIReconciler) reconcileInit( ) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, + condition.CreateServiceReadyErrorMessage, err.Error())) return ctrl.Result{}, err @@ -443,19 +443,19 @@ func (r *OctaviaAPIReconciler) reconcileInit( ctrlResult, err := svc.CreateOrPatch(ctx, helper) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, + condition.CreateServiceReadyErrorMessage, err.Error())) return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, + condition.CreateServiceReadyCondition, condition.RequestedReason, condition.SeverityInfo, - condition.ExposeServiceReadyRunningMessage)) + condition.CreateServiceReadyRunningMessage)) return ctrlResult, nil } // create service - end @@ -471,7 +471,7 @@ func (r *OctaviaAPIReconciler) reconcileInit( return ctrl.Result{}, err } } - instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) // expose service - end // diff --git a/go.mod b/go.mod index 5940e69e..a789df5c 100644 --- a/go.mod +++ b/go.mod @@ -5,26 +5,26 @@ go 1.21 require ( github.com/go-logr/logr v1.4.2 github.com/gophercloud/gophercloud v1.14.1 - github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.4 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 - github.com/openstack-k8s-operators/infra-operator/apis v0.4.1-0.20241015142911-4fc474b2f29c - github.com/openstack-k8s-operators/keystone-operator/api v0.4.1-0.20241013092400-3f9337945472 - github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20241014140317-e5c35d28f3af - github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241015090956-b0954ab72dcd + github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241024081600-3e23dc62002c + github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241023160107-bd8e671350e1 + github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333 + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241025164019-30baa23bf6f1 + github.com/openstack-k8s-operators/mariadb-operator/api v0.5.0 github.com/openstack-k8s-operators/octavia-operator/api v0.0.0-00010101000000-000000000000 - github.com/openstack-k8s-operators/ovn-operator/api v0.4.1-0.20241015160527-e9859771daa8 + github.com/openstack-k8s-operators/ovn-operator/api v0.5.1-0.20241022085743-26ff82e785e2 go.uber.org/zap v1.27.0 - k8s.io/api v0.29.9 - k8s.io/apimachinery v0.29.9 - k8s.io/client-go v0.29.9 + k8s.io/api v0.29.10 + k8s.io/apimachinery v0.29.10 + k8s.io/client-go v0.29.10 sigs.k8s.io/controller-runtime v0.17.6 ) require ( github.com/google/uuid v1.6.0 - github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20241014140317-e5c35d28f3af + github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241025164019-30baa23bf6f1 golang.org/x/crypto v0.28.0 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 ) @@ -78,8 +78,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.9 // indirect - k8s.io/component-base v0.29.9 // indirect + k8s.io/apiextensions-apiserver v0.29.10 // indirect + k8s.io/component-base v0.29.10 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 9e6f86fc..00120cd0 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.4 h1:vn0d/47rWzEHpDl3Ac7MiD9DfKK3gNBezI5khPKE7AQ= -github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.4/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5 h1:CELpSMPSyicFBaVsxROmfrWlu9yr3Dduk+y7vGrIsx8= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -77,20 +77,20 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openstack-k8s-operators/infra-operator/apis v0.4.1-0.20241015142911-4fc474b2f29c h1:leIcJtvFglYJf1vWw3vxwQ3UoHkTuG8jPjFgO0nUEC0= -github.com/openstack-k8s-operators/infra-operator/apis v0.4.1-0.20241015142911-4fc474b2f29c/go.mod h1:J9oUh3eGBvAFfyUMiPxPRBSxAcO8rnwITN4RTh/It+8= -github.com/openstack-k8s-operators/keystone-operator/api v0.4.1-0.20241013092400-3f9337945472 h1:rT7rpeXxzrrgSGmt/FCxpD2iIdUsmuxgJkMq6E/tiis= -github.com/openstack-k8s-operators/keystone-operator/api v0.4.1-0.20241013092400-3f9337945472/go.mod h1:E8azRt+pUzZCAxq2YRWwILorodLbojKBlniPP5qkcQc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af h1:MNNuYPfaXWf14lfsWGay3WjLjIIu9KN5+iHrLdCMQac= -github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20241014140317-e5c35d28f3af/go.mod h1:EKfyFMU2brJYqR4M2Hwfo5ZLBQxAJXAYfHjpdqX6N5g= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20241014140317-e5c35d28f3af h1:fevDUHmqcnI4wDTKupKe/CcgVdgNpZXWkJx8u0/xEXs= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20241014140317-e5c35d28f3af/go.mod h1:djfljx3jfHqywhY3oDvPg/GLKwiFVkds6v7P7/Yg+8g= -github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20241014140317-e5c35d28f3af h1:u5yJ/rV9VBMjKIcsu16/3rhiPcBxZN2sPqxnZHoRrNc= -github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20241014140317-e5c35d28f3af/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241015090956-b0954ab72dcd h1:PqJByNFRxjnhcEMv9vrXX1wH4qOYOt+fSdGmOEkC5gA= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241015090956-b0954ab72dcd/go.mod h1:Uyc8m+72l3rVm6jKb8FRUrQbjMWyifc5m0K+Ge0QV80= -github.com/openstack-k8s-operators/ovn-operator/api v0.4.1-0.20241015160527-e9859771daa8 h1:bOqPl+wgFTwcefb9E1z+lGgfhG4XLOgSQXXD1dLOFtI= -github.com/openstack-k8s-operators/ovn-operator/api v0.4.1-0.20241015160527-e9859771daa8/go.mod h1:UNtD7SqSmflXUs8bzUu+azWxaFSEGPpprb9U86FaGTY= +github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241024081600-3e23dc62002c h1:/TXY/Hux6hfDEx5KIWQIMx7IKs2no4Pa/TsTVKDuLkw= +github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241024081600-3e23dc62002c/go.mod h1:J9oUh3eGBvAFfyUMiPxPRBSxAcO8rnwITN4RTh/It+8= +github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241023160107-bd8e671350e1 h1:he0/o7mLKhXa16QlwajRHtAOjot84Emvgl4jdl3esgU= +github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241023160107-bd8e671350e1/go.mod h1:saoorrsPo3DzDPGM6PJ8sQJBNuNRGCHjRHChRQmkoQ0= +github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333 h1:yejekTWudX5af3mCJQ1MUPLEa0X6sIsklf07o9KilRk= +github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241025164019-30baa23bf6f1 h1:k5aZEt+xNGZXvEYs02FC2nL0I6QLgsWOsDOGwgiQh2E= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241025164019-30baa23bf6f1/go.mod h1:djfljx3jfHqywhY3oDvPg/GLKwiFVkds6v7P7/Yg+8g= +github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241025164019-30baa23bf6f1 h1:7tlv11dDu2fKxjhbEtrjNAPRE7q0FfmCNX0sxOKdMJs= +github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241025164019-30baa23bf6f1/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o= +github.com/openstack-k8s-operators/mariadb-operator/api v0.5.0 h1:XBx1TuyKhgtWAigYVcdqTUzIwWRYHN63pfa0zxHB12M= +github.com/openstack-k8s-operators/mariadb-operator/api v0.5.0/go.mod h1:Uyc8m+72l3rVm6jKb8FRUrQbjMWyifc5m0K+Ge0QV80= +github.com/openstack-k8s-operators/ovn-operator/api v0.5.1-0.20241022085743-26ff82e785e2 h1:DKYNHQtM5QrMhA9owKpqJsRPtSTio1w9pQirZxDwBgo= +github.com/openstack-k8s-operators/ovn-operator/api v0.5.1-0.20241022085743-26ff82e785e2/go.mod h1:Bb6Kv6AzEE1ksesBEYh9O9jhRcOhRrB8AVAxAWkVqbw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -201,16 +201,16 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.9 h1:FwdflpNsfMUYUOblMZNWJ4K/q0OSL5A4jGa0iOgcJco= -k8s.io/api v0.29.9/go.mod h1:fNhmzRfKaSEHCmczA/jRx6CiDKhYOnFLJBERMJAXEk8= -k8s.io/apiextensions-apiserver v0.29.9 h1:EB6RK06kFJjbzBwU1YiVznxrcgBE0hhDWt6EQQIcOy4= -k8s.io/apiextensions-apiserver v0.29.9/go.mod h1:jcaHG6R/bB1iU6XzC1DMhB1x2ktTJLt2KKpg6B65Z2c= -k8s.io/apimachinery v0.29.9 h1:YZ8HUid1TzQVz94cnNlsQjLdH0VoAhWSqz7t0q6B12A= -k8s.io/apimachinery v0.29.9/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= -k8s.io/client-go v0.29.9 h1:4f/Wz6li3rEyIPFj32XAQMtOGMM1tg7KQi1oeS6ibPg= -k8s.io/client-go v0.29.9/go.mod h1:2N1drQEZ5yiYrWVaE2Un8JiISUhl47D8pyZlYLszke4= -k8s.io/component-base v0.29.9 h1:lPENvp3CCwdeMEWGjiTfn5b287qQYuK7gX32OBOovmA= -k8s.io/component-base v0.29.9/go.mod h1:NGDa6Ih0EdcLA2G4K2ZYySoiB+2Tn+rmSqPyudCPgDY= +k8s.io/api v0.29.10 h1:Fao3HOxccbGRC1HZtXD+Y41xJhP0tEToVo5W7EEUBm0= +k8s.io/api v0.29.10/go.mod h1:rF0sRh64w1hMNAVGh4YYniSxODyHye3GLmymAbWBDvY= +k8s.io/apiextensions-apiserver v0.29.10 h1:2k2AHrWgs4S+07HrOjEdcyrQ/UqYlvBi6uon1fox1KE= +k8s.io/apiextensions-apiserver v0.29.10/go.mod h1:3rrJC52bW5frHf10N3JvaDpl++hgrzD9Gs6IZpKWMoc= +k8s.io/apimachinery v0.29.10 h1:57OLNqOJUgp5KlRRY3JOBFOTTa5Rt/LVkmKiiN2cvaQ= +k8s.io/apimachinery v0.29.10/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= +k8s.io/client-go v0.29.10 h1:hPmG1pmKslRhmCIzVd90sA58B0sJwNwduNgXFWsFqhI= +k8s.io/client-go v0.29.10/go.mod h1:gnMCQiRXGL9K0VtlW8gTkhzptGrHm2BJ4qBbujNemc4= +k8s.io/component-base v0.29.10 h1:YQrQ/bpzGPGqIPEPaBzxjH0/1DJOI+yZPZNbbz7ZCBY= +k8s.io/component-base v0.29.10/go.mod h1:IbwsBob2DnYiAONsSHIuYenchqcDycbHSLHrXshuLgM= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/hack/configure_local_webhook.sh b/hack/run_with_local_webhook.sh similarity index 65% rename from hack/configure_local_webhook.sh rename to hack/run_with_local_webhook.sh index 4e1a50c4..7bf80937 100755 --- a/hack/configure_local_webhook.sh +++ b/hack/run_with_local_webhook.sh @@ -1,6 +1,16 @@ #!/bin/bash set -ex +# Define a cleanup function +cleanup() { + echo "Caught signal, cleaning up local webhooks..." + ./hack/clean_local_webhook.sh + exit 0 +} + +# Set trap to catch SIGINT and SIGTERM +trap cleanup SIGINT SIGTERM + TMPDIR=${TMPDIR:-"/tmp/k8s-webhook-server/serving-certs"} SKIP_CERT=${SKIP_CERT:-false} CRC_IP=${CRC_IP:-$(/sbin/ip -o -4 addr list crc | awk '{print $4}' | cut -d/ -f1)} @@ -91,6 +101,23 @@ oc apply -n openstack -f ${TMPDIR}/patch_webhook_configurations.yaml CSV_NAME="$(oc get csv -n openstack-operators -l operators.coreos.com/octavia-operator.openstack-operators -o name)" if [ -n "${CSV_NAME}" ]; then + CUR_REPLICAS=$(oc get -n openstack-operators "${CSV_NAME}" -o=jsonpath='{.spec.install.spec.deployments[0].spec.replicas}') + CUR_WEBHOOK_DEFS=$(oc get -n openstack-operators "${CSV_NAME}" -o=jsonpath='{.spec.webhookdefinitions}') + + # Back-up CSV if it currently uses OLM defaults for deployment replicas or webhook definitions + if [[ "${CUR_REPLICAS}" -gt 0 || ( -n "${CUR_WEBHOOK_DEFS}" && "${CUR_WEBHOOK_DEFS}" != "[]" ) ]]; then + CSV_FILE=$(mktemp -t "$(echo "${CSV_NAME}" | cut -d "/" -f 2).XXXXXX" --suffix .json) + oc get -n openstack-operators "${CSV_NAME}" -o json | \ + jq -r 'del(.metadata.generation, .metadata.resourceVersion, .metadata.uid)' > "${CSV_FILE}" + + printf \ + "\n\tNow patching operator CSV to remove its OLM deployment and associated webhooks. + The original OLM version of the operator's CSV has been copied to %s. To restore it, use: + oc patch -n openstack-operators %s --type=merge --patch-file=%s\n\n" "${CSV_FILE}" "${CSV_NAME}" "${CSV_FILE}" + fi + oc patch "${CSV_NAME}" -n openstack-operators --type=json -p="[{'op': 'replace', 'path': '/spec/install/spec/deployments/0/spec/replicas', 'value': 0}]" oc patch "${CSV_NAME}" -n openstack-operators --type=json -p="[{'op': 'replace', 'path': '/spec/webhookdefinitions', 'value': []}]" fi + +go run ./main.go -metrics-bind-address ":${METRICS_PORT}" -health-probe-bind-address ":${HEALTH_PORT}" diff --git a/pkg/amphoracontrollers/daemonset.go b/pkg/amphoracontrollers/daemonset.go index 852c4c74..63b9f8d6 100644 --- a/pkg/amphoracontrollers/daemonset.go +++ b/pkg/amphoracontrollers/daemonset.go @@ -27,6 +27,12 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + // InitContainerCommand - + InitContainerCommand = "/usr/local/bin/container-scripts/init.sh %s" ) // DaemonSet func @@ -36,8 +42,6 @@ func DaemonSet( labels map[string]string, annotations map[string]string, ) *appsv1.DaemonSet { - runAsUser := int64(0) - privileged := true serviceName := fmt.Sprintf("octavia-%s", instance.Spec.Role) // The API pod has an extra volume so the API and the provider agent can @@ -47,7 +51,7 @@ func DaemonSet( certsSecretName := fmt.Sprintf("%s-certs-secret", parentOctaviaName) volumes = append(volumes, GetCertVolume(certsSecretName)...) - volumeMounts := GetVolumeMounts(serviceName) + volumeMounts := octavia.GetVolumeMounts(serviceName) volumeMounts = append(volumeMounts, GetCertVolumeMount()...) livenessProbe := &corev1.Probe{ @@ -100,6 +104,11 @@ func DaemonSet( volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) } + args := []string{ + "-c", + fmt.Sprintf(InitContainerCommand, instance.Name), + } + // Because we don't use jobboard, we need to ensure that the octavia // controllers are gracefully shutdown, so after they receive the signal, // they need to complete the jobs that are being executed (creating a LB, @@ -112,6 +121,12 @@ func DaemonSet( // re-enabled terminationGracePeriodSeconds := int64(600) + capabilities := []corev1.Capability{"NET_ADMIN", "SYS_ADMIN", "SYS_NICE"} + if instance.Spec.Role == octaviav1.HealthManager { + // NET_RAW is required for IP advertisements + capabilities = append(capabilities, corev1.Capability("NET_RAW")) + } + daemonset := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, @@ -127,25 +142,40 @@ func DaemonSet( Labels: labels, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(octavia.OctaviaUID), + }, TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, ServiceAccountName: instance.Spec.ServiceAccount, Containers: []corev1.Container{ { - Name: serviceName, - Image: instance.Spec.ContainerImage, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + Name: serviceName, + Image: instance.Spec.ContainerImage, + SecurityContext: octavia.GetOctaviaSecurityContext(), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "init", + Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN", "SYS_ADMIN", "SYS_NICE"}, + Add: capabilities, Drop: []corev1.Capability{}, }, - RunAsUser: &runAsUser, - Privileged: &privileged, + RunAsUser: ptr.To(int64(0)), }, + Command: []string{ + "/bin/bash", + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + Args: args, + VolumeMounts: GetInitVolumeMounts(), }, }, Volumes: volumes, @@ -163,15 +193,9 @@ func DaemonSet( }, corev1.LabelHostname, ) - if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + if len(instance.Spec.NodeSelector) > 0 { daemonset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := octavia.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - VolumeMounts: octavia.GetInitVolumeMounts(), - } - daemonset.Spec.Template.Spec.InitContainers = octavia.InitContainer(initContainerDetails) - return daemonset } diff --git a/pkg/amphoracontrollers/volumes.go b/pkg/amphoracontrollers/volumes.go index c9a5ca72..176441c1 100644 --- a/pkg/amphoracontrollers/volumes.go +++ b/pkg/amphoracontrollers/volumes.go @@ -48,9 +48,9 @@ func GetVolumes(name string) []corev1.Volume { ) } -func GetVolumeMounts(serviceName string) []corev1.VolumeMount { +func GetInitVolumeMounts() []corev1.VolumeMount { return append( - octavia.GetVolumeMounts(serviceName), + octavia.GetInitVolumeMounts(), corev1.VolumeMount{ Name: "hm-ports", MountPath: "/var/lib/hmports", diff --git a/pkg/octavia/dbsync.go b/pkg/octavia/dbsync.go index 426e6928..c69c1ec2 100644 --- a/pkg/octavia/dbsync.go +++ b/pkg/octavia/dbsync.go @@ -22,11 +22,12 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) const ( - // DBSyncCommand - - DBSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" + // InitContainerCommand - + InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" ) // DbSyncJob func @@ -35,13 +36,9 @@ func DbSyncJob( labels map[string]string, annotations map[string]string, ) *batchv1.Job { - runAsUser := int64(0) - initVolumeMounts := GetInitVolumeMounts() volumeMounts := GetVolumeMounts("db-sync") volumes := GetVolumes(instance.Name) - args := []string{"-c", DBSyncCommand} - envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -51,6 +48,11 @@ func DbSyncJob( volumeMounts = append(volumeMounts, instance.Spec.OctaviaAPI.TLS.CreateVolumeMounts(nil)...) } + args := []string{ + "-c", + InitContainerCommand, + } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name + "-db-sync", @@ -63,21 +65,30 @@ func DbSyncJob( Annotations: annotations, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(OctaviaUID), + }, RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), Containers: []corev1.Container{ { - Name: ServiceName + "-db-sync", + Name: ServiceName + "-db-sync", + Image: instance.Spec.OctaviaAPI.ContainerImage, + SecurityContext: GetOctaviaSecurityContext(), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "init", + Image: instance.Spec.OctaviaAPI.ContainerImage, + SecurityContext: GetOctaviaSecurityContext(), Command: []string{ "/bin/bash", }, - Args: args, - Image: instance.Spec.OctaviaAPI.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, + Args: args, + VolumeMounts: GetInitVolumeMounts(), }, }, Volumes: volumes, @@ -86,11 +97,5 @@ func DbSyncJob( }, } - initContainerDetails := APIDetails{ - ContainerImage: instance.Spec.OctaviaAPI.ContainerImage, - VolumeMounts: initVolumeMounts, - } - job.Spec.Template.Spec.InitContainers = InitContainer(initContainerDetails) - return job } diff --git a/pkg/octavia/initcontainer.go b/pkg/octavia/initcontainer.go deleted file mode 100644 index 96897589..00000000 --- a/pkg/octavia/initcontainer.go +++ /dev/null @@ -1,56 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package octavia - -import ( - corev1 "k8s.io/api/core/v1" -) - -// APIDetails information -type APIDetails struct { - ContainerImage string - VolumeMounts []corev1.VolumeMount -} - -const ( - // InitContainerCommand - - InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" -) - -// InitContainer - init container for octavia api pods -func InitContainer(init APIDetails) []corev1.Container { - runAsUser := int64(0) - - args := []string{ - "-c", - InitContainerCommand, - } - - return []corev1.Container{ - { - Name: "init", - Image: init.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - }, - Command: []string{ - "/bin/bash", - }, - Args: args, - VolumeMounts: GetInitVolumeMounts(), - }, - } -} diff --git a/pkg/octavia/securitycontext.go b/pkg/octavia/securitycontext.go new file mode 100644 index 00000000..bc09c51e --- /dev/null +++ b/pkg/octavia/securitycontext.go @@ -0,0 +1,34 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package octavia + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +const ( + OctaviaUID int64 = 42437 + OctaviaGID int64 = 42437 +) + +func GetOctaviaSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + RunAsUser: ptr.To(OctaviaUID), + RunAsGroup: ptr.To(OctaviaGID), + RunAsNonRoot: ptr.To(true), + } +} diff --git a/pkg/octaviaapi/deployment.go b/pkg/octaviaapi/deployment.go index 9f909d2c..0efeacbe 100644 --- a/pkg/octaviaapi/deployment.go +++ b/pkg/octaviaapi/deployment.go @@ -35,7 +35,10 @@ import ( const ( // ServiceCommand - - ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" + ServiceCommand = "/usr/local/bin/kolla_start" + + // InitContainerCommand - + InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" ) // Deployment func @@ -45,8 +48,6 @@ func Deployment( labels map[string]string, annotations map[string]string, ) (*appsv1.Deployment, error) { - runAsUser := int64(0) - initVolumeMounts := octavia.GetInitVolumeMounts() livenessProbe := &corev1.Probe{ // TODO might need tuning @@ -133,6 +134,11 @@ func Deployment( serviceName := fmt.Sprintf("%s-api", octavia.ServiceName) + initArgs := []string{ + "-c", + InitContainerCommand, + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, @@ -149,6 +155,9 @@ func Deployment( Labels: labels, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(octavia.OctaviaUID), + }, ServiceAccountName: instance.Spec.ServiceAccount, Containers: []corev1.Container{ { @@ -156,25 +165,36 @@ func Deployment( Command: []string{ "/bin/bash", }, - Args: args, - Image: instance.Spec.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: octavia.GetOctaviaSecurityContext(), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }, { - Name: fmt.Sprintf("%s-provider-agent", serviceName), - Image: instance.Spec.ContainerImage, - Env: env.MergeEnvs([]corev1.EnvVar{}, agentEnvVars), - VolumeMounts: volumeMountsDriverAgent, - Resources: instance.Spec.Resources, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + Name: fmt.Sprintf("%s-provider-agent", serviceName), + Image: instance.Spec.ContainerImage, + SecurityContext: octavia.GetOctaviaSecurityContext(), + Env: env.MergeEnvs([]corev1.EnvVar{}, agentEnvVars), + VolumeMounts: volumeMountsDriverAgent, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "init", + Image: instance.Spec.ContainerImage, + SecurityContext: octavia.GetOctaviaSecurityContext(), + Command: []string{ + "/bin/bash", + }, + Args: initArgs, + VolumeMounts: octavia.GetInitVolumeMounts(), }, }, Volumes: volumes, @@ -192,15 +212,9 @@ func Deployment( }, corev1.LabelHostname, ) - if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + if len(instance.Spec.NodeSelector) > 0 { deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := octavia.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - VolumeMounts: initVolumeMounts, - } - deployment.Spec.Template.Spec.InitContainers = octavia.InitContainer(initContainerDetails) - return deployment, nil } diff --git a/pkg/octaviarsyslog/daemonset.go b/pkg/octaviarsyslog/daemonset.go index a5cf729f..a311c83d 100644 --- a/pkg/octaviarsyslog/daemonset.go +++ b/pkg/octaviarsyslog/daemonset.go @@ -25,6 +25,12 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + // InitContainerCommand - + InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" ) // DaemonSet func @@ -34,18 +40,13 @@ func DaemonSet( labels map[string]string, annotations map[string]string, ) *appsv1.DaemonSet { - runAsUser := int64(0) serviceName := "octavia-rsyslog" // The API pod has an extra volume so the API and the provider agent can // communicate with each other. volumes := GetVolumes(instance.Name) - //parentOctaviaName := octavia.GetOwningOctaviaControllerName(instance) - //certsSecretName := fmt.Sprintf("%s-certs-secret", parentOctaviaName) - //volumes = append(volumes, GetCertVolume(certsSecretName)...) volumeMounts := octavia.GetVolumeMounts(serviceName) - //volumeMounts = append(volumeMounts, GetCertVolumeMount()...) livenessProbe := &corev1.Probe{ // TODO might need tuning @@ -76,6 +77,11 @@ func DaemonSet( }, } + args := []string{ + "-c", + InitContainerCommand, + } + envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -107,9 +113,28 @@ func DaemonSet( Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "init", + // TODO(gthiemonge) Using Octavia HM Container image is a workaround to get a container with pyroute2 + // Replace it by an init container image with pyroute2 when it's available + // OSPRH-8434 + Image: octaviav1.OctaviaHealthManagerContainerImage, SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, + RunAsUser: ptr.To(int64(0)), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW", "SYS_ADMIN", "SYS_NICE"}, + Drop: []corev1.Capability{}, + }, + }, + Command: []string{ + "/bin/bash", }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + Args: args, + VolumeMounts: GetInitVolumeMounts(), }, }, Volumes: volumes, @@ -127,19 +152,9 @@ func DaemonSet( }, corev1.LabelHostname, ) - if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + if len(instance.Spec.NodeSelector) > 0 { daemonset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := APIDetails{ - // TODO(gthiemonge) Using Octavia HM Container image is a workaround to get a container with pyroute2 - // Replace it by an init container image with pyroute2 when it's available - // OSPRH-8434 - ContainerImage: octaviav1.OctaviaHealthManagerContainerImage, - VolumeMounts: octavia.GetInitVolumeMounts(), - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - } - daemonset.Spec.Template.Spec.InitContainers = InitContainer(initContainerDetails) - return daemonset } diff --git a/pkg/octaviarsyslog/initcontainer.go b/pkg/octaviarsyslog/initcontainer.go deleted file mode 100644 index 89aff686..00000000 --- a/pkg/octaviarsyslog/initcontainer.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package octaviarsyslog - -import ( - corev1 "k8s.io/api/core/v1" -) - -// APIDetails information -type APIDetails struct { - ContainerImage string - VolumeMounts []corev1.VolumeMount - Env []corev1.EnvVar -} - -const ( - // InitContainerCommand - - InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" -) - -// InitContainer - init container for octavia api pods -func InitContainer(init APIDetails) []corev1.Container { - runAsUser := int64(0) - privileged := true - - args := []string{ - "-c", - InitContainerCommand, - } - - return []corev1.Container{ - { - Name: "init", - Image: init.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - Privileged: &privileged, - }, - Command: []string{ - "/bin/bash", - }, - Args: args, - Env: init.Env, - VolumeMounts: GetInitVolumeMounts(), - }, - } -} diff --git a/templates/octaviaamphoracontroller/bin/init.sh b/templates/octaviaamphoracontroller/bin/init.sh index 759ad0c9..67f21e42 100755 --- a/templates/octaviaamphoracontroller/bin/init.sh +++ b/templates/octaviaamphoracontroller/bin/init.sh @@ -32,3 +32,27 @@ cp -a ${SVC_CFG} ${SVC_CFG_MERGED} for dir in /var/lib/config-data/default; do merge_config_dir ${dir} done + +# Network configuration +if [ "$MGMT_CIDR" != "" ]; then + /usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$MGMT_CIDR" "$MGMT_GATEWAY" +fi + +idx=0 +while true; do + var_name="MGMT_CIDR${idx}" + cidr="${!var_name}" + if [ "$cidr" = "" ]; then + break + fi + /usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$cidr" "$MGMT_GATEWAY" + idx=$((idx+1)) +done + +if [ "$1" = "octavia-healthmanager" ]; then + /usr/local/bin/container-scripts/setipalias.py octavia hm + /usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia +fi + +# Ignore possible errors +/usr/local/bin/container-scripts/octavia_status.py || true diff --git a/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh index cca4b89f..2d3d7027 100755 --- a/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh +++ b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh @@ -15,27 +15,4 @@ # under the License. set -ex -if [ "$MGMT_CIDR" != "" ]; then - /usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$MGMT_CIDR" "$MGMT_GATEWAY" -fi - -idx=0 -while true; do - var_name="MGMT_CIDR${idx}" - cidr="${!var_name}" - if [ "$cidr" = "" ]; then - break - fi - /usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$cidr" "$MGMT_GATEWAY" - idx=$((idx+1)) -done - -if [ "$1" = "octavia-health-manager" ]; then - /usr/local/bin/container-scripts/setipalias.py octavia hm - /usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia -fi - -# Ignore possible errors -/usr/local/bin/container-scripts/octavia_status.py || true - exec /usr/bin/$1 --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf --config-dir /etc/octavia/octavia.conf.d/ diff --git a/templates/octaviaapi/config/httpd.conf b/templates/octaviaapi/config/httpd.conf index 0720a327..fbf161e5 100644 --- a/templates/octaviaapi/config/httpd.conf +++ b/templates/octaviaapi/config/httpd.conf @@ -22,6 +22,7 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout {{ range $endpt, $vhost := .VHosts }} # {{ $endpt }} vhost {{ $vhost.ServerName }} configuration diff --git a/templates/octaviaapi/config/octavia-api-config.json b/templates/octaviaapi/config/octavia-api-config.json index ad9cec88..2d6245f6 100644 --- a/templates/octaviaapi/config/octavia-api-config.json +++ b/templates/octaviaapi/config/octavia-api-config.json @@ -60,6 +60,10 @@ "path": "/run/octavia", "owner": "octavia:octavia", "recurse": true + }, { + "path": "/etc/httpd/run/", + "owner": "octavia:apache", + "recurse": true } ] } diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index c073bdaf..d7001f6d 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -23,6 +23,11 @@ spec: preserveJobs: false secret: osp-secret serviceUser: octavia + lbMgmtNetwork: + availabilityZones: + - zone-1 + manageLbMgmtNetworks: true + createDefaultLbMgmtNetwork: true octaviaAPI: customServiceConfig: | [DEFAULT] @@ -96,7 +101,7 @@ spec: containers: - args: - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /bin/bash imagePullPolicy: IfNotPresent @@ -121,6 +126,10 @@ spec: periodSeconds: 15 successThreshold: 1 timeoutSeconds: 15 + securityContext: + runAsUser: 42437 + runAsGroup: 42437 + runAsNonRoot: true - env: - name: CONFIG_HASH - name: KOLLA_CONFIG_STRATEGY @@ -155,6 +164,10 @@ spec: imagePullPolicy: IfNotPresent name: init resources: {} + securityContext: + runAsUser: 42437 + runAsGroup: 42437 + runAsNonRoot: true restartPolicy: Always serviceAccount: octavia-octavia serviceAccountName: octavia-octavia @@ -236,3 +249,20 @@ commands: fi done exit 0 + - script: | + # Check that the octavia-healthmanager pod has 2 IP addresses on the octavia interface and a route to lb-mgmt-subunet + pod=$(oc get pods -n $NAMESPACE -l service=octavia-healthmanager --field-selector=status.phase=Running -o name | head -1) + if [ "$pod" = "" ]; then + exit 1 + fi + # Note: $pod already includes the resource type (ex: pod/octavia-healthmanager-xxxxx) + node_name=$(oc get -n $NAMESPACE $pod -o go-template='{{.spec.nodeName}}') + octavia_fixed_ip=$(oc get configmaps -n $NAMESPACE octavia-hmport-map -o json | jq -r '.data["hm_'$node_name'"]') + n_matches=$(oc logs -n $NAMESPACE -c init "$pod" | + grep -e "addr 172.23.*/24 dev octavia" \ + -e "addr $octavia_fixed_ip/32 dev octavia" \ + -e "route 172.24.0.0/16 via 172.23.0.150 dev octavia" | wc -l) + if [ "$n_matches" != "3" ]; then + exit 1 + fi + exit 0 diff --git a/tests/kuttl/tests/octavia_scale/00-assert.yaml b/tests/kuttl/tests/octavia_scale/00-assert.yaml new file mode 100644 index 00000000..d785559b --- /dev/null +++ b/tests/kuttl/tests/octavia_scale/00-assert.yaml @@ -0,0 +1,4 @@ +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: octavia diff --git a/tests/kuttl/tests/octavia_scale/00-test-resources.yaml b/tests/kuttl/tests/octavia_scale/00-test-resources.yaml new file mode 100644 index 00000000..a2932aa7 --- /dev/null +++ b/tests/kuttl/tests/octavia_scale/00-test-resources.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc kustomize ../../../../config/samples/network-attachment-definition | oc apply -n $NAMESPACE -f - + - script: | + oc patch -n $NAMESPACE ovncontrollers ovncontroller-sample --type merge -p ' + spec: + external-ids: + availability-zones: + - zone-1 + ' diff --git a/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml b/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml deleted file mode 120000 index dc5dd67b..00000000 --- a/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../config/samples/octavia_v1beta1_octavia.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml b/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml new file mode 100644 index 00000000..f706c72b --- /dev/null +++ b/tests/kuttl/tests/octavia_scale/01-deploy-octavia.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc kustomize ../../../../config/samples | oc apply -n $NAMESPACE -f - diff --git a/tests/kuttl/tests/octavia_tls/02-assert.yaml b/tests/kuttl/tests/octavia_tls/02-assert.yaml index 6479de08..64ffd378 100644 --- a/tests/kuttl/tests/octavia_tls/02-assert.yaml +++ b/tests/kuttl/tests/octavia_tls/02-assert.yaml @@ -100,7 +100,7 @@ spec: containers: - args: - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /bin/bash imagePullPolicy: IfNotPresent