From 291c3f083f1cfdda2175dfcc9df3bea8379580f9 Mon Sep 17 00:00:00 2001 From: Brent Eagles Date: Wed, 21 Jun 2023 18:24:48 +0000 Subject: [PATCH] Support for amphora provider services Adds support for launching the amphora controller services. Implementation is informed by the fact that the amphora provider specific services share many common configurations and deployment requirements. Note: full deployment support isn't available with this change and would require several manual steps to enable. --- ...enstack.org_octaviaamphoracontrollers.yaml | 257 ++++++++++ api/bases/octavia.openstack.org_octavias.yaml | 459 ++++++++++++++++++ api/v1beta1/amphoracontroller_types.go | 153 ++++++ api/v1beta1/octavia_types.go | 35 +- api/v1beta1/octavia_webhook.go | 16 +- api/v1beta1/zz_generated.deepcopy.go | 122 +++++ ...enstack.org_octaviaamphoracontrollers.yaml | 257 ++++++++++ .../bases/octavia.openstack.org_octavias.yaml | 459 ++++++++++++++++++ config/crd/kustomization.yaml | 1 + ...njection_in_octaviaamphoracontrollers.yaml | 7 + .../webhook_in_octaviaamphoracontrollers.yaml | 16 + ...ctavia-operator.clusterserviceversion.yaml | 4 + .../octaviamphoracontroller_editor_role.yaml | 24 + .../octaviamphoracontroller_viewer_role.yaml | 20 + config/rbac/role.yaml | 38 ++ config/samples/octavia_v1beta1_octavia.yaml | 46 +- controllers/amphoracontroller_controller.go | 359 ++++++++++++++ controllers/octavia_controller.go | 156 ++++++ controllers/octaviaapi_controller.go | 3 +- go.mod | 1 + go.sum | 2 + main.go | 12 + pkg/amphoracontrollers/deployment.go | 139 ++++++ pkg/octavia/dbsync.go | 2 +- pkg/octavia/initcontainer.go | 18 + pkg/octavia/volumes.go | 12 +- pkg/octaviaapi/volumes.go | 8 +- templates/octavia/bin/init.sh | 6 + .../octaviaamphoracontroller/bin/init.sh | 46 ++ .../config/octavia-healthmanager-config.json | 24 + .../config/octavia-housekeeping-config.json | 24 + .../config/octavia-worker-config.json | 24 + .../config/octavia.conf | 83 ++++ .../common/assert_sample_deployment.yaml | 39 ++ .../04-scale-down-zero-octaviaapi.yaml | 3 + 35 files changed, 2848 insertions(+), 27 deletions(-) create mode 100644 api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml create mode 100644 api/v1beta1/amphoracontroller_types.go create mode 100644 config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml create mode 100644 config/crd/patches/cainjection_in_octaviaamphoracontrollers.yaml create mode 100644 config/crd/patches/webhook_in_octaviaamphoracontrollers.yaml create mode 100644 config/rbac/octaviamphoracontroller_editor_role.yaml create mode 100644 config/rbac/octaviamphoracontroller_viewer_role.yaml create mode 100644 controllers/amphoracontroller_controller.go create mode 100644 pkg/amphoracontrollers/deployment.go create mode 100755 templates/octaviaamphoracontroller/bin/init.sh create mode 100644 templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json create mode 100644 templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json create mode 100644 templates/octaviaamphoracontroller/config/octavia-worker-config.json create mode 100644 templates/octaviaamphoracontroller/config/octavia.conf diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml new file mode 100644 index 00000000..eb8062a7 --- /dev/null +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: octaviaamphoracontrollers.octavia.openstack.org +spec: + group: octavia.openstack.org + names: + kind: OctaviaAmphoraController + listKind: OctaviaAmphoraControllerList + plural: octaviaamphoracontrollers + singular: octaviaamphoracontroller + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: OctaviaAmphoraController is the Schema for the octaviaworkers + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OctaviaAmphoraControllerSpec defines common state for all + Octavia Amphora Controllers + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing certs + for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config using + this parameter to change service defaults, or overwrite rendered + information using raw OpenStack config format. The content gets + added to to /etc//.conf.d directory as custom.conf + file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia DB, + defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default config + files like e.g. logging.conf or policy.json. But can also be used + to add additional files. Those get added to the service config dir + in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB and + AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change in + mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, housekeeping, + healthmanager) + type: string + secret: + description: Secret containing OpenStack password information for + octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, do we + need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + status: + description: OctaviaAmphoraControllerStatus defines the observed state + of the Octavia Amphora Controller + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: Octavia Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + readyCount: + description: ReadyCount of Octavia Amphora Controllers + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 25a09258..4489ac6d 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -246,6 +246,456 @@ spec: - secret - serviceAccount type: object + octaviaHealthManager: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + octaviaHousekeeping: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + octaviaWorker: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object passwordSelectors: default: database: OctaviaDatabasePassword @@ -270,6 +720,11 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Octavia + type: string secret: description: Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword @@ -281,6 +736,7 @@ spec: required: - databaseInstance - octaviaAPI + - rabbitMqClusterName - secret type: object status: @@ -349,6 +805,9 @@ spec: description: ReadyCount of octavia Housekeeping instances format: int32 type: integer + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string workerreadyCount: description: ReadyCount of octavia Worker instances format: int32 diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go new file mode 100644 index 00000000..187bdfcb --- /dev/null +++ b/api/v1beta1/amphoracontroller_types.go @@ -0,0 +1,153 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OctaviaAmphoraControllerSpec defines common state for all Octavia Amphora Controllers +type OctaviaAmphoraControllerSpec struct { + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=octavia + // DatabaseUser - optional username used for octavia DB, defaults to octavia + // TODO: -> implement needs work in mariadb-operator, right now only octavia + DatabaseUser string `json:"databaseUser"` + + // +kubebuilder:validation:Optional + // DatabaseHostname - Octavia DB hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=octavia + // ServiceUser - service user name (TODO: beagles, do we need this at all) + ServiceUser string `json:"serviceUser"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide Octavia services the default SA name + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // ContainerImage - Amphora Controller Container Image URL + ContainerImage string `json:"containerImage,omitempty"` + + // +kubebuilder:validation:Required + // Role - the role for the controller (one of worker, housekeeping, healthmanager) + Role string `json:"role"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Maximum=32 + // +kubebuilder:validation:Minimum=0 + // Replicas - Octavia Worker Replicas + Replicas int32 `json:"replicas"` + + // +kubebuilder:validation:Required + // Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword + Secret string `json:"secret"` + + // *kubebuilder:validation:Required + // Secret containing certs for securing communication with amphora based Load Balancers + LoadBalancerCerts string `json:"certssecret"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={database: OctaviaDatabasePassword, service: OctaviaPassword} + // PasswordSelectors - Selectors to identify the DB and AdminUser password from the Secret + PasswordSelectors PasswordSelector `json:"passwordSelectors,omitempty"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="# add your customization here" + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as custom.conf file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // ConfigOverwrite - interface to overwrite default config files like e.g. logging.conf or policy.json. + // But can also be used to add additional files. Those get added to the service config dir in /etc/ . + // TODO: -> implement + DefaultConfigOverwrite map[string]string `json:"defaultConfigOverwrite,omitempty"` + + // +kubebuilder:validation:Optional + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` +} + +// OctaviaAmphoraControllerStatus defines the observed state of the Octavia Amphora Controller +type OctaviaAmphoraControllerStatus struct { + // ReadyCount of Octavia Amphora Controllers + ReadyCount int32 `json:"readyCount,omitempty"` + + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // Octavia Database Hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +//+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" + +// OctaviaAmphoraController is the Schema for the octaviaworkers API +type OctaviaAmphoraController struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OctaviaAmphoraControllerSpec `json:"spec,omitempty"` + Status OctaviaAmphoraControllerStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// OctaviaAmphoraControllerList contains a list of OctaviaWorker +type OctaviaAmphoraControllerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OctaviaAmphoraController `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OctaviaAmphoraController{}, &OctaviaAmphoraControllerList{}) +} + +// IsReady - returns true if service is ready to work +func (instance OctaviaAmphoraController) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) +} diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index f1784910..3ba45920 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -27,6 +27,15 @@ const ( // OctaviaAPIContainerImage is the fall-back container image for OctaviaAPI OctaviaAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-octavia-api:current-podified" + + // OctaviaHousekeepingContainerImage is the fall-back container image for OctaviaHousekeeping + OctaviaHousekeepingContainerImage = "quay.io/podified-antelope-centos9/openstack-octavia-housekeeping:current-podified" + + // OctaviaHealthManagerContainerImage is the fall-back container image for OctaviaHealthManager + OctaviaHealthManagerContainerImage = "quay.io/podified-antelope-centos9/openstack-octavia-health-manager:current-podified" + + // OctaviaWorkerContainerImage is the fall-back container image for OctaviaWorker + OctaviaWorkerContainerImage = "quay.io/podified-antelope-centos9/openstack-octavia-worker:current-podified" ) // OctaviaSpec defines the desired state of Octavia @@ -46,6 +55,12 @@ type OctaviaSpec struct { // TODO: -> implement needs work in mariadb-operator, right now only octavia DatabaseUser string `json:"databaseUser"` + // +kubebuilder:validation:Required + // +kubebuilder:default=rabbitmq + // RabbitMQ instance name + // Needed to request a transportURL that is created and used in Octavia + RabbitMqClusterName string `json:"rabbitMqClusterName"` + // +kubebuilder:validation:Optional // +kubebuilder:default=octavia // ServiceUser - service user name @@ -90,6 +105,18 @@ type OctaviaSpec struct { // +kubebuilder:validation:Required // OctaviaAPI - Spec definition for the API service of the Octavia deployment OctaviaAPI OctaviaAPISpec `json:"octaviaAPI"` + + // +kubebuilder:validation:Optional + // OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment + OctaviaHousekeeping OctaviaAmphoraControllerSpec `json:"octaviaHousekeeping"` + + // +kubebuilder:validation:Optional + // OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment + OctaviaHealthManager OctaviaAmphoraControllerSpec `json:"octaviaHealthManager"` + + // +kubebuilder:validation:Optional + // OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment + OctaviaWorker OctaviaAmphoraControllerSpec `json:"octaviaWorker"` } // PasswordSelector to identify the DB and AdminUser password from the Secret @@ -117,6 +144,9 @@ type OctaviaStatus struct { // Octavia Database Hostname DatabaseHostname string `json:"databaseHostname,omitempty"` + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + // ReadyCount of octavia API instances OctaviaAPIReadyCount int32 `json:"apireadyCount,omitempty"` @@ -169,7 +199,10 @@ func (instance Octavia) IsReady() bool { func SetupDefaults() { // Acquire environmental defaults and initialize Octavia defaults with them octaviaDefaults := OctaviaDefaults{ - ContainerImageURL: util.GetEnvVar("OCTAVIA_API_IMAGE_URL_DEFAULT", OctaviaAPIContainerImage), + APIContainerImageURL: util.GetEnvVar("OCTAVIA_API_IMAGE_URL_DEFAULT", OctaviaAPIContainerImage), + HousekeepingContainerImageURL: util.GetEnvVar("OCTAVIA_HOUSEKEEPING_IMAGE_URL_DEFAULT", OctaviaHousekeepingContainerImage), + HealthManagerContainerImageURL: util.GetEnvVar("OCTAVIA_HEALTHMANAGER_IMAGE_URL_DEFAULT", OctaviaHealthManagerContainerImage), + WorkerContainerImageURL: util.GetEnvVar("OCTAVIA_WORKER_IMAGE_URL_DEFAULT", OctaviaWorkerContainerImage), } SetupOctaviaDefaults(octaviaDefaults) diff --git a/api/v1beta1/octavia_webhook.go b/api/v1beta1/octavia_webhook.go index dae93399..842d7089 100644 --- a/api/v1beta1/octavia_webhook.go +++ b/api/v1beta1/octavia_webhook.go @@ -25,7 +25,10 @@ import ( // OctaviaDefaults - type OctaviaDefaults struct { - ContainerImageURL string + APIContainerImageURL string + HousekeepingContainerImageURL string + HealthManagerContainerImageURL string + WorkerContainerImageURL string } var octaviaDefaults OctaviaDefaults @@ -60,7 +63,16 @@ func (r *Octavia) Default() { // Default - set defaults for this Octavia spec func (spec *OctaviaSpec) Default() { if spec.OctaviaAPI.ContainerImage == "" { - spec.OctaviaAPI.ContainerImage = octaviaDefaults.ContainerImageURL + spec.OctaviaAPI.ContainerImage = octaviaDefaults.APIContainerImageURL + } + if spec.OctaviaHousekeeping.ContainerImage == "" { + spec.OctaviaHousekeeping.ContainerImage = octaviaDefaults.HousekeepingContainerImageURL + } + if spec.OctaviaHealthManager.ContainerImage == "" { + spec.OctaviaHealthManager.ContainerImage = octaviaDefaults.HealthManagerContainerImageURL + } + if spec.OctaviaWorker.ContainerImage == "" { + spec.OctaviaWorker.ContainerImage = octaviaDefaults.WorkerContainerImageURL } } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index bb623e9a..76c8ae79 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -193,6 +193,125 @@ func (in *OctaviaAPIStatus) DeepCopy() *OctaviaAPIStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OctaviaAmphoraController) DeepCopyInto(out *OctaviaAmphoraController) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraController. +func (in *OctaviaAmphoraController) DeepCopy() *OctaviaAmphoraController { + if in == nil { + return nil + } + out := new(OctaviaAmphoraController) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OctaviaAmphoraController) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OctaviaAmphoraControllerList) DeepCopyInto(out *OctaviaAmphoraControllerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OctaviaAmphoraController, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraControllerList. +func (in *OctaviaAmphoraControllerList) DeepCopy() *OctaviaAmphoraControllerList { + if in == nil { + return nil + } + out := new(OctaviaAmphoraControllerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OctaviaAmphoraControllerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OctaviaAmphoraControllerSpec) DeepCopyInto(out *OctaviaAmphoraControllerSpec) { + *out = *in + out.PasswordSelectors = in.PasswordSelectors + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.DefaultConfigOverwrite != nil { + in, out := &in.DefaultConfigOverwrite, &out.DefaultConfigOverwrite + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraControllerSpec. +func (in *OctaviaAmphoraControllerSpec) DeepCopy() *OctaviaAmphoraControllerSpec { + if in == nil { + return nil + } + out := new(OctaviaAmphoraControllerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OctaviaAmphoraControllerStatus) DeepCopyInto(out *OctaviaAmphoraControllerStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraControllerStatus. +func (in *OctaviaAmphoraControllerStatus) DeepCopy() *OctaviaAmphoraControllerStatus { + if in == nil { + return nil + } + out := new(OctaviaAmphoraControllerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OctaviaDefaults) DeepCopyInto(out *OctaviaDefaults) { *out = *in @@ -260,6 +379,9 @@ func (in *OctaviaSpec) DeepCopyInto(out *OctaviaSpec) { } } in.OctaviaAPI.DeepCopyInto(&out.OctaviaAPI) + in.OctaviaHousekeeping.DeepCopyInto(&out.OctaviaHousekeeping) + in.OctaviaHealthManager.DeepCopyInto(&out.OctaviaHealthManager) + in.OctaviaWorker.DeepCopyInto(&out.OctaviaWorker) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaSpec. diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml new file mode 100644 index 00000000..eb8062a7 --- /dev/null +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: octaviaamphoracontrollers.octavia.openstack.org +spec: + group: octavia.openstack.org + names: + kind: OctaviaAmphoraController + listKind: OctaviaAmphoraControllerList + plural: octaviaamphoracontrollers + singular: octaviaamphoracontroller + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: OctaviaAmphoraController is the Schema for the octaviaworkers + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OctaviaAmphoraControllerSpec defines common state for all + Octavia Amphora Controllers + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing certs + for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config using + this parameter to change service defaults, or overwrite rendered + information using raw OpenStack config format. The content gets + added to to /etc//.conf.d directory as custom.conf + file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia DB, + defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default config + files like e.g. logging.conf or policy.json. But can also be used + to add additional files. Those get added to the service config dir + in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB and + AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change in + mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, housekeeping, + healthmanager) + type: string + secret: + description: Secret containing OpenStack password information for + octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, do we + need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + status: + description: OctaviaAmphoraControllerStatus defines the observed state + of the Octavia Amphora Controller + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: Octavia Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + readyCount: + description: ReadyCount of Octavia Amphora Controllers + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 25a09258..4489ac6d 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -246,6 +246,456 @@ spec: - secret - serviceAccount type: object + octaviaHealthManager: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + octaviaHousekeeping: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object + octaviaWorker: + description: OctaviaHousekeeping - Spec definition for the Octavia + Housekeeping agent for the Octavia deployment + properties: + certssecret: + description: '*kubebuilder:validation:Required Secret containing + certs for securing communication with amphora based Load Balancers' + type: string + containerImage: + description: ContainerImage - Amphora Controller Container Image + URL + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseHostname: + description: DatabaseHostname - Octavia DB hostname + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: octavia + description: 'DatabaseUser - optional username used for octavia + DB, defaults to octavia TODO: -> implement needs work in mariadb-operator, + right now only octavia' + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + passwordSelectors: + default: + database: OctaviaDatabasePassword + service: OctaviaPassword + description: PasswordSelectors - Selectors to identify the DB + and AdminUser password from the Secret + properties: + database: + default: OctaviaDatabasePassword + description: 'Database - Selector to get the octavia Database + user password from the Secret TODO: not used, need change + in mariadb-operator' + type: string + service: + default: OctaviaPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - Octavia Worker Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: Role - the role for the controller (one of worker, + housekeeping, healthmanager) + type: string + secret: + description: Secret containing OpenStack password information + for octavia OctaviaDatabasePassword, AdminPassword + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string + serviceUser: + default: octavia + description: 'ServiceUser - service user name (TODO: beagles, + do we need this at all)' + type: string + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - certssecret + - databaseInstance + - role + - secret + - serviceAccount + type: object passwordSelectors: default: database: OctaviaDatabasePassword @@ -270,6 +720,11 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Octavia + type: string secret: description: Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword @@ -281,6 +736,7 @@ spec: required: - databaseInstance - octaviaAPI + - rabbitMqClusterName - secret type: object status: @@ -349,6 +805,9 @@ spec: description: ReadyCount of octavia Housekeeping instances format: int32 type: integer + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string workerreadyCount: description: ReadyCount of octavia Worker instances format: int32 diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ea7cb62e..180c985e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,6 +2,7 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: +- bases/octavia.openstack.org_octaviaamphoracontrollers.yaml - bases/octavia.openstack.org_octaviaapis.yaml - bases/octavia.openstack.org_octavias.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/patches/cainjection_in_octaviaamphoracontrollers.yaml b/config/crd/patches/cainjection_in_octaviaamphoracontrollers.yaml new file mode 100644 index 00000000..8415aed4 --- /dev/null +++ b/config/crd/patches/cainjection_in_octaviaamphoracontrollers.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: octaviaamphoracontrollerss.octavia.openstack.org diff --git a/config/crd/patches/webhook_in_octaviaamphoracontrollers.yaml b/config/crd/patches/webhook_in_octaviaamphoracontrollers.yaml new file mode 100644 index 00000000..0deefe74 --- /dev/null +++ b/config/crd/patches/webhook_in_octaviaamphoracontrollers.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: octaviaamphoracontrollers.octavia.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/manifests/bases/octavia-operator.clusterserviceversion.yaml b/config/manifests/bases/octavia-operator.clusterserviceversion.yaml index e2b31e40..03715be4 100644 --- a/config/manifests/bases/octavia-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/octavia-operator.clusterserviceversion.yaml @@ -12,6 +12,10 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: + - displayName: Octavia Amphora Controller + kind: OctaviaAmphoraController + name: octaviaamphoracontrollers.octavia.openstack.org + version: v1beta1 - description: OctaviaAPI is the Schema for the octaviaapis API displayName: Octavia API kind: OctaviaAPI diff --git a/config/rbac/octaviamphoracontroller_editor_role.yaml b/config/rbac/octaviamphoracontroller_editor_role.yaml new file mode 100644 index 00000000..ef1b72f1 --- /dev/null +++ b/config/rbac/octaviamphoracontroller_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit octaviaamphoracontrollers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: octaviaamphoracontroller-editor-role +rules: +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers/status + verbs: + - get diff --git a/config/rbac/octaviamphoracontroller_viewer_role.yaml b/config/rbac/octaviamphoracontroller_viewer_role.yaml new file mode 100644 index 00000000..752f84d0 --- /dev/null +++ b/config/rbac/octaviamphoracontroller_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view octaviaamphoracontrollers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: octaviaamphoracontroller-viewer-role +rules: +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollerss + verbs: + - get + - list + - watch +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e7aff244..41faccdf 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -131,6 +131,32 @@ rules: - patch - update - watch +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers/finalizers + verbs: + - update +- apiGroups: + - octavia.openstack.org + resources: + - octaviaamphoracontrollers/status + verbs: + - get + - patch + - update - apiGroups: - octavia.openstack.org resources: @@ -191,6 +217,18 @@ rules: - get - list - watch +- apiGroups: + - rabbitmq.openstack.org + resources: + - transporturls + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/config/samples/octavia_v1beta1_octavia.yaml b/config/samples/octavia_v1beta1_octavia.yaml index 5e4ff3d4..59796bbd 100644 --- a/config/samples/octavia_v1beta1_octavia.yaml +++ b/config/samples/octavia_v1beta1_octavia.yaml @@ -3,13 +3,12 @@ kind: Octavia metadata: name: octavia spec: - # TODO(tweining): Add fields here databaseInstance: openstack databaseUser: octavia serviceUser: octavia + rabbitMqClusterName: rabbitmq + replicas: 1 secret: osp-secret - # passwordSelectors: TODO - # nodeSelector: TODO debug: dbSync: false service: false @@ -17,14 +16,51 @@ spec: customServiceConfig: | [DEFAULT] debug = true + octaviaHousekeeping: + databaseInstance: openstack + databaseUser: octavia + serviceUser: octavia + serviceAccount: octavia + role: housekeeping + certssecret: todo + replicas: 1 + secret: osp-secret + preserveJobs: false + customServiceConfig: | + [DEFAULT] + debug = true + octaviaHealthManager: + databaseInstance: openstack + databaseUser: octavia + serviceUser: octavia + serviceAccount: octavia + role: healthmanager + certssecret: todo + replicas: 1 + secret: osp-secret + preserveJobs: false + customServiceConfig: | + [DEFAULT] + debug = true + octaviaWorker: + databaseInstance: openstack + databaseUser: octavia + serviceUser: octavia + serviceAccount: octavia + replicas: 1 + role: worker + certssecret: todo + secret: osp-secret + preserveJobs: false + customServiceConfig: | + [DEFAULT] + debug = true octaviaAPI: databaseInstance: openstack databaseUser: octavia serviceUser: octavia serviceAccount: octavia secret: osp-secret - # passwordSelectors: TODO - # nodeSelector: TODO debug: dbSync: false service: false diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go new file mode 100644 index 00000000..13a0ddc6 --- /dev/null +++ b/controllers/amphoracontroller_controller.go @@ -0,0 +1,359 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" + "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" + "github.com/openstack-k8s-operators/octavia-operator/pkg/amphoracontrollers" + "github.com/openstack-k8s-operators/octavia-operator/pkg/octavia" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// OctaviaAmphoraControllerReconciler reconciles an OctaviaAmmphoraController object +type OctaviaAmphoraControllerReconciler struct { + client.Client + Kclient kubernetes.Interface + Log logr.Logger + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers/finalizers,verbs=update + +// Reconcile implementation of the reconcile loop for amphora +// controllers like the octavia housekeeper, worker and health manager +// services +func (r *OctaviaAmphoraControllerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = r.Log.WithValues("amphoracontroller", req.NamespacedName) + + instance := &octaviav1.OctaviaAmphoraController{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic, use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object, requeue the request + return ctrl.Result{}, err + } + + if instance.Status.Conditions == nil { + // Setup the initial conditions + instance.Status.Conditions = condition.Conditions{} + cl := condition.CreateList( + condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + ) + // TODO(beagles): what other conditions? + instance.Status.Conditions.Init(&cl) + if err := r.Status().Update(ctx, instance); err != nil { + return ctrl.Result{}, err + } + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + r.Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + + if err := helper.SetAfter(instance); err != nil { + util.LogErrorForObject(helper, err, "Set after and calc patch/diff", instance) + } + + if changed := helper.GetChanges()["status"]; changed { + patch := client.MergeFrom(helper.GetBeforeObject()) + + if err := r.Status().Patch(ctx, instance, patch); err != nil && !k8s_errors.IsNotFound(err) { + util.LogErrorForObject(helper, err, "Update status", instance) + } + } + }() + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +func (r *OctaviaAmphoraControllerReconciler) reconcileDelete(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, + helper *helper.Helper) (ctrl.Result, error) { + util.LogForObject(helper, "Reconciling Service delete", instance) + + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + + if err := r.Update(ctx, instance); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + util.LogForObject(helper, "Reconciled Service delete successfully", instance) + return ctrl.Result{}, nil +} + +func (r *OctaviaAmphoraControllerReconciler) reconcileUpdate(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, + helper *helper.Helper) (ctrl.Result, error) { + util.LogForObject(helper, "Reconciling Service update", instance) + util.LogForObject(helper, "Reconciled Service update successfully", instance) + return ctrl.Result{}, nil +} + +func (r *OctaviaAmphoraControllerReconciler) reconcileUpgrade(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, + helper *helper.Helper) (ctrl.Result, error) { + util.LogForObject(helper, "Reconciling Service upgrade", instance) + util.LogForObject(helper, "Reconciled Service upgrade successfully", instance) + return ctrl.Result{}, nil +} + +func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, + helper *helper.Helper) (ctrl.Result, error) { + + util.LogForObject(helper, "Reconciling Service", instance) + if !controllerutil.ContainsFinalizer(instance, helper.GetFinalizer()) { + // If the service object doesn't have our finalizer, add it. + controllerutil.AddFinalizer(instance, helper.GetFinalizer()) + // Register the finalizer immediately to avoid orphaning resources on delete + err := r.Update(ctx, instance) + if err != nil { + return ctrl.Result{}, err + } + } + + // Handle config map + configMapVars := make(map[string]env.Setter) + err := r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + if err != nil { + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // Handle service update + ctrlResult, err := r.reconcileUpdate(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // Handle service upgrade + ctrlResult, err = r.reconcileUpgrade(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // normal reconcile tasks + // + + serviceLabels := map[string]string{ + common.AppSelector: octavia.ServiceName, + } + + // Define a new Deployment object + depl := deployment.NewDeployment( + amphoracontrollers.Deployment( + instance, + inputHash, + serviceLabels), + 5, + ) + + ctrlResult, err = depl.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } + // create Deployment - end + + util.LogForObject(helper, "Reconciled Service successfully", instance) + return ctrl.Result{}, nil +} + +func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps( + ctx context.Context, + instance *octaviav1.OctaviaAmphoraController, + helper *helper.Helper, + envVars *map[string]env.Setter, +) error { + r.Log.Info(fmt.Sprintf("generating service config map for %s (%s)", instance.Name, instance.Kind)) + cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(octavia.ServiceName), map[string]string{}) + customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + for key, data := range instance.Spec.DefaultConfigOverwrite { + customData[key] = data + } + templateParameters := make(map[string]interface{}) + + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, helper, instance.Namespace, map[string]string{}) + if err != nil { + return err + } + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + keystonePublicURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointPublic) + if err != nil { + return err + } + templateParameters["ServiceUser"] = instance.Spec.ServiceUser + templateParameters["KeystoneInternalURL"] = keystoneInternalURL + templateParameters["KeystonePublicURL"] = keystonePublicURL + templateParameters["ServiceRoleName"] = instance.Spec.Role + + // TODO(beagles): populate the template parameters + cms := []util.Template{ + // ScriptsConfigMap + { + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, + Labels: cmLabels, + }, + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: cmLabels, + }, + } + + err = configmap.EnsureConfigMaps(ctx, helper, instance, cms, envVars) + if err != nil { + r.Log.Error(err, "unable to process config map") + return err + } + + r.Log.Info("Service config map generated") + + return nil +} + +func (r *OctaviaAmphoraControllerReconciler) createHashOfInputHashes( + ctx context.Context, + instance *octaviav1.OctaviaAmphoraController, + envVars map[string]env.Setter, +) (string, error) { + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, err + } + + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + if err := r.Client.Status().Update(ctx, instance); err != nil { + return hash, err + } + r.Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *OctaviaAmphoraControllerReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&octaviav1.OctaviaAmphoraController{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). + Owns(&appsv1.Deployment{}). + Complete(r) +} diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index ca90b2ed..452c3bd7 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -22,6 +22,7 @@ import ( "time" "github.com/go-logr/logr" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" @@ -74,6 +75,7 @@ type OctaviaReconciler struct { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update @@ -186,6 +188,7 @@ func (r *OctaviaReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&rabbitmqv1.TransportURL{}). Complete(r) } @@ -373,6 +376,52 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav // ConfigMap configMapVars := make(map[string]env.Setter) + transportURL, op, err := r.transportURLCreateOrUpdate(instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + r.Log.Info(fmt.Sprintf("Waiting for the TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil + } + + transportURLSecret, hash, err := oko_secret.GetSecret(ctx, helper, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("TransportURL secret %s not found", instance.Status.TransportURLSecret) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configMapVars[transportURLSecret.Name] = env.SetValue(hash) + // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // @@ -490,6 +539,57 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Status.Conditions.Set(conditionStatus) } + octaviaHousekeeping, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaHousekeeping, "housekeeping") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "OctaviaHousekeeping error occurred %s", + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("Deployment of OctaviaHousekeeping for %s successfully reconciled - operation: %s", instance.Name, string(op))) + } + + instance.Status.OctaviaHousekeepingReadyCount = octaviaHousekeeping.Status.ReadyCount + + octaviaHealthManager, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaHealthManager, "healthmanager") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "OctaviaHealthManager error occurred %s", + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("Deployment of OctaviaHealthManager for %s successfully reconciled - operation: %s", instance.Name, string(op))) + } + + instance.Status.OctaviaHealthManagerReadyCount = octaviaHealthManager.Status.ReadyCount + + octaviaWorker, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaWorker, "worker") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "OctaviaWorker error occurred %s", + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("Deployment of OctaviaWorker for %s successfully reconciled - operation: %s", instance.Name, string(op))) + } + + instance.Status.OctaviaWorkerReadyCount = octaviaWorker.Status.ReadyCount + // create Deployment - end r.Log.Info("Reconciled Service successfully") @@ -605,3 +705,59 @@ func (r *OctaviaReconciler) apiDeploymentCreateOrUpdate(instance *octaviav1.Octa return deployment, op, err } + +func (r *OctaviaReconciler) transportURLCreateOrUpdate( + instance *octaviav1.Octavia, +) (*rabbitmqv1.TransportURL, + controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-octavia-transport", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.RabbitMqClusterName + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + return transportURL, op, err +} + +func (r *OctaviaReconciler) amphoraControllerDeploymentCreateOrUpdate( + instance *octaviav1.Octavia, + controllerSpec octaviav1.OctaviaAmphoraControllerSpec, + role string, +) (*octaviav1.OctaviaAmphoraController, + controllerutil.OperationResult, error) { + + deployment := &octaviav1.OctaviaAmphoraController{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", instance.Name, role), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, deployment, func() error { + deployment.Spec = controllerSpec + deployment.Spec.Role = role + deployment.Spec.DatabaseInstance = instance.Spec.DatabaseInstance + deployment.Spec.DatabaseHostname = instance.Status.DatabaseHostname + deployment.Spec.DatabaseUser = instance.Spec.DatabaseUser + deployment.Spec.ServiceUser = instance.Spec.ServiceUser + deployment.Spec.Secret = instance.Spec.Secret + deployment.Spec.TransportURLSecret = instance.Status.TransportURLSecret + deployment.Spec.ServiceAccount = instance.RbacResourceName() + if len(deployment.Spec.NodeSelector) == 0 { + deployment.Spec.NodeSelector = instance.Spec.NodeSelector + } + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + return nil + }) + + return deployment, op, err +} diff --git a/controllers/octaviaapi_controller.go b/controllers/octaviaapi_controller.go index f1aac769..519e87db 100644 --- a/controllers/octaviaapi_controller.go +++ b/controllers/octaviaapi_controller.go @@ -576,10 +576,11 @@ func (r *OctaviaAPIReconciler) generateServiceConfigMaps( } err = configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars) - r.Log.Info("Service config map generated") if err != nil { + r.Log.Error(err, "unable to process config map") return err } + r.Log.Info("Service config map generated") return nil } diff --git a/go.mod b/go.mod index a2838b22..3515c78e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 github.com/openshift/api v3.9.0+incompatible + github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230626141355-6d8b643fb532 github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230913075424-2680ce4b6ad2 diff --git a/go.sum b/go.sum index 2d46c1b9..34068043 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxCMwNRnMjhhIDOWHJowi6q8G6koI= github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4= +github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230626141355-6d8b643fb532 h1:jw/s40DA79qOGFcV9Jxfhah+Qb7D57LaZ6RzGcY1USc= +github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230626141355-6d8b643fb532/go.mod h1:uuwrYRvdp5VwsWJpA47MnDT+UQjX1UWixagit7FzwV8= github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 h1:p98vKnS4KzdgU/+vrVKFY3y9n9v1Z6cpo4JvbTNRxlM= github.com/openstack-k8s-operators/keystone-operator/api v0.1.0/go.mod h1:LNJJdteQG4E2fhWDerE+f8S2/ephEJg8yBkH1eqYYOo= github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 h1:/ez+9PSwtucQ9v1I5X72xlP5UJztTMPH4M5gDAJAatc= diff --git a/main.go b/main.go index 8bfdbab4..6c404123 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ovn1beta1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" @@ -54,6 +55,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(mariadbv1.AddToScheme(scheme)) utilruntime.Must(keystonev1.AddToScheme(scheme)) + utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(octaviav1.AddToScheme(scheme)) utilruntime.Must(ovn1beta1.AddToScheme(scheme)) @@ -113,6 +115,16 @@ func main() { os.Exit(1) } + if err = (&controllers.OctaviaAmphoraControllerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + Log: ctrl.Log.WithName("controllers").WithName("OctaviaAmphoraController"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "OctaviaAmphoraController") + os.Exit(1) + } + if err = (&controllers.OctaviaAPIReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/pkg/amphoracontrollers/deployment.go b/pkg/amphoracontrollers/deployment.go new file mode 100644 index 00000000..33b65ad1 --- /dev/null +++ b/pkg/amphoracontrollers/deployment.go @@ -0,0 +1,139 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amphoracontrollers + +import ( + "fmt" + + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" + "github.com/openstack-k8s-operators/octavia-operator/pkg/octavia" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Deployment func +func Deployment( + instance *octaviav1.OctaviaAmphoraController, + configHash string, + labels map[string]string, +) *appsv1.Deployment { + serviceName := fmt.Sprintf("octavia-%s", instance.Spec.Role) + + // The API pod has an extra volume so the API and the provider agent can + // communicate with each other. + volumes := octavia.GetVolumes(instance.Name) + + // TODO(beagles): service debugging + + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 15, + PeriodSeconds: 13, + InitialDelaySeconds: 3, + } + readinessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 15, + PeriodSeconds: 15, + InitialDelaySeconds: 5, + } + + // TODO(beagles): use equivalent's of healthcheck's in tripleo which + // seem to largely based on connections to database. The pgrep's + // could be tightened up too but they seem to be a bit tricky. + + livenessProbe.Exec = &corev1.ExecAction{ + Command: []string{ + "/usr/bin/pgrep", "-r", "DRST", "octavia", + }, + } + + readinessProbe.Exec = &corev1.ExecAction{ + Command: []string{ + "/usr/bin/pgrep", "-r", "DRST", "octavia", + }, + } + + envVars := map[string]env.Setter{} + + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: &instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: serviceName, + Image: instance.Spec.ContainerImage, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: octavia.GetVolumeMounts(serviceName), + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + deployment.Spec.Template.Spec.Affinity = affinity.DistributePods( + common.AppSelector, + []string{ + serviceName, + }, + corev1.LabelHostname, + ) + if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector + } + + initContainerDetails := octavia.APIDetails{ + ContainerImage: instance.Spec.ContainerImage, + DatabaseHost: instance.Spec.DatabaseHostname, + DatabaseUser: instance.Spec.DatabaseUser, + DatabaseName: octavia.DatabaseName, + OSPSecret: instance.Spec.Secret, + DBPasswordSelector: instance.Spec.PasswordSelectors.Database, + UserPasswordSelector: instance.Spec.PasswordSelectors.Service, + VolumeMounts: octavia.GetInitVolumeMounts(), + } + deployment.Spec.Template.Spec.InitContainers = octavia.InitContainer(initContainerDetails) + + return deployment +} diff --git a/pkg/octavia/dbsync.go b/pkg/octavia/dbsync.go index 2495be32..b3846783 100644 --- a/pkg/octavia/dbsync.go +++ b/pkg/octavia/dbsync.go @@ -37,7 +37,7 @@ func DbSyncJob( ) *batchv1.Job { runAsUser := int64(0) initVolumeMounts := GetInitVolumeMounts() - volumeMounts := GetDBSyncVolumeMounts() + volumeMounts := GetVolumeMounts("db-sync") volumes := GetVolumes(instance.Name) args := []string{"-c"} diff --git a/pkg/octavia/initcontainer.go b/pkg/octavia/initcontainer.go index 77e14714..dff67b1c 100644 --- a/pkg/octavia/initcontainer.go +++ b/pkg/octavia/initcontainer.go @@ -27,6 +27,7 @@ type APIDetails struct { DatabaseHost string DatabaseUser string DatabaseName string + TransportURLSecret string OSPSecret string DBPasswordSelector string UserPasswordSelector string @@ -76,6 +77,23 @@ func InitContainer(init APIDetails) []corev1.Container { }, }, } + + // TODO(beagles): should this be conditional? It seems like it should be required. + if init.TransportURLSecret != "" { + envs = append(envs, + corev1.EnvVar{ + Name: "TransportURL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: init.TransportURLSecret, + }, + Key: "transport_url", + }, + }, + }, + ) + } envs = env.MergeEnvs(envs, envVars) return []corev1.Container{ diff --git a/pkg/octavia/volumes.go b/pkg/octavia/volumes.go index 42d2f821..6d4c4618 100644 --- a/pkg/octavia/volumes.go +++ b/pkg/octavia/volumes.go @@ -78,7 +78,7 @@ func GetInitVolumeMounts() []corev1.VolumeMount { } // GetVolumeMounts - general VolumeMounts -func GetVolumeMounts() []corev1.VolumeMount { +func GetVolumeMounts(serviceName string) []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "scripts", @@ -90,19 +90,11 @@ func GetVolumeMounts() []corev1.VolumeMount { MountPath: "/var/lib/config-data/merged", ReadOnly: false, }, - } -} - -// GetDBSyncVolumeMounts - VolumeMounts for db sync -func GetDBSyncVolumeMounts() []corev1.VolumeMount { - volumeMounts := []corev1.VolumeMount{ { Name: "config-data-merged", MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: "db-sync-config.json", + SubPath: serviceName + "-config.json", ReadOnly: true, }, } - - return append(GetVolumeMounts(), volumeMounts...) } diff --git a/pkg/octaviaapi/volumes.go b/pkg/octaviaapi/volumes.go index da629cff..491f5b44 100644 --- a/pkg/octaviaapi/volumes.go +++ b/pkg/octaviaapi/volumes.go @@ -46,12 +46,6 @@ func getVolumeMounts(serviceName string) []corev1.VolumeMount { MountPath: "/run/octavia", ReadOnly: false, }, - { - Name: "config-data-merged", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: serviceName + "-config.json", - ReadOnly: true, - }, } - return append(octavia.GetVolumeMounts(), volumeMounts...) + return append(octavia.GetVolumeMounts(serviceName), volumeMounts...) } diff --git a/templates/octavia/bin/init.sh b/templates/octavia/bin/init.sh index 900b8e1f..e89afcab 100755 --- a/templates/octavia/bin/init.sh +++ b/templates/octavia/bin/init.sh @@ -24,6 +24,7 @@ export DBHOST=${DatabaseHost:?"Please specify a DatabaseHost variable."} export DBUSER=${DatabaseUser:?"Please specify a DatabaseUser variable."} export DBPASSWORD=${DatabasePassword:?"Please specify a DatabasePassword variable."} export DB=${DatabaseName:-"octavia"} +export TRANSPORTURL=${TransportURL:-""} SVC_CFG=/etc/octavia/octavia.conf SVC_CFG_MERGED=/var/lib/config-data/merged/octavia.conf @@ -41,5 +42,10 @@ for dir in /var/lib/config-data/default; do done # set secrets + +# set secrets +if [ -n "$TRANSPORTURL" ]; then + crudini --set /var/lib/config-data/merged/neutron.conf DEFAULT transport_url $TRANSPORTURL +fi crudini --set ${SVC_CFG_MERGED} database connection mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB} crudini --set ${SVC_CFG_MERGED} keystone_authtoken password $PASSWORD diff --git a/templates/octaviaamphoracontroller/bin/init.sh b/templates/octaviaamphoracontroller/bin/init.sh new file mode 100755 index 00000000..2bca6c5a --- /dev/null +++ b/templates/octaviaamphoracontroller/bin/init.sh @@ -0,0 +1,46 @@ +#!/bin//bash +# +# Copyright 2020 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -ex + +# This script generates the octavia.conf/logging.conf file and +# copies the result to the ephemeral /var/lib/config-data/merged volume. +# +# Secrets are obtained from ENV variables. +export PASSWORD=${AdminPassword:?"Please specify a AdminPassword variable."} +export DBHOST=${DatabaseHost:?"Please specify a DatabaseHost variable."} +export DBUSER=${DatabaseUser:?"Please specify a DatabaseUser variable."} +export DBPASSWORD=${DatabasePassword:?"Please specify a DatabasePassword variable."} +export DB=${DatabaseName:-"octavia"} + +SVC_CFG=/etc/octavia/octavia.conf +SVC_CFG_MERGED=/var/lib/config-data/merged/octavia.conf + +# expect that the common.sh is in the same dir as the calling script +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +. ${SCRIPTPATH}/common.sh --source-only + +# Copy default service config from container image as base +cp -a ${SVC_CFG} ${SVC_CFG_MERGED} + +# Merge all templates from config CM +for dir in /var/lib/config-data/default; do + merge_config_dir ${dir} +done + +# set secrets +crudini --set ${SVC_CFG_MERGED} database connection mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB} +crudini --set ${SVC_CFG_MERGED} keystone_authtoken password $PASSWORD +crudini --set ${SVC_CFG_MERGED} service_auth password $PASSWORD diff --git a/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json new file mode 100644 index 00000000..19aa9549 --- /dev/null +++ b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json @@ -0,0 +1,24 @@ +{ + "command": "/usr/bin/octavia-health-manager --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf --log-file /var/log/octavia/healthmanager.log", + "config_files": [ + { + "source": "/var/lib/config-data/merged/octavia.conf", + "dest": "/etc/octavia/octavia.conf", + "owner": "octavia", + "perm": "0600" + }, + { + "source": "/var/lib/config-data/merged/custom.conf", + "dest": "/etc/octavia/octavia.conf.d/custom.conf", + "owner": "octavia", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/octavia", + "owner": "octavia:octavia", + "recurse": true + } + ] +} diff --git a/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json new file mode 100644 index 00000000..4c37a4a2 --- /dev/null +++ b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json @@ -0,0 +1,24 @@ +{ + "command": "/usr/bin/octavia-housekeeping --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf --log-file /var/log/octavia/housekeeping.log", + "config_files": [ + { + "source": "/var/lib/config-data/merged/octavia.conf", + "dest": "/etc/octavia/octavia.conf", + "owner": "octavia", + "perm": "0600" + }, + { + "source": "/var/lib/config-data/merged/custom.conf", + "dest": "/etc/octavia/octavia.conf.d/custom.conf", + "owner": "octavia", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/octavia", + "owner": "octavia:octavia", + "recurse": true + } + ] +} diff --git a/templates/octaviaamphoracontroller/config/octavia-worker-config.json b/templates/octaviaamphoracontroller/config/octavia-worker-config.json new file mode 100644 index 00000000..c9896f1d --- /dev/null +++ b/templates/octaviaamphoracontroller/config/octavia-worker-config.json @@ -0,0 +1,24 @@ +{ + "command": "/usr/bin/octavia-worker --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf --log-file /var/log/octavia/worker.log", + "config_files": [ + { + "source": "/var/lib/config-data/merged/octavia.conf", + "dest": "/etc/octavia/octavia.conf", + "owner": "octavia", + "perm": "0600" + }, + { + "source": "/var/lib/config-data/merged/custom.conf", + "dest": "/etc/octavia/octavia.conf.d/custom.conf", + "owner": "octavia", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/octavia", + "owner": "octavia:octavia", + "recurse": true + } + ] +} diff --git a/templates/octaviaamphoracontroller/config/octavia.conf b/templates/octaviaamphoracontroller/config/octavia.conf new file mode 100644 index 00000000..3c6a9c93 --- /dev/null +++ b/templates/octaviaamphoracontroller/config/octavia.conf @@ -0,0 +1,83 @@ +[DEFAULT] +debug=True +rpc_response_timeout=60 +log_file=/var/log/octavia/{{ .ServiceRoleName }}.log +log_dir=/var/log/octavia +[api_settings] +[database] +[health_manager] +heartbeat_key=tobeconfigured +health_update_threads=4 +stats_update_threads=4 +# heartbeat_key=FIXMEkey1 +[keystone_authtoken] +www_authenticate_uri={{ .KeystonePublicURL }} +auth_url={{ .KeystoneInternalURL }} +username={{ .ServiceUser }} +# password=FIXMEpw3 +project_name=service +project_domain_name=Default +user_domain_name=Default +auth_type=password +# memcache_use_advanced_pool=True +# memcached_servers=FIXMEhost1:11211 +# region_name=regionOne +interface=internal +[certificates] +[compute] +[networking] +port_detach_timeout=300 +[haproxy_amphora] +[controller_worker] +[task_flow] +[oslo_messaging] +# topic=octavia-rpc +[oslo_middleware] +# enable_proxy_headers_parsing=True +[house_keeping] +[amphora_agent] +admin_log_targets= +tenant_log_targets= +user_log_facility=0 +administrative_log_facility=1 +forward_all_logs=True +disable_local_log_storage=False +[keepalived_vrrp] + +[service_auth] +project_domain_name=Default +project_name=service +user_domain_name=Default +password=FIXMEpw3 +username=octavia +auth_type=password +auth_url={{ .KeystonePublicURL }}/v3 +region_name=regionOne + +[nova] +region_name=regionOne +endpoint_type=internalURL + +[cinder] +region_name=regionOne +endpoint_type=internalURL + +[glance] +region_name=regionOne +endpoint_type=internalURL + +[neutron] +region_name=regionOne +endpoint_type=internalURL + +[quotas] +[audit] +[audit_middleware_notifications] +[oslo_messaging_notifications] +# driver=noop + +# [healthcheck] +# TODO + +[oslo_policy] +# policy_file=/etc/octavia/policy.yaml diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 7f51b2d2..d4c54aec 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -46,6 +46,45 @@ spec: replicas: 1 secret: osp-secret serviceUser: octavia + octaviaHousekeeping: + containerImage: quay.io/podified-antelope-centos9/openstack-octavia-housekeeping:current-podified + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + databaseUser: octavia + passwordSelectors: + service: OctaviaPassword + database: OctaviaDatabasePassword + replicas: 1 + secret: osp-secret + serviceUser: octavia + octaviaHealthManager: + containerImage: quay.io/podified-antelope-centos9/openstack-octavia-health-manager:current-podified + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + databaseUser: octavia + passwordSelectors: + service: OctaviaPassword + database: OctaviaDatabasePassword + replicas: 1 + secret: osp-secret + serviceUser: octavia + octaviaWorker: + containerImage: quay.io/podified-antelope-centos9/openstack-octavia-worker:current-podified + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + databaseUser: octavia + passwordSelectors: + service: OctaviaPassword + database: OctaviaDatabasePassword + replicas: 1 + secret: osp-secret + serviceUser: octavia status: databaseHostname: openstack apireadyCount: 1 diff --git a/tests/kuttl/tests/octavia_scale/04-scale-down-zero-octaviaapi.yaml b/tests/kuttl/tests/octavia_scale/04-scale-down-zero-octaviaapi.yaml index 8ab7a782..7566f1f0 100644 --- a/tests/kuttl/tests/octavia_scale/04-scale-down-zero-octaviaapi.yaml +++ b/tests/kuttl/tests/octavia_scale/04-scale-down-zero-octaviaapi.yaml @@ -3,3 +3,6 @@ kind: TestStep commands: - script: | oc patch octavia -n $NAMESPACE octavia --type='json' -p='[{"op": "replace", "path": "/spec/octaviaAPI/replicas", "value":0}]' + oc patch octavia -n $NAMESPACE octavia --type='json' -p='[{"op": "replace", "path": "/spec/octaviaHousekeeping/replicas", "value":0}]' + oc patch octavia -n $NAMESPACE octavia --type='json' -p='[{"op": "replace", "path": "/spec/octaviaWorker/replicas", "value":0}]' + oc patch octavia -n $NAMESPACE octavia --type='json' -p='[{"op": "replace", "path": "/spec/octaviaHealthManager/replicas", "value":0}]'