openstack-k8s-operators · openshift-merge-robot · Feb 17, 2023 · Jan 25, 2023 · Jan 25, 2023 · Feb 1, 2023
diff --git a/README.md b/README.md
@@ -23,6 +23,141 @@ spec:
   secret: keystone-secret
 ```
 
+## Example: configure Keystone with additional networks
+
+The Keystone spec can be used to configure Keystone to have the pods
+being attached to additional networks.
+
+Create a network-attachement-definition which then can be referenced
+from the Keystone API CR.
+
+```
+---
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: storage
+  namespace: openstack
+spec:
+  config: |
+    {
+      "cniVersion": "0.3.1",
+      "name": "storage",
+      "type": "macvlan",
+      "master": "enp7s0.21",
+      "ipam": {
+        "type": "whereabouts",
+        "range": "172.18.0.0/24",
+        "range_start": "172.18.0.50",
+        "range_end": "172.18.0.100"
+      }
+    }
+```
+
+The following represents an example of Keystone resource that can be used
+to trigger the service deployment, and have the service pods attached to
+the storage network using the above NetworkAttachmentDefinition.
+
+```
+apiVersion: keystone.openstack.org/v1beta1
+kind: KeystoneAPI
+metadata:
+  name: keystone
+spec:
+  ...
+  networkAttachents:
+  - storage
+...
+```
+
+When the service is up and running, it will now have an additional nic
+configured for the storage network:
+
+```
+# oc rsh keystone-75f5cd6595-kpfr2
+sh-5.1# ip a
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+    inet6 ::1/128 scope host
+       valid_lft forever preferred_lft forever
+3: eth0@if298: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
+    link/ether 0a:58:0a:82:01:18 brd ff:ff:ff:ff:ff:ff link-netnsid 0
+    inet 10.130.1.24/23 brd 10.130.1.255 scope global eth0
+       valid_lft forever preferred_lft forever
+    inet6 fe80::4cf2:a3ff:feb0:932/64 scope link
+       valid_lft forever preferred_lft forever
+4: net1@if26: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
+    link/ether a2:f1:3b:12:fd:be brd ff:ff:ff:ff:ff:ff link-netnsid 0
+    inet 172.18.0.52/24 brd 172.18.0.255 scope global net1
+       valid_lft forever preferred_lft forever
+    inet6 fe80::a0f1:3bff:fe12:fdbe/64 scope link
+       valid_lft forever preferred_lft forever
+```
+
+## Example: expose Keystone to an isolated network
+
+The Keystone spec can be used to configure Keystone to register e.g.
+the internal endpoint to an isolated network. MetalLB is used for this
+scenario.
+
+As a pre requisite, MetalLB needs to be installed and worker nodes
+prepared to work as MetalLB nodes to serve the LoadBalancer service.
+
+In this example the following MetalLB IPAddressPool is used:
+
+```
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: osp-internalapi
+  namespace: metallb-system
+spec:
+  addresses:
+  - 172.17.0.200-172.17.0.210
+  autoAssign: false
+```
+
+The following represents an example of Keystone resource that can be used
+to trigger the service deployment, and have the internal keystoneAPI endpoint
+registerd as a MetalLB service using the IPAddressPool `osp-internal`,
+request to use the IP `172.17.0.202` as the VIP and the IP is shared with
+other services.
+
+```
+apiVersion: keystone.openstack.org/v1beta1
+kind: KeystoneAPI
+metadata:
+  name: keystone
+spec:
+  ...
+  externalEndpoints:
+  - endpoint: internal
+    ipAddressPool: osp-internalapi
+    loadBalancerIPs:
+    - 172.17.0.202
+    sharedIP: true
+    sharedIPKey: ""
+  ...
+...
+```
+
+The internal keystone endpoint gets registered with its service name. This
+service name needs to resolve to the `LoadBalancerIP` on the isolated network
+either by DNS or via /etc/hosts:
+
+```
+# openstack endpoint list -c 'Service Name' -c Interface -c URL --service keystone
++--------------+-----------+-----------------------------------------------------------------+
+| Service Name | Interface | URL                                                             |
++--------------+-----------+-----------------------------------------------------------------+
+| keystone     | public    | http://keystone-public-openstack.apps.ostest.test.metalkube.org |
+| keystone     | internal  | http://keystone-internal.openstack.svc:5000                     |
++--------------+-----------+-----------------------------------------------------------------+
+```
+
 # Design
 The current design takes care of the following:
 

diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml
@@ -16,9 +16,9 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - description: Networks
-      jsonPath: .status.networks
-      name: Networks
+    - description: NetworkAttachments
+      jsonPath: .spec.networkAttachments
+      name: NetworkAttachments
       type: string
     - description: Status
       jsonPath: .status.conditions[0].status
@@ -109,9 +109,50 @@ spec:
                   to add additional files. Those get added to the service config dir
                   in /etc/<service> . TODO: -> implement'
                 type: object
-              networkAttachmentDefinitions:
-                description: NetworkAttachmentDefinitions list of network attachment
-                  definitions the service pod gets attached to
+              externalEndpoints:
+                description: ExternalEndpoints, expose a VIP using a pre-created IPAddressPool
+                items:
+                  description: MetalLBConfig to configure the MetalLB loadbalancer
+                    service
+                  properties:
+                    endpoint:
+                      description: Endpoint, OpenStack endpoint this service maps
+                        to
+                      enum:
+                      - internal
+                      - public
+                      type: string
+                    ipAddressPool:
+                      description: IPAddressPool expose VIP via MetalLB on the IPAddressPool
+                      minLength: 1
+                      type: string
+                    loadBalancerIPs:
+                      description: LoadBalancerIPs, request given IPs from the pool
+                        if available. Using a list to allow dual stack (IPv4/IPv6)
+                        support
+                      items:
+                        type: string
+                      type: array
+                    sharedIP:
+                      default: true
+                      description: SharedIP if true, VIP/VIPs get shared with multiple
+                        services
+                      type: boolean
+                    sharedIPKey:
+                      default: ""
+                      description: SharedIPKey specifies the sharing key which gets
+                        set as the annotation on the LoadBalancer service. Services
+                        which share the same VIP must have the same SharedIPKey. Defaults
+                        to the IPAddressPool if SharedIP is true, but no SharedIPKey
+                        specified.
+                      type: string
+                  required:
+                  - ipAddressPool
+                  type: object
+                type: array
+              networkAttachments:
+                description: NetworkAttachments is a list of NetworkAttachment resource
+                  names to expose the services to the given network
                 items:
                   type: string
                 type: array
@@ -270,12 +311,13 @@ spec:
                   type: string
                 description: Map of hashes to track e.g. job status
                 type: object
-              networks:
-                description: Networks in addtion to the cluster network, the service
-                  is attached to
-                items:
-                  type: string
-                type: array
+              networkAttachments:
+                additionalProperties:
+                  items:
+                    type: string
+                  type: array
+                description: NetworkAttachments status of the deployment pods
+                type: object
               readyCount:
                 description: ReadyCount of keystone API instances
                 format: int32

diff --git a/api/go.mod b/api/go.mod
@@ -3,8 +3,8 @@ module github.com/openstack-k8s-operators/keystone-operator/api
 go 1.19
 
 require (
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad
-	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba
+	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
 	sigs.k8s.io/controller-runtime v0.14.2

diff --git a/api/go.sum b/api/go.sum
@@ -221,10 +221,10 @@ github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc=
 github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
 github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
 github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad h1:o4wmvVc7y/xGy52kUx493jsPRkC7cVJO9Ly91h/HjzQ=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad/go.mod h1:qV9OlokZRpqbHI3lmeN5EOmIKynWphw6GPl3zP9KOGM=
-github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad h1:2GKSJhKzQFa1AcKUHC++G1O65IkVHyM+KbHRVVR69Jo=
-github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba h1:IIM8K8j1mOJx16Epwrau6bx5DU2rxj8NGTjjxrjlF4I=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba/go.mod h1:+EDQmWZRA8ruHnWPcw9s/el3UMi6u4EZkcSe7dCQ50k=
+github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba h1:gwYazA5cJmHle3bXkpxz/iZcx6IZW5HmKaKQVgZPHxQ=
+github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go
@@ -127,8 +127,41 @@ type KeystoneAPISpec struct {
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
 	// +kubebuilder:validation:Optional
-	// NetworkAttachmentDefinitions list of network attachment definitions the service pod gets attached to
-	NetworkAttachmentDefinitions []string `json:"networkAttachmentDefinitions"`
+	// NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network
+	NetworkAttachments []string `json:"networkAttachments"`
+
+	// +kubebuilder:validation:Optional
+	// ExternalEndpoints, expose a VIP using a pre-created IPAddressPool
+	ExternalEndpoints []MetalLBConfig `json:"externalEndpoints"`
+}
+
+// MetalLBConfig to configure the MetalLB loadbalancer service
+type MetalLBConfig struct {
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Enum=internal;public
+	// Endpoint, OpenStack endpoint this service maps to
+	Endpoint endpoint.Endpoint `json:"endpoint"`
+
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// IPAddressPool expose VIP via MetalLB on the IPAddressPool
+	IPAddressPool string `json:"ipAddressPool"`
+
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=true
+	// SharedIP if true, VIP/VIPs get shared with multiple services
+	SharedIP bool `json:"sharedIP"`
+
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=""
+	// SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service.
+	// Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if
+	// SharedIP is true, but no SharedIPKey specified.
+	SharedIPKey string `json:"sharedIPKey"`
+
+	// +kubebuilder:validation:Optional
+	// LoadBalancerIPs, request given IPs from the pool if available. Using a list to allow dual stack (IPv4/IPv6) support
+	LoadBalancerIPs []string `json:"loadBalancerIPs"`
 }
 
 // PasswordSelector to identify the DB and AdminUser password from the Secret
@@ -177,13 +210,13 @@ type KeystoneAPIStatus struct {
 	// Keystone Database Hostname
 	DatabaseHostname string `json:"databaseHostname,omitempty"`
 
-	// Networks in addtion to the cluster network, the service is attached to
-	Networks []string `json:"networks,omitempty"`
+	// NetworkAttachments status of the deployment pods
+	NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
-//+kubebuilder:printcolumn:name="Networks",type="string",JSONPath=".status.networks",description="Networks"
+//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".spec.networkAttachments",description="NetworkAttachments"
 //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status"
 //+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message"
 

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go