diff --git a/README.md b/README.md index 930fcebf..c051520b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,141 @@ spec: secret: keystone-secret ``` +## Example: configure Keystone with additional networks + +The Keystone spec can be used to configure Keystone to have the pods +being attached to additional networks. + +Create a network-attachement-definition which then can be referenced +from the Keystone API CR. + +``` +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: storage + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "enp7s0.21", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.50", + "range_end": "172.18.0.100" + } + } +``` + +The following represents an example of Keystone resource that can be used +to trigger the service deployment, and have the service pods attached to +the storage network using the above NetworkAttachmentDefinition. + +``` +apiVersion: keystone.openstack.org/v1beta1 +kind: KeystoneAPI +metadata: + name: keystone +spec: + ... + networkAttachents: + - storage +... +``` + +When the service is up and running, it will now have an additional nic +configured for the storage network: + +``` +# oc rsh keystone-75f5cd6595-kpfr2 +sh-5.1# ip a +1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +3: eth0@if298: mtu 1450 qdisc noqueue state UP group default + link/ether 0a:58:0a:82:01:18 brd ff:ff:ff:ff:ff:ff link-netnsid 0 + inet 10.130.1.24/23 brd 10.130.1.255 scope global eth0 + valid_lft forever preferred_lft forever + inet6 fe80::4cf2:a3ff:feb0:932/64 scope link + valid_lft forever preferred_lft forever +4: net1@if26: mtu 1500 qdisc noqueue state UP group default + link/ether a2:f1:3b:12:fd:be brd ff:ff:ff:ff:ff:ff link-netnsid 0 + inet 172.18.0.52/24 brd 172.18.0.255 scope global net1 + valid_lft forever preferred_lft forever + inet6 fe80::a0f1:3bff:fe12:fdbe/64 scope link + valid_lft forever preferred_lft forever +``` + +## Example: expose Keystone to an isolated network + +The Keystone spec can be used to configure Keystone to register e.g. +the internal endpoint to an isolated network. MetalLB is used for this +scenario. + +As a pre requisite, MetalLB needs to be installed and worker nodes +prepared to work as MetalLB nodes to serve the LoadBalancer service. + +In this example the following MetalLB IPAddressPool is used: + +``` +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: osp-internalapi + namespace: metallb-system +spec: + addresses: + - 172.17.0.200-172.17.0.210 + autoAssign: false +``` + +The following represents an example of Keystone resource that can be used +to trigger the service deployment, and have the internal keystoneAPI endpoint +registerd as a MetalLB service using the IPAddressPool `osp-internal`, +request to use the IP `172.17.0.202` as the VIP and the IP is shared with +other services. + +``` +apiVersion: keystone.openstack.org/v1beta1 +kind: KeystoneAPI +metadata: + name: keystone +spec: + ... + externalEndpoints: + - endpoint: internal + ipAddressPool: osp-internalapi + loadBalancerIPs: + - 172.17.0.202 + sharedIP: true + sharedIPKey: "" + ... +... +``` + +The internal keystone endpoint gets registered with its service name. This +service name needs to resolve to the `LoadBalancerIP` on the isolated network +either by DNS or via /etc/hosts: + +``` +# openstack endpoint list -c 'Service Name' -c Interface -c URL --service keystone ++--------------+-----------+-----------------------------------------------------------------+ +| Service Name | Interface | URL | ++--------------+-----------+-----------------------------------------------------------------+ +| keystone | public | http://keystone-public-openstack.apps.ostest.test.metalkube.org | +| keystone | internal | http://keystone-internal.openstack.svc:5000 | ++--------------+-----------+-----------------------------------------------------------------+ +``` + # Design The current design takes care of the following: diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml index 73423ac8..82c04df0 100644 --- a/api/bases/keystone.openstack.org_keystoneapis.yaml +++ b/api/bases/keystone.openstack.org_keystoneapis.yaml @@ -16,9 +16,9 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: Networks - jsonPath: .status.networks - name: Networks + - description: NetworkAttachments + jsonPath: .spec.networkAttachments + name: NetworkAttachments type: string - description: Status jsonPath: .status.conditions[0].status @@ -109,9 +109,50 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - networkAttachmentDefinitions: - description: NetworkAttachmentDefinitions list of network attachment - definitions the service pod gets attached to + externalEndpoints: + description: ExternalEndpoints, expose a VIP using a pre-created IPAddressPool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the pool + if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with multiple + services + type: boolean + sharedIPKey: + default: "" + description: SharedIPKey specifies the sharing key which gets + set as the annotation on the LoadBalancer service. Services + which share the same VIP must have the same SharedIPKey. Defaults + to the IPAddressPool if SharedIP is true, but no SharedIPKey + specified. + type: string + required: + - ipAddressPool + type: object + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network items: type: string type: array @@ -270,12 +311,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object - networks: - description: Networks in addtion to the cluster network, the service - is attached to - items: - type: string - type: array + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object readyCount: description: ReadyCount of keystone API instances format: int32 diff --git a/api/go.mod b/api/go.mod index 4b8e8017..47aae00f 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,8 +3,8 @@ module github.com/openstack-k8s-operators/keystone-operator/api go 1.19 require ( - github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad + github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 sigs.k8s.io/controller-runtime v0.14.2 diff --git a/api/go.sum b/api/go.sum index a0c6e801..dbaa9506 100644 --- a/api/go.sum +++ b/api/go.sum @@ -221,10 +221,10 @@ github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad h1:o4wmvVc7y/xGy52kUx493jsPRkC7cVJO9Ly91h/HjzQ= -github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad/go.mod h1:qV9OlokZRpqbHI3lmeN5EOmIKynWphw6GPl3zP9KOGM= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad h1:2GKSJhKzQFa1AcKUHC++G1O65IkVHyM+KbHRVVR69Jo= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY= +github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba h1:IIM8K8j1mOJx16Epwrau6bx5DU2rxj8NGTjjxrjlF4I= +github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba/go.mod h1:+EDQmWZRA8ruHnWPcw9s/el3UMi6u4EZkcSe7dCQ50k= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba h1:gwYazA5cJmHle3bXkpxz/iZcx6IZW5HmKaKQVgZPHxQ= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go index 3d68b2f0..3d20af98 100644 --- a/api/v1beta1/keystoneapi_types.go +++ b/api/v1beta1/keystoneapi_types.go @@ -127,8 +127,41 @@ type KeystoneAPISpec struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:validation:Optional - // NetworkAttachmentDefinitions list of network attachment definitions the service pod gets attached to - NetworkAttachmentDefinitions []string `json:"networkAttachmentDefinitions"` + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments"` + + // +kubebuilder:validation:Optional + // ExternalEndpoints, expose a VIP using a pre-created IPAddressPool + ExternalEndpoints []MetalLBConfig `json:"externalEndpoints"` +} + +// MetalLBConfig to configure the MetalLB loadbalancer service +type MetalLBConfig struct { + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=internal;public + // Endpoint, OpenStack endpoint this service maps to + Endpoint endpoint.Endpoint `json:"endpoint"` + + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // IPAddressPool expose VIP via MetalLB on the IPAddressPool + IPAddressPool string `json:"ipAddressPool"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // SharedIP if true, VIP/VIPs get shared with multiple services + SharedIP bool `json:"sharedIP"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + // Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + // SharedIP is true, but no SharedIPKey specified. + SharedIPKey string `json:"sharedIPKey"` + + // +kubebuilder:validation:Optional + // LoadBalancerIPs, request given IPs from the pool if available. Using a list to allow dual stack (IPv4/IPv6) support + LoadBalancerIPs []string `json:"loadBalancerIPs"` } // PasswordSelector to identify the DB and AdminUser password from the Secret @@ -177,13 +210,13 @@ type KeystoneAPIStatus struct { // Keystone Database Hostname DatabaseHostname string `json:"databaseHostname,omitempty"` - // Networks in addtion to the cluster network, the service is attached to - Networks []string `json:"networks,omitempty"` + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status -//+kubebuilder:printcolumn:name="Networks",type="string",JSONPath=".status.networks",description="Networks" +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".spec.networkAttachments",description="NetworkAttachments" //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" //+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 4ff69dec..1d80c8e5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -105,11 +105,18 @@ func (in *KeystoneAPISpec) DeepCopyInto(out *KeystoneAPISpec) { } } in.Resources.DeepCopyInto(&out.Resources) - if in.NetworkAttachmentDefinitions != nil { - in, out := &in.NetworkAttachmentDefinitions, &out.NetworkAttachmentDefinitions + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments *out = make([]string, len(*in)) copy(*out, *in) } + if in.ExternalEndpoints != nil { + in, out := &in.ExternalEndpoints, &out.ExternalEndpoints + *out = make([]MetalLBConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneAPISpec. @@ -146,10 +153,20 @@ func (in *KeystoneAPIStatus) DeepCopyInto(out *KeystoneAPIStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Networks != nil { - in, out := &in.Networks, &out.Networks - *out = make([]string, len(*in)) - copy(*out, *in) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } } } @@ -445,6 +462,26 @@ func (in *KeystoneServiceStatus) DeepCopy() *KeystoneServiceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetalLBConfig) DeepCopyInto(out *MetalLBConfig) { + *out = *in + if in.LoadBalancerIPs != nil { + in, out := &in.LoadBalancerIPs, &out.LoadBalancerIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetalLBConfig. +func (in *MetalLBConfig) DeepCopy() *MetalLBConfig { + if in == nil { + return nil + } + out := new(MetalLBConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PasswordSelector) DeepCopyInto(out *PasswordSelector) { *out = *in diff --git a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml index 73423ac8..82c04df0 100644 --- a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml +++ b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml @@ -16,9 +16,9 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: Networks - jsonPath: .status.networks - name: Networks + - description: NetworkAttachments + jsonPath: .spec.networkAttachments + name: NetworkAttachments type: string - description: Status jsonPath: .status.conditions[0].status @@ -109,9 +109,50 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - networkAttachmentDefinitions: - description: NetworkAttachmentDefinitions list of network attachment - definitions the service pod gets attached to + externalEndpoints: + description: ExternalEndpoints, expose a VIP using a pre-created IPAddressPool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the pool + if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with multiple + services + type: boolean + sharedIPKey: + default: "" + description: SharedIPKey specifies the sharing key which gets + set as the annotation on the LoadBalancer service. Services + which share the same VIP must have the same SharedIPKey. Defaults + to the IPAddressPool if SharedIP is true, but no SharedIPKey + specified. + type: string + required: + - ipAddressPool + type: object + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network items: type: string type: array @@ -270,12 +311,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object - networks: - description: Networks in addtion to the cluster network, the service - is attached to - items: - type: string - type: array + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object readyCount: description: ReadyCount of keystone API instances format: int32 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 086443bc..a0918d0c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -41,6 +41,13 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list - apiGroups: - "" resources: @@ -65,6 +72,14 @@ rules: - patch - update - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch - apiGroups: - keystone.openstack.org resources: diff --git a/controllers/keystoneapi_controller.go b/controllers/keystoneapi_controller.go index 15f02e37..865b9e6c 100644 --- a/controllers/keystoneapi_controller.go +++ b/controllers/keystoneapi_controller.go @@ -33,6 +33,7 @@ import ( helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" job "github.com/openstack-k8s-operators/lib-common/modules/common/job" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" database "github.com/openstack-k8s-operators/lib-common/modules/database" @@ -44,6 +45,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -86,10 +88,12 @@ type KeystoneAPIReconciler struct { // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch // Reconcile reconcile keystone API requests func (r *KeystoneAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { @@ -152,7 +156,8 @@ func (r *KeystoneAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.BootstrapReadyCondition, condition.InitReason, condition.BootstrapReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage)) + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage)) instance.Status.Conditions.Init(&cl) @@ -165,6 +170,9 @@ func (r *KeystoneAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) if instance.Status.APIEndpoints == nil { instance.Status.APIEndpoints = map[string]string{} } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } // Handle service delete if !instance.DeletionTimestamp.IsZero() { @@ -216,6 +224,7 @@ func (r *KeystoneAPIReconciler) reconcileInit( instance *keystonev1.KeystoneAPI, helper *helper.Helper, serviceLabels map[string]string, + serviceAnnotations map[string]string, ) (ctrl.Result, error) { r.Log.Info("Reconciling Service init") @@ -282,11 +291,7 @@ func (r *KeystoneAPIReconciler) reconcileInit( // run keystone db sync // dbSyncHash := instance.Status.Hash[keystonev1.DbSyncHash] - jobDef, err := keystone.DbSyncJob(instance, serviceLabels) - if err != nil { - return ctrl.Result{}, err - } - + jobDef := keystone.DbSyncJob(instance, serviceLabels, serviceAnnotations) dbSyncjob := job.NewJob( jobDef, keystonev1.DbSyncHash, @@ -327,9 +332,6 @@ func (r *KeystoneAPIReconciler) reconcileInit( // expose the service (create service, route and return the created endpoint URLs) // var keystonePorts = map[endpoint.Endpoint]endpoint.Data{ - endpoint.EndpointAdmin: endpoint.Data{ - Port: keystone.KeystoneAdminPort, - }, endpoint.EndpointPublic: endpoint.Data{ Port: keystone.KeystonePublicPort, }, @@ -338,6 +340,18 @@ func (r *KeystoneAPIReconciler) reconcileInit( }, } + for _, metallbcfg := range instance.Spec.ExternalEndpoints { + portCfg := keystonePorts[metallbcfg.Endpoint] + portCfg.MetalLB = &endpoint.MetalLBData{ + IPAddressPool: metallbcfg.IPAddressPool, + SharedIP: metallbcfg.SharedIP, + SharedIPKey: metallbcfg.SharedIPKey, + LoadBalancerIPs: metallbcfg.LoadBalancerIPs, + } + + keystonePorts[metallbcfg.Endpoint] = portCfg + } + apiEndpoints, ctrlResult, err := endpoint.ExposeEndpoints( ctx, helper, @@ -378,11 +392,7 @@ func (r *KeystoneAPIReconciler) reconcileInit( // // BootStrap Job // - jobDef, err = keystone.BootstrapJob(instance, serviceLabels, instance.Status.APIEndpoints) - if err != nil { - return ctrl.Result{}, err - } - + jobDef = keystone.BootstrapJob(instance, serviceLabels, serviceAnnotations, instance.Status.APIEndpoints) bootstrapjob := job.NewJob( jobDef, keystonev1.BootstrapHash, @@ -537,8 +547,37 @@ func (r *KeystoneAPIReconciler) reconcileNormal(ctx context.Context, instance *k common.AppSelector: keystone.ServiceName, } + // networks to attach to + for _, netAtt := range instance.Spec.NetworkAttachments { + _, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + serviceAnnotations, err := nad.CreateNetworksAnnotation(instance.Namespace, instance.Spec.NetworkAttachments) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.NetworkAttachments, err) + } + // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -566,11 +605,7 @@ func (r *KeystoneAPIReconciler) reconcileNormal(ctx context.Context, instance *k // // Define a new Deployment object - deplDef, err := keystone.Deployment(instance, inputHash, serviceLabels) - if err != nil { - return ctrl.Result{}, err - } - + deplDef := keystone.Deployment(instance, inputHash, serviceLabels, serviceAnnotations) depl := deployment.NewDeployment( deplDef, time.Duration(5)*time.Second, @@ -594,10 +629,32 @@ func (r *KeystoneAPIReconciler) reconcileNormal(ctx context.Context, instance *k return ctrlResult, nil } instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation(ctx, helper, instance.Spec.NetworkAttachments, serviceLabels, instance.Status.ReadyCount) + if err != nil { + return ctrl.Result{}, err + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + if instance.Status.ReadyCount > 0 { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } - instance.Status.Networks = instance.Spec.NetworkAttachmentDefinitions + // create Deployment - end // diff --git a/go.mod b/go.mod index b45dc76c..a385151a 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa github.com/openstack-k8s-operators/keystone-operator/api v0.0.0-20230213142003-4cb137e438a3 - github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad - github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230213100743-10d19ef192ad - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad + github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba + github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230215134634-d31141e5bbba + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba github.com/openstack-k8s-operators/mariadb-operator/api v0.0.0-20230213105001-8dc6cf6aa963 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.26.1 @@ -20,6 +20,8 @@ require ( sigs.k8s.io/controller-runtime v0.14.2 ) +require github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 84214b3b..80e6187a 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -228,12 +230,12 @@ github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDD github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa h1:HJypldaUFUol3iBea4P6UDRuCvpAVSOtnBqOSKSnW50= github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa/go.mod h1:5kG0Ct412tO3fNkZ5b3/BwwSsV7LkSNfOB/apUlbMJI= -github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad h1:o4wmvVc7y/xGy52kUx493jsPRkC7cVJO9Ly91h/HjzQ= -github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230213100743-10d19ef192ad/go.mod h1:qV9OlokZRpqbHI3lmeN5EOmIKynWphw6GPl3zP9KOGM= -github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230213100743-10d19ef192ad h1:i8va36LFSHTxbOVnDVPAYyzmB2vQy8Ygx/CuZwMrbCI= -github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230213100743-10d19ef192ad/go.mod h1:rONM/XgvFs6putDIxRHNv9/CTGh2afAvJM5wRP2OywY= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad h1:2GKSJhKzQFa1AcKUHC++G1O65IkVHyM+KbHRVVR69Jo= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230213100743-10d19ef192ad/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY= +github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba h1:IIM8K8j1mOJx16Epwrau6bx5DU2rxj8NGTjjxrjlF4I= +github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230215134634-d31141e5bbba/go.mod h1:+EDQmWZRA8ruHnWPcw9s/el3UMi6u4EZkcSe7dCQ50k= +github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230215134634-d31141e5bbba h1:KWjeUGGzfZ7u2cQJNdzpg4UIghURS241up0FOyssLxM= +github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230215134634-d31141e5bbba/go.mod h1:rONM/XgvFs6putDIxRHNv9/CTGh2afAvJM5wRP2OywY= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba h1:gwYazA5cJmHle3bXkpxz/iZcx6IZW5HmKaKQVgZPHxQ= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.0.0-20230215134634-d31141e5bbba/go.mod h1:9tj29SmyP9izLIEKj5E44F7M7a82UwcPdIufc3MQpcY= github.com/openstack-k8s-operators/mariadb-operator/api v0.0.0-20230213105001-8dc6cf6aa963 h1:L8H3qpbsnaMjbCNaSyCyuX2kCsQD7j0S6grOp1ZeYzU= github.com/openstack-k8s-operators/mariadb-operator/api v0.0.0-20230213105001-8dc6cf6aa963/go.mod h1:PM7XY+2Uq+rVQ53I/+IbLFUH22b67xMgnBsKJhQmBcA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/main.go b/main.go index e26ddec5..9900d001 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -51,6 +52,7 @@ func init() { utilruntime.Must(keystonev1.AddToScheme(scheme)) utilruntime.Must(mariadbv1.AddToScheme(scheme)) utilruntime.Must(routev1.AddToScheme(scheme)) + utilruntime.Must(networkv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/keystone/bootstrap.go b/pkg/keystone/bootstrap.go index d1e6731b..9d056f26 100644 --- a/pkg/keystone/bootstrap.go +++ b/pkg/keystone/bootstrap.go @@ -16,14 +16,10 @@ limitations under the License. package keystone import ( - "fmt" - keystonev1beta1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" - "github.com/openstack-k8s-operators/lib-common/modules/common/annotations" "github.com/openstack-k8s-operators/lib-common/modules/common/env" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,8 +34,9 @@ const ( func BootstrapJob( instance *keystonev1beta1.KeystoneAPI, labels map[string]string, + annotations map[string]string, endpoints map[string]string, -) (*batchv1.Job, error) { +) *batchv1.Job { runAsUser := int64(0) args := []string{"-c"} @@ -77,6 +74,9 @@ func BootstrapJob( }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: "OnFailure", ServiceAccountName: ServiceAccount, @@ -114,14 +114,6 @@ func BootstrapJob( job.Spec.Template.Spec.Containers[0].Env = env.MergeEnvs(job.Spec.Template.Spec.Containers[0].Env, envVars) job.Spec.Template.Spec.Volumes = getVolumes(instance.Name) - // networks to attach to - nwAnnotation, err := annotations.GetNADAnnotation(instance.Namespace, instance.Spec.NetworkAttachmentDefinitions) - if err != nil { - return nil, fmt.Errorf("failed create network annotation from %s: %w", - instance.Spec.NetworkAttachmentDefinitions, err) - } - job.Spec.Template.Annotations = util.MergeStringMaps(job.Spec.Template.Annotations, nwAnnotation) - initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, @@ -134,5 +126,5 @@ func BootstrapJob( } job.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) - return job, nil + return job } diff --git a/pkg/keystone/dbsync.go b/pkg/keystone/dbsync.go index 728fddbd..a3f5a71b 100644 --- a/pkg/keystone/dbsync.go +++ b/pkg/keystone/dbsync.go @@ -16,14 +16,10 @@ limitations under the License. package keystone import ( - "fmt" - keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" - "github.com/openstack-k8s-operators/lib-common/modules/common/annotations" "github.com/openstack-k8s-operators/lib-common/modules/common/env" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,7 +34,8 @@ const ( func DbSyncJob( instance *keystonev1.KeystoneAPI, labels map[string]string, -) (*batchv1.Job, error) { + annotations map[string]string, +) *batchv1.Job { runAsUser := int64(0) args := []string{"-c"} @@ -61,6 +58,9 @@ func DbSyncJob( }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: "OnFailure", ServiceAccountName: ServiceAccount, @@ -85,15 +85,6 @@ func DbSyncJob( } job.Spec.Template.Spec.Volumes = getVolumes(ServiceName) - - // networks to attach to - nwAnnotation, err := annotations.GetNADAnnotation(instance.Namespace, instance.Spec.NetworkAttachmentDefinitions) - if err != nil { - return nil, fmt.Errorf("failed create network annotation from %s: %w", - instance.Spec.NetworkAttachmentDefinitions, err) - } - job.Spec.Template.Annotations = util.MergeStringMaps(job.Spec.Template.Annotations, nwAnnotation) - initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, @@ -106,5 +97,5 @@ func DbSyncJob( } job.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) - return job, nil + return job } diff --git a/pkg/keystone/deployment.go b/pkg/keystone/deployment.go index 052f37aa..4db512a0 100644 --- a/pkg/keystone/deployment.go +++ b/pkg/keystone/deployment.go @@ -16,14 +16,10 @@ limitations under the License. package keystone import ( - "fmt" - keystonev1beta1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" - "github.com/openstack-k8s-operators/lib-common/modules/common/annotations" "github.com/openstack-k8s-operators/lib-common/modules/common/env" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -41,7 +37,8 @@ func Deployment( instance *keystonev1beta1.KeystoneAPI, configHash string, labels map[string]string, -) (*appsv1.Deployment, error) { + annotations map[string]string, +) *appsv1.Deployment { runAsUser := int64(0) livenessProbe := &corev1.Probe{ @@ -104,7 +101,8 @@ func Deployment( Replicas: &instance.Spec.Replicas, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Annotations: annotations, + Labels: labels, }, Spec: corev1.PodSpec{ ServiceAccountName: ServiceAccount, @@ -145,14 +143,6 @@ func Deployment( deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - // networks to attach to - nwAnnotation, err := annotations.GetNADAnnotation(instance.Namespace, instance.Spec.NetworkAttachmentDefinitions) - if err != nil { - return nil, fmt.Errorf("failed create network annotation from %s: %w", - instance.Spec.NetworkAttachmentDefinitions, err) - } - deployment.Spec.Template.Annotations = util.MergeStringMaps(deployment.Spec.Template.Annotations, nwAnnotation) - initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, @@ -165,5 +155,5 @@ func Deployment( } deployment.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) - return deployment, nil + return deployment }