diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 50c0e900..01ec82ea 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -106,21 +106,9 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 747ed061..5254e7c8 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -519,21 +519,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -717,21 +705,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -915,21 +891,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -1159,7 +1123,7 @@ spec: tenantName: default: service description: TenantName - the name of the OpenStack tenant that controls - the Octavia resources + the Octavia resources TODO(gthiemonge) same as ServiceAccount? type: string required: - apacheContainerImage @@ -1168,7 +1132,6 @@ spec: - rabbitMqClusterName - secret - serviceAccount - - tenantName type: object status: description: OctaviaStatus defines the observed state of Octavia diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index 1b655807..da8bb010 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -123,8 +123,8 @@ type OctaviaAmphoraControllerSpecCore struct { TenantName string `json:"tenantName"` // +kubebuilder:validation:Optional - // +kubebuilder:default={manageLbMgmtNetworks: true, subnetIpVersion: 4} - LbMgmtNetworks OctaviaLbMgmtNetworks `json:"lbMgmtNetwork"` + // +kubebuilder:default="" + LbMgmtNetworkID string `json:"lbMgmtNetworkID"` // +kubebuilder:validation:Optional // +kubebuilder:default={} diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index 4c6cdb12..3c53a3ee 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -155,6 +155,12 @@ type OctaviaSpecBase struct { // TODO: -> implement DefaultConfigOverwrite map[string]string `json:"defaultConfigOverwrite,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=service + // TenantName - the name of the OpenStack tenant that controls the Octavia resources + // TODO(gthiemonge) same as ServiceAccount? + TenantName string `json:"tenantName"` + // +kubebuilder:validation:Optional // +kubebuilder:default={manageLbMgmtNetworks: true, subnetIpVersion: 4} LbMgmtNetworks OctaviaLbMgmtNetworks `json:"lbMgmtNetwork"` @@ -192,11 +198,6 @@ type OctaviaSpecBase struct { // +kubebuilder:validation:Required // Apache Container Image URL ApacheContainerImage string `json:"apacheContainerImage"` - - // +kubebuilder:validation:Required - // +kubebuilder:default=service - // TenantName - the name of the OpenStack tenant that controls the Octavia resources - TenantName string `json:"tenantName"` } // PasswordSelector to identify the DB and AdminUser password from the Secret diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 72744f46..aca6786a 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -336,7 +336,6 @@ func (in *OctaviaAmphoraControllerSpecCore) DeepCopyInto(out *OctaviaAmphoraCont *out = make([]string, len(*in)) copy(*out, *in) } - out.LbMgmtNetworks = in.LbMgmtNetworks if in.AmphoraCustomFlavors != nil { in, out := &in.AmphoraCustomFlavors, &out.AmphoraCustomFlavors *out = make([]OctaviaAmphoraFlavor, len(*in)) diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 50c0e900..01ec82ea 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -106,21 +106,9 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 747ed061..5254e7c8 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -519,21 +519,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -717,21 +705,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -915,21 +891,9 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object - lbMgmtNetwork: - default: - manageLbMgmtNetworks: true - subnetIpVersion: 4 - description: OctaviaLbMgmtNetworks Settings for Octavia management - networks - properties: - manageLbMgmtNetworks: - default: true - type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer - type: object + lbMgmtNetworkID: + default: "" + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -1159,7 +1123,7 @@ spec: tenantName: default: service description: TenantName - the name of the OpenStack tenant that controls - the Octavia resources + the Octavia resources TODO(gthiemonge) same as ServiceAccount? type: string required: - apacheContainerImage @@ -1168,7 +1132,6 @@ spec: - rabbitMqClusterName - secret - serviceAccount - - tenantName type: object status: description: OctaviaStatus defines the observed state of Octavia diff --git a/config/samples/octavia_v1beta1_octavia.yaml b/config/samples/octavia_v1beta1_octavia.yaml index 20a285fe..a5d5152c 100644 --- a/config/samples/octavia_v1beta1_octavia.yaml +++ b/config/samples/octavia_v1beta1_octavia.yaml @@ -27,6 +27,9 @@ spec: customServiceConfig: | [DEFAULT] debug = true + networkAttachments: + - octavia + octaviaHealthManager: databaseInstance: openstack databaseAccount: octavia @@ -41,6 +44,8 @@ spec: customServiceConfig: | [DEFAULT] debug = true + networkAttachments: + - octavia octaviaWorker: databaseInstance: openstack databaseAccount: octavia @@ -55,6 +60,8 @@ spec: customServiceConfig: | [DEFAULT] debug = true + networkAttachments: + - octavia octaviaAPI: databaseInstance: openstack databaseAccount: octavia diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 4741a893..4516666a 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "sort" "strings" "time" @@ -43,6 +44,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -253,13 +255,6 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context } configMapVars[transportURLSecret.Name] = env.SetValue(hash) - // Create load balancer management network and get its Id - networkID, err := amphoracontrollers.EnsureLbMgmtNetworks(ctx, instance, &r.Log, helper) - if err != nil { - return ctrl.Result{}, err - } - r.Log.Info(fmt.Sprintf("Using management network \"%s\"", networkID)) - defaultFlavorID, err := amphoracontrollers.EnsureFlavors(ctx, instance, &r.Log, helper) if err != nil { return ctrl.Result{}, err @@ -267,7 +262,7 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context r.Log.Info(fmt.Sprintf("Using default flavor \"%s\"", defaultFlavorID)) templateVars := OctaviaTemplateVars{ - LbMgmtNetworkID: networkID, + LbMgmtNetworkID: instance.Spec.LbMgmtNetworkID, AmphoraDefaultFlavorID: defaultFlavorID, } @@ -490,7 +485,7 @@ func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps( return err } - parentOctaviaName := amphoracontrollers.GetOwningOctaviaControllerName( + parentOctaviaName := octavia.GetOwningOctaviaControllerName( instance) serverCAPassSecretName := fmt.Sprintf("%s-ca-passphrase", parentOctaviaName) caPassSecret, _, err := secret.GetSecret( @@ -513,6 +508,48 @@ func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps( err.Error())) return err } + // + // TODO(beagles): Improve this with predictable IPs for the health managers because what is + // going to happen on start up is that health managers will restart each time a new one is deployed. + // The easiest strategy is to create a "hole" in the IP address range and control the + // allocation and configuration of an additional IP on each network attached interface. We will + // need a container in the Pod that has the ip command installed to do this however. + // + healthManagerIPs, err := getPodIPs( + fmt.Sprintf("%s-%s", "octavia", octaviav1.HealthManager), + instance.Namespace, + r.Kclient, + &r.Log, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return err + } + + // TODO(beagles): come up with a way to preallocate or ensure + // a stable list of IPs. + + if len(healthManagerIPs) == 0 { + templateParameters["ControllerIPList"] = "" + if instance.Spec.Role != octaviav1.HealthManager { + + // Only let the health manager get an empty port list until + // we get a way to preallocate some ports. This helps reduce + // churn when the Pods are getting initially created or setup. + return fmt.Errorf("Health manager ports are not ready yet") + } + } else { + withPorts := make([]string, len(healthManagerIPs)) + for idx, val := range healthManagerIPs { + withPorts[idx] = fmt.Sprintf("%s:5555", val) + } + templateParameters["ControllerIPList"] = strings.Join(withPorts, ",") + } spec := instance.Spec templateParameters["ServiceUser"] = spec.ServiceUser @@ -597,3 +634,42 @@ func (r *OctaviaAmphoraControllerReconciler) SetupWithManager(mgr ctrl.Manager) Owns(&appsv1.DaemonSet{}). Complete(r) } + +func listHealthManagerPods(name string, ns string, client kubernetes.Interface, log *logr.Logger) (*corev1.PodList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", common.AppSelector, name), + } + log.Info(fmt.Sprintf("Listing pods using label selector %s", listOptions.LabelSelector)) + pods, err := client.CoreV1().Pods(ns).List(context.Background(), listOptions) + if err != nil { + return nil, err + } + return pods, nil +} + +func getPodIPs(name string, ns string, client kubernetes.Interface, log *logr.Logger) ([]string, error) { + // + // Get the IPs for the network attachments for these PODs. + // + var result []string + pods, err := listHealthManagerPods(name, ns, client, log) + if err != nil { + return nil, err + } + for _, pod := range pods.Items { + annotations := pod.GetAnnotations() + networkStatusList, err := nad.GetNetworkStatusFromAnnotation(annotations) + if err != nil { + log.Error(err, fmt.Sprintf("Unable to get network annotations from %s", annotations)) + return nil, err + } + for _, networkStatus := range networkStatusList { + netAttachName := fmt.Sprintf("%s/%s", ns, octavia.LbNetworkAttachmentName) + if networkStatus.Name == netAttachName { + result = append(result, networkStatus.IPs[0]) + } + } + } + sort.Strings(result) + return result, nil +} diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 6b46adf7..acda675f 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -595,53 +595,84 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Status.Conditions.MarkTrue(octaviav1.OctaviaAPIReadyCondition, condition.DeploymentReadyMessage) } - octaviaHousekeeping, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaHousekeeping, octaviav1.Housekeeping) + // ------------------------------------------------------------------------------------------------------------ + // Amphora reconciliation + // ------------------------------------------------------------------------------------------------------------ + + // Create load balancer management network and get its Id (networkInfo is actually a struct and contains + // multiple details. + networkInfo, err := octavia.EnsureAmphoraManagementNetwork( + ctx, + instance.Namespace, + instance.Spec.TenantName, + &instance.Spec.LbMgmtNetworks, + &Log, + helper, + ) + if err != nil { + return ctrl.Result{}, err + } + Log.Info(fmt.Sprintf("Using management network \"%s\"", networkInfo.TenantNetworkID)) + + octaviaHealthManager, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, networkInfo.TenantNetworkID, + instance.Spec.OctaviaHealthManager, octaviav1.HealthManager) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - amphoraControllerReadyCondition(octaviav1.Housekeeping), + amphoraControllerReadyCondition(octaviav1.HealthManager), condition.ErrorReason, condition.SeverityWarning, - amphoraControllerErrorMessage(octaviav1.Housekeeping), + amphoraControllerErrorMessage(octaviav1.HealthManager), err.Error())) return ctrl.Result{}, err } if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("Deployment of OctaviaHousekeeping for %s successfully reconciled - operation: %s", instance.Name, string(op))) + Log.Info(fmt.Sprintf("Deployment of OctaviaHealthManager for %s successfully reconciled - operation: %s", instance.Name, string(op))) } - instance.Status.OctaviaHousekeepingReadyCount = octaviaHousekeeping.Status.ReadyCount - conditionStatus = octaviaHousekeeping.Status.Conditions.Mirror(amphoraControllerReadyCondition(octaviav1.Housekeeping)) + instance.Status.OctaviaHealthManagerReadyCount = octaviaHealthManager.Status.ReadyCount + conditionStatus = octaviaHealthManager.Status.Conditions.Mirror(amphoraControllerReadyCondition(octaviav1.HealthManager)) if conditionStatus != nil { instance.Status.Conditions.Set(conditionStatus) } else { - instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.Housekeeping), condition.DeploymentReadyMessage) + instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.HealthManager), condition.DeploymentReadyMessage) } - octaviaHealthManager, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaHealthManager, octaviav1.HealthManager) + // + // We do not try and reconcile the other controller PODs until after the health manager Pods are all deployed. + // + if octaviaHealthManager.Status.ReadyCount != octaviaHealthManager.Status.DesiredNumberScheduled { + Log.Info("Health managers are not ready. Housekeeping and Worker services pending") + return ctrl.Result{}, nil + } + + // Skip the other amphora controller pods until the health managers are all up and running. + octaviaHousekeeping, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, networkInfo.TenantNetworkID, + instance.Spec.OctaviaHousekeeping, octaviav1.Housekeeping) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - amphoraControllerReadyCondition(octaviav1.HealthManager), + amphoraControllerReadyCondition(octaviav1.Housekeeping), condition.ErrorReason, condition.SeverityWarning, - amphoraControllerErrorMessage(octaviav1.HealthManager), + amphoraControllerErrorMessage(octaviav1.Housekeeping), err.Error())) return ctrl.Result{}, err } if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("Deployment of OctaviaHealthManager for %s successfully reconciled - operation: %s", instance.Name, string(op))) + Log.Info(fmt.Sprintf("Deployment of OctaviaHousekeeping for %s successfully reconciled - operation: %s", instance.Name, string(op))) } - instance.Status.OctaviaHealthManagerReadyCount = octaviaHealthManager.Status.ReadyCount - conditionStatus = octaviaHealthManager.Status.Conditions.Mirror(amphoraControllerReadyCondition(octaviav1.HealthManager)) + instance.Status.OctaviaHousekeepingReadyCount = octaviaHousekeeping.Status.ReadyCount + conditionStatus = octaviaHousekeeping.Status.Conditions.Mirror(amphoraControllerReadyCondition(octaviav1.Housekeeping)) if conditionStatus != nil { instance.Status.Conditions.Set(conditionStatus) } else { - instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.HealthManager), condition.DeploymentReadyMessage) + instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.Housekeeping), condition.DeploymentReadyMessage) } - octaviaWorker, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaWorker, octaviav1.Worker) + octaviaWorker, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, networkInfo.TenantNetworkID, + instance.Spec.OctaviaWorker, octaviav1.Worker) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( amphoraControllerReadyCondition(octaviav1.Worker), @@ -1195,6 +1226,7 @@ func (r *OctaviaReconciler) redisCreateOrUpdate( func (r *OctaviaReconciler) amphoraControllerDaemonSetCreateOrUpdate( instance *octaviav1.Octavia, + tenantNetworkID string, controllerSpec octaviav1.OctaviaAmphoraControllerSpec, role string, ) (*octaviav1.OctaviaAmphoraController, @@ -1218,8 +1250,7 @@ func (r *OctaviaReconciler) amphoraControllerDaemonSetCreateOrUpdate( daemonset.Spec.Secret = instance.Spec.Secret daemonset.Spec.TransportURLSecret = instance.Status.TransportURLSecret daemonset.Spec.ServiceAccount = instance.RbacResourceName() - daemonset.Spec.LbMgmtNetworks.ManageLbMgmtNetworks = instance.Spec.LbMgmtNetworks.ManageLbMgmtNetworks - daemonset.Spec.LbMgmtNetworks.SubnetIPVersion = instance.Spec.LbMgmtNetworks.SubnetIPVersion + daemonset.Spec.LbMgmtNetworkID = tenantNetworkID daemonset.Spec.AmphoraCustomFlavors = instance.Spec.AmphoraCustomFlavors daemonset.Spec.RedisHostIPs = instance.Status.RedisHostIPs if len(daemonset.Spec.NodeSelector) == 0 { diff --git a/pkg/amphoracontrollers/daemonset.go b/pkg/amphoracontrollers/daemonset.go index 107eff17..cea4da00 100644 --- a/pkg/amphoracontrollers/daemonset.go +++ b/pkg/amphoracontrollers/daemonset.go @@ -43,7 +43,7 @@ func DaemonSet( // The API pod has an extra volume so the API and the provider agent can // communicate with each other. volumes := octavia.GetVolumes(instance.Name) - parentOctaviaName := GetOwningOctaviaControllerName(instance) + parentOctaviaName := octavia.GetOwningOctaviaControllerName(instance) certsSecretName := fmt.Sprintf("%s-certs-secret", parentOctaviaName) volumes = append(volumes, GetCertVolume(certsSecretName)...) diff --git a/pkg/amphoracontrollers/flavors.go b/pkg/amphoracontrollers/flavors.go index 2b4f7eee..360f6cb4 100644 --- a/pkg/amphoracontrollers/flavors.go +++ b/pkg/amphoracontrollers/flavors.go @@ -258,7 +258,7 @@ func ensureFlavors(osclient *openstack.OpenStack, log *logr.Logger, instance *oc // // returns the UUID of the default Nova flavor func EnsureFlavors(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, log *logr.Logger, helper *helper.Helper) (string, error) { - osclient, err := GetOpenstackClient(ctx, instance, helper) + osclient, err := octavia.GetOpenstackClient(ctx, instance.Namespace, helper) if err != nil { return "", fmt.Errorf("Error while getting a service client when creating flavors: %w", err) } diff --git a/pkg/amphoracontrollers/lb_mgmt_network.go b/pkg/amphoracontrollers/lb_mgmt_network.go deleted file mode 100644 index 689a352e..00000000 --- a/pkg/amphoracontrollers/lb_mgmt_network.go +++ /dev/null @@ -1,220 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package amphoracontrollers - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/openstack-k8s-operators/lib-common/modules/common/helper" - octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" - "github.com/openstack-k8s-operators/octavia-operator/pkg/octavia" -) - -func ensureLbMgmtSubnet(client *gophercloud.ServiceClient, instance *octaviav1.OctaviaAmphoraController, log *logr.Logger, serviceTenantID string, lbMgmtNetID string) (*subnets.Subnet, error) { - ipVersion := instance.Spec.LbMgmtNetworks.SubnetIPVersion - - listOpts := subnets.ListOpts{ - Name: LbMgmtSubnetName, - NetworkID: lbMgmtNetID, - TenantID: serviceTenantID, - IPVersion: ipVersion, - } - allPages, err := subnets.List(client, listOpts).AllPages() - if err != nil { - return nil, err - } - allSubnets, err := subnets.ExtractSubnets(allPages) - if err != nil { - return nil, err - } - - var createOpts subnets.CreateOpts - if ipVersion == 6 { - gatewayIP := LbMgmtSubnetIPv6GatewayIP - createOpts = subnets.CreateOpts{ - Name: LbMgmtSubnetName, - Description: LbMgmtSubnetDescription, - NetworkID: lbMgmtNetID, - TenantID: serviceTenantID, - CIDR: LbMgmtSubnetIPv6CIDR, - IPVersion: gophercloud.IPVersion(ipVersion), - IPv6AddressMode: LbMgmtSubnetIPv6AddressMode, - IPv6RAMode: LbMgmtSubnetIPv6RAMode, - AllocationPools: []subnets.AllocationPool{ - { - Start: LbMgmtSubnetIPv6AllocationPoolStart, - End: LbMgmtSubnetIPv6AllocationPoolEnd, - }, - }, - GatewayIP: &gatewayIP, - } - } else { - gatewayIP := LbMgmtSubnetGatewayIP - createOpts = subnets.CreateOpts{ - Name: LbMgmtSubnetName, - Description: LbMgmtSubnetDescription, - NetworkID: lbMgmtNetID, - TenantID: serviceTenantID, - CIDR: LbMgmtSubnetCIDR, - IPVersion: gophercloud.IPVersion(ipVersion), - AllocationPools: []subnets.AllocationPool{ - { - Start: LbMgmtSubnetAllocationPoolStart, - End: LbMgmtSubnetAllocationPoolEnd, - }, - }, - GatewayIP: &gatewayIP, - } - } - - var lbMgmtSubnet *subnets.Subnet - if len(allSubnets) == 0 { - log.Info(fmt.Sprintf("Creating Octavia management subnet \"%s\"", createOpts.Name)) - lbMgmtSubnet, err = subnets.Create(client, createOpts).Extract() - if err != nil { - return nil, err - } - } else { - lbMgmtSubnet = &allSubnets[0] - delta := subnets.UpdateOpts{} - updateNeeded := false - if lbMgmtSubnet.Description != createOpts.Description { - delta.Description = &createOpts.Description - updateNeeded = true - } - if lbMgmtSubnet.AllocationPools[0].Start != createOpts.AllocationPools[0].Start || - lbMgmtSubnet.AllocationPools[0].End != createOpts.AllocationPools[0].End { - delta.AllocationPools = []subnets.AllocationPool{ - { - Start: createOpts.AllocationPools[0].Start, - End: createOpts.AllocationPools[0].End, - }, - } - updateNeeded = true - } - if lbMgmtSubnet.GatewayIP != *createOpts.GatewayIP { - delta.GatewayIP = createOpts.GatewayIP - updateNeeded = true - } - if updateNeeded { - log.Info(fmt.Sprintf("Updating Octavia management subnet \"%s\"", createOpts.Name)) - lbMgmtSubnet, err = subnets.Update(client, lbMgmtSubnet.ID, delta).Extract() - if err != nil { - return nil, err - } - } - } - return lbMgmtSubnet, nil -} - -func getLbMgmtNetwork(client *gophercloud.ServiceClient, instance *octaviav1.OctaviaAmphoraController, serviceTenantID string) (*networks.Network, error) { - listOpts := networks.ListOpts{ - Name: LbMgmtNetName, - TenantID: serviceTenantID, - } - allPages, err := networks.List(client, listOpts).AllPages() - if err != nil { - return nil, err - } - allNetworks, err := networks.ExtractNetworks(allPages) - if err != nil { - return nil, err - } - if len(allNetworks) > 0 { - return &allNetworks[0], nil - } - return nil, nil -} - -func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, instance *octaviav1.OctaviaAmphoraController, log *logr.Logger, serviceTenantID string) (*networks.Network, error) { - lbMgmtNet, err := getLbMgmtNetwork(client, instance, serviceTenantID) - if err != nil { - return nil, err - } - - asu := true - createOpts := networks.CreateOpts{ - Name: LbMgmtNetName, - Description: LbMgmtNetDescription, - AdminStateUp: &asu, - TenantID: serviceTenantID, - } - if lbMgmtNet == nil { - log.Info(fmt.Sprintf("Creating Octavia management network \"%s\"", createOpts.Name)) - lbMgmtNet, err = networks.Create(client, createOpts).Extract() - if err != nil { - return nil, err - } - } else { - emptyOpts := networks.UpdateOpts{} - delta := networks.UpdateOpts{} - if lbMgmtNet.Description != createOpts.Description { - delta.Description = &createOpts.Description - } - if lbMgmtNet.AdminStateUp != *createOpts.AdminStateUp { - delta.AdminStateUp = createOpts.AdminStateUp - } - if delta != emptyOpts { - log.Info(fmt.Sprintf("Updating Octavia management network \"%s\"", createOpts.Name)) - lbMgmtNet, err = networks.Update(client, lbMgmtNet.ID, delta).Extract() - if err != nil { - return nil, err - } - } - } - - _, err = ensureLbMgmtSubnet(client, instance, log, serviceTenantID, lbMgmtNet.ID) - if err != nil { - return nil, err - } - - return lbMgmtNet, nil -} - -// EnsureLbMgmtNetworks - ensure that the Octavia management network is created -// -// returns the UUID of the network -func EnsureLbMgmtNetworks(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, log *logr.Logger, helper *helper.Helper) (string, error) { - o, err := GetOpenstackClient(ctx, instance, helper) - if err != nil { - return "", err - } - client, err := octavia.GetNetworkClient(o) - if err != nil { - return "", err - } - serviceTenant, err := octavia.GetProject(o, instance.Spec.TenantName) - if err != nil { - return "", err - } - var network *networks.Network - if instance.Spec.LbMgmtNetworks.ManageLbMgmtNetworks { - network, err = ensureLbMgmtNetwork(client, instance, log, serviceTenant.ID) - } else { - network, err = getLbMgmtNetwork(client, instance, serviceTenant.ID) - if network == nil { - return "", fmt.Errorf("Cannot find network \"%s\"", LbMgmtNetName) - } - } - if err != nil { - return "", err - } - return network.ID, nil -} diff --git a/pkg/octavia/amphora_ssh.go b/pkg/octavia/amphora_ssh.go index ff34d180..b0747db5 100644 --- a/pkg/octavia/amphora_ssh.go +++ b/pkg/octavia/amphora_ssh.go @@ -113,7 +113,7 @@ func uploadKeypair( instance *octaviav1.Octavia, h *helper.Helper, pubKey string) error { - osClient, err := GetOpenstackClient(ctx, instance, h) + osClient, err := GetOpenstackClient(ctx, instance.Namespace, h) if err != nil { return fmt.Errorf("Error getting openstack client: %w", err) } diff --git a/pkg/octavia/common.go b/pkg/octavia/common.go deleted file mode 100644 index 8fcf2b28..00000000 --- a/pkg/octavia/common.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package octavia - -import ( - "context" - - keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" - "github.com/openstack-k8s-operators/lib-common/modules/common/helper" - "github.com/openstack-k8s-operators/lib-common/modules/openstack" - octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" -) - -// GetOpenstackClient returns an openstack admin service client object -func GetOpenstackClient( - ctx context.Context, - instance *octaviav1.Octavia, - h *helper.Helper, -) (*openstack.OpenStack, error) { - keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) - if err != nil { - return nil, err - } - o, _, err := GetAdminServiceClient(ctx, h, keystoneAPI) - if err != nil { - return nil, err - } - return o, nil -} diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go new file mode 100644 index 00000000..b258b374 --- /dev/null +++ b/pkg/octavia/lb_mgmt_network.go @@ -0,0 +1,612 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +@you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package octavia + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/provider" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" + "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" +) + +type NetworkProvisioningSummary struct { + TenantNetworkID string + TenantSubnetID string + TenantRouterPortID string + ProviderNetworkID string + RouterID string +} + +// +// TODO(beagles) we need to decide what, if any of the results of these methods we want to expose in the controller's +// status. +// + +func findPort(client *gophercloud.ServiceClient, portName string, networkID string, subnetID string, ipAddress string, log *logr.Logger) (*ports.Port, error) { + listOpts := ports.ListOpts{ + NetworkID: networkID, + } + allPages, err := ports.List(client, listOpts).AllPages() + if err != nil { + log.Error(err, fmt.Sprintf("Unable to list ports for %s", networkID)) + return nil, err + } + + allPorts, err := ports.ExtractPorts(allPages) + if err != nil { + log.Error(err, "Unable to extract port information from list") + return nil, err + } + if len(allPorts) > 0 { + for _, port := range allPorts { + if len(port.FixedIPs) > 0 && port.FixedIPs[0].IPAddress == ipAddress { + return &port, nil + } + } + } + return nil, nil +} + +func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, tenantSubnet *subnets.Subnet, log *logr.Logger) (*ports.Port, error) { + ipAddress := LbMgmtRouterPortIPv4 + if tenantSubnet.IPVersion == 6 { + ipAddress = LbMgmtRouterPortIPv6 + } + + p, err := findPort(client, LbMgmtRouterPortName, tenantNetwork.ID, tenantSubnet.ID, ipAddress, log) + if err != nil { + return nil, err + } + if p != nil { + // + // TODO(beagles): reconcile port properties? Is there anything to do? Security groups possibly. + // + return p, nil + } + log.Info("Unable to locate port, creating new one") + asu := true + createOpts := ports.CreateOpts{ + Name: LbMgmtRouterPortName, + AdminStateUp: &asu, + NetworkID: tenantNetwork.ID, + FixedIPs: []ports.IP{ + { + SubnetID: tenantSubnet.ID, + IPAddress: ipAddress, + }, + }, + } + p, err = ports.Create(client, createOpts).Extract() + if err != nil { + log.Error(err, "Error creating port") + return nil, err + } + return p, nil +} + +func ensureSubnet(client *gophercloud.ServiceClient, ipVersion int, createOpts subnets.CreateOpts, log *logr.Logger) (*subnets.Subnet, error) { + listOpts := subnets.ListOpts{ + Name: createOpts.Name, + NetworkID: createOpts.NetworkID, + TenantID: createOpts.TenantID, + IPVersion: ipVersion, + } + allPages, err := subnets.List(client, listOpts).AllPages() + if err != nil { + return nil, err + } + allSubnets, err := subnets.ExtractSubnets(allPages) + if err != nil { + return nil, err + } + + var lbMgmtSubnet *subnets.Subnet + if len(allSubnets) == 0 { + log.Info(fmt.Sprintf("Creating Octavia management subnet \"%s\"", createOpts.Name)) + lbMgmtSubnet, err = subnets.Create(client, createOpts).Extract() + if err != nil { + return nil, err + } + } else { + lbMgmtSubnet = &allSubnets[0] + delta := subnets.UpdateOpts{} + updateNeeded := false + if lbMgmtSubnet.Description != createOpts.Description { + delta.Description = &createOpts.Description + updateNeeded = true + } + if lbMgmtSubnet.AllocationPools[0].Start != createOpts.AllocationPools[0].Start || + lbMgmtSubnet.AllocationPools[0].End != createOpts.AllocationPools[0].End { + delta.AllocationPools = []subnets.AllocationPool{ + { + Start: createOpts.AllocationPools[0].Start, + End: createOpts.AllocationPools[0].End, + }, + } + updateNeeded = true + } + if lbMgmtSubnet.GatewayIP != *createOpts.GatewayIP { + delta.GatewayIP = createOpts.GatewayIP + updateNeeded = true + } + if updateNeeded { + log.Info(fmt.Sprintf("Updating Octavia management subnet \"%s\"", createOpts.Name)) + lbMgmtSubnet, err = subnets.Update(client, lbMgmtSubnet.ID, delta).Extract() + if err != nil { + return nil, err + } + } + } + return lbMgmtSubnet, nil +} + +func getNetwork(client *gophercloud.ServiceClient, networkName string, serviceTenantID string) (*networks.Network, error) { + listOpts := networks.ListOpts{ + Name: networkName, + TenantID: serviceTenantID, + } + allPages, err := networks.List(client, listOpts).AllPages() + if err != nil { + return nil, err + } + allNetworks, err := networks.ExtractNetworks(allPages) + if err != nil { + return nil, err + } + if len(allNetworks) > 0 { + return &allNetworks[0], nil + } + return nil, nil +} + +func getNetworkExt(client *gophercloud.ServiceClient, networkName string, serviceTenantID string) (*networks.Network, error) { + extTrue := true + listOpts := external.ListOptsExt{ + ListOptsBuilder: networks.ListOpts{ + Name: networkName, + TenantID: serviceTenantID, + }, + External: &extTrue, + } + allPages, err := networks.List(client, listOpts).AllPages() + if err != nil { + return nil, err + } + allNetworks, err := networks.ExtractNetworks(allPages) + if err != nil { + return nil, err + } + if len(allNetworks) > 0 { + return &allNetworks[0], nil + } + return nil, nil +} + +func ensureNetwork(client *gophercloud.ServiceClient, createOpts networks.CreateOpts, log *logr.Logger, + serviceTenantID string) (*networks.Network, error) { + foundNetwork, err := getNetwork(client, createOpts.Name, serviceTenantID) + if err != nil { + return nil, err + } + + if foundNetwork == nil { + log.Info(fmt.Sprintf("Creating Octavia network \"%s\"", createOpts.Name)) + foundNetwork, err = networks.Create(client, createOpts).Extract() + if err != nil { + return nil, err + } + } else { + emptyOpts := networks.UpdateOpts{} + delta := networks.UpdateOpts{} + if foundNetwork.Description != createOpts.Description { + delta.Description = &createOpts.Description + } + if foundNetwork.AdminStateUp != *createOpts.AdminStateUp { + delta.AdminStateUp = createOpts.AdminStateUp + } + if delta != emptyOpts { + log.Info(fmt.Sprintf("Updating Octavia management network \"%s\"", createOpts.Name)) + foundNetwork, err = networks.Update(client, foundNetwork.ID, delta).Extract() + if err != nil { + return nil, err + } + } + } + return foundNetwork, nil +} + +func ensureNetworkExt(client *gophercloud.ServiceClient, createOpts networks.CreateOpts, log *logr.Logger, serviceTenantID string) (*networks.Network, error) { + foundNetwork, err := getNetworkExt(client, createOpts.Name, serviceTenantID) + if err != nil { + return nil, err + } + + extTrue := true + if foundNetwork == nil { + segment := []provider.Segment{ + provider.Segment{ + NetworkType: "flat", + PhysicalNetwork: "br-octavia", + }, + } + + providerOpts := provider.CreateOptsExt{ + CreateOptsBuilder: createOpts, + Segments: segment, + } + + extCreateOpts := external.CreateOptsExt{ + CreateOptsBuilder: providerOpts, + External: &extTrue, + } + + log.Info(fmt.Sprintf("Creating Octavia network \"%s\"", createOpts.Name)) + foundNetwork, err = networks.Create(client, extCreateOpts).Extract() + if err != nil { + return nil, err + } + } else { + emptyOpts := networks.UpdateOpts{} + delta := networks.UpdateOpts{} + if foundNetwork.Description != createOpts.Description { + delta.Description = &createOpts.Description + } + if foundNetwork.AdminStateUp != *createOpts.AdminStateUp { + delta.AdminStateUp = createOpts.AdminStateUp + } + if delta != emptyOpts { + log.Info(fmt.Sprintf("Updating Octavia management network \"%s\"", createOpts.Name)) + foundNetwork, err = networks.Update(client, foundNetwork.ID, delta).Extract() + if err != nil { + return nil, err + } + } + } + return foundNetwork, nil +} + +func ensureProvSubnet(client *gophercloud.ServiceClient, providerNetwork *networks.Network, log *logr.Logger) ( + *subnets.Subnet, error) { + gatewayIP := LbProvSubnetGatewayIP + createOpts := subnets.CreateOpts{ + Name: LbProvSubnetName, + Description: LbProvSubnetDescription, + NetworkID: providerNetwork.ID, + TenantID: providerNetwork.TenantID, + CIDR: LbProvSubnetCIDR, + IPVersion: gophercloud.IPVersion(4), + AllocationPools: []subnets.AllocationPool{ + { + Start: LbProvSubnetAllocationPoolStart, + End: LbProvSubnetAllocationPoolEnd, + }, + }, + GatewayIP: &gatewayIP, + } + return ensureSubnet(client, 4, createOpts, log) +} + +func ensureProvNetwork(client *gophercloud.ServiceClient, serviceTenantID string, log *logr.Logger) ( + *networks.Network, error) { + provNet, err := getNetwork(client, LbProvNetName, serviceTenantID) + if err != nil { + return nil, err + } + + asu := true + createOpts := networks.CreateOpts{ + Name: LbProvNetName, + Description: LbProvNetDescription, + AdminStateUp: &asu, + TenantID: serviceTenantID, + } + provNet, err = ensureNetworkExt(client, createOpts, log, serviceTenantID) + if err != nil { + return nil, err + } + + return provNet, nil +} + +func ensureLbMgmtSubnet( + client *gophercloud.ServiceClient, + networkDetails *octaviav1.OctaviaLbMgmtNetworks, + tenantNetwork *networks.Network, + log *logr.Logger, +) (*subnets.Subnet, error) { + ipVersion := networkDetails.SubnetIPVersion + + var createOpts subnets.CreateOpts + if ipVersion == 6 { + gatewayIP := LbMgmtSubnetIPv6GatewayIP + createOpts = subnets.CreateOpts{ + Name: LbMgmtSubnetName, + Description: LbMgmtSubnetDescription, + NetworkID: tenantNetwork.ID, + TenantID: tenantNetwork.TenantID, + CIDR: LbMgmtSubnetIPv6CIDR, + IPVersion: gophercloud.IPVersion(ipVersion), + IPv6AddressMode: LbMgmtSubnetIPv6AddressMode, + IPv6RAMode: LbMgmtSubnetIPv6RAMode, + AllocationPools: []subnets.AllocationPool{ + { + Start: LbMgmtSubnetIPv6AllocationPoolStart, + End: LbMgmtSubnetIPv6AllocationPoolEnd, + }, + }, + GatewayIP: &gatewayIP, + // TODO(beagles): ipv6 host routes + } + } else { + gatewayIP := LbMgmtSubnetGatewayIP + createOpts = subnets.CreateOpts{ + Name: LbMgmtSubnetName, + Description: LbMgmtSubnetDescription, + NetworkID: tenantNetwork.ID, + TenantID: tenantNetwork.TenantID, + CIDR: LbMgmtSubnetCIDR, + IPVersion: gophercloud.IPVersion(ipVersion), + AllocationPools: []subnets.AllocationPool{ + { + Start: LbMgmtSubnetAllocationPoolStart, + End: LbMgmtSubnetAllocationPoolEnd, + }, + }, + HostRoutes: []subnets.HostRoute{ + { + DestinationCIDR: LbProvSubnetCIDR, + NextHop: LbMgmtRouterPortIPv4, + }, + }, + GatewayIP: &gatewayIP, + } + } + return ensureSubnet(client, ipVersion, createOpts, log) +} + +func getLbMgmtNetwork(client *gophercloud.ServiceClient, serviceTenantID string) (*networks.Network, error) { + return getNetwork(client, LbMgmtNetName, serviceTenantID) +} + +func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, networkDetails *octaviav1.OctaviaLbMgmtNetworks, + serviceTenantID string, log *logr.Logger) (*networks.Network, error) { + mgmtNetwork, err := getLbMgmtNetwork(client, serviceTenantID) + if err != nil { + return nil, err + } + + if networkDetails == nil && mgmtNetwork == nil { + return nil, fmt.Errorf("Cannot find network \"%s\"", LbMgmtNetName) + } + + asu := true + createOpts := networks.CreateOpts{ + Name: LbMgmtNetName, + Description: LbMgmtNetDescription, + AdminStateUp: &asu, + TenantID: serviceTenantID, + } + mgmtNetwork, err = ensureNetwork(client, createOpts, log, serviceTenantID) + if err != nil { + return nil, err + } + + return mgmtNetwork, nil +} + +func externalFixedIPs(subnetID string) []routers.ExternalFixedIP { + ips := []routers.ExternalFixedIP{ + routers.ExternalFixedIP{ + IPAddress: LbRouterFixedIPAddress, + SubnetID: subnetID, + }, + } + return ips +} + +func compareExternalFixedIPs(a []routers.ExternalFixedIP, b []routers.ExternalFixedIP) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].IPAddress != b[i].IPAddress { + return false + } + if a[i].SubnetID != b[i].SubnetID { + return false + } + } + return true +} + +// reconcileRouter compares existing router properties against what is expected/desired and updates the router if +// necessary! +func reconcileRouter(client *gophercloud.ServiceClient, router *routers.Router, + gatewayNetwork *networks.Network, + gatewaySubnet *subnets.Subnet, + log *logr.Logger) (*routers.Router, error) { + + if !router.AdminStateUp { + return router, fmt.Errorf("Router %s is not up", router.Name) + } + + // TODO(beagles) check the status string. + // if router.Status == ? + + needsUpdate := false + updateInfo := routers.UpdateOpts{} + enableSNAT := false + fixedIPs := externalFixedIPs(gatewaySubnet.ID) + + // + // TODO(beagles) we don't care about the other fields right now because we + // are just going with neutron defaults but in the future we may care about + // Distributed (oddly HA doesn't seem to be an option) + // + gatewayInfo := router.GatewayInfo + if gatewayNetwork.ID != gatewayInfo.NetworkID || *gatewayInfo.EnableSNAT || + compareExternalFixedIPs(gatewayInfo.ExternalFixedIPs, fixedIPs) { + gwInfo := routers.GatewayInfo{ + NetworkID: gatewayNetwork.ID, + EnableSNAT: &enableSNAT, + ExternalFixedIPs: fixedIPs, + } + updateInfo.GatewayInfo = &gwInfo + needsUpdate = true + } + if needsUpdate { + updatedRouter, err := routers.Update(client, router.ID, updateInfo).Extract() + if err != nil { + return nil, err + } + log.Info(fmt.Sprintf("Updated octavia management router %s", router.ID)) + return updatedRouter, nil + } + + return router, nil +} + +// findRouter is a simple helper method... +func findRouter(client *gophercloud.ServiceClient, log *logr.Logger) (*routers.Router, error) { + listOpts := routers.ListOpts{ + Name: LbRouterName, + } + allPages, err := routers.List(client, listOpts).AllPages() + if err != nil { + log.Error(err, "Unable to list routers") + return nil, err + } + allRouters, err := routers.ExtractRouters(allPages) + if err != nil { + log.Error(err, "Unable to extract router results") + return nil, err + } + if len(allRouters) > 0 { + for _, router := range allRouters { + if router.Name == LbRouterName { + return &router, nil + } + } + } + return nil, nil +} + +// EnsureAmphoraManagementNetwork - retrieve, create and reconcile the Octavia management network for the in cluster link to the +// management tenant network. +func EnsureAmphoraManagementNetwork( + ctx context.Context, + ns string, + tenantName string, + netDetails *octaviav1.OctaviaLbMgmtNetworks, + log *logr.Logger, + helper *helper.Helper, +) (NetworkProvisioningSummary, error) { + o, err := GetOpenstackClient(ctx, ns, helper) + if err != nil { + return NetworkProvisioningSummary{}, err + } + client, err := GetNetworkClient(o) + if err != nil { + return NetworkProvisioningSummary{}, err + } + serviceTenant, err := GetProject(o, tenantName) + if err != nil { + return NetworkProvisioningSummary{}, err + } + + tenantNetwork, err := ensureLbMgmtNetwork(client, netDetails, serviceTenant.ID, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + tenantSubnet, err := ensureLbMgmtSubnet(client, netDetails, tenantNetwork, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + + providerNetwork, err := ensureProvNetwork(client, serviceTenant.ID, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + + providerSubnet, err := ensureProvSubnet(client, providerNetwork, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + + router, err := findRouter(client, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + if router != nil { + log.Info("Router object found, reconciling") + router, err = reconcileRouter(client, router, providerNetwork, providerSubnet, log) + if err != nil { + return NetworkProvisioningSummary{}, err + } + log.Info(fmt.Sprintf("Reconciled router %s (%s)", router.Name, router.ID)) + } else { + log.Info("Creating octavia provider router") + enableSNAT := false + gatewayInfo := routers.GatewayInfo{ + NetworkID: providerNetwork.ID, + EnableSNAT: &enableSNAT, + ExternalFixedIPs: externalFixedIPs(providerSubnet.ID), + } + adminStateUp := true + createOpts := routers.CreateOpts{ + Name: LbRouterName, + AdminStateUp: &adminStateUp, + GatewayInfo: &gatewayInfo, + } + router, err = routers.Create(client, createOpts).Extract() + if err != nil { + log.Error(err, "Unable to create router object") + return NetworkProvisioningSummary{}, err + } + } + if tenantRouterPort.DeviceID == "" { + interfaceOpts := routers.AddInterfaceOpts{ + PortID: tenantRouterPort.ID, + } + _, err := routers.AddInterface(client, router.ID, interfaceOpts).Extract() + if err != nil { + log.Error(err, fmt.Sprintf("Unable to add interface port %s to router %s", tenantRouterPort.ID, router.ID)) + } + } else if tenantRouterPort.DeviceID != router.ID { + return NetworkProvisioningSummary{}, + fmt.Errorf("Port %s has unexpected device ID %s and cannot be added to router %s", tenantRouterPort.ID, + tenantRouterPort.DeviceID, router.ID) + } + return NetworkProvisioningSummary{ + TenantNetworkID: tenantNetwork.ID, + TenantSubnetID: tenantSubnet.ID, + TenantRouterPortID: tenantRouterPort.ID, + ProviderNetworkID: providerNetwork.ID, + RouterID: router.ID, + }, nil +} diff --git a/pkg/amphoracontrollers/const.go b/pkg/octavia/network_consts.go similarity index 51% rename from pkg/amphoracontrollers/const.go rename to pkg/octavia/network_consts.go index fc920ac7..f4b32b19 100644 --- a/pkg/amphoracontrollers/const.go +++ b/pkg/octavia/network_consts.go @@ -13,7 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package amphoracontrollers +package octavia + +// NOTE: Strictly speaking, these don't have to be package scope constants, but having them externally +// accessible might aide constructing functional tests later on. const ( // Common consts for Management network @@ -36,7 +39,7 @@ const ( LbMgmtSubnetCIDR = "172.24.0.0/16" // LbMgmtSubnetAllocationPoolStart - - LbMgmtSubnetAllocationPoolStart = "172.24.0.2" + LbMgmtSubnetAllocationPoolStart = "172.24.0.5" // LbMgmtSubnetAllocationPoolEnd - LbMgmtSubnetAllocationPoolEnd = "172.24.255.254" @@ -52,7 +55,7 @@ const ( LbMgmtSubnetIPv6CIDR = "fd6c:6261:6173:0001::/64" // LbMgmtSubnetIPv6AllocationPoolStart - - LbMgmtSubnetIPv6AllocationPoolStart = "fd6c:6261:6173:0001::2" + LbMgmtSubnetIPv6AllocationPoolStart = "fd6c:6261:6173:0001::5" // LbMgmtSubnetIPv6AllocationPoolEnd - LbMgmtSubnetIPv6AllocationPoolEnd = "fd6c:6261:6173:0001:ffff:ffff:ffff:ffff" @@ -65,4 +68,58 @@ const ( // LbMgmtSubnetIPv6GatewayIP - LbMgmtSubnetIPv6GatewayIP = "" + + // Common consts for Management provider network + + // LbProvNetName - + LbProvNetName = "octavia-provider-net" + + // LbProvNetDescription - + LbProvNetDescription = "LBaaS Management Provider Network" + + // LbProvSubnetName - + LbProvSubnetName = "octavia-provider-subnet" + + // LbProvSubnetDescription - + LbProvSubnetDescription = "LBaaS Management Provider Subnet" + + // IPv4 consts + + // LbProvSubnetCIDR - + LbProvSubnetCIDR = "172.23.0.0/24" + + // LbProvSubnetAllocationPoolStart - + LbProvSubnetAllocationPoolStart = "172.23.0.5" + + // LbProvSubnetAllocationPoolEnd - + LbProvSubnetAllocationPoolEnd = "172.23.0.25" + + // LbProvSubnetGatewayIP - + LbProvSubnetGatewayIP = "172.23.0.1" + + // TODO(beagles): support IPv6 for the provider network. + // LbRouterName - + LbRouterName = "octavia-link-router" + + // LbProvBridgeName - + LbProvBridgeName = "br-octavia" + + // LbProvNetAttachName - + LbProvNetAttachName = "octavia" + + // LbRouterFixedIPAddress + LbRouterFixedIPAddress = "172.23.0.5" + + // LbMgmtRouterPortName + LbMgmtRouterPortName = "lb-mgmt-router-port" + + // LbMgmtRouterPortIPIPv4 + LbMgmtRouterPortIPv4 = "172.24.0.3" + + // LbMgmtRouterPortIPPv6 + LbMgmtRouterPortIPv6 = "fd6c:6261:6173:0001::3" + + // Network attachment details + // LbNetworkAttachmentName + LbNetworkAttachmentName = "octavia" ) diff --git a/pkg/amphoracontrollers/common.go b/pkg/octavia/osclient.go similarity index 79% rename from pkg/amphoracontrollers/common.go rename to pkg/octavia/osclient.go index 8f15573f..6efd5e1f 100644 --- a/pkg/amphoracontrollers/common.go +++ b/pkg/octavia/osclient.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package amphoracontrollers +package octavia import ( "context" @@ -20,23 +20,20 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/openstack" - octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" - "github.com/openstack-k8s-operators/octavia-operator/pkg/octavia" - "sigs.k8s.io/controller-runtime/pkg/client" ) // GetOpenstackClient returns an openstack admin service client object func GetOpenstackClient( ctx context.Context, - instance *octaviav1.OctaviaAmphoraController, + ns string, h *helper.Helper, ) (*openstack.OpenStack, error) { - keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, ns, map[string]string{}) if err != nil { return nil, err } - o, _, err := octavia.GetAdminServiceClient(ctx, h, keystoneAPI) + o, _, err := GetAdminServiceClient(ctx, h, keystoneAPI) if err != nil { return nil, err } diff --git a/templates/octaviaamphoracontroller/config/octavia.conf b/templates/octaviaamphoracontroller/config/octavia.conf index 4cb8c9ec..bbd1830b 100644 --- a/templates/octaviaamphoracontroller/config/octavia.conf +++ b/templates/octaviaamphoracontroller/config/octavia.conf @@ -38,6 +38,7 @@ amp_boot_network_list={{ .LbMgmtNetworkId }} amp_flavor_id={{ .AmpFlavorId }} amp_image_tag=amphora-image amp_ssh_key_name={{ .NovaSshKeyPair }} +controller_ip_port_list={{ .ControllerIPList }} client_ca = /etc/octavia/certs/client_ca.cert.pem [task_flow] persistence_connection = {{ .PersistenceDatabaseConnection }}