From 425fb0fa3232216de60f840c6aad58051a92300e Mon Sep 17 00:00:00 2001
From: Bharat Kunwar <brtknr@bath.edu>
Date: Fri, 26 Jul 2019 12:41:57 +0000
Subject: [PATCH] Add network config to stabilise multi-NIC scenario

When there is more than one NIC attached to an instance, openstack cloud
provider returns a random InternalIP back to the host resulting in instability
with API server which only talks to a default interface.

This patch incorporates the changes made in
https://github.com/kubernetes/cloud-provider-openstack/pull/444 which enables
OpenStack Cloud Controller Manager (OCCM) to respect the
`internal-network-name` in cloud-config file which ensures that InternalIP
remains stable.

Uses a separate cloud-config file for OCCM to ensure in-tree Cinder volumes
remain compatible.

Change-Id: Idfa52ed2d512e7dc383a556371e896205dd542f9
Story: 2005333
Task: 30271
---
 .../fragments/kube-apiserver-to-kubelet-role.sh      |  2 +-
 .../kubernetes/fragments/write-heat-params-master.sh |  1 +
 .../kubernetes/fragments/write-kube-os-config.sh     | 12 +++++++++++-
 .../k8s_fedora_atomic_v1/templates/kubemaster.yaml   |  1 +
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
index 2113c6a6bf..f0d0f4623d 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
@@ -292,7 +292,7 @@ spec:
         command:
         - /bin/openstack-cloud-controller-manager
         - --v=2
-        - --cloud-config=/etc/kubernetes/cloud-config
+        - --cloud-config=/etc/kubernetes/cloud-config-occm
         - --cluster-name=${CLUSTER_UUID}
         - --use-service-account-credentials=true
         - --bind-address=127.0.0.1
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
index 2aa6008498..9629f5d257 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
@@ -32,6 +32,7 @@ HEAT_PARAMS=/etc/sysconfig/heat-params
       ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL"
       USERNAME="$USERNAME"
       PASSWORD="$PASSWORD"
+      CLUSTER_NETWORK="$CLUSTER_NETWORK"
       CLUSTER_SUBNET="$CLUSTER_SUBNET"
       TLS_DISABLED="$TLS_DISABLED"
       KUBE_DASHBOARD_ENABLED="$KUBE_DASHBOARD_ENABLED"
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
index e25634de7f..5a8c3be89a 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
@@ -15,7 +15,7 @@ $ssh_cmd cp /etc/pki/tls/certs/ca-bundle.crt /etc/kubernetes/ca-bundle.crt
 
 # Generate a the configuration for Kubernetes services
 # to talk to OpenStack Neutron and Cinder
-cat > $KUBE_OS_CLOUD_CONFIG <<EOF
+CLOUD_CONFIG=$(cat <<EOF
 [Global]
 auth-url=$AUTH_URL
 user-id=$TRUSTEE_USER_ID
@@ -33,6 +33,8 @@ monitor-max-retries=3
 [BlockStorage]
 bs-version=v2
 EOF
+)
+echo $CLOUD_CONFIG > $KUBE_OS_CLOUD_CONFIG
 
 # Provide optional region parameter if it's set.
 if [ -n "${REGION_NAME}" ]; then
@@ -41,3 +43,11 @@ fi
 
 # backwards compatibility, some apps may expect this file from previous magnum versions.
 $ssh_cmd cp ${KUBE_OS_CLOUD_CONFIG} /etc/kubernetes/kube_openstack_config
+
+# Append additional networking config to config file provided to openstack
+# cloud controller manager (not supported by in-tree Cinder).
+cat > ${KUBE_OS_CLOUD_CONFIG}-occm <<EOF
+$CLOUD_CONFIG
+[Networking]
+internal-network-name=$CLUSTER_NETWORK
+EOF
diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
index e4d612e4ea..362f521528 100644
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@@ -572,6 +572,7 @@ resources:
                   "$AUTH_URL": {get_param: auth_url}
                   "$USERNAME": {get_param: username}
                   "$PASSWORD": {get_param: password}
+                  "$CLUSTER_NETWORK": {get_param: fixed_network}
                   "$CLUSTER_SUBNET": {get_param: fixed_subnet}
                   "$TLS_DISABLED": {get_param: tls_disabled}
                   "$TRAEFIK_INGRESS_CONTROLLER_TAG": {get_param: traefik_ingress_controller_tag}