Fix policy_based_routing.sh script on simple-nva module

modules/cloud-config-container/simple-nva/cloud-config.yaml

@@ -34,16 +34,16 @@ write_files:
       After=network-online.target
       Wants=network-online.target
       [Service]
+      RemainAfterExit=true
       ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh"
-
   - path: /var/run/nva/start-routing.sh
     permissions: 0744
     owner: root
     content: |
       iptables --policy FORWARD ACCEPT
 %{ for interface in network_interfaces ~}
 %{ if enable_health_checks ~}
-      /var/run/nva/policy_based_routing.sh ${interface.name}
+      /var/run/nva/policy_based_routing.sh ${interface.name} &>/dev/null &
 %{ endif ~}
 %{ if interface.enable_masquerading ~}
 %{ for cidr in interface.non_masq_cidrs ~}

modules/cloud-config-container/simple-nva/files/policy_based_routing.sh

@@ -15,20 +15,43 @@
 # limitations under the License.
 
 IF_NAME=$1
-IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ")
-
-# Sleep while there's no load balancer IP route for this IF
-while [ -z $IP_LB ] ; do
-  sleep 2
-  IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ")
-done
-
 IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//)
 IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google")
 IF_IP=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/ip -H "Metadata-Flavor: Google")
 IF_NETMASK=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/subnetmask -H "Metadata-Flavor: Google")
 IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK)
-grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables
-ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME
-ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME
-ip rule add from $IP_LB/32 table hc-$IF_NAME
+
+# Sleep while there's no load balancer IP route for this IF
+while true
+do
+  IPS_LB_STR=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " " |  tr -s '\n' ' ')
+  IPS_LB=($IPS_LB_STR)
+  for IP in "${IPS_LB[@]}"
+  do
+    # Configure hc routing table if not available for this network interface
+    grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || {
+      echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables
+      ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME
+      ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME
+    }
+
+    # configure PBR route for LB
+    ip rule list | grep -qF "$IP" || ip rule add from $IP/32 table hc-$IF_NAME
+  done
+
+  # remove previously configure PBR for old LB removed from network interface
+  # first get list of PBR on this network interface and retrieve LB IP addresses
+  PBR_LB_IPS_STR=$(ip rule list | grep "hc-$IF_NAME" | cut -f 2 -d " " |  tr -s '\n' ' ')
+  PBR_LB_IPS=($PBR_LB_IPS_STR)
+
+  # iterate over PBR LB IP addresses
+  for PBR_IP in "${PBR_LB_IPS[@]}"
+  do
+    # check if the PBR LB IP belongs to the current array of LB IPs attached to the
+    # network interface, if not delete the corresponding PBR rule
+    if [ -z "$IPS_LB" ] || ! echo ${IPS_LB[@]} | grep --quiet "$PBR_IP" ; then
+      ip rule del from $PBR_IP
+    fi
+  done
+  sleep 2
+done