From f8a9ab1180bca8b2c24c3f1c6986fef71e17576f Mon Sep 17 00:00:00 2001 From: Simone Ruffilli Date: Tue, 25 Oct 2022 14:29:04 +0200 Subject: [PATCH 1/3] Fix race condition --- .../simple-nva/cloud-config.yaml | 35 ++++++++++++++----- .../simple-nva/files/policy_based_routing.sh | 23 +++++++----- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index 8d18a3563f..ab7f494038 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -22,17 +22,36 @@ write_files: content: | ${indent(6, data.content)} %{ endfor } - -bootcmd: - - systemctl start node-problem-detector - -runcmd: - - iptables --policy FORWARD ACCEPT + - path: /etc/systemd/system/routing.service + permissions: 0644 + owner: root + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Start routing + After=network-online.target + Wants=network-online.target + [Service] + ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh" + - path: /var/run/nva/start-routing.sh + permissions: 0744 + owner: root + content: | + iptables --policy FORWARD ACCEPT %{ for interface in network_interfaces ~} %{ if enable_health_checks ~} - - /var/run/nva/policy_based_routing.sh ${interface.name} + /var/run/nva/policy_based_routing.sh ${interface.name} %{ endif ~} %{ for route in interface.routes ~} - - ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name} + ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name} %{ endfor ~} %{ endfor ~} + +bootcmd: + - systemctl start node-problem-detector + +runcmd: + - systemctl daemon-reload + - systemctl enable routing + - systemctl start routing \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index 42ed0dcb15..e653af8fa0 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -15,13 +15,18 @@ # limitations under the License. IF_NAME=$1 -IF_NUMBER=$(echo $1 | sed -e s/eth//) -IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google") -IF_IP=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/ip -H "Metadata-Flavor: Google") -IF_NETMASK=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/subnetmask -H "Metadata-Flavor: Google") -IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") -grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables -ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME -ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME -ip rule add from $IP_LB/32 table hc-$IF_NAME + +# If there's a load balancer for this IF... +if [ ! -z $IP_LB ] +then + IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//) + IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google") + IF_IP=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/ip -H "Metadata-Flavor: Google") + IF_NETMASK=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/subnetmask -H "Metadata-Flavor: Google") + IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) + grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables + ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME + ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME + ip rule add from $IP_LB/32 table hc-$IF_NAME +fi \ No newline at end of file From a0444feede53ebe0ff53211985dcd45dfbc6fb15 Mon Sep 17 00:00:00 2001 From: Simone Ruffilli Date: Tue, 25 Oct 2022 14:32:30 +0200 Subject: [PATCH 2/3] Update cloud-config.yaml --- modules/cloud-config-container/simple-nva/cloud-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index ab7f494038..f1d71e8262 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -54,4 +54,5 @@ bootcmd: runcmd: - systemctl daemon-reload - systemctl enable routing - - systemctl start routing \ No newline at end of file + - systemctl start routing + From 5a52ec9c22021e0d6c63390cbcf4c9eebcb653d1 Mon Sep 17 00:00:00 2001 From: Simone Ruffilli Date: Tue, 25 Oct 2022 14:32:41 +0200 Subject: [PATCH 3/3] Update policy_based_routing.sh --- .../simple-nva/files/policy_based_routing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index e653af8fa0..2e1eb15235 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -29,4 +29,4 @@ then ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME ip rule add from $IP_LB/32 table hc-$IF_NAME -fi \ No newline at end of file +fi