packages/nixos: add IMDS setup script

Azure needs special care for enabling IMDS within Peerpods. This adds a script to setup IMDS through Proxy ARP (from Peerpods upstream, see https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/podvm/files/usr/local/bin/setup-nat-for-imds.sh), so that all requests to the IMDS from within the pod are routed through an interface that is peered to the Pod VM. Verified to work in 2 distinct Azure peer pods.

packages/by-name/mkNixosConfig/package.nix

@@ -42,7 +42,11 @@ lib.makeOverridable (
             kernel-podvm-azure
             pause-bundle
             ;
+
           inherit (outerPkgs.kata) kata-agent;
+          inherit (outerPkgs)
+            peerpod-imds-nat
+            ;
         })
       ];
 

packages/by-name/peerpod-imds-nat/package.nix

@@ -0,0 +1,28 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+{
+  writeShellApplication,
+  iproute2,
+  iptables,
+  sysctl,
+  gawk,
+}:
+
+writeShellApplication {
+  name = "peerpod-imds-nat";
+
+  runtimeInputs = [
+    iproute2
+    iptables
+    sysctl
+    gawk
+  ];
+
+  text = builtins.readFile ./setup-nat-for-imds.sh;
+
+  meta = {
+    mainProgram = "peerpod-imds-nat";
+    homepage = "https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/podvm/files/usr/local/bin/setup-nat-for-imds.sh";
+  };
+}

packages/by-name/peerpod-imds-nat/setup-nat-for-imds.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# This script sets up NAT for IMDS on Azure/AWS
+# This is required for IMDS to work on Azure/AWS
+# This script is executed as a oneshot systemd service
+# during first boot
+
+set -euo pipefail
+
+IMDS_IP="169.254.169.254"
+DUMMY_IP="169.254.99.99"
+
+# trap errors
+trap 'echo "Error: $0:$LINENO stopped"; exit 1' ERR INT
+
+# Function to setup veth pair
+function setup_proxy_arp() {
+  local pod_ip
+  pod_ip=$(ip netns exec podns ip route get "$IMDS_IP" | awk '{ for(i=1; i<=NF; i++) { if($i == "src") { print $(i+1); break; } } }')
+
+  ip link add veth2 type veth peer name veth1
+  # Proxy arp does not get enabled when no IP address is assigned
+  ip address add "$DUMMY_IP/32" dev veth1
+  ip link set up dev veth1
+
+  sysctl -w net.ipv4.ip_forward=1
+  sysctl -w net.ipv4.conf.veth1.proxy_arp=1
+  sysctl -w net.ipv4.neigh.veth1.proxy_delay=0
+
+  ip link set veth2 netns podns
+  ip netns exec podns ip link set up dev veth2
+  ip netns exec podns ip route add "$IMDS_IP/32" dev veth2
+
+  ip route add "$pod_ip/32" dev veth1
+
+  local hwaddr
+  hwaddr=$(ip netns exec podns ip -br link show veth2 | awk 'NR==1 { print $3 }')
+  ip neigh replace "$pod_ip" dev veth1 lladdr "$hwaddr"
+
+  iptables -t nat -A POSTROUTING -s "$pod_ip/32" -d "$IMDS_IP/32" -j MASQUERADE
+}
+
+# Execute functions
+setup_proxy_arp

packages/nixos/azure.nix

@@ -72,10 +72,7 @@ in
 
     services.udev.extraRules = azure-storage-rules;
     systemd.services.azure-readiness-report = {
-      wantedBy = [
-        "basic.target"
-        "multi-user.target"
-      ];
+      wantedBy = [ "multi-user.target" ];
       wants = [ "network-online.target" ];
       after = [ "network-online.target" ];
       description = "Azure Readiness Report";
@@ -85,5 +82,21 @@ in
         ExecStart = "${lib.getExe pkgs.azure-no-agent}";
       };
     };
+
+    systemd.services.setup-nat-for-imds = {
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      description = "Setup NAT for IMDS";
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = "yes";
+        # TODO: Find out why just ordering this after network-online.target
+        # isn't sufficient. (Errors with saying that the network is unreachable)
+        Restart = "on-failure";
+        RestartSec = "5s";
+        ExecStart = "${lib.getExe pkgs.peerpod-imds-nat}";
+      };
+    };
   };
 }