baremetal: Run ironic containers on bootstrap VM

This adds ironic to the bootstrap VM for the baremetal IPI platform.

Related: openshift#2060

data/data/baremetal/main.tf

@@ -5,6 +5,7 @@ provider "libvirt" {
 provider "ironic" {
   url          = var.ironic_uri
   microversion = "1.52"
+  timeout      = 1500
 }
 
 module "bootstrap" {

data/data/bootstrap/baremetal/files/usr/local/bin/startironic.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+set -ex
+
+# We should switch to openshift builds of these images when ready ref
+# https://github.com/openshift/installer/issues/2090
+IRONIC_IMAGE=${IRONIC_IMAGE:-"quay.io/metal3-io/ironic:master"}
+IRONIC_INSPECTOR_IMAGE=${IRONIC_INSPECTOR_IMAGE:-"quay.io/metal3-io/ironic-inspector:master"}
+IPA_DOWNLOADER_IMAGE=${IPA_DOWNLOADER_IMAGE:-"quay.io/metal3-io/ironic-ipa-downloader:master"}
+COREOS_DOWNLOADER_IMAGE=${COREOS_DOWNLOADER_IMAGE:-"quay.io/openshift-metal3/rhcos-downloader:master"}
+
+# FIXME this should be provided by the installer
+RHCOS_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/ootpa/410.8.20190520.0"
+
+# First we stop any previously started containers, because ExecStop only runs when the ExecStart process
+# e.g this script is still running, but we exit if *any* of the containers exits unexpectedly
+for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb ipa-downloader coreos-downloader; do
+    podman ps | grep -w "$name$" && podman kill $name
+    podman ps --all | grep -w "$name$" && podman rm $name -f
+done
+
+# Start the provisioning nic if not already started
+# Note removal of the hard-coded subnet tracked via https://github.com/openshift/installer/issues/2091
+PROVISIONING_NIC=ens4
+if ! nmcli -t device | grep "$PROVISIONING_NIC:ethernet:connected:provisioning"; then
+    nmcli c add type ethernet ifname $PROVISIONING_NIC con-name provisioning ip4 172.22.0.2/24 gw4 172.22.0.1
+    nmcli c up provisioning
+fi
+
+# Wait for the interface to come up
+# This is how the ironic container currently detects IRONIC_IP, this could probably be improved by using
+# nmcli show provisioning there instead, but we need to confirm that works with the static-ip-manager
+while [ -z "$(ip -4 address show dev "$PROVISIONING_NIC" | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -n 1)" ]; do
+    sleep 1
+done
+
+# set password for mariadb
+mariadb_password=$(uuidgen -r  | sed "s/-//g")
+
+IRONIC_SHARED_VOLUME="ironic"
+# Ignore errors here so we reuse any existing volume on pod restart
+# this is helpful if an API service causes restart after the images
+# have been downloaded
+podman volume create $IRONIC_SHARED_VOLUME || true
+
+# Apparently network-online doesn't necessarily mean iptables is ready, so wait until it is..
+while ! iptables -L; do
+  sleep 1
+done
+
+# Add firewall rules to ensure the IPA ramdisk can reach httpd, Ironic and the Inspector API on the host
+for port in 80 5050 6385 ; do
+    if ! sudo iptables -C INPUT -i $PROVISIONING_NIC -p tcp -m tcp --dport $port -j ACCEPT > /dev/null 2>&1; then
+        sudo iptables -I INPUT -i $PROVISIONING_NIC -p tcp -m tcp --dport $port -j ACCEPT
+    fi
+done
+
+# Start dnsmasq, http, mariadb, and ironic containers using same image
+# Currently we do this outside of a pod because we need to ensure the images
+# are downloaded before starting the API pods
+podman run -d --net host --privileged --name mariadb \
+     -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/runmariadb \
+     --env MARIADB_PASSWORD=$mariadb_password ${IRONIC_IMAGE}
+
+podman run -d --net host --privileged --name dnsmasq \
+     --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
+     -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE}
+
+podman run -d --net host --privileged --name httpd \
+     --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
+     -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/runhttpd ${IRONIC_IMAGE}
+
+podman run -d --net host --name ipa-downloader \
+     -v $IRONIC_SHARED_VOLUME:/shared:z ${IPA_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh
+
+podman run -d --net host --name coreos-downloader \
+     -v $IRONIC_SHARED_VOLUME:/shared:z ${COREOS_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh $RHCOS_IMAGE_URL
+
+# Wait for images to be downloaded/ready
+podman wait -i 1000 ipa-downloader
+podman wait -i 1000 coreos-downloader
+while ! curl --fail http://localhost:80/images/rhcos-ootpa-latest.qcow2.md5sum ; do sleep 1 ; done
+
+sudo podman run -d --net host --privileged --name ironic-conductor \
+     --env MARIADB_PASSWORD=$mariadb_password \
+     --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
+     --env OS_CONDUCTOR__HEARTBEAT_TIMEOUT=120 \
+     --entrypoint /bin/runironic-conductor \
+     -v $IRONIC_SHARED_VOLUME:/shared:z ${IRONIC_IMAGE}
+
+# We need a better way to wait for the DB sync to happen..
+sleep 10
+
+podman run -d --net host --privileged --name ironic-inspector \
+     --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
+     -v $IRONIC_SHARED_VOLUME:/shared:z "${IRONIC_INSPECTOR_IMAGE}"
+
+sudo podman run -d --net host --privileged --name ironic-api \
+     --env MARIADB_PASSWORD=$mariadb_password \
+     --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
+     --entrypoint /bin/runironic-api \
+     -v $IRONIC_SHARED_VOLUME:/shared:z ${IRONIC_IMAGE}
+
+# Now loop so the service remains active and restart everything should one of the containers exit unexpectedly.
+# The alternative would be RemainAfterExit=yes but then we lose the ability to restart if something crashes.
+while true; do
+    for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb; do
+        podman ps | grep -w "$name$" || exit 1
+    done
+    sleep 10
+done

data/data/bootstrap/baremetal/files/usr/local/bin/stopironic.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -x 
+
+for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb ipa-downloader coreos-downloader; do
+    podman ps | grep -w "$name$" && podman kill $name
+    podman ps --all | grep -w "$name$" && podman rm $name -f
+done

data/data/bootstrap/baremetal/systemd/units/ironic.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Baremetal Deployment Ironic Services
+Wants=network-online.target crio.service
+After=network-online.target crio.service
+
+[Service]
+Type=exec
+ExecStart=/usr/local/bin/startironic.sh
+ExecStop=/usr/local/bin/stopironic.sh
+
+Restart=on-failure
+RestartSec=10
+TimeoutStartSec=600
+
+[Install]
+WantedBy=multi-user.target

docs/user/metal/install_ipi.md

@@ -15,18 +15,6 @@ deployments, see [install_upi.md](install_upi.md).
 
 ## Prerequisites
 
-### Ironic
-
-Currently, the `baremetal` platform requires an existing Ironic environment.
-This will eventually be handled by `openshift-install`, with Ironic being
-deployed onto the bootstrap node. Until then, users of the `baremetal` platform
-should use the
-[openshift-metal3/dev-scripts](https://github.com/openshift-metal3/dev-scripts)
-repository to handle configuration of Ironic.
-
-The following PR contains the WIP changes for automating Ironic from
-`openshift-install`: https://github.com/openshift-metal3/kni-installer/pull/100
-
 ### Network Requirements
 
 It is assumed that all hosts have at least 2 NICs, used for the following
@@ -227,3 +215,18 @@ When an installation fails, `openshift-install` will attempt to gather debug
 information from hosts.  This is not yet supported by the `baremetal` platform.
 
 https://github.com/openshift-metal3/kni-installer/issues/79
+
+### Provisioning subnet not fully configurable
+
+There are some install-config parameters to control templating of the provisioning
+network configuration, but fully supporting alternative subnets for the
+provisioning network is incomplete.
+
+https://github.com/openshift/installer/issues/2091
+
+### Ironic services are using upstream images
+
+We need to move to downstream openshift images for the Ironic containers that are
+started on the boostrap VM
+
+https://github.com/openshift/installer/issues/2090

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -293,6 +293,7 @@ func (a *Bootstrap) addSystemdUnits(uri string, templateData *bootstrapTemplateD
 		// baremetal & openstack platform services
 		"keepalived.service": {},
 		"coredns.service":    {},
+		"ironic.service":     {},
 	}
 
 	directory, err := data.Assets.Open(uri)

pkg/types/baremetal/defaults/platform.go

@@ -11,7 +11,7 @@ import (
 // Defaults for the baremetal platform.
 const (
 	LibvirtURI         = "qemu:///system"
-	IronicURI          = "http://localhost:6385/v1"
+	IronicURI          = "http://172.22.0.2:6385/v1"
 	ExternalBridge     = "baremetal"
 	ProvisioningBridge = "provisioning"
 	HardwareProfile    = "default"