diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 11505406a8..6ce0ac8f17 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -25,6 +25,24 @@ Any additional prerequisite software may be installed with the following script: $ ./tools/install_prerequisites.sh ---- +=== Make me a Gimlet! + +The sled agent expects to manage a real Gimlet. However, until those are built, +developers generally make do with something else, usually a commodity machine. +To make your machine "look" like a Gimlet, the +`./tools/create_virtual_hardware.sh` script can be used. This creates a few +file-based ZFS vdevs and ZFS zpools on top of those, and a couple of VNICs. The +vdevs model the actual U.2s that will be in a Gimlet, and the VNICs model the +two Chelsio NIC ports. + +You can clean up these resources with `./tools/destroy_virtual_hardware.sh`. Be +aware that this is a best effort script. If the sled agent or other Omicron +zones are still running, the zpools cannot be deleted. The script will warn you +about the things it cannot remove, so you can do so yourself. Also note that all +the networking bits are temporary, so a reboot should always clear them. + +Both scripts must be run as root, e.g, `pfexec ./tools/create_virtual_hardware.sh`. + == Deploying Omicron The control plane repository contains a packaging tool which bundles binaries diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh new file mode 100755 index 0000000000..eac2be4fd8 --- /dev/null +++ b/tools/create_virtual_hardware.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Make me a Gimlet! +# +# The entire control plane stack is designed to run and operate on the Oxide +# rack, in each Gimlet. But Gimlet's don't quite exist yet. In the meantime, +# this script can be used to create a few pieces of virtual hardware that +# _simulate_ a Gimlet, allowing us to develop software that approximates the +# eventual operation on Oxide hardware. +# +# See `docs/how-to-run.adoc` section "Make me a Gimlet" for more details. + +set -e +set -u +set -x + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +OMICRON_TOP="$SOURCE_DIR/.." + +# Select the physical link over which to simulate the Chelsio links +if [[ $# -ge 1 ]]; then + PHYSICAL_LINK="$1" +else + PHYSICAL_LINK="$(dladm show-phys -p -o LINK | head -1)" +fi +echo "Using $PHYSICAL_LINK as physical link" + +function success { + echo -e "\e[1;36m$1\e[0m" +} + +# Create the ZFS zpools required for the sled agent, backed by file-based vdevs. +function ensure_zpools { + # Find the list of zpools the sled agent expects, from its configuration + # file. + readarray -t ZPOOLS < <( \ + grep '"oxp_' "$OMICRON_TOP/smf/sled-agent/config.toml" | \ + sed 's/[ ",]//g' \ + ) + for ZPOOL in "${ZPOOLS[@]}"; do + VDEV_PATH="$OMICRON_TOP/$ZPOOL.vdev" + if ! [[ -f "$VDEV_PATH" ]]; then + truncate -s 10GB "$VDEV_PATH" + fi + success "ZFS vdev $VDEV_PATH exists" + if [[ -z "$(zpool list -o name | grep $ZPOOL)" ]]; then + zpool create "$ZPOOL" "$VDEV_PATH" + fi + success "ZFS zpool $ZPOOL exists" + done +} + +# Create VNICs to represent the Chelsio physical links +# +# Arguments: +# $1: Optional name of the physical link to use. If not provided, use the +# first physical link available on the machine. +function ensure_simulated_chelsios { + local PHYSICAL_LINK="$1" + VNIC_NAMES=("vioif0" "vioif1") + for VNIC in "${VNIC_NAMES[@]}"; do + if [[ -z "$(dladm show-vnic -p -o LINK "$VNIC")" ]]; then + dladm create-vnic -t -l "$PHYSICAL_LINK" "$VNIC" + fi + success "VNIC $VNIC exists" + if [[ -z "$(ipadm show-addr -p -o ADDR "$VNIC/v6")" ]]; then + ipadm create-addr -t -T addrconf "$VNIC/v6" + fi + success "IP address $VNIC/v6 exists" + done + + # Create an address on the underlay network + UNDERLAY_ADDR="lo0/underlay" + if [[ -z "$(ipadm show-addr -p -o ADDR "$UNDERLAY_ADDR")" ]]; then + ipadm create-addr -t -T static -a fd00:1::1/64 lo0/underlay + fi + success "IP address $UNDERLAY_ADDR exists" +} + +function ensure_xde_driver { + # Always remove the driver first. There seems to be a bug in the driver, + # preventing it from showing up in `modinfo` on boot, even if it's actually + # installed. + if [[ -z "$(modinfo | grep xde)" ]]; then + rem_drv xde + add_drv xde + fi +} + +function ensure_run_as_root { + if [[ "$(id -u)" -ne 0 ]]; then + echo "This script must be run as root" + exit 1 + fi +} + +ensure_run_as_root +ensure_zpools +ensure_simulated_chelsios "$PHYSICAL_LINK" +ensure_xde_driver diff --git a/tools/destroy_virtual_hardware.sh b/tools/destroy_virtual_hardware.sh new file mode 100755 index 0000000000..13e903a43c --- /dev/null +++ b/tools/destroy_virtual_hardware.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +# Unmake me a Gimlet! +# +# This tool undoes the operations of `./tools/create_virtual_hardware.sh`, +# destroying VNICs and ZFS zpools, to the extent possible. Note that if an +# operation fails, for example because a VNIC link is busy, a warning is +# printed. The user is responsible for deleting these entirely. + +set -u +set -x + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "${SOURCE_DIR}/.." +OMICRON_TOP="$PWD" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "This must be run as root" + exit 1 +fi + +function warn { + echo -e "\e[1;31m$1\e[0m" +} + +function success { + echo -e "\e[1;36m$1\e[0m" +} + +function try_uninstall_xde { + RC=0 + if ! [[ -z "$(modinfo | grep xde)" ]]; then + rem_drv xde + RC=$? + fi + + if [[ $RC -eq 0 ]]; then + success "XDE kernel module uninstalled" + else + warn "Failed to uninstall XDE kernel module" + fi +} + +function try_remove_address { + local ADDRESS="$1" + RC=0 + if [[ "$(ipadm show-addr -p -o addr "$ADDRESS")" ]]; then + ipadm delete-addr "$ADDRESS" + RC=$? + fi + if [[ $RC -eq 0 ]]; then + success "Address $ADDRESS destroyed" + else + warn "Failed to delete address $ADDRESS" + fi +} + +function try_remove_vnic { + local LINK="$1" + RC=0 + if [[ "$(dladm show-vnic -p -o LINK "$LINK")" ]]; then + dladm delete-vnic "$LINK" + RC=$? + fi + if [[ $RC -eq 0 ]]; then + success "VNIC link $LINK destroyed" + else + warn "Failed to delete VNIC link $LINK" + fi +} + +function try_remove_vnics { + try_remove_address "lo0/underlay" + VNIC_LINKS=("vioif0" "vioif1") + for LINK in "${VNIC_LINKS[@]}"; do + try_remove_address "$LINK/v6" + try_remove_vnic "$LINK" + done +} + +function try_destroy_zpools { + readarray -t ZPOOLS < <(zfs list -d 0 -o name | grep "^oxp_") + for ZPOOL in "${ZPOOLS[@]}"; do + RC=0 + VDEV_FILE="$OMICRON_TOP/$ZPOOL.vdev" + zfs destroy -r "$ZPOOL" && zfs unmount "$ZPOOL" && zpool destroy "$ZPOOL" && rm -f "$VDEV_FILE" + RC=$? + + if [[ $RC -eq 0 ]]; then + success "Removed ZFS pool and vdev: $ZPOOL" + else + warn "Failed to remove ZFS pool and vdev: $ZPOOL" + fi + done +} + +try_uninstall_xde && try_remove_vnics +try_destroy_zpools diff --git a/tools/install_opte.sh b/tools/install_opte.sh new file mode 100755 index 0000000000..842eab947a --- /dev/null +++ b/tools/install_opte.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Small tool to install OPTE and the xde kernel driver and ONU bits. + +set -e +set -u +set -x + +if [[ "$(uname)" != "SunOS" ]]; then + echo "This script is intended for Helios only" +fi + +if [[ $(id -u) -ne 0 ]]; then + echo "This must be run as root" + exit 1 +fi + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "${SOURCE_DIR}/.." +OMICRON_TOP="$PWD" +OUT_DIR="$OMICRON_TOP/out" +mkdir -p "$OUT_DIR" + +# Compute the SHA256 of the path in $1, returning just the sum +function file_sha { + sha256sum "$1" | cut -d ' ' -f 1 +} + +# Download a file from $1 and compare its sha256 to the value provided in $2 +function download_and_check_sha { + local URL="$1" + local FILENAME="$(basename "$URL")" + local OUT_PATH="$OUT_DIR/$FILENAME" + local SHA="$2" + + # Check if the file already exists, with the expected SHA + if ! [[ -f "$OUT_PATH" ]] || [[ "$SHA" != "$(file_sha "$OUT_PATH")" ]]; then + curl -L -o "$OUT_PATH" "$URL" 2> /dev/null + local ACTUAL_SHA="$(sha256sum "$OUT_PATH" | cut -d ' ' -f 1)" + if [[ "$ACTUAL_SHA" != "$SHA" ]]; then + echo "SHA mismatch downloding file $FILENAME" + exit 1 + fi + else + echo "File $FILENAME already exists with correct SHA" + fi +} + +# Download a SHA-256 sum output from $1 and return just the SHA +function sha_from_url { + local SHA_URL="$1" + curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1 +} + +OPTE_P5P_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSFZZWPFEQBW7JRS7ST99G/opte-0.1.58.p5p" +OPTE_P5P_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSG01CGP6TH9THNY39G88Z/opte-0.1.58.p5p.sha256" +OPTE_P5P_REPO_PATH="$OUT_DIR/$(basename "$OPTE_P5P_URL")" +XDE_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHNYQ1NS7DBX8VG3JPAP0/xde" +XDE_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHP47353961S3ETXBSD2T/xde.sha256" + +download_and_check_sha "$OPTE_P5P_URL" "$(sha_from_url "$OPTE_P5P_SHA_URL")" +XDE_SHA="$(sha_from_url "$XDE_SHA_URL")" +download_and_check_sha "$XDE_URL" "$XDE_SHA" + +# Move the XDE driver into it the expected location to allow operating on it +# with `add_drv` and `rem_drv` +DRIVER_DIR="/kernel/drv/amd64" +XDE_FILENAME="$(basename "$XDE_URL")" +XDE_PATH="$DRIVER_DIR/$XDE_FILENAME" +if ! [[ -f "$XDE_PATH" ]] || [[ "$XDE_SHA" != "$(file_sha "$XDE_PATH")" ]]; then + echo "Replacing XDE driver" + mv -f "$OUT_DIR/$XDE_FILENAME" "$XDE_PATH" +else + echo "XDE driver already exists with correct SHA" +fi + +# Add the OPTE P5P package repository (at the top of the search order) and +# update the OS packages. This may require a reboot. +pkg set-publisher -p "$OPTE_P5P_REPO_PATH" --search-first +pkg set-publisher --non-sticky helios-dev +RC=0 +pkg update || RC=$?; +if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then + exit 0 +else + exit "$RC" +fi diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh index 8ef1ba5544..a579451c0d 100755 --- a/tools/install_prerequisites.sh +++ b/tools/install_prerequisites.sh @@ -100,6 +100,15 @@ fi ./tools/ci_download_cockroachdb ./tools/ci_download_clickhouse +# Install OPTE +# +# OPTE is a Rust package that is consumed by a kernel module called xde. This +# installs the `xde` driver and some kernel bits required to work with that +# driver. +if [[ "${HOST_OS}" == "SunOS" ]]; then + pfexec ./tools/install_opte.sh +fi + # Validate the PATH: expected_in_path=( 'pg_config'