Add iso builder

.github/workflows/enki.yml

@@ -0,0 +1,24 @@
+---
+name: 'run enki unit tests'
+
+on:
+  pull_request:
+
+concurrency:
+  group: enki-${{ github.ref || github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: earthly/[email protected]
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build
+        run: cd tools-image/enki && earthly -P +test

tools-image/Dockerfile

@@ -3,84 +3,59 @@ ARG LEAP_VERSION=15.4
 ARG LUET_VERSION=0.33.0
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 
-FROM opensuse/leap:$LEAP_VERSION as luet-install
+FROM golang:1.20 as enki
+ENV CGO_ENABLED=0
+COPY ./enki /src/enki
+WORKDIR /src/enki
+RUN go mod download
+# Set arg/env after go mod download, otherwise we invalidate the cached layers due to the commit changing easily
+ARG ENKI_VERSION=0.0.1
+ARG ENKI_COMMIT=""
+ENV ENKI_VERSION=${ENKI_VERSION}
+ENV ENKI_COMMIT=${ENKI_COMMIT}
+RUN go build \
+    -ldflags "-w -s \
+    -X github.com/kairos-io/enki/internal/version.version=$ENKI_VERSION \
+    -X github.com/kairos-io/enki/internal/version.gitCommit=$ENKI_COMMIT" \
+    -o /usr/bin/enki
+
+FROM opensuse/leap:$LEAP_VERSION as default
 COPY --from=luet /usr/bin/luet /usr/bin/luet
 ENV LUET_NOLOCK=true
 ENV TMPDIR=/tmp
 COPY luet.yaml /etc/luet/luet.yaml
-RUN luet install -y system/elemental-cli
-RUN luet install -y livecd/grub2 --system-target /grub2
-RUN luet install -y livecd/grub2-efi-image --system-target /efi
 
-# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
-RUN rm -Rf /grub2/var
-RUN rm -Rf /efi/var
+RUN zypper ref && zypper dup -y
+## ISO+ Arm image + Netboot + cloud images Build depedencies
+RUN zypper ref && zypper in -y bc qemu-tools jq cdrtools docker git curl gptfdisk kpartx sudo xfsprogs parted util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd
 
-## amd64 Live CD artifacts
-FROM quay.io/kairos/packages:grub2-livecd-0.0.6 AS grub2
-FROM quay.io/kairos/packages:grub2-efi-image-livecd-0.0.6 AS efi
+## Live CD artifacts
+RUN luet install -y livecd/grub2 --system-target /grub2
+RUN luet install -y livecd/grub2-efi-image --system-target /efi
 
 ## RPI64
-
-## Firmware is in the amd64 repo (noarch)
-FROM quay.io/kairos/packages:u-boot-rpi64-firmware-2021.01-5.1 AS rpi-u-boot
-FROM quay.io/kairos/packages:raspberrypi-firmware-firmware-2021.03.10-2.1 AS rpi-firmware
-FROM quay.io/kairos/packages:raspberrypi-firmware-config-firmware-2021.03.10-2.1 AS rpi-firmware-config
-FROM quay.io/kairos/packages:raspberrypi-firmware-dt-firmware-2021.03.15-2.1 AS rpi-firmware-dt
+RUN luet install -y firmware/u-boot-rpi64 firmware/raspberrypi-firmware firmware/raspberrypi-firmware-config firmware/raspberrypi-firmware-dt --system-target /rpi/
 
 ## PineBook64 Pro
-FROM quay.io/kairos/packages:u-boot-rockchip-arm-vendor-blob-0.1 AS pinebook-u-boot
-
-## Generic ARM artifacts
-FROM quay.io/kairos/packages-arm64:grub-efi-static-0.2  AS grub-efi
-FROM quay.io/kairos/packages-arm64:grub-config-static-0.3  AS grub-config
-FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.2   AS grub-artifacts
+RUN luet install -y arm-vendor-blob/u-boot-rockchip --system-target /pinebookpro/u-boot
 
  ## RAW images
-FROM quay.io/kairos/packages:grub-efi-static-0.1  AS grub-raw-efi
-FROM quay.io/kairos/packages:grub-config-static-0.1  AS grub-raw-config
-FROM quay.io/kairos/packages:grub-artifacts-static-0.1   AS grub-raw-artifacts
-
-FROM opensuse/leap:$LEAP_VERSION
-COPY --from=luet-install /usr/bin/elemental /usr/bin/elemental
-COPY --from=luet /usr/bin/luet /usr/bin/luet
-
-# ISO files
-COPY --from=luet-install /grub2 /grub2
-COPY --from=luet-install /efi /efi
-
-# RAW images
-COPY --from=grub-raw-efi / /raw/grub
-COPY --from=grub-raw-config / /raw/grubconfig
-COPY --from=grub-raw-artifacts / /raw/grubartifacts
-
-# RPI64
-COPY --from=rpi-u-boot / /rpi/u-boot
-COPY --from=rpi-firmware / /rpi/rpi-firmware
-COPY --from=rpi-firmware-config / /rpi/rpi-firmware-config
-COPY --from=rpi-firmware-dt / /rpi/rpi-firmware-dt
-
-# Pinebook
-COPY --from=pinebook-u-boot / /pinebookpro/u-boot
+RUN luet install -y static/grub-efi --system-target /raw/grub
+RUN luet install -y static/grub-config --system-target /raw/grubconfig
+RUN luet install -y static/grub-artifacts --system-target /raw/grubartifacts
 
-# Generic
-COPY --from=grub-efi / /arm/grub/efi
-COPY --from=grub-config / /arm/grub/config
-COPY --from=grub-artifacts / /arm/grub/artifacts
-
-RUN zypper ref && zypper dup -y
+# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
+# so temporal files are stored in each dir
+RUN rm -Rf /grub2/var/tmp
+RUN rm -Rf /efi/var/tmp
+RUN rm -Rf /rpi/var/tmp
+RUN rm -Rf /pinebookpro/u-boot/var/tmp
+RUN rm -Rf /raw/grub/var/tmp
+RUN rm -Rf /raw/grubconfig/var/tmp
+RUN rm -Rf /raw/grubartifacts/var/tmp
 
-## ISO Build depedencies
-RUN zypper ref && zypper in -y xfsprogs parted util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd
 RUN mkdir /config
 
-# Arm image build deps
-RUN zypper in -y jq docker git curl gptfdisk kpartx sudo
-# Netboot
-RUN zypper in -y cdrtools
-# cloud images
-RUN zypper in -y bc qemu-tools
-
 # ISO build config
 COPY ./config.yaml /config/manifest.yaml
 COPY ./entrypoint.sh /entrypoint.sh
@@ -101,5 +76,6 @@ COPY ./netboot.sh /netboot.sh
 
 COPY defaults.yaml /defaults.yaml
 
+COPY --from=enki /usr/bin/enki /usr/bin/enki
 
 ENTRYPOINT [ "/entrypoint.sh" ]

tools-image/arm/boards/rpi64.sh

@@ -16,9 +16,7 @@ TEMPDIR="$(mktemp -d)"
 echo $TEMPDIR
 mount "${device}p1" "${TEMPDIR}"
 
-for dir in /rpi/u-boot /rpi/rpi-firmware /rpi/rpi-firmware-config /rpi/rpi-firmware-dt
-do
-    cp -rfv ${dir}/* $TEMPDIR
-done
+# Copy all rpi files
+cp -rfv /rpi/* $TEMPDIR
 
 umount "${TEMPDIR}"

tools-image/build-arm-image.sh

@@ -320,9 +320,9 @@ cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img
 tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
 
 # Install real grub config to recovery
-cp -rfv /arm/grub/config/* $RECOVERY
+cp -rfv /raw/grubconfig/* $RECOVERY
 mkdir -p $RECOVERY/grub2/fonts
-cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
+cp -rfv /raw/grubartifacts/* $RECOVERY/grub2
 mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts
 
 sync
@@ -335,7 +335,7 @@ if [ -z "$EFI" ]; then
   exit 1
 fi
 
-cp -rfv /arm/grub/efi/* $EFI
+cp -rfv /efi/* $EFI
 if [ -n "$EFI" ] && [  -n "$efi_dir" ]; then
   echo "Copy $efi_dir to EFI directory"
   cp -rfv $efi_dir/* $EFI

tools-image/enki/Earthfile

@@ -0,0 +1,19 @@
+VERSION 0.7
+
+# renovate: datasource=docker depName=golang
+ARG --global GO_VERSION=1.20-alpine3.18
+
+test:
+    FROM golang:$GO_VERSION
+    RUN apk add rsync gcc musl-dev docker jq
+    WORKDIR /build
+    COPY . .
+    RUN go mod download
+    ARG TEST_PATHS=./...
+    ARG LABEL_FILTER=
+    ENV CGO_ENABLED=1
+    # Some test require the docker sock exposed
+    WITH DOCKER
+        RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "$LABEL_FILTER" -v --fail-fast --race --covermode=atomic --coverprofile=coverage.out --coverpkg=github.com/kairos-io/enki/... -p -r $TEST_PATHS
+    END
+    SAVE ARTIFACT coverage.out AS LOCAL coverage.out

tools-image/enki/cmd/build-iso.go

@@ -0,0 +1,122 @@
+package cmd
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/kairos-io/enki/pkg/action"
+	"github.com/kairos-io/enki/pkg/config"
+	"github.com/kairos-io/enki/pkg/utils"
+	v1 "github.com/kairos-io/kairos-agent/v2/pkg/types/v1"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/mount-utils"
+)
+
+// NewBuildISOCmd returns a new instance of the build-iso subcommand and appends it to
+// the root command.
+func NewBuildISOCmd() *cobra.Command {
+	c := &cobra.Command{
+		Use:   "build-iso SOURCE",
+		Short: "Build bootable installation media ISOs",
+		Long: "Build bootable installation media ISOs\n\n" +
+			"SOURCE - should be provided as uri in following format <sourceType>:<sourceName>\n" +
+			"    * <sourceType> - might be [\"dir\", \"file\", \"oci\", \"docker\"], as default is \"docker\"\n" +
+			"    * <sourceName> - is path to file or directory, image name with tag version",
+		Args: cobra.MaximumNArgs(1),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return CheckRoot()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			path, err := exec.LookPath("mount")
+			if err != nil {
+				return err
+			}
+			mounter := mount.New(path)
+
+			cfg, err := config.ReadConfigBuild(viper.GetString("config-dir"), cmd.Flags(), mounter)
+			if err != nil {
+				cfg.Logger.Errorf("Error reading config: %s\n", err)
+			}
+
+			flags := cmd.Flags()
+
+			// Set this after parsing of the flags, so it fails on parsing and prints usage properly
+			cmd.SilenceUsage = true
+			cmd.SilenceErrors = true // Do not propagate errors down the line, we control them
+			spec, err := config.ReadBuildISO(cfg, flags)
+			if err != nil {
+				cfg.Logger.Errorf("invalid install command setup %v", err)
+				return err
+			}
+
+			if len(args) == 1 {
+				imgSource, err := v1.NewSrcFromURI(args[0])
+				if err != nil {
+					cfg.Logger.Errorf("not a valid rootfs source image argument: %s", args[0])
+					return err
+				}
+				spec.RootFS = []*v1.ImageSource{imgSource}
+			} else if len(spec.RootFS) == 0 {
+				errmsg := "rootfs source image for building ISO was not provided"
+				cfg.Logger.Errorf(errmsg)
+				return fmt.Errorf(errmsg)
+			}
+
+			// Repos and overlays can't be unmarshaled directly as they require
+			// to be merged on top and flags do not match any config value key
+			oRootfs, _ := flags.GetString("overlay-rootfs")
+			oUEFI, _ := flags.GetString("overlay-uefi")
+			oISO, _ := flags.GetString("overlay-iso")
+
+			if oRootfs != "" {
+				if ok, err := utils.Exists(cfg.Fs, oRootfs); ok {
+					spec.RootFS = append(spec.RootFS, v1.NewDirSrc(oRootfs))
+				} else {
+					cfg.Logger.Errorf("Invalid value for overlay-rootfs")
+					return fmt.Errorf("Invalid path '%s': %v", oRootfs, err)
+				}
+			}
+			if oUEFI != "" {
+				if ok, err := utils.Exists(cfg.Fs, oUEFI); ok {
+					spec.UEFI = append(spec.UEFI, v1.NewDirSrc(oUEFI))
+				} else {
+					cfg.Logger.Errorf("Invalid value for overlay-uefi")
+					return fmt.Errorf("Invalid path '%s': %v", oUEFI, err)
+				}
+			}
+			if oISO != "" {
+				if ok, err := utils.Exists(cfg.Fs, oISO); ok {
+					spec.Image = append(spec.Image, v1.NewDirSrc(oISO))
+				} else {
+					cfg.Logger.Errorf("Invalid value for overlay-iso")
+					return fmt.Errorf("Invalid path '%s': %v", oISO, err)
+				}
+			}
+
+			buildISO := action.NewBuildISOAction(cfg, spec)
+			err = buildISO.ISORun()
+			if err != nil {
+				cfg.Logger.Errorf(err.Error())
+				return err
+			}
+
+			return nil
+		},
+	}
+	c.Flags().StringP("name", "n", "", "Basename of the generated ISO file")
+	c.Flags().StringP("output", "o", "", "Output directory (defaults to current directory)")
+	c.Flags().Bool("date", false, "Adds a date suffix into the generated ISO file")
+	c.Flags().String("overlay-rootfs", "", "Path of the overlayed rootfs data")
+	c.Flags().String("overlay-uefi", "", "Path of the overlayed uefi data")
+	c.Flags().String("overlay-iso", "", "Path of the overlayed iso data")
+	c.Flags().String("label", "", "Label of the ISO volume")
+	archType := newEnumFlag([]string{"x86_64", "arm64"}, "x86_64")
+	c.Flags().Bool("squash-no-compression", true, "Disable squashfs compression.")
+	c.Flags().VarP(archType, "arch", "a", "Arch to build the image for")
+	return c
+}
+
+func init() {
+	rootCmd.AddCommand(NewBuildISOCmd())
+}