ci: add workflow for installer tests

.github/workflows/release-installer.yaml

@@ -0,0 +1,113 @@
+# When a third-party action is added (i.e., `uses`), please also add it to `download-licenses` in Makefile.
+# When the installers are ready in installer private bucket, run this workflow to test them.
+# TODO: Add job of uploading installers to Github release when installer tests are stable.
+# "release" runner tag is to make target hosts deterministic for easy clean up.
+# This not only refers to the cleanup described in https://github.com/runfinch/finch/issues/106 for arm hosts.
+# Currently tests on amd64 hosts sometimes also need manual cleanup via SSHing to the host.
+# TODO: Remove the "release" runner tag when installer tests are stable.
+name: Release Installer
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write   # This is required for requesting the JWT
+  contents: read    # This is required for actions/checkout
+
+jobs:
+  # TODO: fix the arm64 installer tests. (https://github.com/runfinch/finch/issues/106)
+  # Currently the first time of calling any Finch command in arm64 hosts triggered by Github action will fail by
+  # the error "Error: Process completed with exit code 137." So the arm64 job will fail.
+  # We temporarily use follow-up manual steps to complete and clean up it.
+  # 1. Ssh to the arm64 hosts.
+  # 2. Run `cd ar/_work/finch/finch`.
+  # 3. Run `INSTALLED=true make test-e2e` to complete the installer e2e tests.
+  # 4. Run `sudo /Applications/Finch/uninstall.sh` to uninstall Finch.
+  # 5. Run `rm -rf Finch-<version>-aarch64.pkg` to delete the pkg.
+  macos-arm64-test-installer:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [[self-hosted, macos, arm64, 11.7, release], [self-hosted, macos, arm64, 12.6, release], [self-hosted, macos, arm64, 13.0, release]]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    env:
+      ACCESS_TOKEN: ${{ secrets.FINCH_BOT_TOKEN }}
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.x
+      - uses: actions/checkout@v2
+      - name: Clean up previous files
+        run: |
+          sudo rm -rf /opt/finch
+          sudo rm -rf ~/.finch
+          if pgrep '^qemu-system'; then
+            sudo pkill '^qemu-system'
+          fi
+          if pgrep '^socket_vmnet'; then
+            sudo pkill '^socket_vmnet'
+          fi
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.ROLE }}
+          role-session-name: download-installer-session
+          aws-region: ${{ secrets.REGION }}
+      - name: Download from S3
+        run: |
+          aws s3 cp s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-${GITHUB_REF_NAME}-aarch64.pkg Finch-${GITHUB_REF_NAME}-aarch64.pkg
+      - name: Silently install
+        # Rosetta is only needed in arm64. Currently the installer command will fail to install without Rosetta.
+        # TODO: Remove Rosetta command when the installer is fixed. (https://github.com/runfinch/finch/issues/105)
+        run: |
+          echo 'A' | sudo softwareupdate --install-rosetta
+          sudo installer -pkg Finch-${GITHUB_REF_NAME}-aarch64.pkg -target /
+      - name: Run e2e tests
+        run: INSTALLED=true make test-e2e
+      - name: Silently uninstall
+        run: echo 'y' | sudo bash /Applications/Finch/uninstall.sh
+      - name: Delete installer
+        run: rm -rf Finch-${GITHUB_REF_NAME}-aarch64.pkg
+
+  macos-amd64-test-installer:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [[self-hosted, macos, amd64, 10.15, release], [self-hosted, macos, amd64, 11.7, release], [self-hosted, macos, amd64, 12.6, release], [self-hosted, macos, amd64, 13.0, release]]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    env:
+      ACCESS_TOKEN: ${{ secrets.FINCH_BOT_TOKEN }}
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+      - uses: actions/checkout@v3
+      - name: Clean up previous files
+        run: |
+          sudo rm -rf /opt/finch
+          sudo rm -rf ~/.finch
+          if pgrep '^qemu-system'; then
+            sudo pkill '^qemu-system'
+          fi
+          if pgrep '^socket_vmnet'; then
+            sudo pkill '^socket_vmnet'
+          fi
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.ROLE }}
+          role-session-name: download-installer-session
+          aws-region: ${{ secrets.REGION }}
+      - name: Download from S3
+        run: |
+          aws s3 cp s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-${GITHUB_REF_NAME}-x86_64.pkg Finch-${GITHUB_REF_NAME}-x86_64.pkg
+      - name: Silently install
+        run: |
+          sudo installer -pkg Finch-${GITHUB_REF_NAME}-x86_64.pkg -target /
+      - name: Run e2e tests
+        run: INSTALLED=true make test-e2e
+      - name: Silently uninstall
+        run: echo 'y' | sudo bash /Applications/Finch/uninstall.sh
+      - name: Delete installer
+        run: rm -rf Finch-${GITHUB_REF_NAME}-x86_64.pkg

Makefile

@@ -31,6 +31,8 @@ LDFLAGS := "-X $(PACKAGE)/pkg/version.Version=$(VERSION)"
 
 .DEFAULT_GOAL := all
 
+INSTALLED ?= false
+
 ifneq (,$(findstring arm64,$(ARCH)))
 	SUPPORTED_ARCH = true
 	LIMA_ARCH = aarch64
@@ -241,7 +243,7 @@ test-unit:
 # test-e2e assumes the VM instance doesn't exist, please make sure to remove it before running.
 .PHONY: test-e2e
 test-e2e:
-	go test -ldflags $(LDFLAGS) -timeout 60m ./e2e/... -test.v -ginkgo.v
+	go test -ldflags $(LDFLAGS) -timeout 60m ./e2e/... -test.v -ginkgo.v --installed="$(INSTALLED)"
 
 .PHONY: gen-code
 # Since different projects may have different versions of tool binaries,

e2e/config_test.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/onsi/ginkgo/v2"
@@ -18,7 +19,7 @@ import (
 
 var finchConfigFilePath = os.Getenv("HOME") + "/.finch/finch.yaml"
 
-const limaConfigFilePath = "../_output/lima/data/_config/override.yaml"
+const defaultLimaConfigFilePath = "../_output/lima/data/_config/override.yaml"
 
 func readFile(filePath string) []byte {
 	out, err := os.ReadFile(filepath.Clean(filePath))
@@ -53,11 +54,20 @@ func updateAndApplyConfig(o *option.Option, configBytes []byte) *gexec.Session {
 // empty and a non-existent Finch config.yaml. Meaning, if you run this without an existing config.yaml,
 // an empty config.yaml will be created after all test cases are run. This currently does not change the behavior
 // of Finch, but may need to be revisited later.
-var testConfig = func(o *option.Option) {
+var testConfig = func(o *option.Option, installed bool) {
 	// These tests are run in serial because we only define one virtual machine instance, and it requires disk I/O.
 	ginkgo.Describe("Config", ginkgo.Serial, func() {
+		var limaConfigFilePath string
 		ginkgo.BeforeEach(func() {
 			origFinchCfg := readFile(finchConfigFilePath)
+			limaConfigFilePath = defaultLimaConfigFilePath
+			if installed {
+				path, err := exec.LookPath(installedTestSubject)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				realFinchPath, err := filepath.EvalSymlinks(path)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				limaConfigFilePath = filepath.Join(realFinchPath, "/../../lima/data/_config/override.yaml")
+			}
 			origLimaCfg := readFile(limaConfigFilePath)
 
 			ginkgo.DeferCleanup(func() {
@@ -74,7 +84,7 @@ var testConfig = func(o *option.Option) {
 			gomega.Expect(startCmdSession).Should(gexec.Exit(0))
 
 			gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile())
-			cfgBuf, err := os.ReadFile(limaConfigFilePath)
+			cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath))
 			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 			gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.ContainSubstring("cpus: 6"), gomega.ContainSubstring("memory: 4GiB")))
 		})
@@ -84,7 +94,7 @@ var testConfig = func(o *option.Option) {
 			gomega.Expect(startCmdSession).Should(gexec.Exit(0))
 
 			gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile())
-			cfgBuf, err := os.ReadFile(limaConfigFilePath)
+			cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath))
 			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 			// 4 CPUs is the default
 			gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.MatchRegexp(`cpus: \d`), gomega.ContainSubstring("memory: 6GiB")))
@@ -95,7 +105,7 @@ var testConfig = func(o *option.Option) {
 			gomega.Expect(startCmdSession).Should(gexec.Exit(0))
 
 			gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile())
-			cfgBuf, err := os.ReadFile(limaConfigFilePath)
+			cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath))
 			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 			gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.MatchRegexp(`cpus: \d`), gomega.MatchRegexp(`memory: \dGiB`)))
 		})

e2e/e2e_test.go

@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"flag"
 	"os"
 	"path/filepath"
 	"testing"
@@ -15,7 +16,12 @@ import (
 	"github.com/runfinch/common-tests/tests"
 )
 
-const virtualMachineRootCmd = "vm"
+const (
+	virtualMachineRootCmd = "vm"
+	installedTestSubject  = "finch"
+)
+
+var installed = flag.Bool("installed", false, "the flag to show whether the tests are ran against installed application")
 
 //nolint:paralleltest // TestE2e is like TestMain for our e2e tests.
 func TestE2e(t *testing.T) {
@@ -26,6 +32,9 @@ func TestE2e(t *testing.T) {
 		t.Fatalf("failed to get the current working directory: %v", err)
 	}
 	subject := filepath.Join(wd, "../_output/bin/finch")
+	if *installed {
+		subject = installedTestSubject
+	}
 
 	o, err := option.New([]string{subject})
 	if err != nil {
@@ -89,7 +98,7 @@ func TestE2e(t *testing.T) {
 		// When running tests in serial sequence and using the local registry, testVirtualMachine needs to run after generic tests finished
 		// since it will remove the VM instance thus removing the local registry.
 		testVirtualMachine(o)
-		testConfig(o)
+		testConfig(o, *installed)
 		testVersion(o)
 	})