diff --git a/.github/workflows/release-installer.yaml b/.github/workflows/release-installer.yaml new file mode 100644 index 000000000..02ddfd8a6 --- /dev/null +++ b/.github/workflows/release-installer.yaml @@ -0,0 +1,113 @@ +# When a third-party action is added (i.e., `uses`), please also add it to `download-licenses` in Makefile. +# When the installers are ready in installer private bucket, run this workflow to test them. +# TODO: Add job of uploading installers to Github release when installer tests are stable. +# "release" runner tag is to make target hosts deterministic for easy clean up. +# This not only refers to the cleanup described in https://github.com/runfinch/finch/issues/106 for arm hosts. +# Currently tests on amd64 hosts sometimes also need manual cleanup via SSHing to the host. +# TODO: Remove the "release" runner tag when installer tests are stable. +name: Release Installer +on: + workflow_dispatch: + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + # TODO: fix the arm64 installer tests. (https://github.com/runfinch/finch/issues/106) + # Currently the first time of calling any Finch command in arm64 hosts triggered by Github action will fail by + # the error "Error: Process completed with exit code 137." So the arm64 job will fail. + # We temporarily use follow-up manual steps to complete and clean up it. + # 1. Ssh to the arm64 hosts. + # 2. Run `cd ar/_work/finch/finch`. + # 3. Run `INSTALLED=true make test-e2e` to complete the installer e2e tests. + # 4. Run `sudo /Applications/Finch/uninstall.sh` to uninstall Finch. + # 5. Run `rm -rf Finch--aarch64.pkg` to delete the pkg. + macos-arm64-test-installer: + strategy: + fail-fast: false + matrix: + os: [[self-hosted, macos, arm64, 11.7, release], [self-hosted, macos, arm64, 12.6, release], [self-hosted, macos, arm64, 13.0, release]] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + env: + ACCESS_TOKEN: ${{ secrets.FINCH_BOT_TOKEN }} + steps: + - uses: actions/setup-go@v2 + with: + go-version: 1.19.x + - uses: actions/checkout@v2 + - name: Clean up previous files + run: | + sudo rm -rf /opt/finch + sudo rm -rf ~/.finch + if pgrep '^qemu-system'; then + sudo pkill '^qemu-system' + fi + if pgrep '^socket_vmnet'; then + sudo pkill '^socket_vmnet' + fi + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: download-installer-session + aws-region: ${{ secrets.REGION }} + - name: Download from S3 + run: | + aws s3 cp s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-${GITHUB_REF_NAME}-aarch64.pkg Finch-${GITHUB_REF_NAME}-aarch64.pkg + - name: Silently install + # Rosetta is only needed in arm64. Currently the installer command will fail to install without Rosetta. + # TODO: Remove Rosetta command when the installer is fixed. (https://github.com/runfinch/finch/issues/105) + run: | + echo 'A' | sudo softwareupdate --install-rosetta + sudo installer -pkg Finch-${GITHUB_REF_NAME}-aarch64.pkg -target / + - name: Run e2e tests + run: INSTALLED=true make test-e2e + - name: Silently uninstall + run: echo 'y' | sudo bash /Applications/Finch/uninstall.sh + - name: Delete installer + run: rm -rf Finch-${GITHUB_REF_NAME}-aarch64.pkg + + macos-amd64-test-installer: + strategy: + fail-fast: false + matrix: + os: [[self-hosted, macos, amd64, 10.15, release], [self-hosted, macos, amd64, 11.7, release], [self-hosted, macos, amd64, 12.6, release], [self-hosted, macos, amd64, 13.0, release]] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + env: + ACCESS_TOKEN: ${{ secrets.FINCH_BOT_TOKEN }} + steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.19.x + - uses: actions/checkout@v3 + - name: Clean up previous files + run: | + sudo rm -rf /opt/finch + sudo rm -rf ~/.finch + if pgrep '^qemu-system'; then + sudo pkill '^qemu-system' + fi + if pgrep '^socket_vmnet'; then + sudo pkill '^socket_vmnet' + fi + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: download-installer-session + aws-region: ${{ secrets.REGION }} + - name: Download from S3 + run: | + aws s3 cp s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-${GITHUB_REF_NAME}-x86_64.pkg Finch-${GITHUB_REF_NAME}-x86_64.pkg + - name: Silently install + run: | + sudo installer -pkg Finch-${GITHUB_REF_NAME}-x86_64.pkg -target / + - name: Run e2e tests + run: INSTALLED=true make test-e2e + - name: Silently uninstall + run: echo 'y' | sudo bash /Applications/Finch/uninstall.sh + - name: Delete installer + run: rm -rf Finch-${GITHUB_REF_NAME}-x86_64.pkg diff --git a/Makefile b/Makefile index 26fab18fa..6fbf97940 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ LDFLAGS := "-X $(PACKAGE)/pkg/version.Version=$(VERSION)" .DEFAULT_GOAL := all +INSTALLED ?= false + ifneq (,$(findstring arm64,$(ARCH))) SUPPORTED_ARCH = true LIMA_ARCH = aarch64 @@ -243,7 +245,7 @@ test-unit: # test-e2e assumes the VM instance doesn't exist, please make sure to remove it before running. .PHONY: test-e2e test-e2e: - go test -ldflags $(LDFLAGS) -timeout 60m ./e2e/... -test.v -ginkgo.v + go test -ldflags $(LDFLAGS) -timeout 60m ./e2e/... -test.v -ginkgo.v --installed="$(INSTALLED)" .PHONY: gen-code # Since different projects may have different versions of tool binaries, diff --git a/e2e/config_test.go b/e2e/config_test.go index 4fe029367..0c4a29d35 100644 --- a/e2e/config_test.go +++ b/e2e/config_test.go @@ -7,6 +7,7 @@ import ( "errors" "io/fs" "os" + "os/exec" "path/filepath" "github.com/onsi/ginkgo/v2" @@ -18,7 +19,7 @@ import ( var finchConfigFilePath = os.Getenv("HOME") + "/.finch/finch.yaml" -const limaConfigFilePath = "../_output/lima/data/_config/override.yaml" +const defaultLimaConfigFilePath = "../_output/lima/data/_config/override.yaml" func readFile(filePath string) []byte { out, err := os.ReadFile(filepath.Clean(filePath)) @@ -53,11 +54,20 @@ func updateAndApplyConfig(o *option.Option, configBytes []byte) *gexec.Session { // empty and a non-existent Finch config.yaml. Meaning, if you run this without an existing config.yaml, // an empty config.yaml will be created after all test cases are run. This currently does not change the behavior // of Finch, but may need to be revisited later. -var testConfig = func(o *option.Option) { +var testConfig = func(o *option.Option, installed bool) { // These tests are run in serial because we only define one virtual machine instance, and it requires disk I/O. ginkgo.Describe("Config", ginkgo.Serial, func() { + var limaConfigFilePath string ginkgo.BeforeEach(func() { origFinchCfg := readFile(finchConfigFilePath) + limaConfigFilePath = defaultLimaConfigFilePath + if installed { + path, err := exec.LookPath(installedTestSubject) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + realFinchPath, err := filepath.EvalSymlinks(path) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + limaConfigFilePath = filepath.Join(realFinchPath, "/../../lima/data/_config/override.yaml") + } origLimaCfg := readFile(limaConfigFilePath) ginkgo.DeferCleanup(func() { @@ -74,7 +84,7 @@ var testConfig = func(o *option.Option) { gomega.Expect(startCmdSession).Should(gexec.Exit(0)) gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile()) - cfgBuf, err := os.ReadFile(limaConfigFilePath) + cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath)) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.ContainSubstring("cpus: 6"), gomega.ContainSubstring("memory: 4GiB"))) }) @@ -84,7 +94,7 @@ var testConfig = func(o *option.Option) { gomega.Expect(startCmdSession).Should(gexec.Exit(0)) gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile()) - cfgBuf, err := os.ReadFile(limaConfigFilePath) + cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath)) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) // 4 CPUs is the default gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.MatchRegexp(`cpus: \d`), gomega.ContainSubstring("memory: 6GiB"))) @@ -95,7 +105,7 @@ var testConfig = func(o *option.Option) { gomega.Expect(startCmdSession).Should(gexec.Exit(0)) gomega.Expect(limaConfigFilePath).Should(gomega.BeARegularFile()) - cfgBuf, err := os.ReadFile(limaConfigFilePath) + cfgBuf, err := os.ReadFile(filepath.Clean(limaConfigFilePath)) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) gomega.Expect(cfgBuf).Should(gomega.SatisfyAll(gomega.MatchRegexp(`cpus: \d`), gomega.MatchRegexp(`memory: \dGiB`))) }) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 078eb85f2..e048471b0 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -4,6 +4,7 @@ package e2e import ( + "flag" "os" "path/filepath" "testing" @@ -15,7 +16,12 @@ import ( "github.com/runfinch/common-tests/tests" ) -const virtualMachineRootCmd = "vm" +const ( + virtualMachineRootCmd = "vm" + installedTestSubject = "finch" +) + +var installed = flag.Bool("installed", false, "the flag to show whether the tests are ran against installed application") //nolint:paralleltest // TestE2e is like TestMain for our e2e tests. func TestE2e(t *testing.T) { @@ -26,6 +32,9 @@ func TestE2e(t *testing.T) { t.Fatalf("failed to get the current working directory: %v", err) } subject := filepath.Join(wd, "../_output/bin/finch") + if *installed { + subject = installedTestSubject + } o, err := option.New([]string{subject}) if err != nil { @@ -89,7 +98,7 @@ func TestE2e(t *testing.T) { // When running tests in serial sequence and using the local registry, testVirtualMachine needs to run after generic tests finished // since it will remove the VM instance thus removing the local registry. testVirtualMachine(o) - testConfig(o) + testConfig(o, *installed) testVersion(o) })