build: improve goreleaser, add testing capabilities, "head" release, …

…docs, fixes ddev#5821, fixes ddev#5811 (ddev#5824)

Co-authored-by: Stanislav Zhuk <[email protected]>

.github/workflows/master-build.yml

@@ -172,6 +172,14 @@ jobs:
     env:
       DDEV_GITHUB_TOKEN: ${{ secrets.DDEV_GITHUB_TOKEN }}
       CHOCOLATEY_API_KEY: ${{ secrets.CHOCOLATEY_API_KEY }}
+      # For goreleaser
+      GITHUB_TOKEN: ${{ secrets.DDEV_GITHUB_TOKEN }}
+      GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+      FURY_ACCOUNT: ${{ secrets.FURY_ACCOUNT }}
+      FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
+      AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
+      AUR_STABLE_GIT_URL: ${{ secrets.AUR_STABLE_GIT_URL }}
+      AUR_EDGE_GIT_URL: ${{ secrets.AUR_EDGE_GIT_URL }}
 
     steps:
       # The upload-artifacts section often fails. This may give us a recovery technique
@@ -222,22 +230,24 @@ jobs:
         run: exit 1
 
       # Goreleaser does GitHub release artifacts, homebrew, AUR, deb/rpm
+
+      # Actual release
       - name: goreleaser
         uses: goreleaser/goreleaser-action@v5
         if: startsWith( github.ref, 'refs/tags/v1')
         with:
-          # either 'goreleaser' (default) or 'goreleaser-pro'
           distribution: goreleaser-pro
           version: latest
           args: release --clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.DDEV_GITHUB_TOKEN }}
-          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
-          FURY_ACCOUNT: ${{ secrets.FURY_ACCOUNT }}
-          FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
-          AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
-          AUR_STABLE_GIT_URL: ${{ secrets.AUR_STABLE_GIT_URL }}
-          AUR_EDGE_GIT_URL: ${{ secrets.AUR_EDGE_GIT_URL }}
+
+      # HEAD release, if we don't have a `v` tag
+      - name: goreleaser
+        uses: goreleaser/goreleaser-action@v5
+        if: startsWith( github.ref, 'refs/tags/v1') != true
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          args: release --clean --nightly
 
       # Do artifacts for upload to workflow URL
       - name: Generate artifacts

.goreleaser.yml

@@ -1,6 +1,14 @@
 ##### BUILDS ######
+nightly:
+  publish_release: true
+  keep_single_release: true
+  tag_name: head
+  name_template: '{{ incpatch .Version }}-{{ .ShortCommit }}-head'
+
 builds:
 - id: ddev
+  main: ./cmd/ddev
+
   # Requires make  darwin_amd64 darwin_arm64 linux_amd64 linux_arm64 windows_amd64 mkcert
   builder: prebuilt
   goos:
@@ -121,7 +129,6 @@ builds:
 ###### Archives ######
 archives:
 - id: ddev
-  rlcp: true
   builds:
   - ddev
   - mkcert
@@ -140,14 +147,12 @@ archives:
   allow_different_binary_count: true
 
 - id: completions-tarball
-  rlcp: true
   builds:
   - completions-tarball
   format: binary
   name_template: ddev_shell_completion_scripts.v{{.Version}}.tar.gz
 
 - id: ddev-windows-installer
-  rlcp: true
   builds:
   - ddev-windows-installer
   format: binary
@@ -168,7 +173,7 @@ brews:
 - name: ddev
   ids:
   - ddev
-  tap:
+  repository:
     owner: "{{ .Env.GITHUB_REPOSITORY_OWNER }}"
     name: homebrew-ddev
   description: DDEV
@@ -206,7 +211,7 @@ brews:
 - name: ddev
   ids:
   - ddev
-  tap:
+  repository:
     owner: "{{ .Env.GITHUB_REPOSITORY_OWNER }}"
     name: homebrew-ddev-edge
   description: DDEV

Makefile

@@ -87,7 +87,7 @@ $(TARGETS): mkcert $(GOFILES)
 
 $(GOTMP)/bin/completions.tar.gz: build
 	$(GOTMP)/bin/$(BUILD_OS)_$(BUILD_ARCH)/ddev_gen_autocomplete
-	tar -C $(GOTMP)/bin/completions -cf $(GOTMP)/bin/completions.tar.gz .
+	tar -C $(GOTMP)/bin/completions -czf $(GOTMP)/bin/completions.tar.gz .
 
 mkcert: $(GOTMP)/bin/darwin_arm64/mkcert $(GOTMP)/bin/darwin_amd64/mkcert $(GOTMP)/bin/linux_arm64/mkcert $(GOTMP)/bin/linux_amd64/mkcert
 

docs/content/developers/building-contributing.md

@@ -8,7 +8,7 @@ search:
 
 There are several ways to use DDEV’s latest-committed HEAD version:
 
-* **Download** the latest master branch artifacts from [nightly.link](https://nightly.link/ddev/ddev/workflows/master-build/master). Each of these is built by the CI system, signed, and notarized. Get the one you need and place it in your `$PATH`.
+* **Download** artifacts from the latest HEAD pre-release in [releases](https://github.com/ddev/ddev/releases), which are signed and notarized. Get the one you need and place it in your `$PATH`.
 * **Homebrew install HEAD**: On macOS and Linux, run `brew unlink ddev && brew install ddev/ddev/ddev --HEAD --fetch-HEAD` to get the latest DDEV commit, even if it’s unreleased. Since you’re building this on your own computer, it’s not signed or notarized, and you’ll get a notification that instrumentation doesn’t work, which is fine. If you’re using Linux/WSL2, you’ll likely need to install build-essential by running the following command: `sudo apt install -y build-essential`.
 * **Build manually**: If you have normal build tools like `make` and `go` installed, you can check out the code and run `make`.
 * **Gitpod** You can use the latest build by visiting DDEV on [Gitpod](https://gitpod.io/#https://github.com/ddev/ddev).

docs/content/developers/release-management.md

@@ -6,7 +6,7 @@ search:
 
 ## Release process and tools
 
-* [GoReleaser Pro](https://goreleaser.com/pro/) is used to do the actual releasing using [.goreleaser.yml](https://github.com/ddev/ddev/blob/master/.goreleaser.yml). GoReleaser Pro is a licensed product that requires a license key, which is in the GitHub Workflow configuration and is available in 1Password to DDEV maintainers who need it.
+* [GoReleaser Pro](https://goreleaser.com/pro/) is used to do the actual releasing using [.goreleaser.yml](https://github.com/ddev/ddev/blob/master/.goreleaser.yml). GoReleaser Pro is a licensed product that requires separate installation and a license key, which is in the GitHub Workflow configuration and is available in 1Password to DDEV maintainers who need it.
 * The [Master Build/Release GitHub Action](https://github.com/ddev/ddev/blob/master/.github/workflows/master-build.yml) does the actual running of the GoReleaser actions and provides the needed secrets.
 
 ## GitHub Actions Required Secrets
@@ -35,11 +35,14 @@ The following “Repository secret” environment variables must be added to <ht
 * `DDEV_GITHUB_TOKEN`: GitHub personal token (`repo` scope, classic PAT) that gives access to create releases and push to the Homebrew repositories.
 * `DDEV_MACOS_APP_PASSWORD`: Password used for notarization, see [signing_tools](https://github.com/ddev/signing_tools).
 * `DDEV_MACOS_SIGNING_PASSWORD`: Password for the macOS signing key, see [signing_tools](https://github.com/ddev/signing_tools).
+* `DDEV_MAIN_REPO_ORGNAME`: The organization to be used for testing, normally `ddev` but it may be `ddev-test` for the test organization.
 * `DDEV_WINDOWS_SIGNING_PASSWORD`: Windows signing password.
 * `FURY_ACCOUNT`: [Gemfury](https://gemfury.com) account that receives package pushes.
 * `FURY_TOKEN`: Push token assigned to the above Gemfury account.
 * `GORELEASER_KEY`: License key for GoReleaser Pro.
-* `SEGMENTKEY`: Key that enables Segment reporting. Environment variable for Make is `SegmentKey`.
+* `HOMEBREW_EDGE_REPOSITORY`: Like `ddev/homebrew-ddev-edge` but may be `ddev-test/homebrew-ddev-edge`.
+* `HOMEBREW_STABLE_REPOSITORY`: Like `ddev/homebrew-ddev-edge` but may be `ddev/homebrew-ddev-edge`.
+* `SEGMENTKEY`: Key that enables Segment reporting. Environment variable for Make is `SegmentKey`. (This will be obsolete in DDEV v1.23.)
 
 ## Creating a Release
 
@@ -201,3 +204,37 @@ The Linux `apt` and `yum`/`rpm` packages are built and pushed by the `nfpms` and
 * The `pkg.ddev.com` domain name is set up as a custom alias for our package repositories; see `https://manage.fury.io/manage/drud/domains`. (Users do not see `drud` anywhere. Although we could have moved to a new organization for this, the existing repositories contain all the historical versions so it made sense to be less disruptive.)
 * The `pkg.ddev.com` `CNAME` is managed in CloudFlare because `ddev.com` is managed there.
 * The fury.io tokens are in DDEV’s shared 1Password account.
+
+## Testing Release Creation
+
+When significant changes are made to the `.goreleaser.yml` or related configuration, it's important to be able to test without actually deploying to `ddev/ddev/releases` of course. We have two ways to test the configuration; we can run `goreleaser` manually for simpler tests, or run a full release on `ddev-test/ddev` where needed.
+
+### Running `goreleaser` manually
+
+This approach is great for seeing what artifacts get created, without deploying them.
+
+Prerequisites:
+
+* GoReleaser Pro must be installed
+* `export GORELEASER_KEY=<key>`
+
+```bash
+export GITHUB_REPOSITORY_OWNER=ddev-test
+git tag <tagname> # Try to include context like PR number, for example v1.22.8-PR5824
+make windows_amd64 darwin_amd64 darwin_arm64 linux_amd64 linux_arm64 completions
+goreleaser release --prepare --nightly --clean
+```
+
+This will create all the artifacts that would have been pushed in the `dist` directory. You can copy Linux packages from there to test them manually, download the built tarballs for use elsewhere, install Homebrew package manually, for example:
+
+```bash
+brew install ./dist/homebrew/Formula/ddev.rb
+```
+
+### Creating a test release on `ddev-test/ddev`
+
+[ddev-test/ddev](https://github.com/ddev-test/ddev) is now set up for actual release testing. It has all or most of the environment variables set up already. It also acts against `ddev-test/homebrew-ddev` and `ddev-test/homebrew-ddev-edge` so you can test Homebrew publishing.
+
+1. Create a branch on `ddev-test/ddev`.
+2. Using the web UI, create a release using that branch as base. The release tag must start with `v1.`. Where possible, please use a release tag that includes context about the PR you are working against, like `v1.28.8-PR2022FixStuff`, and include in the release notes a link to the issue. The tag must be a valid Semantic Version tag, so don't use underscores, etc.
+3. Test out the resulting artifacts that get published or deployed.