fix: use the right installer URL for the canary channel (cashapp#257)

* fix: use the right installer URL for the canary channel

- We make changes for the generated bin/hermit to correctly identity the
download URL for the canary channel. This part was overlooked in an
earlier implementation.
- The `InstallScriptSHAs` map is manually populated for the "canary" and
"stable" channels, respectively, in `env.go`.
- Update `ScriptSHAs` for the updated `bin/hermit` scripts, for "canary"
and "stable" channels.
- Add `InstallerSHA256Sums` map to the `Config` struct of the main Hermit app.
- Update GHA CI step for sanity check of SHA256 sums of install script.
- Fix a buggy integration test.

* Add installer-sha-256 command

This allows us to avoid inspecting the Hermit source for the CI test to
validate checksum coherence in Hermit and in geninstaller. It makes the
CI test more robust and less inelegant.

* Make installer sums app configurable during init

The default configuration for `InstallerSHA256Sum` is
`hermit.InstallScriptSHAs`. This value now can be overridden by the main
application.

* Fix test: envfixture

Provide an empty map as the extra input argument needed by hermit.Init().

* Run gofmt -s on env.go

Reformatting so it's conformant.

.github/workflows/ci.yml

@@ -74,9 +74,11 @@ jobs:
 #          ./run.sh
       - name: Check install script SHA-256 sum
         run: |
-          SHA256SUM_IN_SRC=$(grep "InstallScriptSHA = " env.go | \
-            grep -Eo "[0-9a-f]{64}")
-          SHA256SUM_INSTALLER=$(./bin/go run ./cmd/geninstaller \
-            --dest=$(mktemp) \
-            --dist-url=https://github.com/cashapp/hermit/releases/download/stable)
-          [ "$SHA256SUM_IN_SRC" = "$SHA256SUM_INSTALLER" ] || exit 1
+          for channel in "stable" "canary"; do
+            ./bin/go build -ldflags "-X main.version=checksum -X main.channel=${channel}" -o ./build/hermit ./cmd/hermit
+            SHA256SUM_IN_HERMIT=$(./build/hermit installer-sha-256)
+            SHA256SUM_INSTALLER=$(./bin/go run ./cmd/geninstaller \
+              --dest=$(mktemp) \
+              --dist-url=https://github.com/cashapp/hermit/releases/download/${channel})
+            [ "$SHA256SUM_IN_HERMIT" = "$SHA256SUM_INSTALLER" ] || exit 1
+          done

app/commands.go

@@ -44,6 +44,7 @@ type cliBase struct {
 	Update               updateCmd            `cmd:"" aliases:"sync" help:"Update manifest sources." group:"global"`
 	Search               searchCmd            `cmd:"" help:"Search for packages to install." group:"global"`
 	DumpUserConfigSchema dumpUserConfigSchema `cmd:"" help:"Dump user configuration schema." hidden:""`
+	InstallerSHA256      installersha256Cmd   `cmd:"" help:"Show installer script hash info." group:"global"`
 	kong.Plugins
 }
 

app/init_cmd.go

@@ -13,7 +13,7 @@ type initCmd struct {
 }
 
 func (i *initCmd) Run(w *ui.UI, config Config) error {
-	return hermit.Init(w, i.Dir, config.BaseDistURL, hermit.UserStateDir, hermit.Config{
+	return hermit.Init(w, i.Dir, config.BaseDistURL, config.InstallerSHA256Sums, hermit.UserStateDir, hermit.Config{
 		Sources:     i.Sources,
 		ManageGit:   !i.NoGit,
 		AddIJPlugin: i.Idea,

app/installersha256_cmd.go

@@ -0,0 +1,14 @@
+package app
+
+import (
+	"fmt"
+
+	"github.com/alecthomas/kong"
+)
+
+type installersha256Cmd struct{}
+
+func (v *installersha256Cmd) Run(kctx kong.Vars) error {
+	fmt.Println(kctx["installersha256"])
+	return nil
+}

app/main.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path"
 	"runtime"
 	"runtime/pprof"
 	"time"
@@ -39,11 +40,15 @@ type Config struct {
 	BaseDistURL string
 	// SHA256 checksums for all known versions of per-environment scripts.
 	// If empty shell.ScriptSHAs will be used.
-	SHA256Sums  []string
-	HTTP        func(HTTPTransportConfig) *http.Client
-	State       state.Config
-	KongOptions []kong.Option
-	KongPlugins kong.Plugins
+	SHA256Sums []string
+	// SHA256 checksums for install scripts of known channels - currently
+	// "stable" and "canary".
+	// If empty hermit.InstallScriptSHAs will be used
+	InstallerSHA256Sums map[string]string
+	HTTP                func(HTTPTransportConfig) *http.Client
+	State               state.Config
+	KongOptions         []kong.Option
+	KongPlugins         kong.Plugins
 	// Defaults to cache.GetSource if nil.
 	PackageSourceSelector cache.PackageSourceSelector
 	// True if we're running in CI - disables progress bar.
@@ -83,6 +88,17 @@ func (c Config) defaultHTTPClient(logger ui.Logger) *http.Client {
 	return c.makeHTTPClient(logger, HTTPTransportConfig{})
 }
 
+// Helper function to get installer SHA-256 sum for a channel from a map of
+// channel: sha256sum pairs
+func getChannelInstallerSHA(channel string, m map[string]string) string {
+	InstallScriptSHA, ok := m[channel]
+	if !ok {
+		// Magic string for an unknown channel
+		InstallScriptSHA = "BYPASS"
+	}
+	return InstallScriptSHA
+}
+
 // Main runs the Hermit command-line application with the given config.
 func Main(config Config) {
 	config.LogLevel = ui.AutoLevel(config.LogLevel)
@@ -102,6 +118,11 @@ func Main(config Config) {
 	if len(config.SHA256Sums) == 0 {
 		config.SHA256Sums = hermit.ScriptSHAs
 	}
+
+	if len(config.InstallerSHA256Sums) == 0 {
+		config.InstallerSHA256Sums = hermit.InstallScriptSHAs
+	}
+
 	var (
 		err         error
 		p           *ui.UI
@@ -196,8 +217,9 @@ func Main(config Config) {
 			}
 		}),
 		kong.Vars{
-			"version": config.Version,
-			"env":     envPath,
+			"version":         config.Version,
+			"installersha256": getChannelInstallerSHA(path.Base(config.BaseDistURL), config.InstallerSHA256Sums),
+			"env":             envPath,
 		},
 		kong.HelpOptions{
 			Compact: true,

env.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -56,14 +57,17 @@ var (
 		"4a4353c2147bb92bc2407a4b94817a90048c39d5f3cb08de807e7860517444c0",
 		"4b9ed62ae1a7101db99ab7997a67f47a8bac39eeac75b99a515d2545645b1c21",
 		"9413f2347c5f70e6a004e62b7faac99d3bb1666f86451ed1f6e05a679e3bc27c",
-		"661f265a25a112a8c95af6b6fdbe6d5709f725b82588208acf7246db1dc88de9",
-		"321d06280fa194b0a6f01220f6e42ca77d3092ac29015f6d0f59f41180b56db4",
+		"49f331b75e8886b5f55d331ad36c0b8304e4402b2ffe7b5e2aa711463babb299",
+		"52946535460cff969184ffa495886d1cfd6e1b9169027e161325cdba95a017e7",
 	}
 
-	// InstallScriptSHA is the SHA256 value of the install script, which
-	// can be obtained by running `cmd/geninstaller` with the
-	// `--print-sha-256` option
-	InstallScriptSHA = "180e997dd837f839a3072a5e2f558619b6d12555cd5452d3ab19d87720704e38"
+	// InstallScriptSHAs are the SHA256 values of the install script for
+	// known channels, which can be obtained by running `cmd/geninstaller`
+	// with `--dist-url=https://github.com/cashapp/hermit/releases/download/<channel>`
+	InstallScriptSHAs = map[string]string{
+		"stable": "180e997dd837f839a3072a5e2f558619b6d12555cd5452d3ab19d87720704e38",
+		"canary": "7637176267f01d11dc5725e4b976e4de26d7b9233589438e40f77d18b225d573",
+	}
 
 	//go:embed files
 	files embed.FS
@@ -134,9 +138,16 @@ type Env struct {
 var extDepData []byte
 
 // Init a new Env.
-func Init(l *ui.UI, env string, distURL string, stateDir string, config Config) error {
+func Init(l *ui.UI, env string, distURL string, installScriptSHAs map[string]string, stateDir string, config Config) error {
 	env = util.RealPath(env)
 	l.Infof("Creating new Hermit environment in %s", env)
+	channel := path.Base(distURL)
+	InstallScriptSHA, ok := installScriptSHAs[channel]
+	if !ok {
+		// Magic string to signal bypass of verification for an unknown channel
+		InstallScriptSHA = "BYPASS"
+	}
+
 	vars := map[string]string{
 		"HERMIT_DEFAULT_DIST_URL":      distURL,
 		"HERMIT_INSTALL_SCRIPT_SHA256": InstallScriptSHA,

files/hermit

@@ -25,12 +25,16 @@ if [ ! -x "${HERMIT_EXE}" ]; then
   INSTALL_SCRIPT="$(mktemp)"
   # This value must match that of the install script
   INSTALL_SCRIPT_SHA256="HERMIT_INSTALL_SCRIPT_SHA256"
-  # Install script is versioned by its sha256sum value
-  curl -fsSL "${HERMIT_DIST_URL}/install-${INSTALL_SCRIPT_SHA256}.sh" -o "${INSTALL_SCRIPT}"
-  # Verify install script's sha256sum
-  openssl dgst -sha256 "${INSTALL_SCRIPT}" | \
-    awk -v EXPECTED="$INSTALL_SCRIPT_SHA256" \
-    '$2!=EXPECTED {print "Install script sha256 " $2 " does not match " EXPECTED; exit 1}'
+  if [ "${INSTALL_SCRIPT_SHA256}" = "BYPASS" ]; then
+    curl -fsSL "${HERMIT_DIST_URL}/install.sh" -o "${INSTALL_SCRIPT}"
+  else
+    # Install script is versioned by its sha256sum value
+    curl -fsSL "${HERMIT_DIST_URL}/install-${INSTALL_SCRIPT_SHA256}.sh" -o "${INSTALL_SCRIPT}"
+    # Verify install script's sha256sum
+    openssl dgst -sha256 "${INSTALL_SCRIPT}" | \
+      awk -v EXPECTED="$INSTALL_SCRIPT_SHA256" \
+      '$2!=EXPECTED {print "Install script sha256 " $2 " does not match " EXPECTED; exit 1}'
+  fi
   /bin/bash "${INSTALL_SCRIPT}" 1>&2
 fi
 

hermittest/envfixture.go

@@ -45,7 +45,7 @@ func NewEnvTestFixture(t *testing.T, handler http.Handler) *EnvTestFixture {
 
 	log, buf := ui.NewForTesting()
 
-	err = hermit.Init(log, envDir, "", stateDir, hermit.Config{})
+	err = hermit.Init(log, envDir, "", map[string]string{}, stateDir, hermit.Config{})
 	require.NoError(t, err)
 
 	server := httptest.NewServer(handler)
@@ -98,7 +98,7 @@ func (f *EnvTestFixture) NewEnv() *hermit.Env {
 	envDir, err := ioutil.TempDir("", "")
 	require.NoError(f.t, err)
 	log, _ := ui.NewForTesting()
-	err = hermit.Init(log, envDir, "", f.State.Root(), hermit.Config{})
+	err = hermit.Init(log, envDir, "", map[string]string{}, f.State.Root(), hermit.Config{})
 	require.NoError(f.t, err)
 	env, err := hermit.OpenEnv(envDir, f.State, f.Cache.GetSource, envars.Envars{}, f.Server.Client(), nil)
 	require.NoError(f.t, err)

it/common.sh

@@ -17,9 +17,12 @@ if [ ! -z $"HERMIT_EXE" ]; then
   unset HERMIT_EXE
 fi
 
-# Creates a "fake" release directory and sets HERMIT_DIST_URL to refer to it
+# Creates a "fake" release directory and sets HERMIT_DIST_URL to refer to it.
+# This function expects the basename of the first argument to be the
+# channel name, e.g., "release/canary" or "release/stable"
 fakeRelease() {
   DIR=$1
+  CHANNEL="$(basename "${DIR}")"
 
   echo "Compiling hermit"
   (
@@ -34,7 +37,7 @@ fakeRelease() {
   gzip -c hermit > "$DIR/hermit-${OS}-${ARCH}.gz"
   INSTALLER_VERSION=$(../../.hermit/go/bin/geninstaller \
     --dest="${DIR}/install.sh" \
-    --dist-url=https://github.com/cashapp/hermit/releases/download/stable)
+    --dist-url=https://github.com/cashapp/hermit/releases/download/"${CHANNEL}")
   cp "${DIR}/install.sh" "${DIR}/install-${INSTALLER_VERSION}.sh"
 
   export HERMIT_DIST_URL=file://$PWD/$DIR