From 903f846682a22e372a271743830cb83998b4ef99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 26 Aug 2021 17:24:52 +0200 Subject: [PATCH] Update to Go 1.17 Go 1.17 now lazy-loads dependencies when `go.mod` is version `go17`. This does not work for us for new projects started with `hugo mod init`. Before this commit, starting a project with Go 1.17 with `hugo mod init` and then start adding dependencies with transitive dependenies to `config.toml` would treat the transitive dependencies as new, and you would potentially get a too recent version of those. Note that this does not effect existing projects, where all dependencies are already recorded in `go.mod`. Fixes #8930 --- .circleci/config.yml | 2 +- .github/workflows/test.yml | 2 +- htesting/test_helpers.go | 27 ++++++++++ htesting/test_helpers_test.go | 31 +++++++++++ modules/client.go | 98 +++++++++++++++++++++++++++++------ modules/collect.go | 9 +++- snap/snapcraft.yaml | 2 +- 7 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 htesting/test_helpers_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 900d973f2bf..9dd73a51806 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ defaults: &defaults docker: - - image: bepsays/ci-goreleaser:1.16.7 + - image: bepsays/ci-goreleaser:1.17.0 environment: CGO_ENABLED: "0" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20e5e8c10fe..32ff044973a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: GO111MODULE: on strategy: matrix: - go-version: [1.16.x] + go-version: [1.16.x, 1.17.x] os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index 813c9bd04ca..9a1fe86efa3 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -16,7 +16,9 @@ package htesting import ( "math/rand" "os" + "regexp" "runtime" + "strconv" "strings" "time" @@ -103,3 +105,28 @@ func IsGitHubAction() bool { func SupportsAll() bool { return IsGitHubAction() } + +// GoMinorVersion returns the minor version of the current Go version, +// e.g. 16 for Go 1.16. +func GoMinorVersion() int { + return extractMinorVersionFromGoTag(runtime.Version()) +} + +var goMinorVersionRe = regexp.MustCompile(`go1.(\d*)`) + +func extractMinorVersionFromGoTag(tag string) int { + // The tag may be on the form go1.17, go1.17.5 go1.17rc2 -- or just a commit hash. + match := goMinorVersionRe.FindStringSubmatch(tag) + + if len(match) == 2 { + i, err := strconv.Atoi(match[1]) + if err != nil { + return -1 + } + return i + } + + // a commit hash, not useful. + return -1 + +} diff --git a/htesting/test_helpers_test.go b/htesting/test_helpers_test.go new file mode 100644 index 00000000000..3e767ac9d85 --- /dev/null +++ b/htesting/test_helpers_test.go @@ -0,0 +1,31 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package htesting + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestExtractMinorVersionFromGoTag(t *testing.T) { + + c := qt.New(t) + + c.Assert(extractMinorVersionFromGoTag("go1.17"), qt.Equals, 17) + c.Assert(extractMinorVersionFromGoTag("go1.16.7"), qt.Equals, 16) + c.Assert(extractMinorVersionFromGoTag("go1.17beta1"), qt.Equals, 17) + c.Assert(extractMinorVersionFromGoTag("asdfadf"), qt.Equals, -1) + +} diff --git a/modules/client.go b/modules/client.go index 73c3242a805..be40663aae4 100644 --- a/modules/client.go +++ b/modules/client.go @@ -318,7 +318,7 @@ func (c *Client) Get(args ...string) error { } var args []string if update { - args = []string{"-u"} + args = append(args, "-u") } args = append(args, m.Path) if err := c.get(args...); err != nil { @@ -333,6 +333,18 @@ func (c *Client) Get(args ...string) error { } func (c *Client) get(args ...string) error { + var hasD bool + for _, arg := range args { + if arg == "-d" { + hasD = true + break + } + } + if !hasD { + // go get without the -d flag does not make sense to us, as it + // it will try to build and install go packages. + args = append([]string{"-d"}, args...) + } if err := c.runGo(context.Background(), c.logger.Out(), append([]string{"get"}, args...)...); err != nil { errors.Wrapf(err, "failed to get %q", args) } @@ -425,31 +437,83 @@ func (c *Client) listGoMods() (goModules, error) { return nil, nil } - out := ioutil.Discard - err := c.runGo(context.Background(), out, "mod", "download") - if err != nil { - return nil, errors.Wrap(err, "failed to download modules") + downloadModules := func(modules ...string) error { + args := []string{"mod", "download"} + args = append(args, modules...) + out := ioutil.Discard + err := c.runGo(context.Background(), out, args...) + if err != nil { + return errors.Wrap(err, "failed to download modules") + } + return nil } - b := &bytes.Buffer{} - err = c.runGo(context.Background(), b, "list", "-m", "-json", "all") - if err != nil { - return nil, errors.Wrap(err, "failed to list modules") + if err := downloadModules(); err != nil { + return nil, err } - var modules goModules + listAndDecodeModules := func(handle func(m *goModule) error, modules ...string) error { + b := &bytes.Buffer{} + args := []string{"list", "-m", "-json"} + if len(modules) > 0 { + args = append(args, modules...) + } else { + args = append(args, "all") + } + err := c.runGo(context.Background(), b, args...) + if err != nil { + return errors.Wrap(err, "failed to list modules") + } + + dec := json.NewDecoder(b) + for { + m := &goModule{} + if err := dec.Decode(m); err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "failed to decode modules list") + } - dec := json.NewDecoder(b) - for { - m := &goModule{} - if err := dec.Decode(m); err != nil { - if err == io.EOF { - break + if err := handle(m); err != nil { + return err } - return nil, errors.Wrap(err, "failed to decode modules list") } + return nil + } + var modules goModules + err := listAndDecodeModules(func(m *goModule) error { modules = append(modules, m) + return nil + }) + if err != nil { + return nil, err + } + + // From Go 1.17, go lazy loads transitive dependencies. + // That does not work for us. + // So, download these modules and update the Dir in the modules list. + var modulesToDownload []string + for _, m := range modules { + if m.Dir == "" { + modulesToDownload = append(modulesToDownload, fmt.Sprintf("%s@%s", m.Path, m.Version)) + } + } + + if len(modulesToDownload) > 0 { + if err := downloadModules(modulesToDownload...); err != nil { + return nil, err + } + err := listAndDecodeModules(func(m *goModule) error { + if mm := modules.GetByPath(m.Path); mm != nil { + mm.Dir = m.Dir + } + return nil + }, modulesToDownload...) + if err != nil { + return nil, err + } } return modules, err diff --git a/modules/collect.go b/modules/collect.go index 29bf057a684..8798f45b118 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -252,15 +252,22 @@ func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool } if moduleDir == "" { + var versionQuery string mod = c.gomods.GetByPath(modulePath) if mod != nil { moduleDir = mod.Dir + versionQuery = mod.Version } if moduleDir == "" { if c.GoModulesFilename != "" && isProbablyModule(modulePath) { // Try to "go get" it and reload the module configuration. - if err := c.Get(modulePath); err != nil { + if versionQuery == "" { + // See https://golang.org/ref/mod#version-queries + // This will select the latest release-version (not beta etc.). + versionQuery = "upgrade" + } + if err := c.Get(fmt.Sprintf("%s@%s", modulePath, versionQuery)); err != nil { return nil, err } if err := c.loadModules(); err != nil { diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ecf7063efeb..6eac98c2dc0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -31,7 +31,7 @@ parts: hugo: plugin: nil - build-snaps: [go/1.16/stable] + build-snaps: [go/1.17/stable] source: . override-build: | set -ex