From 236cf195ccc8419246462bf80da23697687e73cb Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Wed, 7 Dec 2022 10:22:25 +0100 Subject: [PATCH] clusterctl: fix goproxy to also return versions for major > 1 Also: * moves goproxy functionality to an internal package * reuses the package also for releaselink * fixes releaselink references for aws --- cmd/clusterctl/client/repository/goproxy.go | 163 -------------- .../client/repository/goproxy_test.go | 100 --------- .../client/repository/repository_github.go | 24 +- .../repository/repository_github_test.go | 26 ++- docs/book/src/clusterctl/commands/init.md | 2 + docs/book/src/clusterctl/overview.md | 2 + docs/book/src/clusterctl/provider-contract.md | 2 + docs/book/src/user/quick-start.md | 6 +- hack/tools/mdbook/releaselink/releaselink.go | 32 +-- internal/goproxy/doc.go | 18 ++ internal/goproxy/goproxy.go | 192 ++++++++++++++++ internal/goproxy/goproxy_test.go | 210 ++++++++++++++++++ 12 files changed, 481 insertions(+), 296 deletions(-) delete mode 100644 cmd/clusterctl/client/repository/goproxy.go delete mode 100644 cmd/clusterctl/client/repository/goproxy_test.go create mode 100644 internal/goproxy/doc.go create mode 100644 internal/goproxy/goproxy.go create mode 100644 internal/goproxy/goproxy_test.go diff --git a/cmd/clusterctl/client/repository/goproxy.go b/cmd/clusterctl/client/repository/goproxy.go deleted file mode 100644 index 08e8e0fc1b4e..000000000000 --- a/cmd/clusterctl/client/repository/goproxy.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package repository - -import ( - "context" - "io" - "net/http" - "net/url" - "path" - "path/filepath" - "sort" - "strings" - - "github.com/blang/semver" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/wait" -) - -const ( - defaultGoProxyHost = "proxy.golang.org" -) - -type goproxyClient struct { - scheme string - host string -} - -func newGoproxyClient(scheme, host string) *goproxyClient { - return &goproxyClient{ - scheme: scheme, - host: host, - } -} - -func (g *goproxyClient) getVersions(ctx context.Context, base, owner, repository string) ([]string, error) { - // A goproxy is also able to handle the github repository path instead of the actual go module name. - gomodulePath := path.Join(base, owner, repository) - - rawURL := url.URL{ - Scheme: g.scheme, - Host: g.host, - Path: path.Join(gomodulePath, "@v", "/list"), - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL.String(), http.NoBody) - if err != nil { - return nil, errors.Wrapf(err, "failed to get versions: failed to create request") - } - - var rawResponse []byte - var retryError error - _ = wait.PollImmediateWithContext(ctx, retryableOperationInterval, retryableOperationTimeout, func(ctx context.Context) (bool, error) { - retryError = nil - - resp, err := http.DefaultClient.Do(req) - if err != nil { - retryError = errors.Wrapf(err, "failed to get versions: failed to do request") - return false, nil - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - retryError = errors.Errorf("failed to get versions: response status code %d", resp.StatusCode) - return false, nil - } - - rawResponse, err = io.ReadAll(resp.Body) - if err != nil { - retryError = errors.Wrap(err, "failed to get versions: error reading goproxy response body") - return false, nil - } - return true, nil - }) - if retryError != nil { - return nil, retryError - } - - parsedVersions := semver.Versions{} - for _, s := range strings.Split(string(rawResponse), "\n") { - if s == "" { - continue - } - parsedVersion, err := semver.ParseTolerant(s) - if err != nil { - // Discard releases with tags that are not a valid semantic versions (the user can point explicitly to such releases). - continue - } - parsedVersions = append(parsedVersions, parsedVersion) - } - - sort.Sort(parsedVersions) - - versions := []string{} - for _, v := range parsedVersions { - versions = append(versions, "v"+v.String()) - } - - return versions, nil -} - -// getGoproxyHost detects and returns the scheme and host for goproxy requests. -// It returns empty strings if goproxy is disabled via `off` or `direct` values. -func getGoproxyHost(goproxy string) (string, string, error) { - // Fallback to default - if goproxy == "" { - return "https", defaultGoProxyHost, nil - } - - var goproxyHost, goproxyScheme string - // xref https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/proxy.go - for goproxy != "" { - var rawURL string - if i := strings.IndexAny(goproxy, ",|"); i >= 0 { - rawURL = goproxy[:i] - goproxy = goproxy[i+1:] - } else { - rawURL = goproxy - goproxy = "" - } - - rawURL = strings.TrimSpace(rawURL) - if rawURL == "" { - continue - } - if rawURL == "off" || rawURL == "direct" { - // Return nothing to fallback to github repository client without an error. - return "", "", nil - } - - // Single-word tokens are reserved for built-in behaviors, and anything - // containing the string ":/" or matching an absolute file path must be a - // complete URL. For all other paths, implicitly add "https://". - if strings.ContainsAny(rawURL, ".:/") && !strings.Contains(rawURL, ":/") && !filepath.IsAbs(rawURL) && !path.IsAbs(rawURL) { - rawURL = "https://" + rawURL - } - - parsedURL, err := url.Parse(rawURL) - if err != nil { - return "", "", errors.Wrapf(err, "parse GOPROXY url %q", rawURL) - } - goproxyHost = parsedURL.Host - goproxyScheme = parsedURL.Scheme - // A host was found so no need to continue. - break - } - - return goproxyScheme, goproxyHost, nil -} diff --git a/cmd/clusterctl/client/repository/goproxy_test.go b/cmd/clusterctl/client/repository/goproxy_test.go deleted file mode 100644 index 884ef24e362a..000000000000 --- a/cmd/clusterctl/client/repository/goproxy_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package repository - -import ( - "testing" - "time" -) - -func Test_getGoproxyHost(t *testing.T) { - retryableOperationInterval = 200 * time.Millisecond - retryableOperationTimeout = 1 * time.Second - - tests := []struct { - name string - envvar string - wantScheme string - wantHost string - wantErr bool - }{ - { - name: "defaulting", - envvar: "", - wantScheme: "https", - wantHost: "proxy.golang.org", - wantErr: false, - }, - { - name: "direct falls back to empty strings", - envvar: "direct", - wantScheme: "", - wantHost: "", - wantErr: false, - }, - { - name: "off falls back to empty strings", - envvar: "off", - wantScheme: "", - wantHost: "", - wantErr: false, - }, - { - name: "other goproxy", - envvar: "foo.bar.de", - wantScheme: "https", - wantHost: "foo.bar.de", - wantErr: false, - }, - { - name: "other goproxy comma separated, return first", - envvar: "foo.bar,foobar.barfoo", - wantScheme: "https", - wantHost: "foo.bar", - wantErr: false, - }, - { - name: "other goproxy including https scheme", - envvar: "https://foo.bar", - wantScheme: "https", - wantHost: "foo.bar", - wantErr: false, - }, - { - name: "other goproxy including http scheme", - envvar: "http://foo.bar", - wantScheme: "http", - wantHost: "foo.bar", - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotScheme, gotHost, err := getGoproxyHost(tt.envvar) - if (err != nil) != tt.wantErr { - t.Errorf("getGoproxyHost() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotScheme != tt.wantScheme { - t.Errorf("getGoproxyHost() = %v, wantScheme %v", gotScheme, tt.wantScheme) - } - if gotHost != tt.wantHost { - t.Errorf("getGoproxyHost() = %v, wantHost %v", gotHost, tt.wantHost) - } - }) - } -} diff --git a/cmd/clusterctl/client/repository/repository_github.go b/cmd/clusterctl/client/repository/repository_github.go index c7f97ae36597..ac747c20792e 100644 --- a/cmd/clusterctl/client/repository/repository_github.go +++ b/cmd/clusterctl/client/repository/repository_github.go @@ -23,10 +23,12 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strings" "time" + "github.com/blang/semver" "github.com/google/go-github/v45/github" "github.com/pkg/errors" "golang.org/x/oauth2" @@ -36,6 +38,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + "sigs.k8s.io/cluster-api/internal/goproxy" ) const ( @@ -70,7 +73,7 @@ type gitHubRepository struct { rootPath string componentsPath string injectClient *github.Client - injectGoproxyClient *goproxyClient + injectGoproxyClient *goproxy.Client } var _ Repository = &gitHubRepository{} @@ -83,7 +86,7 @@ func injectGithubClient(c *github.Client) githubRepositoryOption { } } -func injectGoproxyClient(c *goproxyClient) githubRepositoryOption { +func injectGoproxyClient(c *goproxy.Client) githubRepositoryOption { return func(g *gitHubRepository) { g.injectGoproxyClient = c } @@ -110,11 +113,20 @@ func (g *gitHubRepository) GetVersions() ([]string, error) { var versions []string if goProxyClient != nil { - versions, err = goProxyClient.getVersions(context.TODO(), githubDomain, g.owner, g.repository) + // A goproxy is also able to handle the github repository path instead of the actual go module name. + gomodulePath := path.Join(githubDomain, g.owner, g.repository) + + var parsedVersions semver.Versions + parsedVersions, err = goProxyClient.GetVersions(context.TODO(), gomodulePath) + // Log the error before fallback to github repository client happens. if err != nil { log.V(5).Info("error using Goproxy client to list versions for repository, falling back to github client", "owner", g.owner, "repository", g.repository, "error", err) } + + for _, v := range parsedVersions { + versions = append(versions, "v"+v.String()) + } } // Fallback to github repository client if goProxyClient is nil or an error occurred. @@ -239,11 +251,11 @@ func (g *gitHubRepository) getClient() *github.Client { // getGoproxyClient returns a go proxy client. // It returns nil, nil if the environment variable is set to `direct` or `off` // to skip goproxy requests. -func (g *gitHubRepository) getGoproxyClient() (*goproxyClient, error) { +func (g *gitHubRepository) getGoproxyClient() (*goproxy.Client, error) { if g.injectGoproxyClient != nil { return g.injectGoproxyClient, nil } - scheme, host, err := getGoproxyHost(os.Getenv("GOPROXY")) + scheme, host, err := goproxy.GetSchemeAndHost(os.Getenv("GOPROXY")) if err != nil { return nil, err } @@ -251,7 +263,7 @@ func (g *gitHubRepository) getGoproxyClient() (*goproxyClient, error) { if scheme == "" && host == "" { return nil, nil } - return newGoproxyClient(scheme, host), nil + return goproxy.NewClient(scheme, host), nil } // setClientToken sets authenticatingHTTPClient field of gitHubRepository struct. diff --git a/cmd/clusterctl/client/repository/repository_github_test.go b/cmd/clusterctl/client/repository/repository_github_test.go index 356523891e83..49c5eac8217a 100644 --- a/cmd/clusterctl/client/repository/repository_github_test.go +++ b/cmd/clusterctl/client/repository/repository_github_test.go @@ -31,6 +31,7 @@ import ( clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test" + "sigs.k8s.io/cluster-api/internal/goproxy" ) func Test_gitHubRepository_GetVersions(t *testing.T) { @@ -63,6 +64,21 @@ func Test_gitHubRepository_GetVersions(t *testing.T) { fmt.Fprint(w, "v0.3.1\n") }) + // setup an handler for returning 3 different major fake releases + muxGoproxy.HandleFunc("/github.com/o/r3/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v1.0.0\n") + fmt.Fprint(w, "v0.1.0\n") + }) + muxGoproxy.HandleFunc("/github.com/o/r3/v2/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v2.0.0\n") + }) + muxGoproxy.HandleFunc("/github.com/o/r3/v3/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v3.0.0\n") + }) + configVariablesClient := test.NewFakeVariableClient() tests := []struct { @@ -83,6 +99,12 @@ func Test_gitHubRepository_GetVersions(t *testing.T) { want: []string{"v0.3.1", "v0.3.2", "v0.4.0", "v0.5.0"}, wantErr: false, }, + { + name: "use goproxy having multiple majors", + providerConfig: config.NewProvider("test", "https://github.com/o/r3/releases/v3.0.0/path", clusterctlv1.CoreProviderType), + want: []string{"v0.1.0", "v1.0.0", "v2.0.0", "v3.0.0"}, + wantErr: false, + }, { name: "failure", providerConfig: config.NewProvider("test", "https://github.com/o/unknown/releases/v0.4.0/path", clusterctlv1.CoreProviderType), @@ -812,7 +834,7 @@ func resetCaches() { // newFakeGoproxy sets up a test HTTP server along with a github.Client that is // configured to talk to that test server. Tests should register handlers on // mux which provide mock responses for the API method being tested. -func newFakeGoproxy() (client *goproxyClient, mux *http.ServeMux, teardown func()) { +func newFakeGoproxy() (client *goproxy.Client, mux *http.ServeMux, teardown func()) { // mux is the HTTP request multiplexer used with the test server. mux = http.NewServeMux() @@ -824,5 +846,5 @@ func newFakeGoproxy() (client *goproxyClient, mux *http.ServeMux, teardown func( // client is the GitHub client being tested and is configured to use test server. url, _ := url.Parse(server.URL + "/") - return &goproxyClient{scheme: url.Scheme, host: url.Host}, mux, server.Close + return goproxy.NewClient(url.Scheme, url.Host), mux, server.Close } diff --git a/docs/book/src/clusterctl/commands/init.md b/docs/book/src/clusterctl/commands/init.md index d8dac4655fd8..ee4b9501f084 100644 --- a/docs/book/src/clusterctl/commands/init.md +++ b/docs/book/src/clusterctl/commands/init.md @@ -117,6 +117,8 @@ API calls to the GitHub API. It is possible to configure the go proxy url using for go itself (defaults to `https://proxy.golang.org`). To immediately fallback to the GitHub client and not use a go proxy, the environment variable could get set to `GOPROXY=off` or `GOPROXY=direct`. +If a provider does not follow Go's semantic versioning, `clusterctl` may fail when detecting the correct version. +In such cases, disabling the go proxy functionality via `GOPROXY=off` should be considered. See [clusterctl configuration](../configuration.md) for more info about provider repository configurations. diff --git a/docs/book/src/clusterctl/overview.md b/docs/book/src/clusterctl/overview.md index 84d58ed8c97d..e4cbaaf15117 100644 --- a/docs/book/src/clusterctl/overview.md +++ b/docs/book/src/clusterctl/overview.md @@ -33,6 +33,8 @@ API calls to the GitHub API. It is possible to configure the go proxy url using for go itself (defaults to `https://proxy.golang.org`). To immediately fallback to the GitHub client and not use a go proxy, the environment variable could get set to `GOPROXY=off` or `GOPROXY=direct`. +If a provider does not follow Go's semantic versioning, `clusterctl` may fail when detecting the correct version. +In such cases, disabling the go proxy functionality via `GOPROXY=off` should be considered. # Installing clusterctl Instructions are available in the [Quick Start](../user/quick-start.md#install-clusterctl). diff --git a/docs/book/src/clusterctl/provider-contract.md b/docs/book/src/clusterctl/provider-contract.md index ba132d0ceeb8..841e2082b126 100644 --- a/docs/book/src/clusterctl/provider-contract.md +++ b/docs/book/src/clusterctl/provider-contract.md @@ -54,6 +54,8 @@ API calls to the GitHub API. It is possible to configure the go proxy url using for go itself (defaults to `https://proxy.golang.org`). To immediately fallback to the GitHub client and not use a go proxy, the environment variable could get set to `GOPROXY=off` or `GOPROXY=direct`. +If a provider does not follow Go's semantic versioning, `clusterctl` may fail when detecting the correct version. +In such cases, disabling the go proxy functionality via `GOPROXY=off` should be considered. #### Creating a provider repository on GitLab diff --git a/docs/book/src/user/quick-start.md b/docs/book/src/user/quick-start.md index 419fdb7fd578..e1a21f8793b9 100644 --- a/docs/book/src/user/quick-start.md +++ b/docs/book/src/user/quick-start.md @@ -262,7 +262,7 @@ Download the latest binary of `clusterawsadm` from the [AWS provider releases]. Download the latest release; on linux, type: ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-linux-amd64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-linux-amd64" version:">=1.0.0"}} -o clusterawsadm ``` Make it executable @@ -284,12 +284,12 @@ clusterawsadm version Download the latest release; on macOs, type: ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-amd64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-amd64" version:">=1.0.0"}} -o clusterawsadm ``` Or if your Mac has an M1 CPU (”Apple Silicon”): ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-arm64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-arm64" version:">=1.0.0"}} -o clusterawsadm ``` Make it executable diff --git a/hack/tools/mdbook/releaselink/releaselink.go b/hack/tools/mdbook/releaselink/releaselink.go index c18b1cfd0fc0..4bd707e196e6 100644 --- a/hack/tools/mdbook/releaselink/releaselink.go +++ b/hack/tools/mdbook/releaselink/releaselink.go @@ -21,20 +21,18 @@ limitations under the License. package main import ( + "context" "fmt" - "io" "log" - "net/http" - "net/url" "os" - "path" "reflect" - "sort" "strings" "github.com/blang/semver" "golang.org/x/tools/go/vcs" "sigs.k8s.io/kubebuilder/docs/book/utils/plugin" + + "sigs.k8s.io/cluster-api/internal/goproxy" ) // ReleaseLink responds to {{#releaselink }} input. It asks for a `gomodule` parameter @@ -58,36 +56,26 @@ func (l ReleaseLink) Process(input *plugin.Input) error { versionRange := semver.MustParseRange(tags.Get("version")) includePrereleases := tags.Get("prereleases") == "true" - repo, err := vcs.RepoRootForImportPath(gomodule, false) + scheme, host, err := goproxy.GetSchemeAndHost(os.Getenv("GOPROXY")) if err != nil { return "", err } - - rawURL := url.URL{ - Scheme: "https", - Host: "proxy.golang.org", - Path: path.Join(gomodule, "@v", "/list"), + if scheme == "" || host == "" { + return "", fmt.Errorf("releaselink does not support disabling the go proxy: GOPROXY=%q", os.Getenv("GOPROXY")) } - resp, err := http.Get(rawURL.String()) //nolint:noctx // NB: as we're just implementing an external interface we won't be able to get a context here. + goproxyClient := goproxy.NewClient(scheme, host) + + repo, err := vcs.RepoRootForImportPath(gomodule, false) if err != nil { return "", err } - defer resp.Body.Close() - out, err := io.ReadAll(resp.Body) + parsedTags, err := goproxyClient.GetVersions(context.Background(), gomodule) if err != nil { return "", err } - parsedTags := semver.Versions{} - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, "v") { - parsedTags = append(parsedTags, semver.MustParse(strings.TrimPrefix(line, "v"))) - } - } - sort.Sort(parsedTags) - var picked semver.Version for i, tag := range parsedTags { if !includePrereleases && len(tag.Pre) > 0 { diff --git a/internal/goproxy/doc.go b/internal/goproxy/doc.go new file mode 100644 index 000000000000..3fb5d94638be --- /dev/null +++ b/internal/goproxy/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package goproxy implements a goproxy client. +package goproxy diff --git a/internal/goproxy/goproxy.go b/internal/goproxy/goproxy.go new file mode 100644 index 000000000000..d79958a27771 --- /dev/null +++ b/internal/goproxy/goproxy.go @@ -0,0 +1,192 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goproxy + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/blang/semver" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + defaultGoProxyHost = "proxy.golang.org" +) + +var ( + retryableOperationInterval = 10 * time.Second + retryableOperationTimeout = 1 * time.Minute +) + +// Client is a client to query versions from a goproxy instance. +type Client struct { + scheme string + host string +} + +// NewClient returns a new goproxyClient instance. +func NewClient(scheme, host string) *Client { + return &Client{ + scheme: scheme, + host: host, + } +} + +// GetVersions returns the a sorted list of semantical versions which exist for a go module. +func (g *Client) GetVersions(ctx context.Context, gomodulePath string) (semver.Versions, error) { + parsedVersions := semver.Versions{} + + majorVersionNumber := 1 + var majorVersion string + for { + if majorVersionNumber > 1 { + majorVersion = fmt.Sprintf("v%d", majorVersionNumber) + } + rawURL := url.URL{ + Scheme: g.scheme, + Host: g.host, + Path: path.Join(gomodulePath, majorVersion, "@v", "/list"), + } + majorVersionNumber++ + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL.String(), http.NoBody) + if err != nil { + return nil, errors.Wrapf(err, "failed to get versions: failed to create request") + } + + var rawResponse []byte + var responseStatusCode int + var retryError error + _ = wait.PollImmediateWithContext(ctx, retryableOperationInterval, retryableOperationTimeout, func(ctx context.Context) (bool, error) { + retryError = nil + + resp, err := http.DefaultClient.Do(req) + if err != nil { + retryError = errors.Wrapf(err, "failed to get versions: failed to do request") + return false, nil + } + defer resp.Body.Close() + + responseStatusCode = resp.StatusCode + + // Status codes OK and NotFound are expected results: + // * OK indicates that we got a list of versions to read. + // * NotFound indicates that there are no versions for this module / modules major version. + if responseStatusCode != http.StatusOK && responseStatusCode != http.StatusNotFound { + retryError = errors.Errorf("failed to get versions: response status code %d", resp.StatusCode) + return false, nil + } + + // only read the response for http.StatusOK + if responseStatusCode == http.StatusOK { + rawResponse, err = io.ReadAll(resp.Body) + if err != nil { + retryError = errors.Wrap(err, "failed to get versions: error reading goproxy response body") + return false, nil + } + } + return true, nil + }) + if retryError != nil { + return nil, retryError + } + + // Don't try to read the versions if status was not found. + if responseStatusCode == http.StatusNotFound { + break + } + + for _, s := range strings.Split(string(rawResponse), "\n") { + if s == "" { + continue + } + parsedVersion, err := semver.ParseTolerant(s) + if err != nil { + // Discard releases with tags that are not a valid semantic versions (the user can point explicitly to such releases). + continue + } + parsedVersions = append(parsedVersions, parsedVersion) + } + } + + if len(parsedVersions) == 0 { + return nil, fmt.Errorf("no versions found for go module %q", gomodulePath) + } + + sort.Sort(parsedVersions) + + return parsedVersions, nil +} + +// GetSchemeAndHost detects and returns the scheme and host for goproxy requests. +// It returns empty strings if goproxy is disabled via `off` or `direct` values. +func GetSchemeAndHost(goproxy string) (string, string, error) { + // Fallback to default + if goproxy == "" { + return "https", defaultGoProxyHost, nil + } + + var goproxyHost, goproxyScheme string + // xref https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/proxy.go + for goproxy != "" { + var rawURL string + if i := strings.IndexAny(goproxy, ",|"); i >= 0 { + rawURL = goproxy[:i] + goproxy = goproxy[i+1:] + } else { + rawURL = goproxy + goproxy = "" + } + + rawURL = strings.TrimSpace(rawURL) + if rawURL == "" { + continue + } + if rawURL == "off" || rawURL == "direct" { + // Return nothing to fallback to github repository client without an error. + return "", "", nil + } + + // Single-word tokens are reserved for built-in behaviors, and anything + // containing the string ":/" or matching an absolute file path must be a + // complete URL. For all other paths, implicitly add "https://". + if strings.ContainsAny(rawURL, ".:/") && !strings.Contains(rawURL, ":/") && !filepath.IsAbs(rawURL) && !path.IsAbs(rawURL) { + rawURL = "https://" + rawURL + } + + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", "", errors.Wrapf(err, "parse GOPROXY url %q", rawURL) + } + goproxyHost = parsedURL.Host + goproxyScheme = parsedURL.Scheme + // A host was found so no need to continue. + break + } + + return goproxyScheme, goproxyHost, nil +} diff --git a/internal/goproxy/goproxy_test.go b/internal/goproxy/goproxy_test.go new file mode 100644 index 000000000000..c914fa6f26e1 --- /dev/null +++ b/internal/goproxy/goproxy_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goproxy + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/blang/semver" + . "github.com/onsi/gomega" +) + +func TestClient_GetVersions(t *testing.T) { + retryableOperationInterval = 200 * time.Millisecond + retryableOperationTimeout = 1 * time.Second + + clientGoproxy, muxGoproxy, teardownGoproxy := NewFakeGoproxy() + defer teardownGoproxy() + + // setup an handler for returning 2 fake releases + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v1.1.0\n") + fmt.Fprint(w, "v0.2.0\n") + }) + + // setup an handler for returning 2 fake releases for v1 + muxGoproxy.HandleFunc("/github.com/o/r2/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v1.1.0\n") + fmt.Fprint(w, "v0.2.0\n") + }) + // setup an handler for returning 2 fake releases for v2 + muxGoproxy.HandleFunc("/github.com/o/r2/v2/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v2.0.1\n") + fmt.Fprint(w, "v2.0.0\n") + }) + + tests := []struct { + name string + gomodulePath string + want semver.Versions + wantErr bool + }{ + { + "No versions", + "github.com/o/doesntexist", + nil, + true, + }, + { + "Two versions < v2", + "github.com/o/r1", + semver.Versions{ + semver.MustParse("0.2.0"), + semver.MustParse("1.1.0"), + }, + false, + }, + { + "Multiple versiosn including > v1", + "github.com/o/r2", + semver.Versions{ + semver.MustParse("0.2.0"), + semver.MustParse("1.1.0"), + semver.MustParse("2.0.0"), + semver.MustParse("2.0.1"), + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + got, err := clientGoproxy.GetVersions(ctx, tt.gomodulePath) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(got).To(BeEquivalentTo(tt.want)) + }) + } +} + +func Test_GetGoproxyHost(t *testing.T) { + retryableOperationInterval = 200 * time.Millisecond + retryableOperationTimeout = 1 * time.Second + + tests := []struct { + name string + envvar string + wantScheme string + wantHost string + wantErr bool + }{ + { + name: "defaulting", + envvar: "", + wantScheme: "https", + wantHost: "proxy.golang.org", + wantErr: false, + }, + { + name: "direct falls back to empty strings", + envvar: "direct", + wantScheme: "", + wantHost: "", + wantErr: false, + }, + { + name: "off falls back to empty strings", + envvar: "off", + wantScheme: "", + wantHost: "", + wantErr: false, + }, + { + name: "other goproxy", + envvar: "foo.bar.de", + wantScheme: "https", + wantHost: "foo.bar.de", + wantErr: false, + }, + { + name: "other goproxy comma separated, return first", + envvar: "foo.bar,foobar.barfoo", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including https scheme", + envvar: "https://foo.bar", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including http scheme", + envvar: "http://foo.bar", + wantScheme: "http", + wantHost: "foo.bar", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + gotScheme, gotHost, err := GetSchemeAndHost(tt.envvar) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(gotScheme).To(Equal(tt.wantScheme)) + g.Expect(gotHost).To(Equal(tt.wantHost)) + }) + } +} + +// NewFakeGoproxy sets up a test HTTP server along with a github.Client that is +// configured to talk to that test server. Tests should register handlers on +// mux which provide mock responses for the API method being tested. +func NewFakeGoproxy() (client *Client, mux *http.ServeMux, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + // client is the GitHub client being tested and is configured to use test server. + url, _ := url.Parse(server.URL + "/") + return NewClient(url.Scheme, url.Host), mux, server.Close +} + +func testMethod(t *testing.T, r *http.Request, want string) { + t.Helper() + + if got := r.Method; got != want { + t.Errorf("Request method: %v, want %v", got, want) + } +}