From 3203984ad1a323bf972cd03a92e019aaefc1b3bb Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 9 Dec 2022 12:42:46 +0100 Subject: [PATCH] move goproxy getVersions to a separate package and reuse at releaselink --- .../client/repository/goproxy_test.go | 100 --------- .../client/repository/repository_github.go | 22 +- .../repository/repository_github_test.go | 5 +- docs/book/src/user/quick-start.md | 6 +- hack/tools/mdbook/releaselink/releaselink.go | 29 +-- internal/goproxy/doc.go | 18 ++ .../goproxy}/goproxy.go | 32 +-- internal/goproxy/goproxy_test.go | 210 ++++++++++++++++++ 8 files changed, 273 insertions(+), 149 deletions(-) delete mode 100644 cmd/clusterctl/client/repository/goproxy_test.go create mode 100644 internal/goproxy/doc.go rename {cmd/clusterctl/client/repository => internal/goproxy}/goproxy.go (87%) create mode 100644 internal/goproxy/goproxy_test.go diff --git a/cmd/clusterctl/client/repository/goproxy_test.go b/cmd/clusterctl/client/repository/goproxy_test.go deleted file mode 100644 index 884ef24e362a..000000000000 --- a/cmd/clusterctl/client/repository/goproxy_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package repository - -import ( - "testing" - "time" -) - -func Test_getGoproxyHost(t *testing.T) { - retryableOperationInterval = 200 * time.Millisecond - retryableOperationTimeout = 1 * time.Second - - tests := []struct { - name string - envvar string - wantScheme string - wantHost string - wantErr bool - }{ - { - name: "defaulting", - envvar: "", - wantScheme: "https", - wantHost: "proxy.golang.org", - wantErr: false, - }, - { - name: "direct falls back to empty strings", - envvar: "direct", - wantScheme: "", - wantHost: "", - wantErr: false, - }, - { - name: "off falls back to empty strings", - envvar: "off", - wantScheme: "", - wantHost: "", - wantErr: false, - }, - { - name: "other goproxy", - envvar: "foo.bar.de", - wantScheme: "https", - wantHost: "foo.bar.de", - wantErr: false, - }, - { - name: "other goproxy comma separated, return first", - envvar: "foo.bar,foobar.barfoo", - wantScheme: "https", - wantHost: "foo.bar", - wantErr: false, - }, - { - name: "other goproxy including https scheme", - envvar: "https://foo.bar", - wantScheme: "https", - wantHost: "foo.bar", - wantErr: false, - }, - { - name: "other goproxy including http scheme", - envvar: "http://foo.bar", - wantScheme: "http", - wantHost: "foo.bar", - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotScheme, gotHost, err := getGoproxyHost(tt.envvar) - if (err != nil) != tt.wantErr { - t.Errorf("getGoproxyHost() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotScheme != tt.wantScheme { - t.Errorf("getGoproxyHost() = %v, wantScheme %v", gotScheme, tt.wantScheme) - } - if gotHost != tt.wantHost { - t.Errorf("getGoproxyHost() = %v, wantHost %v", gotHost, tt.wantHost) - } - }) - } -} diff --git a/cmd/clusterctl/client/repository/repository_github.go b/cmd/clusterctl/client/repository/repository_github.go index c7f97ae36597..ecb32a8e2995 100644 --- a/cmd/clusterctl/client/repository/repository_github.go +++ b/cmd/clusterctl/client/repository/repository_github.go @@ -23,6 +23,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strings" "time" @@ -36,6 +37,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + "sigs.k8s.io/cluster-api/internal/goproxy" ) const ( @@ -70,7 +72,7 @@ type gitHubRepository struct { rootPath string componentsPath string injectClient *github.Client - injectGoproxyClient *goproxyClient + injectGoproxyClient *goproxy.Client } var _ Repository = &gitHubRepository{} @@ -83,7 +85,7 @@ func injectGithubClient(c *github.Client) githubRepositoryOption { } } -func injectGoproxyClient(c *goproxyClient) githubRepositoryOption { +func injectGoproxyClient(c *goproxy.Client) githubRepositoryOption { return func(g *gitHubRepository) { g.injectGoproxyClient = c } @@ -110,11 +112,19 @@ func (g *gitHubRepository) GetVersions() ([]string, error) { var versions []string if goProxyClient != nil { - versions, err = goProxyClient.getVersions(context.TODO(), githubDomain, g.owner, g.repository) + // A goproxy is also able to handle the github repository path instead of the actual go module name. + gomodulePath := path.Join(githubDomain, g.owner, g.repository) + + parsedVersions, err := goProxyClient.GetVersions(context.TODO(), gomodulePath) + // Log the error before fallback to github repository client happens. if err != nil { log.V(5).Info("error using Goproxy client to list versions for repository, falling back to github client", "owner", g.owner, "repository", g.repository, "error", err) } + + for _, v := range parsedVersions { + versions = append(versions, "v"+v.String()) + } } // Fallback to github repository client if goProxyClient is nil or an error occurred. @@ -239,11 +249,11 @@ func (g *gitHubRepository) getClient() *github.Client { // getGoproxyClient returns a go proxy client. // It returns nil, nil if the environment variable is set to `direct` or `off` // to skip goproxy requests. -func (g *gitHubRepository) getGoproxyClient() (*goproxyClient, error) { +func (g *gitHubRepository) getGoproxyClient() (*goproxy.Client, error) { if g.injectGoproxyClient != nil { return g.injectGoproxyClient, nil } - scheme, host, err := getGoproxyHost(os.Getenv("GOPROXY")) + scheme, host, err := goproxy.GetSchemeAndHost(os.Getenv("GOPROXY")) if err != nil { return nil, err } @@ -251,7 +261,7 @@ func (g *gitHubRepository) getGoproxyClient() (*goproxyClient, error) { if scheme == "" && host == "" { return nil, nil } - return newGoproxyClient(scheme, host), nil + return goproxy.NewGoproxyClient(scheme, host), nil } // setClientToken sets authenticatingHTTPClient field of gitHubRepository struct. diff --git a/cmd/clusterctl/client/repository/repository_github_test.go b/cmd/clusterctl/client/repository/repository_github_test.go index 417115081dae..0816a562b37c 100644 --- a/cmd/clusterctl/client/repository/repository_github_test.go +++ b/cmd/clusterctl/client/repository/repository_github_test.go @@ -31,6 +31,7 @@ import ( clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test" + "sigs.k8s.io/cluster-api/internal/goproxy" ) func Test_gitHubRepository_GetVersions(t *testing.T) { @@ -833,7 +834,7 @@ func resetCaches() { // newFakeGoproxy sets up a test HTTP server along with a github.Client that is // configured to talk to that test server. Tests should register handlers on // mux which provide mock responses for the API method being tested. -func newFakeGoproxy() (client *goproxyClient, mux *http.ServeMux, teardown func()) { +func newFakeGoproxy() (client *goproxy.Client, mux *http.ServeMux, teardown func()) { // mux is the HTTP request multiplexer used with the test server. mux = http.NewServeMux() @@ -845,5 +846,5 @@ func newFakeGoproxy() (client *goproxyClient, mux *http.ServeMux, teardown func( // client is the GitHub client being tested and is configured to use test server. url, _ := url.Parse(server.URL + "/") - return &goproxyClient{scheme: url.Scheme, host: url.Host}, mux, server.Close + return goproxy.NewGoproxyClient(url.Scheme, url.Host), mux, server.Close } diff --git a/docs/book/src/user/quick-start.md b/docs/book/src/user/quick-start.md index 69f114a119d6..e0cfa25cf37e 100644 --- a/docs/book/src/user/quick-start.md +++ b/docs/book/src/user/quick-start.md @@ -262,7 +262,7 @@ Download the latest binary of `clusterawsadm` from the [AWS provider releases]. Download the latest release; on Linux, type: ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-linux-amd64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-linux-amd64" version:">=1.0.0"}} -o clusterawsadm ``` Make it executable @@ -284,12 +284,12 @@ clusterawsadm version Download the latest release; on macOs, type: ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-amd64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-amd64" version:">=1.0.0"}} -o clusterawsadm ``` Or if your Mac has an M1 CPU (”Apple Silicon”): ``` -curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-arm64" version:"1.x"}} -o clusterawsadm +curl -L {{#releaselink gomodule:"sigs.k8s.io/cluster-api-provider-aws" asset:"clusterawsadm-darwin-arm64" version:">=1.0.0"}} -o clusterawsadm ``` Make it executable diff --git a/hack/tools/mdbook/releaselink/releaselink.go b/hack/tools/mdbook/releaselink/releaselink.go index c18b1cfd0fc0..69e2ff2fc2c2 100644 --- a/hack/tools/mdbook/releaselink/releaselink.go +++ b/hack/tools/mdbook/releaselink/releaselink.go @@ -21,19 +21,16 @@ limitations under the License. package main import ( + "context" "fmt" - "io" "log" - "net/http" - "net/url" "os" - "path" "reflect" - "sort" "strings" "github.com/blang/semver" "golang.org/x/tools/go/vcs" + "sigs.k8s.io/cluster-api/internal/goproxy" "sigs.k8s.io/kubebuilder/docs/book/utils/plugin" ) @@ -58,36 +55,22 @@ func (l ReleaseLink) Process(input *plugin.Input) error { versionRange := semver.MustParseRange(tags.Get("version")) includePrereleases := tags.Get("prereleases") == "true" - repo, err := vcs.RepoRootForImportPath(gomodule, false) + scheme, host, err := goproxy.GetSchemeAndHost(os.Getenv("GOPROXY")) if err != nil { return "", err } + goproxyClient := goproxy.NewGoproxyClient(scheme, host) - rawURL := url.URL{ - Scheme: "https", - Host: "proxy.golang.org", - Path: path.Join(gomodule, "@v", "/list"), - } - - resp, err := http.Get(rawURL.String()) //nolint:noctx // NB: as we're just implementing an external interface we won't be able to get a context here. + repo, err := vcs.RepoRootForImportPath(gomodule, false) if err != nil { return "", err } - defer resp.Body.Close() - out, err := io.ReadAll(resp.Body) + parsedTags, err := goproxyClient.GetVersions(context.Background(), gomodule) if err != nil { return "", err } - parsedTags := semver.Versions{} - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, "v") { - parsedTags = append(parsedTags, semver.MustParse(strings.TrimPrefix(line, "v"))) - } - } - sort.Sort(parsedTags) - var picked semver.Version for i, tag := range parsedTags { if !includePrereleases && len(tag.Pre) > 0 { diff --git a/internal/goproxy/doc.go b/internal/goproxy/doc.go new file mode 100644 index 000000000000..3fb5d94638be --- /dev/null +++ b/internal/goproxy/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package goproxy implements a goproxy client. +package goproxy diff --git a/cmd/clusterctl/client/repository/goproxy.go b/internal/goproxy/goproxy.go similarity index 87% rename from cmd/clusterctl/client/repository/goproxy.go rename to internal/goproxy/goproxy.go index bd0f415b7db8..e1e07cd1a553 100644 --- a/cmd/clusterctl/client/repository/goproxy.go +++ b/internal/goproxy/goproxy.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package repository +package goproxy import ( "context" @@ -26,6 +26,7 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/blang/semver" "github.com/pkg/errors" @@ -36,21 +37,27 @@ const ( defaultGoProxyHost = "proxy.golang.org" ) -type goproxyClient struct { +var ( + retryableOperationInterval = 10 * time.Second + retryableOperationTimeout = 1 * time.Minute +) + +// Client is a client to query versions from a goproxy instance. +type Client struct { scheme string host string } -func newGoproxyClient(scheme, host string) *goproxyClient { - return &goproxyClient{ +// NewGoproxyClient returns a new goproxyClient instance. +func NewGoproxyClient(scheme, host string) *Client { + return &Client{ scheme: scheme, host: host, } } -func (g *goproxyClient) getVersions(ctx context.Context, base, owner, repository string) ([]string, error) { - // A goproxy is also able to handle the github repository path instead of the actual go module name. - gomodulePath := path.Join(base, owner, repository) +// GetVersions returns the a sorted list of semantical versions which exist for a go module. +func (g *Client) GetVersions(ctx context.Context, gomodulePath string) (semver.Versions, error) { parsedVersions := semver.Versions{} majorVersionNumber := 1 @@ -132,17 +139,12 @@ func (g *goproxyClient) getVersions(ctx context.Context, base, owner, repository sort.Sort(parsedVersions) - versions := []string{} - for _, v := range parsedVersions { - versions = append(versions, "v"+v.String()) - } - - return versions, nil + return parsedVersions, nil } -// getGoproxyHost detects and returns the scheme and host for goproxy requests. +// GetSchemeAndHost detects and returns the scheme and host for goproxy requests. // It returns empty strings if goproxy is disabled via `off` or `direct` values. -func getGoproxyHost(goproxy string) (string, string, error) { +func GetSchemeAndHost(goproxy string) (string, string, error) { // Fallback to default if goproxy == "" { return "https", defaultGoProxyHost, nil diff --git a/internal/goproxy/goproxy_test.go b/internal/goproxy/goproxy_test.go new file mode 100644 index 000000000000..29600fc2e29d --- /dev/null +++ b/internal/goproxy/goproxy_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goproxy + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" + "time" + + "github.com/blang/semver" +) + +func TestClient_GetVersions(t *testing.T) { + retryableOperationInterval = 200 * time.Millisecond + retryableOperationTimeout = 1 * time.Second + + clientGoproxy, muxGoproxy, teardownGoproxy := NewFakeGoproxy() + defer teardownGoproxy() + + // setup an handler for returning 2 fake releases + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v1.1.0\n") + fmt.Fprint(w, "v0.2.0\n") + }) + + // setup an handler for returning 2 fake releases for v1 + muxGoproxy.HandleFunc("/github.com/o/r2/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v1.1.0\n") + fmt.Fprint(w, "v0.2.0\n") + }) + // setup an handler for returning 2 fake releases for v2 + muxGoproxy.HandleFunc("/github.com/o/r2/v2/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v2.0.1\n") + fmt.Fprint(w, "v2.0.0\n") + }) + + tests := []struct { + name string + gomodulePath string + want semver.Versions + wantErr bool + }{ + { + "No versions", + "github.com/o/doesntexist", + nil, + true, + }, + { + "Two versions < v2", + "github.com/o/r1", + semver.Versions{ + semver.MustParse("0.2.0"), + semver.MustParse("1.1.0"), + }, + false, + }, + { + "Multiple versiosn including > v1", + "github.com/o/r2", + semver.Versions{ + semver.MustParse("0.2.0"), + semver.MustParse("1.1.0"), + semver.MustParse("2.0.0"), + semver.MustParse("2.0.1"), + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + got, err := clientGoproxy.GetVersions(ctx, tt.gomodulePath) + if (err != nil) != tt.wantErr { + t.Errorf("Client.GetVersions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.GetVersions() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_GetGoproxyHost(t *testing.T) { + retryableOperationInterval = 200 * time.Millisecond + retryableOperationTimeout = 1 * time.Second + + tests := []struct { + name string + envvar string + wantScheme string + wantHost string + wantErr bool + }{ + { + name: "defaulting", + envvar: "", + wantScheme: "https", + wantHost: "proxy.golang.org", + wantErr: false, + }, + { + name: "direct falls back to empty strings", + envvar: "direct", + wantScheme: "", + wantHost: "", + wantErr: false, + }, + { + name: "off falls back to empty strings", + envvar: "off", + wantScheme: "", + wantHost: "", + wantErr: false, + }, + { + name: "other goproxy", + envvar: "foo.bar.de", + wantScheme: "https", + wantHost: "foo.bar.de", + wantErr: false, + }, + { + name: "other goproxy comma separated, return first", + envvar: "foo.bar,foobar.barfoo", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including https scheme", + envvar: "https://foo.bar", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including http scheme", + envvar: "http://foo.bar", + wantScheme: "http", + wantHost: "foo.bar", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotScheme, gotHost, err := GetSchemeAndHost(tt.envvar) + if (err != nil) != tt.wantErr { + t.Errorf("getGoproxyHost() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotScheme != tt.wantScheme { + t.Errorf("getGoproxyHost() = %v, wantScheme %v", gotScheme, tt.wantScheme) + } + if gotHost != tt.wantHost { + t.Errorf("getGoproxyHost() = %v, wantHost %v", gotHost, tt.wantHost) + } + }) + } +} + +// NewFakeGoproxy sets up a test HTTP server along with a github.Client that is +// configured to talk to that test server. Tests should register handlers on +// mux which provide mock responses for the API method being tested. +func NewFakeGoproxy() (client *Client, mux *http.ServeMux, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + // client is the GitHub client being tested and is configured to use test server. + url, _ := url.Parse(server.URL + "/") + return NewGoproxyClient(url.Scheme, url.Host), mux, server.Close +} + +func testMethod(t *testing.T, r *http.Request, want string) { + t.Helper() + + if got := r.Method; got != want { + t.Errorf("Request method: %v, want %v", got, want) + } +}