diff --git a/cmd/clusterctl/client/cluster/cert_manager.go b/cmd/clusterctl/client/cluster/cert_manager.go index 5539845eee7c..f37761a21019 100644 --- a/cmd/clusterctl/client/cluster/cert_manager.go +++ b/cmd/clusterctl/client/cluster/cert_manager.go @@ -19,6 +19,7 @@ package cluster import ( "context" _ "embed" + "strings" "time" "github.com/blang/semver" @@ -401,7 +402,9 @@ func (cm *certManagerClient) getManifestObjs(certManagerConfig config.CertManage // Given that cert manager components yaml are stored in a repository like providers components yaml, // we are using the same machinery to retrieve the file by using a fake provider object using // the cert manager repository url. - certManagerFakeProvider := config.NewProvider("cert-manager", certManagerConfig.URL(), "") + // We replace "latest" in the url by the wanted version to disable auto version-discovery in repositoryClientFactory. + url := strings.Replace(certManagerConfig.URL(), "latest", certManagerConfig.Version(), 1) + certManagerFakeProvider := config.NewProvider("cert-manager", url, "") certManagerRepository, err := cm.repositoryClientFactory(certManagerFakeProvider, cm.configClient) if err != nil { return nil, err diff --git a/cmd/clusterctl/client/repository/goproxy.go b/cmd/clusterctl/client/repository/goproxy.go new file mode 100644 index 000000000000..e4fed1b07e44 --- /dev/null +++ b/cmd/clusterctl/client/repository/goproxy.go @@ -0,0 +1,125 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +type goproxyClient struct { + scheme string + host string +} + +func newGoproxyClient(scheme, host string) *goproxyClient { + return &goproxyClient{ + scheme: scheme, + host: host, + } +} + +func (g *goproxyClient) List(owner, repository string) ([]byte, error) { + // build the naive path + var gomodulePath string + switch owner { + case "kubernetes-sigs": + gomodulePath = path.Join("sigs.k8s.io", repository) + case "kubernetes": + gomodulePath = path.Join("k8s.io", repository) + default: + gomodulePath = path.Join("github.com", owner, repository) + } + + rawURL := url.URL{ + Scheme: g.scheme, + Host: g.host, + Path: path.Join(gomodulePath, "@v", "/list"), + } + + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, rawURL.String(), http.NoBody) + if err != nil { + return nil, errors.Wrapf(err, "failed to get versions for go module %q via goproxy %q: failed to create request", gomodulePath, g.host) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "failed to get versions for go module %q via goproxy %q", gomodulePath, g.host) + } + defer resp.Body.Close() + + rawResponse, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "error read goproxy response body") + } + return rawResponse, nil +} + +func getGoproxyHost(goproxy string) (string, string, error) { + // fallback default + if goproxy == "" { + return "https", "proxy.golang.org", nil + } + + var goproxyHost, goproxyScheme string + // Use url from GOPROXY variable if set. + // xref https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/proxy.go + for goproxy != "" { + var rawURL string + if i := strings.IndexAny(goproxy, ",|"); i >= 0 { + rawURL = goproxy[:i] + goproxy = goproxy[i+1:] + } else { + rawURL = goproxy + goproxy = "" + } + + rawURL = strings.TrimSpace(rawURL) + if rawURL == "" { + continue + } + if rawURL == "off" || rawURL == "direct" { + return "", "", fmt.Errorf("expected GOPROXY environment variable to not be set to off or direct") + } + + // Single-word tokens are reserved for built-in behaviors, and anything + // containing the string ":/" or matching an absolute file path must be a + // complete URL. For all other paths, implicitly add "https://". + if strings.ContainsAny(rawURL, ".:/") && !strings.Contains(rawURL, ":/") && !filepath.IsAbs(rawURL) && !path.IsAbs(rawURL) { + rawURL = "https://" + rawURL + } + + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", "", errors.Wrapf(err, "parse GOPROXY url %q", rawURL) + } + goproxyHost = parsedURL.Host + goproxyScheme = parsedURL.Scheme + // A host was found so no need to continue. + break + } + + return goproxyScheme, goproxyHost, nil +} diff --git a/cmd/clusterctl/client/repository/goproxy_test.go b/cmd/clusterctl/client/repository/goproxy_test.go new file mode 100644 index 000000000000..2946357c0018 --- /dev/null +++ b/cmd/clusterctl/client/repository/goproxy_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import "testing" + +func Test_getGoproxyHost(t *testing.T) { + tests := []struct { + name string + envvar string + wantScheme string + wantHost string + wantErr bool + }{ + { + name: "defaulting", + envvar: "", + wantScheme: "https", + wantHost: "proxy.golang.org", + wantErr: false, + }, + { + name: "direct is unsupported", + envvar: "direct", + wantScheme: "", + wantHost: "", + wantErr: true, + }, + { + name: "off is unsupported", + envvar: "off", + wantScheme: "", + wantHost: "", + wantErr: true, + }, + { + name: "other goproxy", + envvar: "foo.bar.de", + wantScheme: "https", + wantHost: "foo.bar.de", + wantErr: false, + }, + { + name: "other goproxy comma separated, return first", + envvar: "foo.bar,foobar.barfoo", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including https scheme", + envvar: "https://foo.bar", + wantScheme: "https", + wantHost: "foo.bar", + wantErr: false, + }, + { + name: "other goproxy including http scheme", + envvar: "http://foo.bar", + wantScheme: "http", + wantHost: "foo.bar", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotScheme, gotHost, err := getGoproxyHost(tt.envvar) + if (err != nil) != tt.wantErr { + t.Errorf("getGoproxyHost() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotScheme != tt.wantScheme { + t.Errorf("getGoproxyHost() = %v, wantScheme %v", gotScheme, tt.wantScheme) + } + if gotHost != tt.wantHost { + t.Errorf("getGoproxyHost() = %v, wantHost %v", gotHost, tt.wantHost) + } + }) + } +} diff --git a/cmd/clusterctl/client/repository/repository_github.go b/cmd/clusterctl/client/repository/repository_github.go index 3e5d506ba095..bc98e6b02073 100644 --- a/cmd/clusterctl/client/repository/repository_github.go +++ b/cmd/clusterctl/client/repository/repository_github.go @@ -22,14 +22,16 @@ import ( "io" "net/http" "net/url" + "os" "path/filepath" + "sort" "strings" "time" + "github.com/blang/semver" "github.com/google/go-github/v45/github" "github.com/pkg/errors" "golang.org/x/oauth2" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -68,6 +70,7 @@ type gitHubRepository struct { rootPath string componentsPath string injectClient *github.Client + injectGoproxyClient *goproxyClient } var _ Repository = &gitHubRepository{} @@ -80,6 +83,12 @@ func injectGithubClient(c *github.Client) githubRepositoryOption { } } +func injectGoproxyClient(c *goproxyClient) githubRepositoryOption { + return func(g *gitHubRepository) { + g.injectGoproxyClient = c + } +} + // DefaultVersion returns defaultVersion field of gitHubRepository struct. func (g *gitHubRepository) DefaultVersion() string { return g.defaultVersion @@ -201,6 +210,18 @@ func (g *gitHubRepository) getClient() *github.Client { return github.NewClient(g.authenticatingHTTPClient) } +// getGoproxyClient returns a go proxy client. +func (g *gitHubRepository) getGoproxyClient() (*goproxyClient, error) { + if g.injectGoproxyClient != nil { + return g.injectGoproxyClient, nil + } + scheme, host, err := getGoproxyHost(os.Getenv("GOPROXY")) + if err != nil { + return nil, err + } + return newGoproxyClient(scheme, host), nil +} + // setClientToken sets authenticatingHTTPClient field of gitHubRepository struct. func (g *gitHubRepository) setClientToken(token string) { ts := oauth2.StaticTokenSource( @@ -216,21 +237,20 @@ func (g *gitHubRepository) getVersions() ([]string, error) { return versions, nil } - client := g.getClient() + versionClient, err := g.getGoproxyClient() + if err != nil { + return nil, errors.Wrap(err, "get versions client") + } // get all the releases // NB. currently Github API does not support result ordering, so it not possible to limit results - var releases []*github.RepositoryRelease + var rawResponse []byte var retryError error _ = wait.PollImmediate(retryableOperationInterval, retryableOperationTimeout, func() (bool, error) { - var listReleasesErr error - releases, _, listReleasesErr = client.Repositories.ListReleases(context.TODO(), g.owner, g.repository, nil) - if listReleasesErr != nil { - retryError = g.handleGithubErr(listReleasesErr, "failed to get the list of releases") - // return immediately if we are rate limited - if _, ok := listReleasesErr.(*github.RateLimitError); ok { - return false, retryError - } + var retryErr error + rawResponse, retryErr = versionClient.List(g.owner, g.repository) + if retryErr != nil { + retryError = errors.Wrapf(retryErr, "failed to get versions owner=%s, repository=%s", g.owner, g.repository) return false, nil } retryError = nil @@ -239,18 +259,25 @@ func (g *gitHubRepository) getVersions() ([]string, error) { if retryError != nil { return nil, retryError } - versions := []string{} - for _, r := range releases { - r := r // pin - if r.TagName == nil { + + parsedVersions := semver.Versions{} + + for _, s := range strings.Split(string(rawResponse), "\n") { + if s == "" { continue } - tagName := *r.TagName - if _, err := version.ParseSemantic(tagName); err != nil { + parsedVersion, err := semver.Parse(strings.TrimPrefix(s, "v")) + if err != nil { // Discard releases with tags that are not a valid semantic versions (the user can point explicitly to such releases). continue } - versions = append(versions, tagName) + parsedVersions = append(parsedVersions, parsedVersion) + } + sort.Sort(parsedVersions) + + versions := []string{} + for _, v := range parsedVersions { + versions = append(versions, "v"+v.String()) } cacheVersions[cacheID] = versions diff --git a/cmd/clusterctl/client/repository/repository_github_test.go b/cmd/clusterctl/client/repository/repository_github_test.go index faef2373a8e5..ac0ec525a950 100644 --- a/cmd/clusterctl/client/repository/repository_github_test.go +++ b/cmd/clusterctl/client/repository/repository_github_test.go @@ -19,6 +19,8 @@ package repository import ( "fmt" "net/http" + "net/http/httptest" + "net/url" "testing" "time" @@ -235,19 +237,18 @@ func Test_githubRepository_getFile(t *testing.T) { func Test_gitHubRepository_getVersions(t *testing.T) { retryableOperationInterval = 200 * time.Millisecond retryableOperationTimeout = 1 * time.Second - client, mux, teardown := test.NewFakeGitHub() - defer teardown() + + clientGoproxy, muxGoproxy, teardownGoproxy := newFakeGoproxy() + defer teardownGoproxy() // setup an handler for returning 5 fake releases - mux.HandleFunc("/repos/o/r1/releases", func(w http.ResponseWriter, r *http.Request) { + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") - fmt.Fprint(w, `[`) - fmt.Fprint(w, `{"id":1, "tag_name": "v0.4.0"},`) - fmt.Fprint(w, `{"id":2, "tag_name": "v0.4.1"},`) - fmt.Fprint(w, `{"id":3, "tag_name": "v0.4.2"},`) - fmt.Fprint(w, `{"id":4, "tag_name": "v0.4.3-alpha"},`) // prerelease - fmt.Fprint(w, `{"id":5, "tag_name": "foo"}`) // no semantic version tag - fmt.Fprint(w, `]`) + fmt.Fprint(w, "v0.4.0\n") + fmt.Fprint(w, "v0.4.1\n") + fmt.Fprint(w, "v0.4.2\n") + fmt.Fprint(w, "v0.4.3-alpha\n") // prerelease + fmt.Fprint(w, "foo") // no semantic version tag }) configVariablesClient := test.NewFakeVariableClient() @@ -275,7 +276,7 @@ func Test_gitHubRepository_getVersions(t *testing.T) { g := NewWithT(t) resetCaches() - gitHub, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGithubClient(client)) + gitHub, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGoproxyClient(clientGoproxy)) g.Expect(err).NotTo(HaveOccurred()) got, err := gitHub.(*gitHubRepository).getVersions() @@ -296,17 +297,6 @@ func Test_gitHubRepository_getLatestContractRelease(t *testing.T) { client, mux, teardown := test.NewFakeGitHub() defer teardown() - // setup an handler for returning 3 fake releases - mux.HandleFunc("/repos/o/r1/releases", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, "GET") - fmt.Fprint(w, `[`) - fmt.Fprint(w, `{"id":1, "tag_name": "v0.5.0", "assets": [{"id": 1, "name": "metadata.yaml"}]},`) - fmt.Fprint(w, `{"id":2, "tag_name": "v0.4.0", "assets": [{"id": 1, "name": "metadata.yaml"}]},`) - fmt.Fprint(w, `{"id":3, "tag_name": "v0.3.2", "assets": [{"id": 1, "name": "metadata.yaml"}]},`) - fmt.Fprint(w, `{"id":4, "tag_name": "v0.3.1", "assets": [{"id": 1, "name": "metadata.yaml"}]}`) - fmt.Fprint(w, `]`) - }) - // test.NewFakeGitHub and handler for returning a fake release mux.HandleFunc("/repos/o/r1/releases/tags/v0.5.0", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") @@ -321,6 +311,18 @@ func Test_gitHubRepository_getLatestContractRelease(t *testing.T) { fmt.Fprint(w, "apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3\nreleaseSeries:\n - major: 0\n minor: 4\n contract: v1alpha4\n - major: 0\n minor: 5\n contract: v1alpha4\n - major: 0\n minor: 3\n contract: v1alpha3\n") }) + clientGoproxy, muxGoproxy, teardownGoproxy := newFakeGoproxy() + defer teardownGoproxy() + + // setup an handler for returning 4 fake releases + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + fmt.Fprint(w, "v0.5.0\n") + fmt.Fprint(w, "v0.4.0\n") + fmt.Fprint(w, "v0.3.2\n") + fmt.Fprint(w, "v0.3.1\n") + }) + configVariablesClient := test.NewFakeVariableClient() type field struct { @@ -366,7 +368,7 @@ func Test_gitHubRepository_getLatestContractRelease(t *testing.T) { g := NewWithT(t) resetCaches() - gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGithubClient(client)) + gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGithubClient(client), injectGoproxyClient(clientGoproxy)) g.Expect(err).NotTo(HaveOccurred()) got, err := latestContractRelease(gRepo, tt.contract) @@ -381,34 +383,30 @@ func Test_gitHubRepository_getLatestContractRelease(t *testing.T) { } func Test_gitHubRepository_getLatestRelease(t *testing.T) { - client, mux, teardown := test.NewFakeGitHub() - defer teardown() + clientGoproxy, muxGoproxy, teardownGoproxy := newFakeGoproxy() + defer teardownGoproxy() // setup an handler for returning 4 fake releases - mux.HandleFunc("/repos/o/r1/releases", func(w http.ResponseWriter, r *http.Request) { + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") - fmt.Fprint(w, `[`) - fmt.Fprint(w, `{"id":1, "tag_name": "v0.4.1"},`) - fmt.Fprint(w, `{"id":2, "tag_name": "v0.4.2"},`) - fmt.Fprint(w, `{"id":3, "tag_name": "v0.4.3-alpha"},`) // prerelease - fmt.Fprint(w, `{"id":4, "tag_name": "foo"}`) // no semantic version tag - fmt.Fprint(w, `]`) + fmt.Fprint(w, "v0.4.1\n") + fmt.Fprint(w, "v0.4.2\n") + fmt.Fprint(w, "v0.4.3-alpha\n") // prerelease + fmt.Fprint(w, "foo\n") // no semantic version tag }) // setup an handler for returning no releases - mux.HandleFunc("/repos/o/r2/releases", func(w http.ResponseWriter, r *http.Request) { + muxGoproxy.HandleFunc("/github.com/o/r2/@v/list", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") // no releases }) // setup an handler for returning fake prereleases only - mux.HandleFunc("/repos/o/r3/releases", func(w http.ResponseWriter, r *http.Request) { + muxGoproxy.HandleFunc("/github.com/o/r3/@v/list", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") - fmt.Fprint(w, `[`) - fmt.Fprint(w, `{"id":1, "tag_name": "v0.1.0-alpha.0"},`) - fmt.Fprint(w, `{"id":2, "tag_name": "v0.1.0-alpha.1"},`) - fmt.Fprint(w, `{"id":3, "tag_name": "v0.1.0-alpha.2"}`) - fmt.Fprint(w, `]`) + fmt.Fprint(w, "v0.1.0-alpha.0\n") + fmt.Fprint(w, "v0.1.0-alpha.1\n") + fmt.Fprint(w, "v0.1.0-alpha.2\n") }) configVariablesClient := test.NewFakeVariableClient() @@ -452,7 +450,7 @@ func Test_gitHubRepository_getLatestRelease(t *testing.T) { g := NewWithT(t) resetCaches() - gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGithubClient(client)) + gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGoproxyClient(clientGoproxy)) g.Expect(err).NotTo(HaveOccurred()) got, err := latestRelease(gRepo) @@ -468,17 +466,15 @@ func Test_gitHubRepository_getLatestRelease(t *testing.T) { } func Test_gitHubRepository_getLatestPatchRelease(t *testing.T) { - client, mux, teardown := test.NewFakeGitHub() - defer teardown() + clientGoproxy, muxGoproxy, teardownGoproxy := newFakeGoproxy() + defer teardownGoproxy() - // setup an handler for returning 3 fake releases - mux.HandleFunc("/repos/o/r1/releases", func(w http.ResponseWriter, r *http.Request) { + // setup an handler for returning 4 fake releases + muxGoproxy.HandleFunc("/github.com/o/r1/@v/list", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") - fmt.Fprint(w, `[`) - fmt.Fprint(w, `{"id":1, "tag_name": "v0.4.0"},`) - fmt.Fprint(w, `{"id":2, "tag_name": "v0.3.2"},`) - fmt.Fprint(w, `{"id":3, "tag_name": "v1.3.2"}`) - fmt.Fprint(w, `]`) + fmt.Fprint(w, "v0.4.0\n") + fmt.Fprint(w, "v0.3.2\n") + fmt.Fprint(w, "v1.3.2\n") }) major0 := uint(0) @@ -534,7 +530,7 @@ func Test_gitHubRepository_getLatestPatchRelease(t *testing.T) { g := NewWithT(t) resetCaches() - gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGithubClient(client)) + gRepo, err := NewGitHubRepository(tt.field.providerConfig, configVariablesClient, injectGoproxyClient(clientGoproxy)) g.Expect(err).NotTo(HaveOccurred()) got, err := latestPatchRelease(gRepo, tt.major, tt.minor) @@ -732,3 +728,21 @@ func resetCaches() { cacheReleases = map[string]*github.RepositoryRelease{} cacheFiles = map[string][]byte{} } + +// newFakeGoproxy sets up a test HTTP server along with a github.Client that is +// configured to talk to that test server. Tests should register handlers on +// mux which provide mock responses for the API method being tested. +func newFakeGoproxy() (client *goproxyClient, mux *http.ServeMux, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + // client is the GitHub client being tested and is configured to use test server. + url, _ := url.Parse(server.URL + "/") + return &goproxyClient{scheme: url.Scheme, host: url.Host}, mux, server.Close +}