Reduce github api requests in clusterctl

* Removes additional cert-manager latest version detection because it always gets overwritten.
* Uses goproxy instead of github api for listing repository versions.

cmd/clusterctl/client/cluster/cert_manager.go

@@ -19,6 +19,7 @@ package cluster
 import (
 	"context"
 	_ "embed"
+	"strings"
 	"time"
 
 	"github.com/blang/semver"
@@ -401,7 +402,9 @@ func (cm *certManagerClient) getManifestObjs(certManagerConfig config.CertManage
 	// Given that cert manager components yaml are stored in a repository like providers components yaml,
 	// we are using the same machinery to retrieve the file by using a fake provider object using
 	// the cert manager repository url.
-	certManagerFakeProvider := config.NewProvider("cert-manager", certManagerConfig.URL(), "")
+	// In order to avoid unnecessary calls to GitHub API, "latest" is replace with the version currently installed by clusterctl.
+	url := strings.Replace(certManagerConfig.URL(), "latest", certManagerConfig.Version(), 1)
+	certManagerFakeProvider := config.NewProvider("cert-manager", url, "")
 	certManagerRepository, err := cm.repositoryClientFactory(certManagerFakeProvider, cm.configClient)
 	if err != nil {
 		return nil, err

cmd/clusterctl/client/repository/goproxy.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repository
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	defaultGoProxyHost = "proxy.golang.org"
+)
+
+type goproxyClient struct {
+	scheme string
+	host   string
+}
+
+func newGoproxyClient(scheme, host string) *goproxyClient {
+	return &goproxyClient{
+		scheme: scheme,
+		host:   host,
+	}
+}
+
+func (g *goproxyClient) getVersions(owner, repository string) ([]string, error) {
+	// A goproxy is also able to handle the github repository path instead of the actual go module name.
+	gomodulePath := path.Join("github.com", owner, repository)
+
+	rawURL := url.URL{
+		Scheme: g.scheme,
+		Host:   g.host,
+		Path:   path.Join(gomodulePath, "@v", "/list"),
+	}
+
+	var rawResponse []byte
+	var retryError error
+	_ = wait.PollImmediate(retryableOperationInterval, retryableOperationTimeout, func() (bool, error) {
+		req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, rawURL.String(), http.NoBody)
+		if err != nil {
+			return false, errors.Wrapf(err, "failed to create request")
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			retryError = errors.Wrapf(err, "failed to get versions")
+			return false, nil
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != 200 {
+			retryError = errors.Errorf("Request status code %d: %s", resp.StatusCode, resp.Status)
+			return false, nil
+		}
+
+		rawResponse, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return false, errors.Wrap(err, "error reading goproxy response body")
+		}
+		return true, nil
+	})
+	if retryError != nil {
+		return nil, retryError
+	}
+
+	parsedVersions := semver.Versions{}
+	for _, s := range strings.Split(string(rawResponse), "\n") {
+		if s == "" {
+			continue
+		}
+		parsedVersion, err := semver.ParseTolerant(s)
+		if err != nil {
+			// Discard releases with tags that are not a valid semantic versions (the user can point explicitly to such releases).
+			continue
+		}
+		parsedVersions = append(parsedVersions, parsedVersion)
+	}
+
+	sort.Sort(parsedVersions)
+
+	versions := []string{}
+	for _, v := range parsedVersions {
+		versions = append(versions, "v"+v.String())
+	}
+
+	return versions, nil
+}
+
+// getGoproxyHost detects and returns the scheme and host for goproxy requests.
+// It returns empty strings if goproxy is disabled via `off` or `direct` values.
+func getGoproxyHost(goproxy string) (string, string, error) {
+	// Fallback to default
+	if goproxy == "" {
+		return "https", defaultGoProxyHost, nil
+	}
+
+	var goproxyHost, goproxyScheme string
+	// xref https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/proxy.go
+	for goproxy != "" {
+		var rawURL string
+		if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
+			rawURL = goproxy[:i]
+			goproxy = goproxy[i+1:]
+		} else {
+			rawURL = goproxy
+			goproxy = ""
+		}
+
+		rawURL = strings.TrimSpace(rawURL)
+		if rawURL == "" {
+			continue
+		}
+		if rawURL == "off" || rawURL == "direct" {
+			// Return nothing to fallback to github repository client without an error.
+			return "", "", nil
+		}
+
+		// Single-word tokens are reserved for built-in behaviors, and anything
+		// containing the string ":/" or matching an absolute file path must be a
+		// complete URL. For all other paths, implicitly add "https://".
+		if strings.ContainsAny(rawURL, ".:/") && !strings.Contains(rawURL, ":/") && !filepath.IsAbs(rawURL) && !path.IsAbs(rawURL) {
+			rawURL = "https://" + rawURL
+		}
+
+		parsedURL, err := url.Parse(rawURL)
+		if err != nil {
+			return "", "", errors.Wrapf(err, "parse GOPROXY url %q", rawURL)
+		}
+		goproxyHost = parsedURL.Host
+		goproxyScheme = parsedURL.Scheme
+		// A host was found so no need to continue.
+		break
+	}
+
+	return goproxyScheme, goproxyHost, nil
+}

cmd/clusterctl/client/repository/goproxy_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repository
+
+import (
+	"testing"
+	"time"
+)
+
+func Test_getGoproxyHost(t *testing.T) {
+	retryableOperationInterval = 200 * time.Millisecond
+	retryableOperationTimeout = 1 * time.Second
+
+	tests := []struct {
+		name       string
+		envvar     string
+		wantScheme string
+		wantHost   string
+		wantErr    bool
+	}{
+		{
+			name:       "defaulting",
+			envvar:     "",
+			wantScheme: "https",
+			wantHost:   "proxy.golang.org",
+			wantErr:    false,
+		},
+		{
+			name:       "direct falls back to empty strings",
+			envvar:     "direct",
+			wantScheme: "",
+			wantHost:   "",
+			wantErr:    false,
+		},
+		{
+			name:       "off falls back to empty strings",
+			envvar:     "off",
+			wantScheme: "",
+			wantHost:   "",
+			wantErr:    false,
+		},
+		{
+			name:       "other goproxy",
+			envvar:     "foo.bar.de",
+			wantScheme: "https",
+			wantHost:   "foo.bar.de",
+			wantErr:    false,
+		},
+		{
+			name:       "other goproxy comma separated, return first",
+			envvar:     "foo.bar,foobar.barfoo",
+			wantScheme: "https",
+			wantHost:   "foo.bar",
+			wantErr:    false,
+		},
+		{
+			name:       "other goproxy including https scheme",
+			envvar:     "https://foo.bar",
+			wantScheme: "https",
+			wantHost:   "foo.bar",
+			wantErr:    false,
+		},
+		{
+			name:       "other goproxy including http scheme",
+			envvar:     "http://foo.bar",
+			wantScheme: "http",
+			wantHost:   "foo.bar",
+			wantErr:    false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotScheme, gotHost, err := getGoproxyHost(tt.envvar)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getGoproxyHost() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotScheme != tt.wantScheme {
+				t.Errorf("getGoproxyHost() = %v, wantScheme %v", gotScheme, tt.wantScheme)
+			}
+			if gotHost != tt.wantHost {
+				t.Errorf("getGoproxyHost() = %v, wantHost %v", gotHost, tt.wantHost)
+			}
+		})
+	}
+}