Use all kops mirrors to determine artifacts hashes

hack/.packages

@@ -202,6 +202,7 @@ k8s.io/kops/util/pkg/env
 k8s.io/kops/util/pkg/exec
 k8s.io/kops/util/pkg/hashing
 k8s.io/kops/util/pkg/maps
+k8s.io/kops/util/pkg/mirrors
 k8s.io/kops/util/pkg/proxy
 k8s.io/kops/util/pkg/reflectutils
 k8s.io/kops/util/pkg/slice

pkg/assets/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//pkg/kubemanifest:go_default_library",
         "//pkg/values:go_default_library",
         "//util/pkg/hashing:go_default_library",
+        "//util/pkg/mirrors:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/blang/semver/v4:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",

pkg/assets/builder.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/values"
 	"k8s.io/kops/util/pkg/hashing"
+	"k8s.io/kops/util/pkg/mirrors"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -311,27 +312,30 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
 		}
 
 		for _, ext := range []string{".sha256", ".sha1"} {
-			hashURL := u.String() + ext
-			b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff))
-			if err != nil {
-				// Try to log without being too alarming - issue #7550
-				if ext == ".sha256" {
-					klog.V(2).Infof("unable to read new sha256 hash file (is this an older/unsupported kubernetes release?) %q: %v", hashURL, err)
-				} else {
-					klog.V(2).Infof("unable to read hash file %q: %v", hashURL, err)
+			for _, mirror := range mirrors.FindUrlMirrors(u.String()) {
+				hashURL := mirror + ext
+				klog.V(3).Infof("Trying to read hash fie: %q", hashURL)
+				b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff))
+				if err != nil {
+					// Try to log without being too alarming - issue #7550
+					if ext == ".sha256" {
+						klog.V(2).Infof("Unable to read new sha256 hash file (is this an older/unsupported kubernetes release?) %q: %v", hashURL, err)
+					} else {
+						klog.V(2).Infof("Unable to read hash file %q: %v", hashURL, err)
+					}
+					continue
 				}
-				continue
-			}
-			hashString := strings.TrimSpace(string(b))
-			klog.V(2).Infof("Found hash %q for %q", hashString, u)
-
-			// Accept a hash string that is `<hash> <filename>`
-			fields := strings.Fields(hashString)
-			if len(fields) == 0 {
-				klog.Infof("hash file was empty %q", hashURL)
-				continue
+				hashString := strings.TrimSpace(string(b))
+				klog.V(2).Infof("Found hash %q for %q", hashString, u)
+
+				// Accept a hash string that is `<hash> <filename>`
+				fields := strings.Fields(hashString)
+				if len(fields) == 0 {
+					klog.Infof("Hash file was empty %q", hashURL)
+					continue
+				}
+				return hashing.FromString(fields[0])
 			}
-			return hashing.FromString(fields[0])
 		}
 	}
 

upup/pkg/fi/cloudup/BUILD.bazel

@@ -78,6 +78,7 @@ go_library(
         "//util/pkg/architectures:go_default_library",
         "//util/pkg/env:go_default_library",
         "//util/pkg/hashing:go_default_library",
+        "//util/pkg/mirrors:go_default_library",
         "//util/pkg/reflectutils:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/Masterminds/sprig:go_default_library",

upup/pkg/fi/cloudup/urls.go

@@ -21,14 +21,14 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"sort"
 	"strings"
 
 	"k8s.io/klog/v2"
 	"k8s.io/kops"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/hashing"
+	"k8s.io/kops/util/pkg/mirrors"
 )
 
 const (
@@ -47,23 +47,6 @@ type mirror struct {
 	Replace map[string]string
 }
 
-// GitHub releases have special artifact names because full paths cannot be used
-// and artifact names must remain easy to read for humans.
-var gitHubReplace = map[string]string{
-	"/":                  "-",
-	"linux-amd64-nodeup": "nodeup-linux-amd64",
-	"linux-arm64-nodeup": "nodeup-linux-arm64",
-}
-
-// defaultKopsMirrors is a list of our well-known mirrors
-// Note that we download in order
-var defaultKopsMirrors = []mirror{
-	{URL: "https://artifacts.k8s.io/binaries/kops/%s/"},
-	{URL: "https://github.com/kubernetes/kops/releases/download/v%s/", Replace: gitHubReplace},
-	// We do need to include defaultKopsMirrorBase - the list replaces the base url
-	{URL: "https://kubeupv2.s3.amazonaws.com/kops/%s/"},
-}
-
 var kopsBaseURL *url.URL
 
 // nodeUpAsset caches the nodeup download urls/hash
@@ -251,30 +234,12 @@ func BuildMirroredAsset(u *url.URL, hash *hashing.Hash) *MirroredAsset {
 		Hash: hash,
 	}
 
-	urlString := u.String()
-	a.Locations = []string{urlString}
-
-	// Look at mirrors
-	if strings.HasPrefix(urlString, baseURLString) {
+	a.Locations = []string{u.String()}
+	if strings.HasPrefix(u.String(), baseURLString) {
 		if hash == nil {
 			klog.Warningf("not using mirrors for asset %s as it does not have a known hash", u.String())
 		} else {
-			suffix := strings.TrimPrefix(urlString, baseURLString)
-			// This is under our base url - add our well-known mirrors
-			a.Locations = []string{}
-			for _, m := range defaultKopsMirrors {
-				keys := make([]string, len(m.Replace))
-				for k := range m.Replace {
-					keys = append(keys, k)
-				}
-				sort.Strings(keys)
-				filename := suffix
-				for _, k := range keys {
-					filename = strings.Replace(filename, k, m.Replace[k], -1)
-				}
-				base := fmt.Sprintf(m.URL, kops.Version)
-				a.Locations = append(a.Locations, base+filename)
-			}
+			a.Locations = mirrors.FindUrlMirrors(u.String())
 		}
 	}
 

util/pkg/mirrors/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["mirrors.go"],
+    importpath = "k8s.io/kops/util/pkg/mirrors",
+    visibility = ["//visibility:public"],
+    deps = ["//:go_default_library"],
+)

util/pkg/mirrors/mirrors.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mirrors
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/kops"
+)
+
+const (
+	defaultKopsMirrorBase    = "https://kubeupv2.s3.amazonaws.com/kops/%s/"
+	githubKopsMirrorBase     = "https://github.com/kubernetes/kops/releases/download/v%s/"
+	kubernetesKopsMirrorBase = "https://artifacts.k8s.io/binaries/kops/%s/"
+)
+
+func FindUrlMirrors(u string) []string {
+	// Use the mirrors to also find hashes.
+	baseURLString := fmt.Sprintf(defaultKopsMirrorBase, kops.Version)
+	if !strings.HasSuffix(baseURLString, "/") {
+		baseURLString += "/"
+	}
+
+	var mirrors []string
+	if strings.HasPrefix(u, baseURLString) {
+		suffix := strings.TrimPrefix(u, baseURLString)
+		// artifacts.k8s.io is the official and preferred mirror.
+		mirrors = append(mirrors, fmt.Sprintf(kubernetesKopsMirrorBase, kops.Version)+suffix)
+		// GitHub artifact names are quite different, because the suffix path is collapsed.
+		githubSuffix := strings.ReplaceAll(suffix, "/", "-")
+		githubSuffix = strings.ReplaceAll(githubSuffix, "linux-amd64-nodeup", "nodeup-linux-amd64")
+		githubSuffix = strings.ReplaceAll(githubSuffix, "linux-arm64-nodeup", "nodeup-linux-arm64")
+		mirrors = append(mirrors, fmt.Sprintf(githubKopsMirrorBase, kops.Version)+githubSuffix)
+	}
+	// Finally append the original URL to the list of mirrored URLs.
+	// In case this is a custom URL, there won't be any mirrors before it,
+	// otherwise it will be the last mirror URL, because it's now a legacy location.
+	mirrors = append(mirrors, u)
+
+	return mirrors
+}

util/pkg/vfs/context.go

@@ -280,7 +280,7 @@ func RetryWithBackoff(backoff wait.Backoff, condition func() (bool, error)) (boo
 		}
 
 		if noMoreRetries {
-			klog.Infof("hit maximum retries %d with error %v", i, err)
+			klog.V(2).Infof("hit maximum retries %d with error %v", i, err)
 			return done, err
 		}
 	}