diff --git a/pkg/assets/BUILD.bazel b/pkg/assets/BUILD.bazel index 95aa44ccce06a..e91789ada42d0 100644 --- a/pkg/assets/BUILD.bazel +++ b/pkg/assets/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//pkg/kubemanifest:go_default_library", "//pkg/values:go_default_library", "//util/pkg/hashing:go_default_library", + "//util/pkg/mirrors:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/github.com/blang/semver/v4:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 82cecb487dfc1..683280c30e732 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -34,6 +34,7 @@ import ( "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/values" "k8s.io/kops/util/pkg/hashing" + "k8s.io/kops/util/pkg/mirrors" "k8s.io/kops/util/pkg/vfs" ) @@ -311,27 +312,30 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) { } for _, ext := range []string{".sha256", ".sha1"} { - hashURL := u.String() + ext - b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff)) - if err != nil { - // Try to log without being too alarming - issue #7550 - if ext == ".sha256" { - klog.V(2).Infof("unable to read new sha256 hash file (is this an older/unsupported kubernetes release?) %q: %v", hashURL, err) - } else { - klog.V(2).Infof("unable to read hash file %q: %v", hashURL, err) + for _, mirror := range mirrors.FindUrlMirrors(u.String()) { + hashURL := mirror + ext + klog.V(3).Infof("Trying to read hash fie: %q", hashURL) + b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff)) + if err != nil { + // Try to log without being too alarming - issue #7550 + if ext == ".sha256" { + klog.V(2).Infof("Unable to read new sha256 hash file (is this an older/unsupported kubernetes release?) %q: %v", hashURL, err) + } else { + klog.V(2).Infof("Unable to read hash file %q: %v", hashURL, err) + } + continue } - continue - } - hashString := strings.TrimSpace(string(b)) - klog.V(2).Infof("Found hash %q for %q", hashString, u) - - // Accept a hash string that is ` ` - fields := strings.Fields(hashString) - if len(fields) == 0 { - klog.Infof("hash file was empty %q", hashURL) - continue + hashString := strings.TrimSpace(string(b)) + klog.V(2).Infof("Found hash %q for %q", hashString, u) + + // Accept a hash string that is ` ` + fields := strings.Fields(hashString) + if len(fields) == 0 { + klog.Infof("Hash file was empty %q", hashURL) + continue + } + return hashing.FromString(fields[0]) } - return hashing.FromString(fields[0]) } } diff --git a/upup/pkg/fi/cloudup/urls.go b/upup/pkg/fi/cloudup/urls.go index ab43990537113..4f09ac022f540 100644 --- a/upup/pkg/fi/cloudup/urls.go +++ b/upup/pkg/fi/cloudup/urls.go @@ -21,7 +21,6 @@ import ( "net/url" "os" "path" - "sort" "strings" "k8s.io/klog/v2" @@ -29,6 +28,7 @@ import ( "k8s.io/kops/pkg/assets" "k8s.io/kops/util/pkg/architectures" "k8s.io/kops/util/pkg/hashing" + "k8s.io/kops/util/pkg/mirrors" ) const ( @@ -47,23 +47,6 @@ type mirror struct { Replace map[string]string } -// GitHub releases have special artifact names because full paths cannot be used -// and artifact names must remain easy to read for humans. -var gitHubReplace = map[string]string{ - "/": "-", - "linux-amd64-nodeup": "nodeup-linux-amd64", - "linux-arm64-nodeup": "nodeup-linux-arm64", -} - -// defaultKopsMirrors is a list of our well-known mirrors -// Note that we download in order -var defaultKopsMirrors = []mirror{ - {URL: "https://artifacts.k8s.io/binaries/kops/%s/"}, - {URL: "https://github.com/kubernetes/kops/releases/download/v%s/", Replace: gitHubReplace}, - // We do need to include defaultKopsMirrorBase - the list replaces the base url - {URL: "https://kubeupv2.s3.amazonaws.com/kops/%s/"}, -} - var kopsBaseURL *url.URL // nodeUpAsset caches the nodeup download urls/hash @@ -251,30 +234,12 @@ func BuildMirroredAsset(u *url.URL, hash *hashing.Hash) *MirroredAsset { Hash: hash, } - urlString := u.String() - a.Locations = []string{urlString} - - // Look at mirrors - if strings.HasPrefix(urlString, baseURLString) { + a.Locations = []string{u.String()} + if strings.HasPrefix(u.String(), baseURLString) { if hash == nil { klog.Warningf("not using mirrors for asset %s as it does not have a known hash", u.String()) } else { - suffix := strings.TrimPrefix(urlString, baseURLString) - // This is under our base url - add our well-known mirrors - a.Locations = []string{} - for _, m := range defaultKopsMirrors { - keys := make([]string, len(m.Replace)) - for k := range m.Replace { - keys = append(keys, k) - } - sort.Strings(keys) - filename := suffix - for _, k := range keys { - filename = strings.Replace(filename, k, m.Replace[k], -1) - } - base := fmt.Sprintf(m.URL, kops.Version) - a.Locations = append(a.Locations, base+filename) - } + a.Locations = mirrors.FindUrlMirrors(u.String()) } } diff --git a/util/pkg/mirrors/BUILD.bazel b/util/pkg/mirrors/BUILD.bazel new file mode 100644 index 0000000000000..37d47ff436fe1 --- /dev/null +++ b/util/pkg/mirrors/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["mirrors.go"], + importpath = "k8s.io/kops/util/pkg/mirrors", + visibility = ["//visibility:public"], + deps = ["//:go_default_library"], +) diff --git a/util/pkg/mirrors/mirrors.go b/util/pkg/mirrors/mirrors.go new file mode 100644 index 0000000000000..e935fa6f2a113 --- /dev/null +++ b/util/pkg/mirrors/mirrors.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirrors + +import ( + "fmt" + "strings" + + "k8s.io/kops" +) + +const ( + defaultKopsMirrorBase = "https://kubeupv2.s3.amazonaws.com/kops/%s/" + githubKopsMirrorBase = "https://github.com/kubernetes/kops/releases/download/v%s/" + kubernetesKopsMirrorBase = "https://artifacts.k8s.io/binaries/kops/%s/" +) + +func FindUrlMirrors(u string) []string { + // Use the mirrors to also find hashes. + baseURLString := fmt.Sprintf(defaultKopsMirrorBase, kops.Version) + if !strings.HasSuffix(baseURLString, "/") { + baseURLString += "/" + } + + var mirrors []string + if strings.HasPrefix(u, baseURLString) { + suffix := strings.TrimPrefix(u, baseURLString) + // artifacts.k8s.io is the official and preferred mirror. + mirrors = append(mirrors, fmt.Sprintf(kubernetesKopsMirrorBase, kops.Version)+suffix) + // GitHub artifact names are quite different, because the suffix path is collapsed. + githubSuffix := strings.ReplaceAll(suffix, "/", "-") + githubSuffix = strings.ReplaceAll(githubSuffix, "linux-amd64-nodeup", "nodeup-linux-amd64") + githubSuffix = strings.ReplaceAll(githubSuffix, "linux-arm64-nodeup", "nodeup-linux-arm64") + mirrors = append(mirrors, fmt.Sprintf(githubKopsMirrorBase, kops.Version)+githubSuffix) + } + // Finally append the original URL to the list of mirrored URLs. + // In case this is a custom URL, there won't be any mirrors before it, + // otherwise it will be the last mirror URL, because it's now a legacy location. + mirrors = append(mirrors, u) + + return mirrors +} diff --git a/util/pkg/vfs/context.go b/util/pkg/vfs/context.go index c33f84bc8eadc..62b1addc36684 100644 --- a/util/pkg/vfs/context.go +++ b/util/pkg/vfs/context.go @@ -280,7 +280,7 @@ func RetryWithBackoff(backoff wait.Backoff, condition func() (bool, error)) (boo } if noMoreRetries { - klog.Infof("hit maximum retries %d with error %v", i, err) + klog.V(2).Infof("hit maximum retries %d with error %v", i, err) return done, err } }