Merge pull request #9373 from johngmyers/automated-cherry-pick-of-#93…

…55-upstream-release-1.18

Automated cherry pick of #9355: Move kube-apiserver-healthcheck to port 3990

cmd/kube-apiserver-healthcheck/BUILD.bazel

@@ -5,7 +5,10 @@ go_library(
     srcs = ["main.go"],
     importpath = "k8s.io/kops/cmd/kube-apiserver-healthcheck",
     visibility = ["//visibility:private"],
-    deps = ["//vendor/k8s.io/klog:go_default_library"],
+    deps = [
+        "//pkg/wellknownports:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
 )
 
 go_binary(

cmd/kube-apiserver-healthcheck/README.md

@@ -4,7 +4,7 @@ This is a small sidecar container that allows for health-checking the
 kube-apiserver without enabling anonymous authentication and without
 enabling the unauthenticated port.
 
-It listens on port 8080 (http), and proxies a few known-safe requests
+It listens on port 3990 (http), and proxies a few known-safe requests
 to the real apiserver listening on 443.  It uses a client certificate
 to authenticate itself to the apiserver.
 
@@ -14,5 +14,5 @@ it also lets us have better load-balancer health-checks.
 Because it runs as a sidecar next to kube-apiserver, it is in the same
 network namespace, and thus it can reach apiserver on
 https://127.0.0.1 .  The kube-apiserver-healthcheck process listens on
-8080, but the health checks for the apiserver container are configured
+3990, but the health checks for the apiserver container are configured
 for :8080 and actually go via the sidecar.

cmd/kube-apiserver-healthcheck/main.go

@@ -28,6 +28,7 @@ import (
 	"os"
 
 	"k8s.io/klog"
+	"k8s.io/kops/pkg/wellknownports"
 )
 
 // healthCheckServer is the http server
@@ -129,7 +130,7 @@ func (s *healthCheckServer) proxyRequest(w http.ResponseWriter, forwardRequest *
 }
 
 func run() error {
-	listen := ":8080"
+	listen := fmt.Sprintf(":%d", wellknownports.KubeAPIServerHealthCheck)
 
 	clientCert := ""
 	clientKey := ""

cmd/kube-apiserver-healthcheck/proxy_test.go

@@ -27,19 +27,19 @@ func TestBuildProxyRequest(t *testing.T) {
 		In  string
 		Out string
 	}{
-		{In: "http://127.0.0.1:8080/readyz", Out: "https://127.0.0.1/readyz"},
-		{In: "http://127.0.0.1:8080/livez", Out: "https://127.0.0.1/livez"},
-		{In: "http://127.0.0.1:8080/healthz", Out: "https://127.0.0.1/healthz"},
-		{In: "http://127.0.0.1:8080/ready", Out: ""},
-		{In: "http://127.0.0.1:8080/", Out: ""},
-		{In: "http://127.0.0.1:8080/readyz/foo", Out: ""},
-		{In: "http://127.0.0.1:8080/readyzfoo", Out: ""},
-		{In: "http://127.0.0.1:8080/readyz?", Out: "https://127.0.0.1/readyz"},
-		{In: "http://127.0.0.1:8080/readyz?foo=bar", Out: "https://127.0.0.1/readyz"},
-		{In: "http://127.0.0.1:8080/readyz?exclude=1", Out: "https://127.0.0.1/readyz?exclude=1"},
-		{In: "http://127.0.0.1:8080/readyz?exclude=1&exclude=2", Out: "https://127.0.0.1/readyz?exclude=1&exclude=2"},
-		{In: "http://127.0.0.1:8080/readyz?exclude=1&verbose", Out: "https://127.0.0.1/readyz?exclude=1"},
-		{In: "http://127.0.0.1:8080/readyz?exclude", Out: "https://127.0.0.1/readyz?exclude="},
+		{In: "http://127.0.0.1:3990/readyz", Out: "https://127.0.0.1/readyz"},
+		{In: "http://127.0.0.1:3990/livez", Out: "https://127.0.0.1/livez"},
+		{In: "http://127.0.0.1:3990/healthz", Out: "https://127.0.0.1/healthz"},
+		{In: "http://127.0.0.1:3990/ready", Out: ""},
+		{In: "http://127.0.0.1:3990/", Out: ""},
+		{In: "http://127.0.0.1:3990/readyz/foo", Out: ""},
+		{In: "http://127.0.0.1:3990/readyzfoo", Out: ""},
+		{In: "http://127.0.0.1:3990/readyz?", Out: "https://127.0.0.1/readyz"},
+		{In: "http://127.0.0.1:3990/readyz?foo=bar", Out: "https://127.0.0.1/readyz"},
+		{In: "http://127.0.0.1:3990/readyz?exclude=1", Out: "https://127.0.0.1/readyz?exclude=1"},
+		{In: "http://127.0.0.1:3990/readyz?exclude=1&exclude=2", Out: "https://127.0.0.1/readyz?exclude=1&exclude=2"},
+		{In: "http://127.0.0.1:3990/readyz?exclude=1&verbose", Out: "https://127.0.0.1/readyz?exclude=1"},
+		{In: "http://127.0.0.1:3990/readyz?exclude", Out: "https://127.0.0.1/readyz?exclude="},
 	}
 
 	for _, g := range grid {

docs/boot-sequence.md

@@ -67,11 +67,6 @@ that have hostNetwork will work - so all the "core" containers run with hostNetw
 
 ## api-server bringup
 
-The api-server will listen on localhost:8080 on the master.  This is an unsecured endpoint,
-but is only reachable from the master, and only for pods running with hostNetwork=true.  This
-is how components like kube-scheduler and kube-controller-manager can reach the API without
-requiring a token.
-
 APIServer also listens on the HTTPS port (443) on all interfaces.  This is a secured endpoint,
 and requires valid authentication/authorization to use it.  This is the endpoint that node kubelets
 will reach, and also that end-users will reach.

nodeup/pkg/model/BUILD.bazel

@@ -60,6 +60,7 @@ go_library(
         "//pkg/rbac:go_default_library",
         "//pkg/systemd:go_default_library",
         "//pkg/tokens:go_default_library",
+        "//pkg/wellknownports:go_default_library",
         "//pkg/wellknownusers:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",

nodeup/pkg/model/kube_apiserver.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/pkg/wellknownusers"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -409,10 +410,10 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	probeAction := &v1.HTTPGetAction{
 		Host: "127.0.0.1",
 		Path: "/healthz",
-		Port: intstr.FromInt(8080),
+		Port: intstr.FromInt(wellknownports.KubeAPIServerHealthCheck),
 	}
 	if useHealthcheckProxy {
-		// kube-apiserver-healthcheck sidecar container runs on port 8080
+		// kube-apiserver-healthcheck sidecar container runs on port 3990
 	} else if kubeAPIServer.InsecurePort != 0 {
 		probeAction.Port = intstr.FromInt(int(kubeAPIServer.InsecurePort))
 	} else if kubeAPIServer.SecurePort != 0 {

pkg/model/components/apiserver.go

@@ -232,7 +232,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
 	c.AnonymousAuth = fi.Bool(false)
 
 	if b.IsKubernetesGTE("1.17") {
-		// We query via the kube-apiserver-healthcheck proxy, which listens on port 8080
+		// We query via the kube-apiserver-healthcheck proxy, which listens on port 3990
 		c.InsecurePort = 0
 	} else {
 		// Older versions of kubernetes continue to rely on the insecure port: kubernetes issue #43784

pkg/model/components/kubeapiserver/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "//pkg/assets:go_default_library",
         "//pkg/k8scodecs:go_default_library",
         "//pkg/model:go_default_library",
+        "//pkg/wellknownports:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/fitasks:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

pkg/model/components/kubeapiserver/model.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
 )
@@ -100,7 +101,7 @@ spec:
         # The sidecar serves a healthcheck on the same port,
         # but with a .kube-apiserver-healthcheck prefix
         path: /.kube-apiserver-healthcheck/healthz
-        port: 8080
+        port: %d
         host: 127.0.0.1
       initialDelaySeconds: 5
       timeoutSeconds: 5
@@ -126,7 +127,7 @@ spec:
 func (b *KubeApiserverBuilder) buildHealthcheckSidecar() (*corev1.Pod, error) {
 	// TODO: pull from bundle
 	bundle := "(embedded kube-apiserver-healthcheck manifest)"
-	manifest := []byte(defaultManifest)
+	manifest := []byte(fmt.Sprintf(defaultManifest, wellknownports.KubeAPIServerHealthCheck))
 
 	var pod *corev1.Pod
 	var container *corev1.Container

pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml

@@ -18,7 +18,7 @@ Contents:
           httpGet:
             host: 127.0.0.1
             path: /.kube-apiserver-healthcheck/healthz
-            port: 8080
+            port: 3990
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: healthcheck

pkg/wellknownports/wellknownports.go

@@ -43,6 +43,14 @@ const (
 
 	// DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip
 	DNSControllerGossipMemberlist = 3993
+
+	// 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium
+
+	// KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens.
+	KubeAPIServerHealthCheck = 3990
+
+	// NodeLocalDNSHealthCheck is the port where the node-local-dns health check listens.
+	NodeLocalDNSHealthCheck = 3989
 )
 
 type PortRange struct {

upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template

@@ -55,7 +55,7 @@ data:
           force_tcp
         }
         prometheus :9253
-        health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080
+        health {{ KubeDNS.NodeLocalDNS.LocalIP }}:{{ NodeLocalDNSHealthCheck }}
     }
     in-addr.arpa:53 {
         errors
@@ -154,7 +154,7 @@ spec:
           httpGet:
             host: {{ .KubeDNS.NodeLocalDNS.LocalIP }}
             path: /health
-            port: 8080
+            port: {{ NodeLocalDNSHealthCheck }}
           initialDelaySeconds: 60
           timeoutSeconds: 5
         volumeMounts:

upup/pkg/fi/cloudup/template_functions.go

@@ -111,6 +111,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 			return tf.cluster.Spec.KubeDNS.ServerIP
 		}
 	}
+	dest["NodeLocalDNSHealthCheck"] = func() string {
+		return fmt.Sprintf("%d", wellknownports.NodeLocalDNSHealthCheck)
+	}
 
 	dest["KopsControllerArgv"] = tf.KopsControllerArgv
 	dest["KopsControllerConfig"] = tf.KopsControllerConfig