Merge pull request #1287 from soider/tunneled-network-from-node-to-ap…

…i-server

.github/workflows/go.yml

@@ -166,6 +166,7 @@ jobs:
           - check-upgrade
           - check-psp
           - check-statussocket
+          - check-tunneledkas
           # skipped, originally titled "Smoke-test for network":
           # - check-etcd
 

cmd/controller/controller.go

@@ -214,13 +214,17 @@ func (c *CmdOpts) startController(ctx context.Context) error {
 		})
 	}
 
-	if c.NodeConfig.Spec.API.ExternalAddress != "" {
+	if c.NodeConfig.Spec.API.TunneledNetworkingMode {
+		c.ClusterComponents.Add(ctx, controller.NewTunneledEndpointReconciler(leaderElector,
+			adminClientFactory))
+	}
+
+	if c.NodeConfig.Spec.API.ExternalAddress != "" && !c.NodeConfig.Spec.API.TunneledNetworkingMode {
 		c.ClusterComponents.Add(ctx, controller.NewEndpointReconciler(
 			leaderElector,
 			adminClientFactory,
 		))
 	}
-
 	if !stringslice.Contains(c.DisableComponents, constant.CsrApproverComponentName) {
 		c.NodeComponents.Add(ctx, controller.NewCSRApprover(c.NodeConfig,
 			leaderElector,

embedded-bins/Makefile.variables

@@ -41,8 +41,8 @@ etcd_build_go_cgo_enabled = 0
 etcd_build_go_ldflags = "-w -s"
 #etcd_build_go_ldflags_extra =
 
-konnectivity_version = 0.0.25
 konnectivity_buildimage = golang:$(go_version)-alpine
+konnectivity_version = 0.0.27-k0s1
 #konnectivity_build_go_tags =
 konnectivity_build_go_cgo_enabled = 0
 konnectivity_build_go_flags = "-a"

embedded-bins/konnectivity/Dockerfile

@@ -10,7 +10,7 @@ ARG BUILD_GO_LDFLAGS_EXTRA
 
 RUN apk add build-base git make protoc
 
-RUN git clone -b v$VERSION --depth=1 https://github.com/kubernetes-sigs/apiserver-network-proxy.git /apiserver-network-proxy
+RUN git clone -b v$VERSION --depth=1 https://github.com/k0sproject/apiserver-network-proxy.git /apiserver-network-proxy
 WORKDIR /apiserver-network-proxy
 RUN go version
 RUN GO111MODULE=on go get github.com/golang/mock/[email protected] github.com/golang/protobuf/[email protected] && \

go.sum

@@ -1644,7 +1644,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=

inttest/Makefile.variables

@@ -23,5 +23,5 @@ smoketests := \
 	check-upgrade \
 	check-psp \
 	check-defaultstorage \
-	check-statussocket
-
+	check-statussocket \
+	check-tunneledkas

inttest/common/footloosesuite.go

@@ -841,7 +841,7 @@ func (s *FootlooseSuite) createConfig() config.Config {
 
 	cfg := config.Config{
 		Cluster: config.Cluster{
-			Name:       s.T().Name(),
+			Name:       s.T().Name() + "DEBUGBRANCH",
 			PrivateKey: path.Join(s.keyDir, "id_rsa"),
 		},
 		Machines: []config.MachineReplicas{

inttest/tunneledkas/suite_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2021 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package tunneledkas
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/k0sproject/k0s/inttest/common"
+)
+
+type Suite struct {
+	common.FootlooseSuite
+}
+
+const config = `
+spec:
+  api:
+    tunneledNetworkingMode: true
+`
+
+func (s *Suite) TestK0sTunneledKasMode() {
+	s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", config)
+	s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml"))
+
+	token, err := s.GetJoinToken("worker")
+	s.NoError(err)
+	s.NoError(s.RunWorkersWithToken(token))
+
+	// out of cluster client
+	kc, err := s.KubeClient(s.ControllerNode(0))
+	s.NoError(err)
+
+	err = s.WaitForNodeReady(s.WorkerNode(0), kc)
+	s.NoError(err)
+	err = s.WaitForNodeReady(s.WorkerNode(1), kc)
+	s.NoError(err)
+	eps, err := kc.CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", v1.GetOptions{})
+	s.NoError(err)
+
+	nodes, err := kc.CoreV1().Nodes().List(context.Background(), v1.ListOptions{})
+	s.NoError(err)
+
+	s.Assert().Equal(1, len(eps.Subsets))
+	s.Assert().Equal(len(nodes.Items), len(eps.Subsets[0].Addresses))
+
+	svc, err := kc.CoreV1().Services("default").Get(context.Background(), "kubernetes", v1.GetOptions{})
+	s.NoError(err)
+	s.Equal("Local", string(*svc.Spec.InternalTrafficPolicy))
+
+	kubeConfig, err := s.GetKubeConfig(s.ControllerNode(0))
+	s.NoError(err)
+
+	// for each node try to call konnectivity-agent directly
+	// nodes IPs are not in the config.spec.api.sans
+	// so skip x509 verification here for the sake of the test
+	kubeConfig.TLSClientConfig.Insecure = true
+	kubeConfig.TLSClientConfig.CAData = nil
+	for _, addr := range eps.Subsets[0].Addresses {
+		kubeConfig.Host = fmt.Sprintf("https://%s:6443", addr.IP)
+		nodeLocalClient, err := kubernetes.NewForConfig(kubeConfig)
+		s.Require().NoError(err)
+		_, err = nodeLocalClient.CoreV1().Nodes().List(context.Background(), v1.ListOptions{})
+		s.Require().NoError(err)
+	}
+}
+
+func TestK0sTunneledKasModeSuite(t *testing.T) {
+	s := Suite{
+		common.FootlooseSuite{
+			ControllerCount: 1,
+			WorkerCount:     2,
+		},
+	}
+	suite.Run(t, &s)
+}

pkg/apis/k0s.k0sproject.io/v1beta1/api.go

@@ -33,7 +33,8 @@ type APISpec struct {
 
 	// The loadbalancer address (for k0s controllers running behind a loadbalancer)
 	ExternalAddress string `json:"externalAddress,omitempty"`
-
+	// TunneledNetworkingMode indicates if we access to KAS through konnectivity tunnel
+	TunneledNetworkingMode bool `json:"tunneledNetworkingMode"`
 	// Map of key-values (strings) for any extra arguments to pass down to Kubernetes api-server process
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 	// Custom port for k0s-api server to listen on (default: 9443)
@@ -52,11 +53,12 @@ func DefaultAPISpec() *APISpec {
 	addresses, _ := iface.AllAddresses()
 	publicAddress, _ := iface.FirstPublicAddress()
 	return &APISpec{
-		Port:       6443,
-		K0sAPIPort: 9443,
-		SANs:       addresses,
-		Address:    publicAddress,
-		ExtraArgs:  make(map[string]string),
+		Port:                   6443,
+		K0sAPIPort:             9443,
+		SANs:                   addresses,
+		Address:                publicAddress,
+		ExtraArgs:              make(map[string]string),
+		TunneledNetworkingMode: false,
 	}
 }
 

pkg/apis/k0s.k0sproject.io/v1beta1/dualstack.go

@@ -7,20 +7,6 @@ type DualStack struct {
 	IPv6ServiceCIDR string `json:"IPv6serviceCIDR,omitempty"`
 }
 
-// EnableDualStackFeatureGate adds ipv6 feature gate to the given args colllection
-func (ds DualStack) EnableDualStackFeatureGate(args map[string]string) {
-	if !ds.Enabled {
-		return
-	}
-	fg, found := args["feature-gates"]
-	if !found {
-		args["feature-gates"] = "IPv6DualStack=true"
-	} else {
-		fg = fg + ",IPv6DualStack=true"
-		args["feature-gates"] = fg
-	}
-}
-
 // DefaultDualStack builds default values
 func DefaultDualStack() DualStack {
 	return DualStack{}

pkg/apis/k0s.k0sproject.io/v1beta1/feature_gates.go

@@ -0,0 +1,25 @@
+package v1beta1
+
+import (
+	"fmt"
+
+	"github.com/k0sproject/k0s/internal/pkg/stringmap"
+)
+
+const (
+	ServiceInternalTrafficPolicyFeatureGate = "ServiceInternalTrafficPolicy"
+	DualStackFeatureGate                    = "IPv6DualStack"
+)
+
+// EnableFeatureGate enables given feature gate in the arguments
+func EnableFeatureGate(args stringmap.StringMap, gateName string) stringmap.StringMap {
+	gateString := fmt.Sprintf("%s=true", gateName)
+	fg, found := args["feature-gates"]
+	if !found {
+		args["feature-gates"] = gateString
+	} else {
+		fg = fg + "," + gateString
+		args["feature-gates"] = fg
+	}
+	return args
+}

pkg/apis/k0s.k0sproject.io/v1beta1/feature_gates_test.go

@@ -0,0 +1,31 @@
+package v1beta1
+
+import (
+	"testing"
+
+	"github.com/k0sproject/k0s/internal/pkg/stringmap"
+	"github.com/stretchr/testify/require"
+)
+
+func TestArgsFeatureGates(t *testing.T) {
+	t.Run("if_no_feature_gates_add_new_one", func(t *testing.T) {
+		args := stringmap.StringMap{}
+		EnableFeatureGate(args, DualStackFeatureGate)
+		require.Equal(t, "IPv6DualStack=true", args["feature-gates"])
+	})
+	t.Run("if_args_has_some_argument_preserve_it", func(t *testing.T) {
+		args := stringmap.StringMap{
+			"some-argument": "value",
+		}
+		EnableFeatureGate(args, DualStackFeatureGate)
+		require.Equal(t, "IPv6DualStack=true", args["feature-gates"])
+		require.Equal(t, "value", args["some-argument"])
+	})
+	t.Run("merge_new_feature_gate_with_the_current", func(t *testing.T) {
+		args := stringmap.StringMap{
+			"feature-gates": "Magic=true",
+		}
+		EnableFeatureGate(args, DualStackFeatureGate)
+		require.Equal(t, "Magic=true,IPv6DualStack=true", args["feature-gates"])
+	})
+}

pkg/apis/k0s.k0sproject.io/v1beta1/images_test.go

@@ -38,7 +38,7 @@ func TestImagesRepoOverrideInConfiguration(t *testing.T) {
 			cfg.Spec.Images.Repository = "my.repo"
 			var testingConfig *ClusterConfig
 			require.NoError(t, yaml.Unmarshal(getConfigYAML(t, cfg), &testingConfig))
-			require.Equal(t, fmt.Sprintf("my.repo/kas-network-proxy/proxy-agent:%s", constant.KonnectivityImageVersion), testingConfig.Spec.Images.Konnectivity.URI())
+			require.Equal(t, fmt.Sprintf("my.repo/k0sproject/apiserver-network-proxy-agent:%s", constant.KonnectivityImageVersion), testingConfig.Spec.Images.Konnectivity.URI())
 			require.Equal(t, fmt.Sprintf("my.repo/metrics-server/metrics-server:%s", constant.MetricsImageVersion), testingConfig.Spec.Images.MetricsServer.URI())
 			require.Equal(t, fmt.Sprintf("my.repo/kube-proxy:%s", constant.KubeProxyImageVersion), testingConfig.Spec.Images.KubeProxy.URI())
 			require.Equal(t, fmt.Sprintf("my.repo/coredns/coredns:%s", constant.CoreDNSImageVersion), testingConfig.Spec.Images.CoreDNS.URI())