diff --git a/Gopkg.lock b/Gopkg.lock index 77f095cd9418..82fd032079ad 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -115,10 +115,11 @@ "eksconfig", "ekstester", "etcdconfig/plugins", + "kubeadmconfig/plugins", "pkg/awsapi/ec2", ] - revision = "1cd3e8ae2b5c41662eb4008bbb3fb9736dc651c7" - version = "0.1.2" + revision = "7a149356ae339b531b6189f4a801378b62c7b98b" + version = "0.1.3" [[projects]] name = "github.com/aws/aws-sdk-go" @@ -1270,6 +1271,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a6d1c21e9d72d50a0968342de074652f4e589e061d7d52e66c1d71cf7d6dc534" + inputs-digest = "4e42140600055c4abd3f81c94fe0a73c70bfd520954970ff07bbfb3545331f2a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml index 3086f37baa0e..5d1da6f346b7 100644 --- a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml +++ b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml @@ -4,6 +4,10 @@ presets: # URL to download the latest 'aws-k8s-tester' release - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_URL value: https://github.com/aws/aws-k8s-tester/releases/download/0.1.3/aws-k8s-tester-0.1.3-linux-amd64 + # URL to download 'kubectl', required for 'kubectl' calls to EKS + # TODO: use upstream 'kubectl' + - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL + value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl # URL to download 'aws-iam-authenticator', required for 'kubectl' calls to EKS - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_URL value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator @@ -25,13 +29,21 @@ presets: # 'true' to open port 22 in security group, and enable SSH for log dumper - name: AWS_K8S_TESTER_EKS_ENABLE_NODE_SSH value: "true" - value: "true" - # 'true' to upload 'aws-k8s-tester' logs to S3 buckets + # 'true' to enable S3 Access Logs and AWS ALB Access Logs + # use it for debug, dump cluster log already handles log artifacts + - name: AWS_K8S_TESTER_EKS_LOG_ACCESS + value: "false" + # 'true' to upload 'aws-k8s-tester' logs to S3 buckets, in addition to log dumper + # use it for debug, dump cluster log already handles log artifacts - name: AWS_K8S_TESTER_EKS_UPLOAD_TESTER_LOGS - value: "true" - # 'true' to upload worker node logs to S3 + value: "false" + # 'true' to upload worker node logs to S3, in addition to log dumper + # use it for debug, dump cluster log already handles worker node logs - name: AWS_K8S_TESTER_EKS_UPLOAD_WORKER_NODE_LOGS - value: "true" + value: "false" + # Amazon EKS-optimized AMI (non-GPU, https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) + - name: AWS_K8S_TESTER_EKS_WORKER_NODE_AMI + value: ami-0f54a2f7d2e9c88b3 # worker node EC2 instance type - name: AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE value: m3.xlarge @@ -44,9 +56,6 @@ presets: # 'true' to enable debug level logs - name: AWS_K8S_TESTER_EKS_LOG_DEBUG value: "false" - # 'true' to open port 22 in security group, and enable SSH for log dumper - - name: AWS_K8S_TESTER_EKS_LOG_ACCESS - value: "true" # 'true' to create AWS ALB - name: AWS_K8S_TESTER_EKS_ALB_ENABLE value: "false" @@ -66,7 +75,7 @@ presets: periodics: # Run Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod labels: preset-service-account: "true" @@ -89,8 +98,33 @@ periodics: - --test_args=--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\] --minStartupPods=8 - --timeout=180m +# Run Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 +# run conformance tests +- interval: 2h + name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + labels: + preset-service-account: "true" + preset-ci-kubernetes-e2e-aws-eks-1-10: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:latest-master + imagePullPolicy: Always + args: + - --timeout=200 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-version-skew=false + - --deployment=eks + - --provider=eks + - --gce-ssh= + - --extract=ci/latest-1.10 + - --ginkgo-parallel=30 + - --test_args=--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\] --minStartupPods=8 + - --timeout=180m + # Run Kubernetes stable e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod labels: preset-service-account: "true" @@ -114,7 +148,7 @@ periodics: - --timeout=180m # Run Kubernetes latest e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-latest-aws-eks-1-10-prod labels: preset-service-account: "true" diff --git a/kubetest/eks/eks.go b/kubetest/eks/eks.go index 837b875fca79..4f1ad782df96 100644 --- a/kubetest/eks/eks.go +++ b/kubetest/eks/eks.go @@ -45,7 +45,6 @@ type deployer struct { stopc chan struct{} cfg *eksconfig.Config awsK8sTesterPath string - kubectlPath string ctrl *process.Control } @@ -84,47 +83,21 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error dp.awsK8sTesterPath, err = exec.LookPath("aws-k8s-tester") if err != nil { - dp.awsK8sTesterPath = filepath.Join(os.TempDir(), "aws-k8s-tester") var f *os.File - f, err = os.Create(dp.awsK8sTesterPath) + f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester") if err != nil { return nil, fmt.Errorf("failed to create %q (%v)", dp.awsK8sTesterPath, err) } - defer f.Close() + dp.awsK8sTesterPath = f.Name() + dp.awsK8sTesterPath, _ = filepath.Abs(dp.awsK8sTesterPath) if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil { return nil, err } + f.Close() if err = util.EnsureExecutable(dp.awsK8sTesterPath); err != nil { return nil, err } } - - dp.kubectlPath, err = exec.LookPath("kubectl") - if err != nil { - return nil, fmt.Errorf("cannot find 'kubectl' executable (%v)", err) - } - - // TODO(gyuho): replace this kubernetes native Go client - _, err = exec.LookPath("aws-iam-authenticator") - if err != nil { - bin := filepath.Join(os.TempDir(), "aws-iam-authenticator") - var f *os.File - f, err = os.Create(bin) - if err != nil { - return nil, fmt.Errorf("failed to create %q (%v)", bin, err) - } - defer f.Close() - if err = httpRead(cfg.AWSIAMAuthenticatorDownloadURL, f); err != nil { - return nil, err - } - if err = util.EnsureExecutable(bin); err != nil { - return nil, err - } - if err = os.Rename(bin, "/usr/local/bin/aws-iam-authenticator"); err != nil { - return nil, err - } - } - return dp, nil } diff --git a/testgrid/config.yaml b/testgrid/config.yaml index b967715ee98a..103a528d270f 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -2172,6 +2172,8 @@ test_groups: # EKS e2e results - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-1-10-aws-eks-1-10-prod +- name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance - name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-stable-aws-eks-1-10-prod - name: ci-kubernetes-e2e-latest-aws-eks-1-10-prod @@ -5709,6 +5711,9 @@ dashboards: - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod test_group_name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod description: Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 + - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + test_group_name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + description: Kubernetes 1.10 branch e2e tests with EKS prod build 1.10, Conformance tests - name: sig-aws-eks-ci-kubernetes-e2e-stable dashboard_tab: - name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod diff --git a/vendor/BUILD.bazel b/vendor/BUILD.bazel index e4173598e1cc..f1cb64e5b823 100644 --- a/vendor/BUILD.bazel +++ b/vendor/BUILD.bazel @@ -34,6 +34,7 @@ filegroup( "//vendor/github.com/aws/aws-k8s-tester/eksconfig:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/ekstester:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:all-srcs", + "//vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/pkg/awsapi/ec2:all-srcs", "//vendor/github.com/aws/aws-sdk-go/aws:all-srcs", "//vendor/github.com/aws/aws-sdk-go/internal/shareddefaults:all-srcs", diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go index bbeed2163f23..171777d43c6c 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go @@ -121,8 +121,8 @@ type Config struct { SubnetIDs []string `json:"subnet-ids,omitempty"` SubnetIDToAvailibilityZone map[string]string `json:"subnet-id-to-availability-zone,omitempty"` // read-only to user - // IngressCIDRs is a map from TCP port to CIDR to allow via security groups. - IngressCIDRs map[int64]string `json:"ingress-cidrs,omitempty"` + // IngressRulesTCP is a map from TCP port range to CIDR to allow via security groups. + IngressRulesTCP map[string]string `json:"ingress-rules-tcp,omitempty"` // SecurityGroupIDs is the list of security group IDs. // Leave empty to create a temporary one. @@ -227,15 +227,8 @@ var defaultConfig = Config{ UserName: "ec2-user", Plugins: []string{ "update-amazon-linux-2", - "install-go1.11.2", - "install-docker-amazon-linux-2", }, - // Ubuntu Server 16.04 LTS (HVM), SSD Volume Type - // ImageID: "ami-ba602bc2", - // UserName: "ubuntu", - // Plugins: []string{"update-ubuntu"}, - // 4 vCPU, 15 GB RAM InstanceType: "m3.xlarge", ClusterSize: 1, @@ -243,11 +236,11 @@ var defaultConfig = Config{ AssociatePublicIPAddress: true, VPCCIDR: "192.168.0.0/16", - IngressCIDRs: map[int64]string{ - 22: "0.0.0.0/0", + IngressRulesTCP: map[string]string{ + "22": "0.0.0.0/0", }, - Wait: false, + Wait: true, } // UpdateFromEnvs updates fields from environmental variables. @@ -314,15 +307,11 @@ func (cfg *Config) UpdateFromEnvs() error { case reflect.Map: ss := strings.Split(sv, ",") switch fieldName { - case "IngressCIDRs": - m := reflect.MakeMap(reflect.TypeOf(map[int64]string{})) + case "IngressRulesTCP": + m := reflect.MakeMap(reflect.TypeOf(map[string]string{})) for i := range ss { fields := strings.Split(ss[i], "=") - nv, nerr := strconv.ParseInt(fields[0], 10, 64) - if nerr != nil { - return fmt.Errorf("failed to parse IngressTCPPort %s (%v)", fields[0], nerr) - } - m.SetMapIndex(reflect.ValueOf(nv), reflect.ValueOf(fields[1])) + m.SetMapIndex(reflect.ValueOf(fields[0]), reflect.ValueOf(fields[1])) } vv.Field(i).Set(m) @@ -483,7 +472,7 @@ func (cfg *Config) SSHCommands() (s string) { s += fmt.Sprintf(`ssh -o "StrictHostKeyChecking no" -i %s %s@%s `, cfg.KeyPath, cfg.UserName, v.PublicDNSName) } - return s + return s + "\n" } // Sync persists current configuration and states to disk. diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel index 8fb713fc863d..92f64297a1fa 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel @@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = [ - "doc.go", - "plugins.go", - ], + srcs = ["plugins.go"], importmap = "k8s.io/test-infra/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins", importpath = "github.com/aws/aws-k8s-tester/ec2config/plugins", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:go_default_library"], + deps = [ + "//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:go_default_library", + "//vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins:go_default_library", + ], ) filegroup( diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go deleted file mode 100644 index 888b347e3812..000000000000 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package plugins defines various plugins to install on EC2 creation, -// using init scripts or EC2 user data. -package plugins diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go index b5e9f4fc7a08..7787e14ad759 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go @@ -1,3 +1,5 @@ +// Package plugins defines various plugins to install on EC2 creation, +// using init scripts or EC2 user data. package plugins import ( @@ -11,6 +13,7 @@ import ( "text/template" etcdplugin "github.com/aws/aws-k8s-tester/etcdconfig/plugins" + kubeadmplugin "github.com/aws/aws-k8s-tester/kubeadmconfig/plugins" ) // headerBash is the bash script header. @@ -31,19 +34,18 @@ func (ss scripts) Swap(i, j int) { ss[i], ss[j] = ss[j], ss[i] } func (ss scripts) Less(i, j int) bool { return keyPriorities[ss[i].key] < keyPriorities[ss[j].key] } var keyPriorities = map[string]int{ // in the order of: - "update-amazon-linux-2": 1, - "update-ubuntu": 2, - "set-env-aws-cred": 3, // TODO: use instance role instead - "mount-aws-cred": 4, // TODO: use instance role instead - "install-go": 5, - "install-csi": 6, - "install-etcd": 7, - "install-aws-k8s-tester": 8, - "install-wrk": 9, - "install-alb": 10, - "install-kubeadm-ubuntu": 11, - "install-docker-amazon-linux-2": 12, - "install-docker-ubuntu": 13, + "update-amazon-linux-2": 1, + "update-ubuntu": 2, + "set-env-aws-cred": 3, // TODO: use instance role instead + "mount-aws-cred": 4, // TODO: use instance role instead + "install-go": 5, + "install-csi": 6, + "install-etcd": 7, + "install-aws-k8s-tester": 8, + "install-wrk": 9, + "install-alb": 10, + "install-start-docker-amazon-linux-2": 11, + "install-start-kubeadm-amazon-linux-2": 12, } func convertToScript(userName, plugin string) (script, error) { @@ -113,10 +115,11 @@ cat << EOT > /home/%s/.aws/credentials EOT`, userName, userName, string(d)), }, nil - case plugin == "install-go1.11.2": + case strings.HasPrefix(plugin, "install-go-"): + goVer := strings.Replace(plugin, "install-go-", "", -1) s, err := createInstallGo(goInfo{ UserName: userName, - GoVersion: "1.11.2", + GoVersion: goVer, }) if err != nil { return script{}, err @@ -190,23 +193,19 @@ make server } return script{key: "install-alb", data: s}, nil - case plugin == "install-kubeadm-ubuntu": + case plugin == "install-start-docker-amazon-linux-2": return script{ key: plugin, - data: installKubeadmnUbuntu, + data: installStartDockerAmazonLinux2, }, nil - case plugin == "install-docker-amazon-linux-2": - return script{ - key: plugin, - data: installDockerAmazonLinux2, - }, nil - - case plugin == "install-docker-ubuntu": - return script{ - key: plugin, - data: installDockerUbuntu, - }, nil + case strings.HasPrefix(plugin, "install-start-kubeadm-amazon-linux-2-"): + id := strings.Replace(plugin, "install-start-kubeadm-amazon-linux-2-", "", -1) + s, err := kubeadmplugin.CreateInstallStart(id) + if err != nil { + return script{}, err + } + return script{key: "install-start-kubeadm-amazon-linux-2", data: s}, nil } return script{}, fmt.Errorf("unknown plugin %q", plugin) @@ -221,7 +220,6 @@ func Create(userName string, plugins []string) (data string, err error) { return "", fmt.Errorf("'update-ubuntu' requires 'ubuntu' user name, got %q", userName) } } - script, err := convertToScript(userName, plugin) if err != nil { return "", err @@ -344,9 +342,7 @@ DOWNLOAD_URL=${GOOGLE_URL} sudo curl -s ${DOWNLOAD_URL}/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz mkdir -p ${GOPATH}/bin/ -mkdir -p ${GOPATH}/src/github.com/kubernetes-sigs -mkdir -p ${GOPATH}/src/k8s.io -mkdir -p ${GOPATH}/src/sigs.k8s.io +mkdir -p ${GOPATH}/src/ if grep -q GOPATH "${HOME}/.bashrc"; then echo "bashrc already has GOPATH"; @@ -371,34 +367,6 @@ go version ` -const installKubeadmnUbuntu = ` - -################################## install kubeadm on Ubuntu - -cd ${HOME} - -sudo apt-get update -y && sudo apt-get install -y apt-transport-https curl -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - - -cat </tmp/kubernetes.list -deb https://apt.kubernetes.io/ kubernetes-$(lsb_release -cs) main -EOF - -sudo cp /tmp/kubernetes.list /etc/apt/sources.list.d/kubernetes.list - -sudo apt-get update -y -sudo apt-get install -y kubelet kubeadm kubectl || true -sudo apt-mark hold kubelet kubeadm kubectl || true - -sudo systemctl enable kubelet -sudo systemctl start kubelet - -sudo journalctl --no-pager --output=cat -u kubelet - -################################## - -` - func createInstallEtcd(g etcdInfo) (string, error) { tpl := template.Must(template.New("installEtcdTemplate").Parse(installEtcdTemplate)) buf := bytes.NewBuffer(nil) @@ -542,50 +510,52 @@ pwd ` -const installDockerUbuntu = ` - -################################## install Docker on Ubuntu -sudo apt update -y -sudo apt install -y apt-transport-https ca-certificates curl software-properties-common - -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - -sudo apt update -y -apt-cache policy docker-ce || true -sudo apt install -y docker-ce - -sudo systemctl start docker || true -sudo systemctl status docker --full --no-pager || true -sudo usermod -aG docker ubuntu || true - -# su - ubuntu -# or logout and login to use docker without 'sudo' - -id -nG -sudo docker version -sudo docker info -################################## - -` - // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-basics.html -const installDockerAmazonLinux2 = ` +// https://kubernetes.io/docs/setup/cri/#docker +const installStartDockerAmazonLinux2 = ` ################################## install Docker on Amazon Linux 2 + sudo yum update -y sudo yum install -y docker +sudo yum install -y yum-utils device-mapper-persistent-data lvm2 + +sudo yum-config-manager \ + --add-repo \ + https://download.docker.com/linux/centos/docker-ce.repo + +sudo yum update && sudo yum install -y docker-ce-18.06.1.ce +sudo mkdir -p /etc/docker + +cat > /etc/docker/daemon.json < /tmp/kubernetes.repo +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +exclude=kube* +EOF +sudo cp /tmp/kubernetes.repo /etc/yum.repos.d/kubernetes.repo + +cat < /tmp/k8s.conf +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +EOF +sudo cp /tmp/k8s.conf /etc/sysctl.d/k8s.conf +sudo sysctl --system +sudo sysctl net.bridge.bridge-nf-call-iptables=1 + +# Set SELinux in permissive mode (effectively disabling it) +setenforce 0 +sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config + +sudo yum install -y cri-tools ebtables kubernetes-cni socat iproute-tc + +RELEASE=v{{ .Version }} + +cd /usr/bin +sudo rm -f /usr/bin/{kubeadm,kubelet,kubectl} + +sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +sudo chmod +x {kubeadm,kubelet,kubectl} + +curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" > /tmp/kubelet.service +cat /tmp/kubelet.service + +# curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" > /tmp/10-kubeadm.conf +# sudo sed -i 's/cgroup-driver=cgroupfs/cgroup-driver=systemd/' /tmp/10-kubeadm.conf + +# delete cni binary +# https://github.com/coreos/coreos-kubernetes/issues/874 +cat << EOT > /tmp/10-kubeadm.conf +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" +Environment="KUBELET_NETWORK_ARGS=" +Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" +Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt" +# Value should match Docker daemon settings. +# Defaults are "cgroupfs" for Debian/Ubuntu/OpenSUSE and "systemd" for Fedora/CentOS/RHEL +Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd" +Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" +Environment="KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true" +ExecStart= +ExecStart=/usr/bin/kubelet __KUBELET_KUBECONFIG_ARGS __KUBELET_SYSTEM_PODS_ARGS __KUBELET_NETWORK_ARGS __KUBELET_DNS_ARGS __KUBELET_AUTHZ_ARGS __KUBELET_CGROUP_ARGS __KUBELET_CADVISOR_ARGS __KUBELET_CERTIFICATE_ARGS __KUBELET_EXTRA_ARGS +EOT +cat /tmp/10-kubeadm.conf +sed -i.bak 's|__KUBELET|\$KUBELET|g' /tmp/10-kubeadm.conf +cat /tmp/10-kubeadm.conf + +sudo mkdir -p /etc/systemd/system/kubelet.service.d +sudo cp /tmp/kubelet.service /etc/systemd/system/kubelet.service +sudo cp /tmp/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +sudo systemctl daemon-reload +sudo systemctl cat kubelet.service +sudo systemctl enable kubelet && sudo systemctl restart kubelet +sudo systemctl status kubelet --full --no-pager || true +sudo journalctl --no-pager --output=cat -u kubelet + +kubeadm version +kubelet --version +kubectl version --client +crictl --version + +################################## + +`