From dfa1aa68edcce17612e79fcd426fbb465eec62dc Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 8 Nov 2018 00:10:09 -0800 Subject: [PATCH 1/3] config/jobs: disalbe S3 log uploads, add conformance tests Now basic e2e tests are working (thanks SIG testing team!) We'd like to add another test to run conformance. And disable log S3 uploads, which was enabled for initial debugging purposes. Signed-off-by: Gyuho Lee --- .../sig-aws/eks/k8s-aws-eks-1.10.yaml | 56 +++++++++++++++---- testgrid/config.yaml | 5 ++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml index 3086f37baa0e..5d1da6f346b7 100644 --- a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml +++ b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml @@ -4,6 +4,10 @@ presets: # URL to download the latest 'aws-k8s-tester' release - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_URL value: https://github.com/aws/aws-k8s-tester/releases/download/0.1.3/aws-k8s-tester-0.1.3-linux-amd64 + # URL to download 'kubectl', required for 'kubectl' calls to EKS + # TODO: use upstream 'kubectl' + - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL + value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl # URL to download 'aws-iam-authenticator', required for 'kubectl' calls to EKS - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_URL value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator @@ -25,13 +29,21 @@ presets: # 'true' to open port 22 in security group, and enable SSH for log dumper - name: AWS_K8S_TESTER_EKS_ENABLE_NODE_SSH value: "true" - value: "true" - # 'true' to upload 'aws-k8s-tester' logs to S3 buckets + # 'true' to enable S3 Access Logs and AWS ALB Access Logs + # use it for debug, dump cluster log already handles log artifacts + - name: AWS_K8S_TESTER_EKS_LOG_ACCESS + value: "false" + # 'true' to upload 'aws-k8s-tester' logs to S3 buckets, in addition to log dumper + # use it for debug, dump cluster log already handles log artifacts - name: AWS_K8S_TESTER_EKS_UPLOAD_TESTER_LOGS - value: "true" - # 'true' to upload worker node logs to S3 + value: "false" + # 'true' to upload worker node logs to S3, in addition to log dumper + # use it for debug, dump cluster log already handles worker node logs - name: AWS_K8S_TESTER_EKS_UPLOAD_WORKER_NODE_LOGS - value: "true" + value: "false" + # Amazon EKS-optimized AMI (non-GPU, https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) + - name: AWS_K8S_TESTER_EKS_WORKER_NODE_AMI + value: ami-0f54a2f7d2e9c88b3 # worker node EC2 instance type - name: AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE value: m3.xlarge @@ -44,9 +56,6 @@ presets: # 'true' to enable debug level logs - name: AWS_K8S_TESTER_EKS_LOG_DEBUG value: "false" - # 'true' to open port 22 in security group, and enable SSH for log dumper - - name: AWS_K8S_TESTER_EKS_LOG_ACCESS - value: "true" # 'true' to create AWS ALB - name: AWS_K8S_TESTER_EKS_ALB_ENABLE value: "false" @@ -66,7 +75,7 @@ presets: periodics: # Run Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod labels: preset-service-account: "true" @@ -89,8 +98,33 @@ periodics: - --test_args=--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\] --minStartupPods=8 - --timeout=180m +# Run Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 +# run conformance tests +- interval: 2h + name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + labels: + preset-service-account: "true" + preset-ci-kubernetes-e2e-aws-eks-1-10: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:latest-master + imagePullPolicy: Always + args: + - --timeout=200 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-version-skew=false + - --deployment=eks + - --provider=eks + - --gce-ssh= + - --extract=ci/latest-1.10 + - --ginkgo-parallel=30 + - --test_args=--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\] --minStartupPods=8 + - --timeout=180m + # Run Kubernetes stable e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod labels: preset-service-account: "true" @@ -114,7 +148,7 @@ periodics: - --timeout=180m # Run Kubernetes latest e2e tests with EKS prod build 1.10 -- interval: 1h +- interval: 2h name: ci-kubernetes-e2e-latest-aws-eks-1-10-prod labels: preset-service-account: "true" diff --git a/testgrid/config.yaml b/testgrid/config.yaml index b967715ee98a..103a528d270f 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -2172,6 +2172,8 @@ test_groups: # EKS e2e results - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-1-10-aws-eks-1-10-prod +- name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance - name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-stable-aws-eks-1-10-prod - name: ci-kubernetes-e2e-latest-aws-eks-1-10-prod @@ -5709,6 +5711,9 @@ dashboards: - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod test_group_name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod description: Kubernetes 1.10 branch e2e tests with EKS prod build 1.10 + - name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + test_group_name: ci-kubernetes-e2e-1-10-aws-eks-1-10-prod-conformance + description: Kubernetes 1.10 branch e2e tests with EKS prod build 1.10, Conformance tests - name: sig-aws-eks-ci-kubernetes-e2e-stable dashboard_tab: - name: ci-kubernetes-e2e-stable-aws-eks-1-10-prod From 85fa5f0e9bf00b519bdac99e7e87f6b055aa81b8 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 9 Nov 2018 11:30:30 -0800 Subject: [PATCH 2/3] kubetest/eks: move "aws-iam-authenticator" download code to "aws-k8s-tester" Signed-off-by: Gyuho Lee --- kubetest/eks/eks.go | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/kubetest/eks/eks.go b/kubetest/eks/eks.go index 837b875fca79..4f1ad782df96 100644 --- a/kubetest/eks/eks.go +++ b/kubetest/eks/eks.go @@ -45,7 +45,6 @@ type deployer struct { stopc chan struct{} cfg *eksconfig.Config awsK8sTesterPath string - kubectlPath string ctrl *process.Control } @@ -84,47 +83,21 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error dp.awsK8sTesterPath, err = exec.LookPath("aws-k8s-tester") if err != nil { - dp.awsK8sTesterPath = filepath.Join(os.TempDir(), "aws-k8s-tester") var f *os.File - f, err = os.Create(dp.awsK8sTesterPath) + f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester") if err != nil { return nil, fmt.Errorf("failed to create %q (%v)", dp.awsK8sTesterPath, err) } - defer f.Close() + dp.awsK8sTesterPath = f.Name() + dp.awsK8sTesterPath, _ = filepath.Abs(dp.awsK8sTesterPath) if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil { return nil, err } + f.Close() if err = util.EnsureExecutable(dp.awsK8sTesterPath); err != nil { return nil, err } } - - dp.kubectlPath, err = exec.LookPath("kubectl") - if err != nil { - return nil, fmt.Errorf("cannot find 'kubectl' executable (%v)", err) - } - - // TODO(gyuho): replace this kubernetes native Go client - _, err = exec.LookPath("aws-iam-authenticator") - if err != nil { - bin := filepath.Join(os.TempDir(), "aws-iam-authenticator") - var f *os.File - f, err = os.Create(bin) - if err != nil { - return nil, fmt.Errorf("failed to create %q (%v)", bin, err) - } - defer f.Close() - if err = httpRead(cfg.AWSIAMAuthenticatorDownloadURL, f); err != nil { - return nil, err - } - if err = util.EnsureExecutable(bin); err != nil { - return nil, err - } - if err = os.Rename(bin, "/usr/local/bin/aws-iam-authenticator"); err != nil { - return nil, err - } - } - return dp, nil } From 5ce9af5ee79a7a1337ed6de3207527310e3d7dc6 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 9 Nov 2018 11:42:07 -0800 Subject: [PATCH 3/3] vendor: upgrade "aws-k8s-tester" to 0.1.3 Signed-off-by: Gyuho Lee --- Gopkg.lock | 7 +- vendor/BUILD.bazel | 1 + .../aws/aws-k8s-tester/ec2config/config.go | 29 ++-- .../ec2config/plugins/BUILD.bazel | 10 +- .../aws-k8s-tester/ec2config/plugins/doc.go | 3 - .../ec2config/plugins/plugins.go | 148 +++++++----------- .../aws/aws-k8s-tester/eksconfig/config.go | 51 +++--- .../kubeadmconfig/plugins/BUILD.bazel | 23 +++ .../kubeadmconfig/plugins/plugins.go | 107 +++++++++++++ 9 files changed, 238 insertions(+), 141 deletions(-) delete mode 100644 vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go create mode 100644 vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins/BUILD.bazel create mode 100644 vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins/plugins.go diff --git a/Gopkg.lock b/Gopkg.lock index 77f095cd9418..82fd032079ad 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -115,10 +115,11 @@ "eksconfig", "ekstester", "etcdconfig/plugins", + "kubeadmconfig/plugins", "pkg/awsapi/ec2", ] - revision = "1cd3e8ae2b5c41662eb4008bbb3fb9736dc651c7" - version = "0.1.2" + revision = "7a149356ae339b531b6189f4a801378b62c7b98b" + version = "0.1.3" [[projects]] name = "github.com/aws/aws-sdk-go" @@ -1270,6 +1271,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a6d1c21e9d72d50a0968342de074652f4e589e061d7d52e66c1d71cf7d6dc534" + inputs-digest = "4e42140600055c4abd3f81c94fe0a73c70bfd520954970ff07bbfb3545331f2a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/BUILD.bazel b/vendor/BUILD.bazel index e4173598e1cc..f1cb64e5b823 100644 --- a/vendor/BUILD.bazel +++ b/vendor/BUILD.bazel @@ -34,6 +34,7 @@ filegroup( "//vendor/github.com/aws/aws-k8s-tester/eksconfig:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/ekstester:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:all-srcs", + "//vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins:all-srcs", "//vendor/github.com/aws/aws-k8s-tester/pkg/awsapi/ec2:all-srcs", "//vendor/github.com/aws/aws-sdk-go/aws:all-srcs", "//vendor/github.com/aws/aws-sdk-go/internal/shareddefaults:all-srcs", diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go index bbeed2163f23..171777d43c6c 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go @@ -121,8 +121,8 @@ type Config struct { SubnetIDs []string `json:"subnet-ids,omitempty"` SubnetIDToAvailibilityZone map[string]string `json:"subnet-id-to-availability-zone,omitempty"` // read-only to user - // IngressCIDRs is a map from TCP port to CIDR to allow via security groups. - IngressCIDRs map[int64]string `json:"ingress-cidrs,omitempty"` + // IngressRulesTCP is a map from TCP port range to CIDR to allow via security groups. + IngressRulesTCP map[string]string `json:"ingress-rules-tcp,omitempty"` // SecurityGroupIDs is the list of security group IDs. // Leave empty to create a temporary one. @@ -227,15 +227,8 @@ var defaultConfig = Config{ UserName: "ec2-user", Plugins: []string{ "update-amazon-linux-2", - "install-go1.11.2", - "install-docker-amazon-linux-2", }, - // Ubuntu Server 16.04 LTS (HVM), SSD Volume Type - // ImageID: "ami-ba602bc2", - // UserName: "ubuntu", - // Plugins: []string{"update-ubuntu"}, - // 4 vCPU, 15 GB RAM InstanceType: "m3.xlarge", ClusterSize: 1, @@ -243,11 +236,11 @@ var defaultConfig = Config{ AssociatePublicIPAddress: true, VPCCIDR: "192.168.0.0/16", - IngressCIDRs: map[int64]string{ - 22: "0.0.0.0/0", + IngressRulesTCP: map[string]string{ + "22": "0.0.0.0/0", }, - Wait: false, + Wait: true, } // UpdateFromEnvs updates fields from environmental variables. @@ -314,15 +307,11 @@ func (cfg *Config) UpdateFromEnvs() error { case reflect.Map: ss := strings.Split(sv, ",") switch fieldName { - case "IngressCIDRs": - m := reflect.MakeMap(reflect.TypeOf(map[int64]string{})) + case "IngressRulesTCP": + m := reflect.MakeMap(reflect.TypeOf(map[string]string{})) for i := range ss { fields := strings.Split(ss[i], "=") - nv, nerr := strconv.ParseInt(fields[0], 10, 64) - if nerr != nil { - return fmt.Errorf("failed to parse IngressTCPPort %s (%v)", fields[0], nerr) - } - m.SetMapIndex(reflect.ValueOf(nv), reflect.ValueOf(fields[1])) + m.SetMapIndex(reflect.ValueOf(fields[0]), reflect.ValueOf(fields[1])) } vv.Field(i).Set(m) @@ -483,7 +472,7 @@ func (cfg *Config) SSHCommands() (s string) { s += fmt.Sprintf(`ssh -o "StrictHostKeyChecking no" -i %s %s@%s `, cfg.KeyPath, cfg.UserName, v.PublicDNSName) } - return s + return s + "\n" } // Sync persists current configuration and states to disk. diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel index 8fb713fc863d..92f64297a1fa 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/BUILD.bazel @@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = [ - "doc.go", - "plugins.go", - ], + srcs = ["plugins.go"], importmap = "k8s.io/test-infra/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins", importpath = "github.com/aws/aws-k8s-tester/ec2config/plugins", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:go_default_library"], + deps = [ + "//vendor/github.com/aws/aws-k8s-tester/etcdconfig/plugins:go_default_library", + "//vendor/github.com/aws/aws-k8s-tester/kubeadmconfig/plugins:go_default_library", + ], ) filegroup( diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go deleted file mode 100644 index 888b347e3812..000000000000 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package plugins defines various plugins to install on EC2 creation, -// using init scripts or EC2 user data. -package plugins diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go index b5e9f4fc7a08..7787e14ad759 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go @@ -1,3 +1,5 @@ +// Package plugins defines various plugins to install on EC2 creation, +// using init scripts or EC2 user data. package plugins import ( @@ -11,6 +13,7 @@ import ( "text/template" etcdplugin "github.com/aws/aws-k8s-tester/etcdconfig/plugins" + kubeadmplugin "github.com/aws/aws-k8s-tester/kubeadmconfig/plugins" ) // headerBash is the bash script header. @@ -31,19 +34,18 @@ func (ss scripts) Swap(i, j int) { ss[i], ss[j] = ss[j], ss[i] } func (ss scripts) Less(i, j int) bool { return keyPriorities[ss[i].key] < keyPriorities[ss[j].key] } var keyPriorities = map[string]int{ // in the order of: - "update-amazon-linux-2": 1, - "update-ubuntu": 2, - "set-env-aws-cred": 3, // TODO: use instance role instead - "mount-aws-cred": 4, // TODO: use instance role instead - "install-go": 5, - "install-csi": 6, - "install-etcd": 7, - "install-aws-k8s-tester": 8, - "install-wrk": 9, - "install-alb": 10, - "install-kubeadm-ubuntu": 11, - "install-docker-amazon-linux-2": 12, - "install-docker-ubuntu": 13, + "update-amazon-linux-2": 1, + "update-ubuntu": 2, + "set-env-aws-cred": 3, // TODO: use instance role instead + "mount-aws-cred": 4, // TODO: use instance role instead + "install-go": 5, + "install-csi": 6, + "install-etcd": 7, + "install-aws-k8s-tester": 8, + "install-wrk": 9, + "install-alb": 10, + "install-start-docker-amazon-linux-2": 11, + "install-start-kubeadm-amazon-linux-2": 12, } func convertToScript(userName, plugin string) (script, error) { @@ -113,10 +115,11 @@ cat << EOT > /home/%s/.aws/credentials EOT`, userName, userName, string(d)), }, nil - case plugin == "install-go1.11.2": + case strings.HasPrefix(plugin, "install-go-"): + goVer := strings.Replace(plugin, "install-go-", "", -1) s, err := createInstallGo(goInfo{ UserName: userName, - GoVersion: "1.11.2", + GoVersion: goVer, }) if err != nil { return script{}, err @@ -190,23 +193,19 @@ make server } return script{key: "install-alb", data: s}, nil - case plugin == "install-kubeadm-ubuntu": + case plugin == "install-start-docker-amazon-linux-2": return script{ key: plugin, - data: installKubeadmnUbuntu, + data: installStartDockerAmazonLinux2, }, nil - case plugin == "install-docker-amazon-linux-2": - return script{ - key: plugin, - data: installDockerAmazonLinux2, - }, nil - - case plugin == "install-docker-ubuntu": - return script{ - key: plugin, - data: installDockerUbuntu, - }, nil + case strings.HasPrefix(plugin, "install-start-kubeadm-amazon-linux-2-"): + id := strings.Replace(plugin, "install-start-kubeadm-amazon-linux-2-", "", -1) + s, err := kubeadmplugin.CreateInstallStart(id) + if err != nil { + return script{}, err + } + return script{key: "install-start-kubeadm-amazon-linux-2", data: s}, nil } return script{}, fmt.Errorf("unknown plugin %q", plugin) @@ -221,7 +220,6 @@ func Create(userName string, plugins []string) (data string, err error) { return "", fmt.Errorf("'update-ubuntu' requires 'ubuntu' user name, got %q", userName) } } - script, err := convertToScript(userName, plugin) if err != nil { return "", err @@ -344,9 +342,7 @@ DOWNLOAD_URL=${GOOGLE_URL} sudo curl -s ${DOWNLOAD_URL}/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz mkdir -p ${GOPATH}/bin/ -mkdir -p ${GOPATH}/src/github.com/kubernetes-sigs -mkdir -p ${GOPATH}/src/k8s.io -mkdir -p ${GOPATH}/src/sigs.k8s.io +mkdir -p ${GOPATH}/src/ if grep -q GOPATH "${HOME}/.bashrc"; then echo "bashrc already has GOPATH"; @@ -371,34 +367,6 @@ go version ` -const installKubeadmnUbuntu = ` - -################################## install kubeadm on Ubuntu - -cd ${HOME} - -sudo apt-get update -y && sudo apt-get install -y apt-transport-https curl -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - - -cat </tmp/kubernetes.list -deb https://apt.kubernetes.io/ kubernetes-$(lsb_release -cs) main -EOF - -sudo cp /tmp/kubernetes.list /etc/apt/sources.list.d/kubernetes.list - -sudo apt-get update -y -sudo apt-get install -y kubelet kubeadm kubectl || true -sudo apt-mark hold kubelet kubeadm kubectl || true - -sudo systemctl enable kubelet -sudo systemctl start kubelet - -sudo journalctl --no-pager --output=cat -u kubelet - -################################## - -` - func createInstallEtcd(g etcdInfo) (string, error) { tpl := template.Must(template.New("installEtcdTemplate").Parse(installEtcdTemplate)) buf := bytes.NewBuffer(nil) @@ -542,50 +510,52 @@ pwd ` -const installDockerUbuntu = ` - -################################## install Docker on Ubuntu -sudo apt update -y -sudo apt install -y apt-transport-https ca-certificates curl software-properties-common - -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - -sudo apt update -y -apt-cache policy docker-ce || true -sudo apt install -y docker-ce - -sudo systemctl start docker || true -sudo systemctl status docker --full --no-pager || true -sudo usermod -aG docker ubuntu || true - -# su - ubuntu -# or logout and login to use docker without 'sudo' - -id -nG -sudo docker version -sudo docker info -################################## - -` - // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-basics.html -const installDockerAmazonLinux2 = ` +// https://kubernetes.io/docs/setup/cri/#docker +const installStartDockerAmazonLinux2 = ` ################################## install Docker on Amazon Linux 2 + sudo yum update -y sudo yum install -y docker +sudo yum install -y yum-utils device-mapper-persistent-data lvm2 + +sudo yum-config-manager \ + --add-repo \ + https://download.docker.com/linux/centos/docker-ce.repo + +sudo yum update && sudo yum install -y docker-ce-18.06.1.ce +sudo mkdir -p /etc/docker + +cat > /etc/docker/daemon.json < /tmp/kubernetes.repo +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +exclude=kube* +EOF +sudo cp /tmp/kubernetes.repo /etc/yum.repos.d/kubernetes.repo + +cat < /tmp/k8s.conf +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +EOF +sudo cp /tmp/k8s.conf /etc/sysctl.d/k8s.conf +sudo sysctl --system +sudo sysctl net.bridge.bridge-nf-call-iptables=1 + +# Set SELinux in permissive mode (effectively disabling it) +setenforce 0 +sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config + +sudo yum install -y cri-tools ebtables kubernetes-cni socat iproute-tc + +RELEASE=v{{ .Version }} + +cd /usr/bin +sudo rm -f /usr/bin/{kubeadm,kubelet,kubectl} + +sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +sudo chmod +x {kubeadm,kubelet,kubectl} + +curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" > /tmp/kubelet.service +cat /tmp/kubelet.service + +# curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" > /tmp/10-kubeadm.conf +# sudo sed -i 's/cgroup-driver=cgroupfs/cgroup-driver=systemd/' /tmp/10-kubeadm.conf + +# delete cni binary +# https://github.com/coreos/coreos-kubernetes/issues/874 +cat << EOT > /tmp/10-kubeadm.conf +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" +Environment="KUBELET_NETWORK_ARGS=" +Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" +Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt" +# Value should match Docker daemon settings. +# Defaults are "cgroupfs" for Debian/Ubuntu/OpenSUSE and "systemd" for Fedora/CentOS/RHEL +Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd" +Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" +Environment="KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true" +ExecStart= +ExecStart=/usr/bin/kubelet __KUBELET_KUBECONFIG_ARGS __KUBELET_SYSTEM_PODS_ARGS __KUBELET_NETWORK_ARGS __KUBELET_DNS_ARGS __KUBELET_AUTHZ_ARGS __KUBELET_CGROUP_ARGS __KUBELET_CADVISOR_ARGS __KUBELET_CERTIFICATE_ARGS __KUBELET_EXTRA_ARGS +EOT +cat /tmp/10-kubeadm.conf +sed -i.bak 's|__KUBELET|\$KUBELET|g' /tmp/10-kubeadm.conf +cat /tmp/10-kubeadm.conf + +sudo mkdir -p /etc/systemd/system/kubelet.service.d +sudo cp /tmp/kubelet.service /etc/systemd/system/kubelet.service +sudo cp /tmp/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +sudo systemctl daemon-reload +sudo systemctl cat kubelet.service +sudo systemctl enable kubelet && sudo systemctl restart kubelet +sudo systemctl status kubelet --full --no-pager || true +sudo journalctl --no-pager --output=cat -u kubelet + +kubeadm version +kubelet --version +kubectl version --client +crictl --version + +################################## + +`