diff --git a/install/check.go b/install/check.go index 396df8126ee..15d702e09e2 100644 --- a/install/check.go +++ b/install/check.go @@ -21,8 +21,7 @@ func (s *SealosInstaller) CheckValid() { // 所有node节点 //nodes := append(Nodes, ParseIPs(NodeIPs)...) //hosts := append(masters, nodes...) - var hosts []string - hosts = append(s.Masters, s.Nodes...) + var hosts = append(s.Masters, s.Nodes...) if len(s.Hosts) == 0 && len(hosts) == 0 { s.Print("Fail") logger.Error("hosts not allow empty") diff --git a/install/clean.go b/install/clean.go index e5189d72bad..203c693e4b5 100644 --- a/install/clean.go +++ b/install/clean.go @@ -155,7 +155,7 @@ func (s *SealosClean) cleanMaster(master string) { func clean(host string) { cmd := "kubeadm reset -f " + vlogToStr() _ = SSHConfig.CmdAsync(host, cmd) - cmd = fmt.Sprintf(`sed -i '/kubectl/d;/sealos/d' /root/.bashrc`) + cmd = `sed -i '/kubectl/d;/sealos/d' /root/.bashrc` _ = SSHConfig.CmdAsync(host, cmd) cmd = "modprobe -r ipip && lsmod" _ = SSHConfig.CmdAsync(host, cmd) @@ -171,13 +171,13 @@ func clean(host string) { _ = SSHConfig.CmdAsync(host, cmd) cmd = fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts ", ApiServer) _ = SSHConfig.CmdAsync(host, cmd) - cmd = fmt.Sprint("rm -rf ~/kube") + cmd = "rm -rf ~/kube" _ = SSHConfig.CmdAsync(host, cmd) //clean pki certs - cmd = fmt.Sprint("rm -rf /etc/kubernetes/pki") + cmd = "rm -rf /etc/kubernetes/pki" _ = SSHConfig.CmdAsync(host, cmd) //clean sealos in /usr/bin/ except exec sealos - cmd = fmt.Sprint("ps -ef |grep -v 'grep'|grep sealos >/dev/null || rm -rf /usr/bin/sealos") + cmd = "ps -ef |grep -v 'grep'|grep sealos >/dev/null || rm -rf /usr/bin/sealos" _ = SSHConfig.CmdAsync(host, cmd) } diff --git a/install/constants.go b/install/constants.go index df1f671df23..e4e6fa54a0d 100644 --- a/install/constants.go +++ b/install/constants.go @@ -29,75 +29,47 @@ const ( DefaultContainerdCRISocket = "/run/containerd/containerd.sock" DefaultCgroupDriver = "cgroupfs" DefaultSystemdCgroupDriver = "systemd" + + KubeadmV1beta1 = "kubeadm.k8s.io/v1beta1" + KubeadmV1beta2 = "kubeadm.k8s.io/v1beta2" + KubeadmV1beta3 = "kubeadm.k8s.io/v1beta3" +/* +A list of changes since v1beta1: + +`certificateKey" field is added to InitConfiguration and JoinConfiguration. +"ignorePreflightErrors" field is added to the NodeRegistrationOptions. +The JSON "omitempty" tag is used in a more places where appropriate. +The JSON "omitempty" tag of the "taints" field (inside NodeRegistrationOptions) is removed. See the Kubernetes 1.15 changelog for further details. + + +A list of changes since v1beta2: + +The deprecated ClusterConfiguration.useHyperKubeImage field has been removed. Kubeadm no longer supports the hyperkube image. +The ClusterConfiguration.dns.type field has been removed since CoreDNS is the only supported DNS server type by kubeadm. +Include "datapolicy" tags on the fields that hold secrets. This would result in the field values to be omitted when API structures are printed with klog. +Add InitConfiguration.skipPhases, JoinConfiguration.skipPhases to allow skipping a list of phases during kubeadm init/join command execution. +Add InitConfiguration.nodeRegistration.imagePullPolicy" andJoinConfiguration.nodeRegistration.imagePullPolicy` to allow specifying the images pull policy during kubeadm "init" and "join". The value must be one of "Always", "Never" or "IfNotPresent". "IfNotPresent" is the default, which has been the existing behavior prior to this addition. +Add InitConfiguration.patches.directory, JoinConfiguration.patches.directory to allow the user to configure a directory from which to take patches for components deployed by kubeadm. +Move the BootstrapToken∗ API and related utilities out of the "kubeadm" API group to a new group "bootstraptoken". The kubeadm API version v1beta3 no longer contains the BootstrapToken∗ structures. + +*/ + Bootstraptokenv1 = "bootstraptoken/v1" ) -const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{.Master0}} - bindPort: 6443 ---- -apiVersion: kubeadm.k8s.io/v1beta1 -kind: ClusterConfiguration -kubernetesVersion: {{.Version}} -controlPlaneEndpoint: "{{.ApiServer}}:6443" -imageRepository: {{.Repo}} -networking: - # dnsDomain: cluster.local - podSubnet: {{.PodCIDR}} - serviceSubnet: {{.SvcCIDR}} -apiServer: - certSANs: - - 127.0.0.1 - - {{.ApiServer}} - {{range .Masters -}} - - {{.}} - {{end -}} - {{range .CertSANS -}} - - {{.}} - {{end -}} - - {{.VIP}} - extraArgs: - feature-gates: TTLAfterFinished=true - extraVolumes: - - name: localtime - hostPath: /etc/localtime - mountPath: /etc/localtime - readOnly: true - pathType: File -controllerManager: - extraArgs: - feature-gates: TTLAfterFinished=true - experimental-cluster-signing-duration: 876000h -{{- if eq .Network "cilium" }} - allocate-node-cidrs: \"true\" -{{- end }} - extraVolumes: - - hostPath: /etc/localtime - mountPath: /etc/localtime - name: localtime - readOnly: true - pathType: File -scheduler: - extraArgs: - feature-gates: TTLAfterFinished=true - extraVolumes: - - hostPath: /etc/localtime - mountPath: /etc/localtime - name: localtime - readOnly: true - pathType: File ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -mode: "ipvs" -ipvs: - excludeCIDRs: - - "{{.VIP}}/32" ---- -` + kubeletConfigDefault) +const ( + InitTemplateText = string(InitConfigurationDefault + + ClusterConfigurationDefault + + kubeproxyConfigDefault + + kubeletConfigDefault) + JoinCPTemplateText = string(bootstrapTokenDefault + + JoinConfigurationDefault + + kubeletConfigDefault) -const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2 + bootstrapTokenDefault = `{{- if eq .BootstrapApi "bootstraptoken/v1" }} +apiVersion: {{.BootstrapApi}} + {{- else}} +apiVersion: {{.KubeadmApi}} +{{- end}} caCertPath: /etc/kubernetes/pki/ca.crt discovery: bootstrapToken: @@ -110,6 +82,17 @@ discovery: caCertHashes: - {{.TokenDiscoveryCAHash}} timeout: 5m0s +` + InitConfigurationDefault = `apiVersion: {{.KubeadmApi}} +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {{.Master0}} + bindPort: 6443 +nodeRegistration: + criSocket: {{.CriSocket}} +` + + JoinConfigurationDefault = ` kind: JoinConfiguration {{- if .Master }} controlPlane: @@ -119,18 +102,10 @@ controlPlane: {{- end}} nodeRegistration: criSocket: {{.CriSocket}} ---- -` + kubeletConfigDefault) +` -const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{.Master0}} - bindPort: 6443 -nodeRegistration: - criSocket: /run/containerd/containerd.sock ---- -apiVersion: kubeadm.k8s.io/v1beta2 + ClusterConfigurationDefault = `--- +apiVersion: {{.KubeadmApi}} kind: ClusterConfiguration kubernetesVersion: {{.Version}} controlPlaneEndpoint: "{{.ApiServer}}:6443" @@ -180,6 +155,8 @@ scheduler: name: localtime readOnly: true pathType: File +` + kubeproxyConfigDefault = ` --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration @@ -187,20 +164,10 @@ mode: "ipvs" ipvs: excludeCIDRs: - "{{.VIP}}/32" +` + kubeletConfigDefault = ` --- -` + kubeletConfigDefault) - -const ( - ContainerdShell = `if grep "SystemdCgroup = true" /etc/containerd/config.toml &> /dev/null; then -driver=systemd -else -driver=cgroupfs -fi -echo ${driver}` - DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}") - echo "${driver}"` - - kubeletConfigDefault = `apiVersion: kubelet.config.k8s.io/v1beta1 +apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration authentication: anonymous: @@ -269,4 +236,13 @@ staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 4h0m0s syncFrequency: 1m0s volumeStatsAggPeriod: 1m0s` + + ContainerdShell = `if grep "SystemdCgroup = true" /etc/containerd/config.toml &> /dev/null; then +driver=systemd +else +driver=cgroupfs +fi +echo ${driver}` + DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}") + echo "${driver}"` ) diff --git a/install/etcd_save.go b/install/etcd_save.go index 774311f5e1b..c69dddfd17c 100644 --- a/install/etcd_save.go +++ b/install/etcd_save.go @@ -193,12 +193,11 @@ func GetEtcdClient(ep []string) (*clientv3.Client, error) { if err != nil { return nil, err } - cli, err := clientv3.New(clientv3.Config{ + return clientv3.New(clientv3.Config{ Endpoints: ep, DialTimeout: 5 * time.Second, TLS: clientTLS, }) - return cli, nil } type epHealth struct { diff --git a/install/generator.go b/install/generator.go index 2e5e4434d66..61f96eb32c8 100644 --- a/install/generator.go +++ b/install/generator.go @@ -11,6 +11,29 @@ import ( var ConfigType string +func setKubeadmApi(version string) { + major, _ := GetMajorMinorInt(version) + switch { + // + case major < 120: + KubeadmApi = KubeadmV1beta1 + CriSocket = DefaultDockerCRISocket + case major < 123 && major >= 120: + KubeadmApi = KubeadmV1beta2 + CriSocket = DefaultContainerdCRISocket + case major >= 123: + KubeadmApi = KubeadmV1beta3 + CriSocket = DefaultContainerdCRISocket + BootstrapApi = Bootstraptokenv1 + default: + KubeadmApi = KubeadmV1beta3 + CriSocket = DefaultContainerdCRISocket + BootstrapApi = Bootstraptokenv1 + } + logger.Debug("KubeadmApi: %s", KubeadmApi) + logger.Debug("CriSocket: %s", CriSocket) +} + func Config() { switch ConfigType { case "kubeadm": @@ -24,7 +47,7 @@ func Config() { func joinKubeadmConfig() string { var sb strings.Builder - sb.Write([]byte(JoinCPTemplateTextV1beta2)) + sb.Write([]byte(JoinCPTemplateText)) return sb.String() } @@ -34,13 +57,7 @@ func printlnJoinKubeadmConfig() { func kubeadmConfig() string { var sb strings.Builder - // kubernetes gt 1.20, use Containerd instead of docker - if For120(Version) { - sb.Write([]byte(InitTemplateTextV1bate2)) - } else { - sb.Write([]byte(InitTemplateTextV1beta1)) - } - + sb.Write([]byte(InitTemplateText)) return sb.String() } @@ -59,6 +76,7 @@ func JoinTemplate(ip string, cgroup string) []byte { } func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte { + setKubeadmApi(Version) tmpl, err := template.New("text").Parse(templateContent) defer func() { if r := recover(); r != nil { @@ -74,11 +92,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte envMap["TokenDiscovery"] = JoinToken envMap["TokenDiscoveryCAHash"] = TokenCaCertHash envMap["VIP"] = VIP - if For120(Version) { - CriSocket = DefaultContainerdCRISocket - } else { - CriSocket = DefaultDockerCRISocket - } + envMap["KubeadmApi"] = KubeadmApi envMap["CriSocket"] = CriSocket envMap["CgroupDriver"] = cgroup var buffer bytes.Buffer @@ -87,6 +101,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte } func TemplateFromTemplateContent(templateContent string) []byte { + setKubeadmApi(Version) tmpl, err := template.New("text").Parse(templateContent) defer func() { if r := recover(); r != nil { @@ -113,6 +128,8 @@ func TemplateFromTemplateContent(templateContent string) []byte { envMap["Master0"] = IpFormat(MasterIPs[0]) envMap["Network"] = Network envMap["CgroupDriver"] = CgroupDriver + envMap["KubeadmApi"] = KubeadmApi + envMap["CriSocket"] = CriSocket var buffer bytes.Buffer _ = tmpl.Execute(&buffer, envMap) return buffer.Bytes() diff --git a/install/generator_test.go b/install/generator_test.go index 371dfc5420c..ce173eac02c 100644 --- a/install/generator_test.go +++ b/install/generator_test.go @@ -32,6 +32,7 @@ func TestNetCiliumTemplate(t *testing.T) { ApiServer = "apiserver.cluster.local" Version = "1.20.5" Network = "cilium" + CgroupDriver = DefaultCgroupDriver t.Log(string(Template())) Network = "calico" t.Log(string(Template())) @@ -39,6 +40,7 @@ func TestNetCiliumTemplate(t *testing.T) { Network = "cilium" t.Log(string(Template())) Network = "calico" + CgroupDriver = DefaultSystemdCgroupDriver t.Log(string(Template())) } @@ -148,19 +150,3 @@ func TestJoinTemplate(t *testing.T) { config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa") t.Log(string(JoinTemplate("", "cgroupfs"))) } - -var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2 -caCertPath: /etc/kubernetes/pki/ca.crt -discovery: - bootstrapToken: - apiServerEndpoint: {{.Master0}}:6443 - token: {{.TokenDiscovery}} - caCertHashes: - - {{.TokenDiscoveryCAHash}} - timeout: 5m0s -kind: JoinConfiguration -controlPlane: - localAPIEndpoint: - advertiseAddress: {{.Master}} - bindPort: 6443 -` diff --git a/install/init.go b/install/init.go index 5eeb6a663cd..de19e964d50 100644 --- a/install/init.go +++ b/install/init.go @@ -193,7 +193,7 @@ func (s *SealosInstaller) InstallMaster0() { decodeOutput(output) cmd = `mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config && chmod 600 /root/.kube/config` - output = SSHConfig.Cmd(s.Masters[0], cmd) + SSHConfig.Cmd(s.Masters[0], cmd) if WithoutCNI { logger.Info("--without-cni is true, so we not install calico or flannel, install it by yourself") @@ -236,7 +236,7 @@ func (s *SealosInstaller) InstallMaster0() { configYamlDir := filepath.Join(home, ".sealos", "cni.yaml") ioutil.WriteFile(configYamlDir, []byte(netyaml), 0755) SSHConfig.Copy(s.Masters[0], configYamlDir, "/tmp/cni.yaml") - output = SSHConfig.Cmd(s.Masters[0], "kubectl apply -f /tmp/cni.yaml") + SSHConfig.Cmd(s.Masters[0], "kubectl apply -f /tmp/cni.yaml") } //SendKubeConfigs diff --git a/install/join.go b/install/join.go index a0f67f6d4c2..f08fb529f60 100644 --- a/install/join.go +++ b/install/join.go @@ -92,8 +92,7 @@ func (s *SealosInstaller) sendJoinCPConfig(joinMaster []string) { wg.Add(1) go func(master string) { defer wg.Done() - var cgroup string - cgroup = s.getCgroupDriverFromShell(master) + cgroup := s.getCgroupDriverFromShell(master) templateData := string(JoinTemplate(IpFormat(master), cgroup)) cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData) _ = SSHConfig.CmdAsync(master, cmd) @@ -149,8 +148,7 @@ func (s *SealosInstaller) JoinNodes() { go func(node string) { defer wg.Done() // send join node config - var cgroup string - cgroup = s.getCgroupDriverFromShell(node) + cgroup := s.getCgroupDriverFromShell(node) templateData := string(JoinTemplate("", cgroup)) cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData) _ = SSHConfig.CmdAsync(node, cmdJoinConfig) diff --git a/install/upgrade.go b/install/upgrade.go index ea9d9a0ebc0..515ba4d705a 100644 --- a/install/upgrade.go +++ b/install/upgrade.go @@ -125,7 +125,7 @@ func (u *SealosUpgrade) upgradeNodes(hostnames []string, isMaster bool) { os.Exit(1) } } else { - cmdUpgrade = fmt.Sprintf("kubeadm upgrade node --certificate-renewal=false") + cmdUpgrade = "kubeadm upgrade node --certificate-renewal=false" err = SSHConfig.CmdAsync(ip, cmdUpgrade) if err != nil { logger.Error("kubeadm upgrade err: ", err) diff --git a/install/utils.go b/install/utils.go index a3109514168..7dfc05347f8 100644 --- a/install/utils.go +++ b/install/utils.go @@ -159,8 +159,7 @@ func IpFormat(host string) string { // RandString 生成随机字符串 func RandString(len int) string { - var r *rand.Rand - r = rand.New(rand.NewSource(time.Now().Unix())) + var r *rand.Rand = rand.New(rand.NewSource(time.Now().Unix())) bytes := make([]byte, len) for i := 0; i < len; i++ { b := r.Intn(26) + 65 diff --git a/install/vars.go b/install/vars.go index 052d5edd222..674a2e9fd3c 100644 --- a/install/vars.go +++ b/install/vars.go @@ -29,8 +29,10 @@ var ( EtcdKey = cert.SealosConfigDir + "/pki/etcd/healthcheck-client.key" //criSocket - CriSocket string + CriSocket string CgroupDriver string + KubeadmApi string + BootstrapApi string VIP string PkgUrl string