diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index 94ff7957b59..ee20b5ed12e 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -23,7 +23,6 @@ import ( "net/url" "os" "path/filepath" - "strconv" "strings" "time" @@ -34,7 +33,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" - "github.com/openyurtio/openyurt/pkg/util/templates" + tmplutil "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" @@ -46,6 +45,8 @@ const ( fileMode = 0666 DefaultRootDir = "/var/lib" DefaultCaPath = "/etc/kubernetes/pki/ca.crt" + yurthubYurtStaticSetName = "yurthub" + defaultConfigmapPath = "/data" ) type yurtHubOperator struct { @@ -77,20 +78,8 @@ func (op *yurtHubOperator) Install() error { // 1. put yurt-hub yaml into /etc/kubernetes/manifests klog.Infof("setting up yurthub on node") - // 1-1. replace variables in yaml file - klog.Infof("setting up yurthub apiServer addr") - yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, map[string]string{ - "yurthubBindingAddr": constants.DefaultYurtHubServerAddr, - "kubernetesServerAddr": op.apiServerAddr, - "image": op.yurthubImage, - "bootstrapFile": constants.YurtHubBootstrapConfig, - "workingMode": string(op.workingMode), - "enableDummyIf": strconv.FormatBool(op.enableDummyIf), - "enableNodePool": strconv.FormatBool(op.enableNodePool), - }) - if err != nil { - return err - } + // 1-1. get configmap data path + configMapDataPath := filepath.Join(defaultConfigmapPath, yurthubYurtStaticSetName) // 1-2. create /var/lib/yurthub/bootstrap-hub.conf if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil { @@ -106,10 +95,24 @@ func (op *yurtHubOperator) Install() error { if err := enutil.EnsureDir(podManifestPath); err != nil { return err } - if err := os.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), fileMode); err != nil { + content, err := os.ReadFile(configMapDataPath) + if err != nil { + return fmt.Errorf("failed to read source file %s: %w", configMapDataPath, err) + } + klog.Infof("yurt-hub.yaml apiServerAddr: %+v", op.apiServerAddr) + yssYurtHub, err := tmplutil.SubsituteTemplate(string(content), map[string]string{ + "kubernetesServerAddr": op.apiServerAddr, + }) + if err != nil { return err } + if err = os.WriteFile(getYurthubYaml(podManifestPath), []byte(yssYurtHub), fileMode); err != nil { + return err + } + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) + klog.Infof("yurt-hub.yaml: %+v", configMapDataPath) + klog.Infof("yurt-hub.yaml content: %+v", yssYurtHub) // 2. wait yurthub pod to be ready return hubHealthcheck(op.yurthubHealthCheckTimeout) diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go index b91d0b4ed0b..f71cf62dd92 100644 --- a/pkg/node-servant/constant.go +++ b/pkg/node-servant/constant.go @@ -42,6 +42,10 @@ spec: hostPath: path: / type: Directory + - name: configmap + configMap: + defaultMode: 420 + name: {{.configmap_name}} containers: - name: node-servant-servant image: {{.node_servant_image}} @@ -56,6 +60,8 @@ spec: volumeMounts: - mountPath: /openyurt name: host-root + - mountPath: /openyurt/data + name: configmap env: - name: NODE_NAME valueFrom: diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go index 5d4300a9876..228bc16efad 100644 --- a/test/e2e/cmd/init/constants/constants.go +++ b/test/e2e/cmd/init/constants/constants.go @@ -187,4 +187,137 @@ data: discardcloudservice: "" masterservice: "" ` + + YurthubCloudYurtStaticSet = ` +apiVersion: apps.openyurt.io/v1alpha1 +kind: YurtStaticSet +metadata: + name: yurt-hub-cloud + namespace: "kube-system" +spec: + staticPodManifest: yurthub + template: + metadata: + labels: + k8s-app: yurt-hub-cloud + spec: + volumes: + - name: hub-dir + hostPath: + path: /var/lib/yurthub + type: DirectoryOrCreate + - name: kubernetes + hostPath: + path: /etc/kubernetes + type: Directory + containers: + - name: yurt-hub + image: {{.yurthub_image}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: hub-dir + mountPath: /var/lib/yurthub + - name: kubernetes + mountPath: /etc/kubernetes + command: + - yurthub + - --v=2 + - --bind-address=127.0.0.1 + - --server-addr={{.kubernetesServerAddr}} + - --node-name=$(NODE_NAME) + - --bootstrap-file=/var/lib/yurthub/bootstrap-hub.conf + - --working-mode=cloud + - --namespace="kube-system" + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /v1/healthz + port: 10267 + initialDelaySeconds: 300 + periodSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 150m + memory: 150Mi + limits: + memory: 300Mi + securityContext: + capabilities: + add: [ "NET_ADMIN", "NET_RAW" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + hostNetwork: true + priorityClassName: system-node-critical + priority: 2000001000 +` + YurthubYurtStaticSet = ` +apiVersion: apps.openyurt.io/v1alpha1 +kind: YurtStaticSet +metadata: + name: yurt-hub + namespace: "kube-system" +spec: + staticPodManifest: yurthub + template: + metadata: + labels: + k8s-app: yurt-hub + spec: + volumes: + - name: hub-dir + hostPath: + path: /var/lib/yurthub + type: DirectoryOrCreate + - name: kubernetes + hostPath: + path: /etc/kubernetes + type: Directory + containers: + - name: yurt-hub + image: {{.yurthub_image}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: hub-dir + mountPath: /var/lib/yurthub + - name: kubernetes + mountPath: /etc/kubernetes + command: + - yurthub + - --v=2 + - --bind-address=127.0.0.1 + - --server-addr={{.kubernetesServerAddr}} + - --node-name=$(NODE_NAME) + - --bootstrap-file=/var/lib/yurthub/bootstrap-hub.conf + - --working-mode=edge + - --namespace="kube-system" + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /v1/healthz + port: 10267 + initialDelaySeconds: 300 + periodSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 150m + memory: 150Mi + limits: + memory: 300Mi + securityContext: + capabilities: + add: [ "NET_ADMIN", "NET_RAW" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + hostNetwork: true + priorityClassName: system-node-critical + priority: 2000001000 +` ) diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go index 9108248b90e..65adca57541 100644 --- a/test/e2e/cmd/init/converter.go +++ b/test/e2e/cmd/init/converter.go @@ -39,7 +39,9 @@ import ( nodeservant "github.com/openyurtio/openyurt/pkg/node-servant" kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo" strutil "github.com/openyurtio/openyurt/pkg/util/strings" + tmplutil "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurthub/util" + "github.com/openyurtio/openyurt/test/e2e/cmd/init/constants" "github.com/openyurtio/openyurt/test/e2e/cmd/init/lock" kubeutil "github.com/openyurtio/openyurt/test/e2e/cmd/init/util/kubernetes" ) @@ -47,6 +49,8 @@ import ( const ( // defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase defaultYurthubHealthCheckTimeout = 2 * time.Minute + yssYurtHubCloudName = "yurt-static-set-yurt-hub-cloud" + yssYurtHubName = "yurt-static-set-yurt-hub" ) type ClusterConverter struct { @@ -124,14 +128,45 @@ func (c *ClusterConverter) deployYurthub() error { // The node-servant will detect the kubeadm_conf_path automatically // It will be either "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" // or "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf". - "kubeadm_conf_path": "", - "working_mode": string(util.WorkingModeEdge), - "enable_dummy_if": strconv.FormatBool(c.EnableDummyIf), + "kubeadm_conf_path": "", + "working_mode": string(util.WorkingModeEdge), + "enable_dummy_if": strconv.FormatBool(c.EnableDummyIf), + "kubernetesServerAddr": "{{.kubernetesServerAddr}}", } if c.YurthubHealthCheckTimeout != defaultYurthubHealthCheckTimeout { convertCtx["yurthub_healthcheck_timeout"] = c.YurthubHealthCheckTimeout.String() } + // create the yurthub-cloud and yurthub yss + tempDir, err := os.MkdirTemp(c.RootDir, "yurt-hub") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + tempFile := filepath.Join(tempDir, "yurthub-cloud-yurtstaticset.yaml") + yssYurtHubCloud, err := tmplutil.SubsituteTemplate(constants.YurthubCloudYurtStaticSet, convertCtx) + if err != nil { + return err + } + if err = os.WriteFile(tempFile, []byte(yssYurtHubCloud), 0644); err != nil { + return err + } + if err = c.ComponentsBuilder.InstallComponents(tempFile, false); err != nil { + return err + } + + tempFile = filepath.Join(tempDir, "yurthub-yurtstaticset.yaml") + yssYurtHub, err := tmplutil.SubsituteTemplate(constants.YurthubYurtStaticSet, convertCtx) + if err != nil { + return err + } + if err = os.WriteFile(tempFile, []byte(yssYurtHub), 0644); err != nil { + return err + } + if err = c.ComponentsBuilder.InstallComponents(tempFile, false); err != nil { + return err + } + npExist, err := nodePoolResourceExists(c.ClientSet) if err != nil { return err @@ -141,6 +176,7 @@ func (c *ClusterConverter) deployYurthub() error { if len(c.EdgeNodes) != 0 { convertCtx["working_mode"] = string(util.WorkingModeEdge) + convertCtx["configmap_name"] = yssYurtHubName if err = kubeutil.RunServantJobs(c.ClientSet, c.WaitServantJobTimeout, func(nodeName string) (*batchv1.Job, error) { return nodeservant.RenderNodeServantJob("convert", convertCtx, nodeName) }, c.EdgeNodes, os.Stderr, false); err != nil { @@ -175,6 +211,7 @@ func (c *ClusterConverter) deployYurthub() error { // deploy yurt-hub and reset the kubelet service on cloud nodes convertCtx["working_mode"] = string(util.WorkingModeCloud) + convertCtx["configmap_name"] = yssYurtHubCloudName klog.Infof("convert context for cloud nodes(%q): %#+v", c.CloudNodes, convertCtx) if err = kubeutil.RunServantJobs(c.ClientSet, c.WaitServantJobTimeout, func(nodeName string) (*batchv1.Job, error) { return nodeservant.RenderNodeServantJob("convert", convertCtx, nodeName)