diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 61cc881be1c..618f3253930 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "text/template" @@ -39,8 +40,6 @@ type bootstrapTemplateData struct { EtcdCertSignerImage string EtcdCluster string EtcdctlImage string - HyperkubeImage string - KubeCoreRenderImage string ReleaseImage string } @@ -75,7 +74,6 @@ func (a *Bootstrap) Dependencies() []asset.Asset { &kubeconfig.Kubelet{}, &manifests.Manifests{}, &manifests.Tectonic{}, - &manifests.KubeCoreOperator{}, } } @@ -97,6 +95,7 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error { a.addBootstrapFiles(dependencies) a.addBootkubeFiles(dependencies, templateData) + a.addTemporaryBootkubeFiles(templateData) a.addTectonicFiles(dependencies, templateData) a.addTLSCertFiles(dependencies) @@ -160,30 +159,23 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst CloudProvider: getCloudProvider(installConfig), CloudProviderConfig: getCloudProviderConfig(installConfig), DebugConfig: "", - KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:375423a332f2c12b79438fc6a6da6e448e28ec0f", EtcdCertSignerImage: "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6", EtcdctlImage: "quay.io/coreos/etcd:v3.2.14", BootkubeImage: "quay.io/coreos/bootkube:v0.10.0", ReleaseImage: releaseImage, - HyperkubeImage: "openshift/origin-node:latest", EtcdCluster: strings.Join(etcdEndpoints, ","), }, nil } func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) { kubeletKubeconfig := &kubeconfig.Kubelet{} - kubeCoreOperator := &manifests.KubeCoreOperator{} - dependencies.Get(kubeletKubeconfig, kubeCoreOperator) + dependencies.Get(kubeletKubeconfig) a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FileFromBytes("/etc/kubernetes/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), ignition.FileFromBytes("/var/lib/kubelet/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), ) - a.Config.Storage.Files = append( - a.Config.Storage.Files, - ignition.FilesFromAsset(rootDir, 0644, kubeCoreOperator)..., - ) a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FileFromString("/opt/tectonic/report-progress.sh", 0555, content.ReportShFileContents), @@ -191,6 +183,7 @@ func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) { } func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) { + bootkubeConfigOverridesDir := filepath.Join(rootDir, "bootkube-config-overrides") adminKubeconfig := &kubeconfig.Admin{} manifests := &manifests.Manifests{} dependencies.Get(adminKubeconfig, manifests) @@ -199,6 +192,12 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b a.Config.Storage.Files, ignition.FileFromString("/opt/tectonic/bootkube.sh", 0555, applyTemplateData(content.BootkubeShFileTemplate, templateData)), ) + for _, o := range content.BootkubeConfigOverrides { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(bootkubeConfigOverridesDir, o.Name()), 0600, applyTemplateData(o, templateData)), + ) + } a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0600, adminKubeconfig)..., @@ -209,6 +208,36 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b ) } +func (a *Bootstrap) addTemporaryBootkubeFiles(templateData *bootstrapTemplateData) { + podCheckpointerBootstrapDir := filepath.Join(rootDir, "pod-checkpointer-operator-bootstrap") + for name, data := range content.PodCheckpointerBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(podCheckpointerBootstrapDir, name), 0644, data), + ) + } + + kubeProxyBootstrapDir := filepath.Join(rootDir, "kube-proxy-operator-bootstrap") + for name, data := range content.KubeProxyBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeProxyBootstrapDir, name), 0644, data), + ) + } + + kubeDNSBootstrapDir := filepath.Join(rootDir, "kube-dns-operator-bootstrap") + for name, data := range content.KubeDNSBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, name), 0644, data), + ) + } + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, "kube-dns-svc.yaml"), 0644, applyTemplateData(content.BootkubeKubeDNSService, templateData)), + ) +} + func (a *Bootstrap) addTectonicFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) { tectonic := &manifests.Tectonic{} dependencies.Get(tectonic) diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 7a906eb4367..46cd07244f5 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -31,7 +31,7 @@ var ( BootkubeShFileTemplate = template.Must(template.New("bootkube.sh").Parse(`#!/usr/bin/env bash set -e -mkdir --parents /etc/kubernetes/manifests/ +mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests} MACHINE_CONFIG_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-operator) MACHINE_CONFIG_CONTROLLER_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-controller) @@ -42,6 +42,9 @@ KUBE_APISERVER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster- KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-controller-manager-operator) KUBE_SCHEDULER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-scheduler-operator) +OPENSHIFT_HYPERSHIFT_IMAGE=$(podman run --rm {{.ReleaseImage}} image hypershift) +OPENSHIFT_HYPERKUBE_IMAGE=$(podman run --rm {{.ReleaseImage}} image hyperkube) + if [ ! -d cvo-bootstrap ] then echo "Rendering Cluster Version Operator Manifests..." @@ -57,22 +60,7 @@ then cp --recursive cvo-bootstrap/manifests . fi -if [ ! -d kco-bootstrap ] -then - echo "Rendering Kubernetes core manifests..." - - # shellcheck disable=SC2154 - podman run \ - --volume "$PWD:/assets:z" \ - --volume /etc/kubernetes:/etc/kubernetes:z \ - "{{.KubeCoreRenderImage}}" \ - --config=/assets/kco-config.yaml \ - --output=/assets/kco-bootstrap - - cp --recursive kco-bootstrap/bootstrap-configs /etc/kubernetes/bootstrap-configs - cp --recursive kco-bootstrap/bootstrap-manifests . - cp --recursive kco-bootstrap/manifests . -fi +mkdir --parents ./{bootstrap-manifests,manifests} if [ ! -d kube-apiserver-bootstrap ] then @@ -84,14 +72,16 @@ then "${KUBE_APISERVER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-apiserver-operator render \ --manifest-etcd-serving-ca=etcd-client-ca.crt \ + --manifest-etcd-server-urls={{.EtcdCluster}} \ + --manifest-image=${OPENSHIFT_HYPERSHIFT_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-apiserver-bootstrap \ - --config-output-file=/assets/kube-apiserver-bootstrap/config + --config-output-file=/assets/kube-apiserver-bootstrap/config \ + --config-override-files=/assets/bootkube-config-overrides/kube-apiserver-config-overrides.yaml - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-apiserver-bootstrap/manifests/00_openshift-kube-apiserver-ns.yaml manifests/00_openshift-kube-apiserver-ns.yaml - cp --recursive kube-apiserver-bootstrap/manifests/secret-* manifests/ - cp --recursive kube-apiserver-bootstrap/manifests/configmap-* manifests/ + cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml + cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-apiserver-bootstrap/manifests/* manifests/ fi if [ ! -d kube-controller-manager-bootstrap ] @@ -103,13 +93,15 @@ then --volume "$PWD:/assets:z" \ "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-controller-manager-operator render \ + --manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-controller-manager-bootstrap \ - --config-output-file=/assets/kube-controller-manager-bootstrap/config + --config-output-file=/assets/kube-controller-manager-bootstrap/config \ + --config-override-files=/assets/bootkube-config-overrides/kube-controller-manager-config-overrides.yaml - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-controller-manager-bootstrap/manifests/00_openshift-kube-controller-manager-ns.yaml manifests/00_openshift-kube-controller-manager-ns.yaml - cp --recursive kube-controller-manager-bootstrap/manifests/configmap-* manifests/ + cp kube-controller-manager-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-controller-manager-config.yaml + cp kube-controller-manager-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-controller-manager-bootstrap/manifests/* manifests/ fi if [ ! -d kube-scheduler-bootstrap ] @@ -121,15 +113,22 @@ then --volume "$PWD:/assets:z" \ "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-scheduler-operator render \ + --manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-scheduler-bootstrap \ --config-output-file=/assets/kube-scheduler-bootstrap/config - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-scheduler-bootstrap/manifests/00_openshift-kube-scheduler-ns.yaml manifests/00_openshift-kube-scheduler-ns.yaml - cp --recursive kube-scheduler-bootstrap/manifests/configmap-* manifests/ + cp kube-scheduler-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-scheduler-config.yaml + cp kube-scheduler-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-scheduler-bootstrap/manifests/* manifests/ fi +# TODO: Remove this when checkpointer, kube-proxy and kube-dns are properly rendered by corresponding operators. +echo "Installing temporary bootstrap manifests..." +cp pod-checkpointer-operator-bootstrap/* manifests/ +cp kube-proxy-operator-bootstrap/* manifests/ +cp kube-dns-operator-bootstrap/* manifests/ + if [ ! -d mco-bootstrap ] then echo "Rendering MCO manifests..." @@ -152,8 +151,8 @@ then # 1. read the controller config rendered by MachineConfigOperator # 2. read the default MachineConfigPools rendered by MachineConfigOperator # 3. read any additional MachineConfigs that are needed for the default MachineConfigPools. - mkdir --parents /etc/mcc/bootstrap/ - cp --recursive mco-bootstrap/manifests /etc/mcc/bootstrap/manifests + mkdir --parents /etc/mcc/bootstrap/manifests /etc/kubernetes/manifests/ + cp mco-bootstrap/manifests/* /etc/mcc/bootstrap/manifests/ cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. @@ -232,3 +231,30 @@ podman run \ start --asset-dir=/assets `)) ) + +var ( + // BootkubeConfigOverrides contains the configuration override files passed to the render commands of the components. + // These are supposed to be customized by the installer where the config differs from the operator render default. + BootkubeConfigOverrides = []*template.Template{ + KubeApiserverConfigOverridesTemplate, + KubeControllerManagerConfigOverridesTemplate, + } +) + +var ( + // KubeApiserverConfigOverridesTemplate are overrides that the installer passes to the default config of the + // kube-apiserver rendered by the cluster-kube-apiserver-operator. + KubeApiserverConfigOverridesTemplate = template.Must(template.New("kube-apiserver-config-overrides.yaml").Parse(` +apiVersion: kubecontrolplane.config.openshift.io/v1 +kind: KubeAPIServerConfig +kubeletClientInfo: + ca: "" # kubelet uses self-signed serving certs. TODO: fix kubelet pki +`)) + + // KubeControllerManagerConfigOverridesTemplate are overrides that the installer passes to the default config of the + // kube-controller-manager rendered by the cluster-kube-controller-manager-operator. + KubeControllerManagerConfigOverridesTemplate = template.Must(template.New("kube-controller-manager-config-overrides.yaml").Parse(` +apiVersion: kubecontrolplane.config.openshift.io/v1 +kind: KubeControllerManagerConfig +`)) +) diff --git a/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go b/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go new file mode 100644 index 00000000000..7a75ecf8d8b --- /dev/null +++ b/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go @@ -0,0 +1,431 @@ +package content + +import "text/template" + +// PodCheckpointerBootkubeManifests is a map of manifests needed by pod checkpointer to install. +// TODO: This must move to pod-checkpointer-operator renderer. +var PodCheckpointerBootkubeManifests = map[string]string{ + "checkpointer-role.yaml": bootkubeCheckpointerRole, + "checkpointer-role-binding.yaml": bootkubeCheckpointerRoleBinding, + "checkpointer-sa.yaml": bootkubeCheckpointerSA, + "checkpointer-daemonset.yaml": bootkubeCheckpointerDaemonset, +} + +// KubeProxyBootkubeManifests is a map of manifests needed by kube-proxy to install. +// TODO: This must move to networking operator renderer. +var KubeProxyBootkubeManifests = map[string]string{ + "kube-proxy-kube-system-rbac-role-binding.yaml": bootkubeKubeSystemRBACRoleBinding, + "kube-proxy-role-binding.yaml": bootkubeKubeProxyRoleBinding, + "kube-proxy-service-account.yaml": bootkubeKubeProxySA, + "kube-proxy-daemonset.yaml": bootkubeKubeProxyDaemonset, +} + +// KubeDNSBootkubeManifests is a map of manifests needed by kube-dns to install. +// TODO: This must move to networking operator renderer. +var KubeDNSBootkubeManifests = map[string]string{ + "kube-dns-deployment.yaml": bootkubeKubeDNSDeployment, +} + +// BootkubeKubeDNSService is a template for kube-dns service. +var BootkubeKubeDNSService = template.Must(template.New("bootkube.sh").Parse(` +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: KubeDNS +spec: + selector: + k8s-app: kube-dns + clusterIP: {{.ClusterDNSIP}} + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 +`)) + +const ( + bootkubeCheckpointerRole = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pod-checkpointer + namespace: kube-system +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] # "" indicates the core API group + resources: ["secrets", "configmaps"] + verbs: ["get"] +` + + bootkubeCheckpointerRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-checkpointer + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-checkpointer +subjects: +- kind: ServiceAccount + name: pod-checkpointer + namespace: kube-system +` + + bootkubeCheckpointerSA = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: pod-checkpointer +` + + bootkubeCheckpointerDaemonset = ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: pod-checkpointer + tier: control-plane + name: pod-checkpointer + namespace: kube-system +spec: + selector: + matchLabels: + tier: control-plane + k8s-app: pod-checkpointer + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + checkpointer.alpha.coreos.com/checkpoint: "true" + labels: + k8s-app: pod-checkpointer + tier: control-plane + spec: + containers: + - command: + - /checkpoint + - --lock-file=/var/run/lock/pod-checkpointer.lock + - --kubeconfig=/etc/checkpointer/kubeconfig + - --checkpoint-grace-period=5m + - --container-runtime-endpoint=unix:///var/run/crio/crio.sock + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: quay.io/coreos/pod-checkpointer:9dc83e1ab3bc36ca25c9f7c18ddef1b91d4a0558 + imagePullPolicy: Always + name: pod-checkpointer + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/checkpointer + name: kubeconfig + - mountPath: /etc/kubernetes + name: etc-kubernetes + - mountPath: /var/run + name: var-run + serviceAccountName: pod-checkpointer + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + restartPolicy: Always + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + volumes: + - name: kubeconfig + secret: + secretName: controller-manager-kubeconfig + - hostPath: + path: /etc/kubernetes + name: etc-kubernetes + - hostPath: + path: /var/run + name: var-run + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +` + + bootkubeKubeSystemRBACRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:default-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: default + namespace: kube-system +` + + bootkubeKubeProxySA = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: kube-proxy +` + + bootkubeKubeProxyDaemonset = ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + tier: node + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + tier: node + template: + metadata: + labels: + k8s-app: kube-proxy + tier: node + spec: + containers: + - command: + - ./hyperkube + - proxy + - --cluster-cidr=10.3.0.0/16 + - --hostname-override=$(NODE_NAME) + - --kubeconfig=/etc/kubernetes/kubeconfig + - --proxy-mode=iptables + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/coreos/hyperkube:v1.9.3_coreos.0 + name: kube-proxy + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /etc/kubernetes + name: kubeconfig + readOnly: true + hostNetwork: true + serviceAccountName: kube-proxy + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /etc/ssl/certs + name: ssl-certs-host + - name: kubeconfig + secret: + defaultMode: 420 + secretName: controller-manager-kubeconfig + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +` + + bootkubeKubeProxyRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier # Automatically created system role. +subjects: +- kind: ServiceAccount + name: kube-proxy + namespace: kube-system +` + bootkubeKubeDNSDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + containers: + - name: kubedns + image: "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8" + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain=cluster.local. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config + - name: dnsmasq + image: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8" + livenessProbe: + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --log-facility=- + - --server=/cluster.local/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8" + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 20Mi + cpu: 10m + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + node-role.kubernetes.io/master: "" + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true +` +) diff --git a/pkg/asset/manifests/content/bootkube/cvo-overrides.go b/pkg/asset/manifests/content/bootkube/cvo-overrides.go index efcdff7e9b6..ed29b8b9389 100644 --- a/pkg/asset/manifests/content/bootkube/cvo-overrides.go +++ b/pkg/asset/manifests/content/bootkube/cvo-overrides.go @@ -19,18 +19,6 @@ upstream: http://localhost:8080/graph channel: fast clusterID: {{.CVOClusterID}} overrides: -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-apiserver-operator - unmanaged: true -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-scheduler-operator - unmanaged: true -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-controller-manager-operator - unmanaged: true - kind: Deployment # this conflicts with kube-core-operator namespace: openshift-cluster-network-operator name: cluster-network-operator diff --git a/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go b/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go deleted file mode 100644 index a67ba4d09ae..00000000000 --- a/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go +++ /dev/null @@ -1,21 +0,0 @@ -package tectonic - -const ( - // AppVersionKubeCore is the variable/constant representing the contents of the respective file - AppVersionKubeCore = ` ---- -apiVersion: tco.coreos.com/v1 -kind: AppVersion -metadata: - name: kube-core - namespace: tectonic-system - labels: - managed-by-channel-operator: "true" -spec: - paused: false -status: - paused: false -upgradereq: 0 -upgradecomp: 0 -` -) diff --git a/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go b/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go deleted file mode 100644 index 909177805d6..00000000000 --- a/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go +++ /dev/null @@ -1,65 +0,0 @@ -package tectonic - -import ( - "text/template" -) - -var ( - // KubeCoreOperator is the variable/constant representing the contents of the respective file - KubeCoreOperator = template.Must(template.New("kube-core-00-operator.yaml").Parse(` -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: kube-core-operator - namespace: kube-system - labels: - k8s-app: kube-core-operator - managed-by-channel-operator: "true" -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: kube-core-operator - template: - metadata: - labels: - k8s-app: kube-core-operator - tectonic-app-version-name: kube-core - spec: - containers: - - name: kube-core-operator - image: {{.KubeCoreOperatorImage}} - imagePullPolicy: Always - args: - - --config=/etc/cluster-config/kco-config.yaml - resources: - limits: - cpu: 20m - memory: 50Mi - requests: - cpu: 20m - memory: 50Mi - volumeMounts: - - name: cluster-config - mountPath: /etc/cluster-config - imagePullSecrets: - - name: coreos-pull-secret - nodeSelector: - node-role.kubernetes.io/master: "" - restartPolicy: Always - securityContext: - runAsNonRoot: true - runAsUser: 65534 - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" - volumes: - - name: cluster-config - configMap: - name: cluster-config-v1 - items: - - key: kco-config - path: kco-config.yaml -`)) -) diff --git a/pkg/asset/manifests/kube-core-operator.go b/pkg/asset/manifests/kube-core-operator.go deleted file mode 100644 index 3b384b2c8d4..00000000000 --- a/pkg/asset/manifests/kube-core-operator.go +++ /dev/null @@ -1,133 +0,0 @@ -package manifests - -import ( - "fmt" - "strings" - - "github.com/apparentlymart/go-cidr/cidr" - kubecore "github.com/coreos/tectonic-config/config/kube-core" - "github.com/ghodss/yaml" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift/installer/pkg/asset" - "github.com/openshift/installer/pkg/asset/installconfig" - "github.com/openshift/installer/pkg/types" -) - -const ( - authConfigOIDCClientID = "tectonic-kubectl" - authConfigOIDCGroupsClaim = "groups" - authConfigOIDCUsernameClaim = "email" - networkConfigAdvertiseAddress = "0.0.0.0" -) - -// KubeCoreOperator generates the kube-core-operator.yaml files -type KubeCoreOperator struct { - Config *kubecore.OperatorConfig - File *asset.File -} - -var _ asset.WritableAsset = (*KubeCoreOperator)(nil) - -// Name returns a human friendly name for the operator -func (kco *KubeCoreOperator) Name() string { - return "Kube Core Operator" -} - -// Dependencies returns all of the dependencies directly needed by an -// KubeCoreOperator asset. -func (kco *KubeCoreOperator) Dependencies() []asset.Asset { - return []asset.Asset{ - &installconfig.InstallConfig{}, - } -} - -// Generate generates the kube-core-operator-config.yml files -func (kco *KubeCoreOperator) Generate(dependencies asset.Parents) error { - installConfig := &installconfig.InstallConfig{} - dependencies.Get(installConfig) - - clusterIP, err := cidr.Host(&installConfig.Config.Networking.ServiceCIDR.IPNet, 10) - if err != nil { - return errors.Wrapf(err, "failed to create %s config from InstallConfig", kco.Name()) - } - - kco.Config = &kubecore.OperatorConfig{ - TypeMeta: metav1.TypeMeta{ - APIVersion: kubecore.APIVersion, - Kind: kubecore.Kind, - }, - ClusterConfig: kubecore.ClusterConfig{ - APIServerURL: getAPIServerURL(installConfig.Config), - }, - AuthConfig: kubecore.AuthConfig{ - OIDCClientID: authConfigOIDCClientID, - OIDCIssuerURL: getOicdIssuerURL(installConfig.Config), - OIDCGroupsClaim: authConfigOIDCGroupsClaim, - OIDCUsernameClaim: authConfigOIDCUsernameClaim, - }, - DNSConfig: kubecore.DNSConfig{ - ClusterIP: clusterIP.String(), - }, - CloudProviderConfig: kubecore.CloudProviderConfig{ - CloudConfigPath: "", - CloudProviderProfile: k8sCloudProvider(installConfig.Config.Platform), - }, - RoutingConfig: kubecore.RoutingConfig{ - Subdomain: getBaseAddress(installConfig.Config), - }, - NetworkConfig: kubecore.NetworkConfig{ - ClusterCIDR: installConfig.Config.Networking.PodCIDR.String(), - ServiceCIDR: installConfig.Config.Networking.ServiceCIDR.String(), - AdvertiseAddress: networkConfigAdvertiseAddress, - EtcdServers: strings.Join(getEtcdServersURLs(installConfig.Config), ","), - }, - } - - data, err := yaml.Marshal(kco.Config) - if err != nil { - return errors.Wrapf(err, "failed to create %s config from InstallConfig", kco.Name()) - } - kco.File = &asset.File{ - Filename: "kco-config.yaml", - Data: data, - } - - return nil -} - -// Files returns the files generated by the asset. -func (kco *KubeCoreOperator) Files() []*asset.File { - if kco.File != nil { - return []*asset.File{kco.File} - } - return []*asset.File{} -} - -func getEtcdServersURLs(ic *types.InstallConfig) []string { - var urls []string - for i := 0; i < ic.MasterCount(); i++ { - urls = append(urls, fmt.Sprintf("https://%s-etcd-%d.%s:2379", ic.ObjectMeta.Name, i, ic.BaseDomain)) - } - return urls -} - -func getOicdIssuerURL(ic *types.InstallConfig) string { - return fmt.Sprintf("https://%s.%s/identity", ic.ObjectMeta.Name, ic.BaseDomain) -} - -func getBaseAddress(ic *types.InstallConfig) string { - return fmt.Sprintf("%s.%s", ic.ObjectMeta.Name, ic.BaseDomain) -} - -// Converts a platform to the cloudProvider that k8s understands -func k8sCloudProvider(platform types.Platform) string { - if platform.AWS != nil { - return "aws" - } - if platform.Libvirt != nil { - //return "libvirt" - } - return "" -} diff --git a/pkg/asset/manifests/operators.go b/pkg/asset/manifests/operators.go index c56a1ac4631..bc58a3fbe03 100644 --- a/pkg/asset/manifests/operators.go +++ b/pkg/asset/manifests/operators.go @@ -44,7 +44,6 @@ func (m *Manifests) Name() string { func (m *Manifests) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, - &KubeCoreOperator{}, &networkOperator{}, &kubeAddonOperator{}, &machineAPIOperator{}, @@ -70,16 +69,14 @@ func (m *Manifests) Dependencies() []asset.Asset { // Generate generates the respective operator config.yml files func (m *Manifests) Generate(dependencies asset.Parents) error { - kco := &KubeCoreOperator{} no := &networkOperator{} addon := &kubeAddonOperator{} mao := &machineAPIOperator{} installConfig := &installconfig.InstallConfig{} - dependencies.Get(kco, no, addon, mao, installConfig) + dependencies.Get(no, addon, mao, installConfig) - // kco+no+mao go to kube-system config map + // no+mao go to kube-system config map m.KubeSysConfig = configMap("kube-system", "cluster-config-v1", genericData{ - "kco-config": string(kco.Files()[0].Data), "network-config": string(no.Files()[0].Data), "install-config": string(installConfig.Files()[0].Data), "mao-config": string(mao.Files()[0].Data), diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index 61fd65fba3c..c040993cb1c 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -63,8 +63,6 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { "99_binding-discovery.yaml": []byte(content.BindingDiscovery), "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), - "99_kube-core-00-appversion.yaml": []byte(content.AppVersionKubeCore), - "99_kube-core-00-operator.yaml": applyTemplateData(content.KubeCoreOperator, templateData), "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, diff --git a/pkg/asset/tls/kubeletcertkey.go b/pkg/asset/tls/kubeletcertkey.go index b6d41889c96..b8632e2025e 100644 --- a/pkg/asset/tls/kubeletcertkey.go +++ b/pkg/asset/tls/kubeletcertkey.go @@ -29,7 +29,9 @@ func (a *KubeletCertKey) Generate(dependencies asset.Parents) error { dependencies.Get(kubeCA) cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system"}}, + // system:masters is a hack to get the kubelet up without kube-core + // TODO(node): make kubelet bootstrapping secure with minimal permissions eventually switching to system:node:* CommonName + Subject: pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system", "system:masters"}}, KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, Validity: ValidityThirtyMinutes,