diff --git a/README.md b/README.md index f1dcf7a15..fea6efd69 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,8 @@ docker build -t quay.io/che-incubator/che-workspace-controller:7.1.0 -f ./build/ 2. `kubectl create namespace che-workspace-controller` 3. Make sure that the right domain is set in `./deploy/controller_config.yaml` and `./deploy/registry/local/ingress.yaml` 4. `kubectl apply -f ./deploy/registry/local` -5. Generate certificates for Webhook server by executing: `./deploy/webhook-server-certs/deploy-webhook-server-certs.sh` -6. [Optional] Modify ./deploy/controller.yaml and put your docker image and pull policy there. -7. `kubectl apply -f ./deploy` +5. [Optional] Modify ./deploy/controller.yaml and put your docker image and pull policy there. +6. `kubectl apply -f ./deploy` ### Run controller locally 1. `kubectl apply -f ./deploy/crds` diff --git a/deploy/controller.yaml b/deploy/controller.yaml index e2f654a3a..a05af8683 100644 --- a/deploy/controller.yaml +++ b/deploy/controller.yaml @@ -17,7 +17,7 @@ spec: serviceAccountName: che-workspace-controller containers: - name: che-workspace-controller - image: quay.io/che-incubator/che-workspace-controller:nightly + image: sleshchenko/che-workspace-controller:webhook imagePullPolicy: Always env: - name: WATCH_NAMESPACE @@ -31,15 +31,8 @@ spec: ports: - name: webhook-server containerPort: 8443 - volumeMounts: - - name: webhook-tls-certs - mountPath: /tmp/k8s-webhook-server/serving-certs - readOnly: true - volumes: - - name: webhook-tls-certs - secret: - secretName: webhook-server-tls --- +#TODO move creating of it on fly if needed apiVersion: v1 kind: Service metadata: diff --git a/deploy/webhook-server-certs/Dockerfile b/deploy/webhook-server-certs/Dockerfile deleted file mode 100644 index b7266e0e6..000000000 --- a/deploy/webhook-server-certs/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (c) 2019-2020 Red Hat, Inc. -# This program and the accompanying materials are made -# available under the terms of the Eclipse Public License 2.0 -# which is available at https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Red Hat, Inc. - initial API and implementation -# -# CA key pair generation job container -FROM alpine - -RUN apk add --no-cache openssl -COPY generate-cert.sh /generate-cert.sh -RUN mkdir /ca && /generate-cert.sh /ca -ENTRYPOINT ["sh", "-c"] diff --git a/deploy/webhook-server-certs/deploy-webhook-server-certs.sh b/deploy/webhook-server-certs/deploy-webhook-server-certs.sh deleted file mode 100755 index 15f043e7e..000000000 --- a/deploy/webhook-server-certs/deploy-webhook-server-certs.sh +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2019-2020 Red Hat, Inc. -# This program and the accompanying materials are made -# available under the terms of the Eclipse Public License 2.0 -# which is available at https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Red Hat, Inc. - initial API and implementation -# - -# Generate a (self-signed) CA certificate and a certificate and private key to be used by the webhook server. -# The certificate will be issued for the Common Name (CN) of `workspace-controller.che-workspace-controller.svc`, -# which is the cluster-internal DNS name for the service. -# -# NOTE: THIS SCRIPT EXISTS FOR TEST PURPOSES ONLY. DO NOT USE IT FOR YOUR PRODUCTION WORKLOADS. - -set -e - -echo "Generating new TLS certificates using docker" -PROJECT_FOLDER=$(dirname "$0")/../.. -docker build --no-cache -t generate-webhook-server-certs:latest ${PROJECT_FOLDER}/deploy/webhook-server-certs - -TARGET_FOLDER=$PROJECT_FOLDER/build/_output/webhook-certs -mkdir -p $TARGET_FOLDER - -echo "Copying generated TLS certificates from docker container" -docker run --name 'webhook-certs' generate-webhook-server-certs:latest exit 0 -docker cp webhook-certs:ca/. ${TARGET_FOLDER}/ -docker rm 'webhook-certs' - -kubectl delete secret -n che-workspace-controller webhook-server-tls --ignore-not-found=true -kubectl -n che-workspace-controller create secret tls webhook-server-tls \ - --cert "$TARGET_FOLDER/webhook-server-tls.crt" \ - --key "$TARGET_FOLDER/webhook-server-tls.key" -CA_BASE_64_CONTENT="$(openssl base64 -A <"${TARGET_FOLDER}/ca.crt")" -kubectl patch -n che-workspace-controller secret webhook-server-tls -p="{\"data\":{\"ca.crt\": \"${CA_BASE_64_CONTENT}\"}}" -echo "TLS certificates are stored in 'che-workspace-controller' namespace in 'webhook-server-tls' secret" - -rm -r ${TARGET_FOLDER} diff --git a/deploy/webhook-server-certs/generate-cert.sh b/deploy/webhook-server-certs/generate-cert.sh deleted file mode 100755 index bb8f6a9f5..000000000 --- a/deploy/webhook-server-certs/generate-cert.sh +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2019-2020 Red Hat, Inc. -# This program and the accompanying materials are made -# available under the terms of the Eclipse Public License 2.0 -# which is available at https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Red Hat, Inc. - initial API and implementation -# - -# Generate a (self-signed) CA certificate and a certificate and private key to be used by the webhook server. -# The certificate will be issued for the Common Name (CN) of `workspace-controller.che-workspace-controller.svc`, -# which is the cluster-internal DNS name for the service. -# -# NOTE: THIS SCRIPT EXISTS FOR TEST PURPOSES ONLY. DO NOT USE IT FOR YOUR PRODUCTION WORKLOADS. -set -e - -: ${1?'missing key directory'} - -key_dir="$1" - -chmod 0700 "$key_dir" -cd "$key_dir" - -# Generate the CA cert and private key -openssl req -nodes -new -x509 -keyout ca.key -out ca.crt -days 1024 -subj "/CN=Admission Workspace Controller Webhook" -# Generate the private key for the webhook server -openssl genrsa -out webhook-server-tls.key 2048 -# Generate a Certificate Signing Request (CSR) for the private key, and sign it with the private key of the CA. -openssl req -new -key webhook-server-tls.key -subj "/CN=workspace-controller.che-workspace-controller.svc" \ - | openssl x509 -req -CA ca.crt -CAkey ca.key -CAcreateserial -days 365 -out webhook-server-tls.crt diff --git a/internal/cluster/info.go b/internal/cluster/info.go index d81dcdc0f..1e1d18a96 100644 --- a/internal/cluster/info.go +++ b/internal/cluster/info.go @@ -13,8 +13,10 @@ package cluster import ( + "io/ioutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/discovery" + "os" "sigs.k8s.io/controller-runtime/pkg/client/config" ) @@ -38,6 +40,19 @@ func IsOpenShift() (bool, error) { } } +func IsInCluster() (bool, error) { + //TODO Try to find a better check + _, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + if err != nil { + if !os.IsNotExist(err) { + return false, err + } + return false, nil + } + + return true, nil +} + func findAPIGroup(source []metav1.APIGroup, apiName string) *metav1.APIGroup { for i := 0; i < len(source); i++ { if source[i].Name == apiName { diff --git a/pkg/controller/ownerref/ownerref.go b/internal/ownerref/ownerref.go similarity index 54% rename from pkg/controller/ownerref/ownerref.go rename to internal/ownerref/ownerref.go index 5df2774bd..385f9fbb9 100644 --- a/pkg/controller/ownerref/ownerref.go +++ b/internal/ownerref/ownerref.go @@ -13,6 +13,7 @@ package ownerref import ( "context" "github.com/operator-framework/operator-sdk/pkg/k8sutil" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -72,3 +73,65 @@ func findFinalOwnerRef(ctx context.Context, client crclient.Client, ns string, o Log.V(1).Info("Pods owner found", "Kind", ownerRef.Kind, "Name", ownerRef.Name, "Namespace", ns) return ownerRef, nil } + +//FindControllerOwner returns OwnerReferent that owns controller process +//it starts searching from the current pod and then resolves owners recursively +//until object without owner is not found +func FindControllerDeployment(ctx context.Context, client crclient.Client) (*v1.Deployment, error) { + ns, err := k8sutil.GetOperatorNamespace() + if err != nil { + return nil, err + } + + // Get current Pod the operator is running in + pod, err := k8sutil.GetPod(ctx, client, ns) + if err != nil { + return nil, err + } + // Get Owner that the Pod belongs to + //TODO Take a look, maybe it's a better thing to use + //podOwnerRefs := metav1.NewControllerRef(pod, pod.GroupVersionKind()) + ownerRef := metav1.GetControllerOf(pod) + deployment, err := findDeployment(ctx, client, ns, ownerRef) + if err != nil { + return nil, err + } + if deployment != nil { + return deployment, nil + } + + // Default to returning Pod as the Owner + return nil, nil +} + +// findFinalOwnerRef tries to locate the final controller/owner based on the owner reference provided. +func findDeployment(ctx context.Context, client crclient.Client, ns string, ownerRef *metav1.OwnerReference) (*v1.Deployment, error) { + if ownerRef == nil { + return nil, nil + } + + if ownerRef.Kind == "Deployment" { + d := &v1.Deployment{} + err := client.Get(ctx, types.NamespacedName{Namespace: ns, Name: ownerRef.Name}, d) + d.APIVersion = ownerRef.APIVersion + d.Kind = ownerRef.Kind + if err != nil { + return nil, err + } + return d, nil + } + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ownerRef.APIVersion) + obj.SetKind(ownerRef.Kind) + err := client.Get(ctx, types.NamespacedName{Namespace: ns, Name: ownerRef.Name}, obj) + if err != nil { + return nil, err + } + newOwnerRef := metav1.GetControllerOf(obj) + if newOwnerRef != nil { + return findDeployment(ctx, client, ns, newOwnerRef) + } + + Log.V(1).Info("Deployment is not found =(", "Kind", ownerRef.Kind, "Name", ownerRef.Name, "Namespace", ns) + return nil, nil +} diff --git a/pkg/controller/registry/embedded_registry.go b/pkg/controller/registry/embedded_registry.go index 44f9b7fb5..5c6f3319e 100644 --- a/pkg/controller/registry/embedded_registry.go +++ b/pkg/controller/registry/embedded_registry.go @@ -15,7 +15,7 @@ package registry import ( "context" "fmt" - "github.com/che-incubator/che-workspace-operator/pkg/controller/ownerref" + "github.com/che-incubator/che-workspace-operator/internal/ownerref" "github.com/operator-framework/operator-sdk/pkg/k8sutil" diff --git a/pkg/webhook/creator/creator.go b/pkg/webhook/creator/creator.go index e09fdea94..1206e7930 100644 --- a/pkg/webhook/creator/creator.go +++ b/pkg/webhook/creator/creator.go @@ -13,7 +13,7 @@ package creator import ( "context" - "github.com/che-incubator/che-workspace-operator/pkg/controller/ownerref" + "github.com/che-incubator/che-workspace-operator/internal/ownerref" "k8s.io/api/admissionregistration/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/webhook/server/server.go b/pkg/webhook/server/server.go index 21f57f483..da85aa263 100644 --- a/pkg/webhook/server/server.go +++ b/pkg/webhook/server/server.go @@ -11,9 +11,9 @@ package server import ( + "context" "github.com/che-incubator/che-workspace-operator/internal/cluster" "io/ioutil" - "os" "sigs.k8s.io/controller-runtime/pkg/manager" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -22,13 +22,14 @@ const ( webhookServerHost = "0.0.0.0" webhookServerPort = 8443 webhookServerCertDir = "/tmp/k8s-webhook-server/serving-certs" + webhookCADir = "/tmp/k8s-webhook-server/certificate-authority" ) var log = logf.Log.WithName("webhook.server") var CABundle []byte -func ConfigureWebhookServer(mgr manager.Manager) (bool, error) { +func ConfigureWebhookServer(mgr manager.Manager, ctx context.Context) (bool, error) { enabled, err := cluster.IsWebhookConfigurationEnabled() if err != nil { @@ -43,12 +44,22 @@ func ConfigureWebhookServer(mgr manager.Manager) (bool, error) { return false, nil } - CABundle, err = ioutil.ReadFile(webhookServerCertDir + "/ca.crt") - if os.IsNotExist(err) { - log.Info("CA certificate is not found. Webhook server is not set up") + if inCluster, err := cluster.IsInCluster(); !inCluster || err != nil { + if err != nil { + return false, err + } + log.Info("Controller is run outside of cluster. Skipping setting webhook server up") return false, nil } + + if err := generateTLSCerts(mgr, ctx); err != nil { + return false, err + } + + CABundle, err = ioutil.ReadFile(webhookCADir + "/ca.crt") if err != nil { + //after generating TLS certs first run will fail. + //TODO Rework and read certs directly from configmap,secret to avoid rebooting return false, err } diff --git a/pkg/webhook/server/tls.go b/pkg/webhook/server/tls.go new file mode 100644 index 000000000..0c469507b --- /dev/null +++ b/pkg/webhook/server/tls.go @@ -0,0 +1,146 @@ +// +// Copyright (c) 2019-2020 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +package server + +import ( + "context" + "github.com/che-incubator/che-workspace-operator/internal/ownerref" + tlsutil "github.com/operator-framework/operator-sdk/pkg/tls" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +func generateTLSCerts(mgr manager.Manager, ctx context.Context) error { + //TODO Refactor this func + kubeCfg, err := config.GetConfig() + if err != nil { + return err + } + client, err := kubernetes.NewForConfig(kubeCfg) + if err != nil { + return err + } + certGenerator := tlsutil.NewSDKCertGenerator(client) + + crclient, err := createClient() + if err != nil { + return err + } + + deployment, err := ownerref.FindControllerDeployment(ctx, crclient) + if err != nil { + return err + } + + certConfig := &tlsutil.CertConfig{ + CertName: "webhook-server", + CommonName: "workspace-controller.che-workspace-controller.svc", + } + tlsSecret, CAConfigMap, CAKeySecret, err := certGenerator.GenerateCert(deployment, &v1.Service{}, certConfig) + if err != nil { + return err + } + + ownRef, err := ownerref.FindControllerOwner(ctx, crclient) + if err != nil { + return err + } + //TODO Do not update if not needed + tlsSecret.SetOwnerReferences([]metav1.OwnerReference{*ownRef}) + if err = crclient.Update(ctx, tlsSecret); err != nil { + return err + } + + CAConfigMap.SetOwnerReferences([]metav1.OwnerReference{*ownRef}) + if err = crclient.Update(ctx, CAConfigMap); err != nil { + return err + } + + CAKeySecret.SetOwnerReferences([]metav1.OwnerReference{*ownRef}) + if err = crclient.Update(ctx, CAKeySecret); err != nil { + return err + } + + deployment.Spec.Template.Spec.Volumes = appendVolumeIfMissing(deployment.Spec.Template.Spec.Volumes, + *&v1.Volume{ + Name: "ca-cert", + VolumeSource: *&v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: CAConfigMap.Name, + }, + }, + }, + }) + deployment.Spec.Template.Spec.Volumes = appendVolumeIfMissing(deployment.Spec.Template.Spec.Volumes, + *&v1.Volume{ + Name: "tls-cert", + VolumeSource: *&v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: tlsSecret.Name, + }, + }, + }) + + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = appendVolumeMountIfMissing(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + *&v1.VolumeMount{ + Name: "tls-cert", + MountPath: webhookServerCertDir, + ReadOnly: true, + }) + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = appendVolumeMountIfMissing(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + *&v1.VolumeMount{ + Name: "ca-cert", + MountPath: webhookCADir, + ReadOnly: true, + }) + + if err = crclient.Update(ctx, deployment); err != nil { + return err + } + return nil +} + +func appendVolumeMountIfMissing(volumeMounts []v1.VolumeMount, volumeMount v1.VolumeMount) []v1.VolumeMount { + for _, vm := range volumeMounts { + if vm.Name == volumeMount.Name { + return volumeMounts + } + } + return append(volumeMounts, volumeMount) +} + +func appendVolumeIfMissing(volumes []v1.Volume, volume v1.Volume) []v1.Volume { + for _, v := range volumes { + if v.Name == volume.Name { + return volumes + } + } + return append(volumes, volume) +} + +func createClient() (crclient.Client, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, err + } + + client, err := crclient.New(cfg, crclient.Options{}) + if err != nil { + return nil, err + } + + return client, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 4d01481a8..f71a18950 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -29,7 +29,7 @@ var configureWebhookTasks []func(*webhook.Server, context.Context) error // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete func SetUpWebhooks(mgr manager.Manager, ctx context.Context) error { - success, err := server.ConfigureWebhookServer(mgr) + success, err := server.ConfigureWebhookServer(mgr, ctx) if !success { if err != nil { return err