k8s_test_setup example

create_kind_cluster.sh

@@ -10,6 +10,7 @@
 # governing permissions and limitations under the License.
 
 set -o errexit
+
 # desired cluster name; default is "kind"
 KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
 
@@ -19,19 +20,26 @@ reg_port='5000'
 running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
 if [ "${running}" != 'true' ]; then
   docker run \
-    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
     registry:2
 fi
-reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
 
 # create a cluster with the local registry enabled in containerd
-cat <<EOF | kind create cluster --name "${KIND_CLUSTER_NAME}" --image "kindest/node:v1.18.15" --config=-
+# --image "kindest/node:v1.19.7" \
+cat <<EOF | kind create cluster \
+  --name "${KIND_CLUSTER_NAME}" \
+  --image "kindest/node:v1.18.15" \
+  --config=-
+---
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_ip}:${reg_port}"]
+    endpoint = ["http://${reg_name}:${reg_port}"]
 EOF
 
+# connect the registry to the cluster network
+# (the network may already be connected)
+docker network connect "${KIND_CLUSTER_NAME}" "${reg_name}" || true
 

delete_kind_cluster.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2020 Adobe. All rights reserved.
+# This file is licensed to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+# OF ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+set -o errexit
+
+# desired cluster name; default is "kind"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
+
+# delete kind cluster
+kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
+
+# deete registry container
+echo "Deleting kind-registry..."
+docker container rm --force "kind-registry" || true
+
+# delete kind cluster network
+echo "Deleting kind network..."
+docker network rm "kind" || true
+

examples/.bazelrc

@@ -1,2 +1,5 @@
+# print out tests output in case of error
+test --test_output=errors
+
 # required for rules_docker 0.15
 common --incompatible_restrict_string_escapes=false

examples/WORKSPACE

@@ -32,3 +32,20 @@ load(
 )
 
 go_image_repositories()
+
+# this is required to load the kubeconfig repository_rule
+# http_archive(
+#     name = "io_bazel_rules_k8s",
+#     sha256 = "cc75cf0d86312e1327d226e980efd3599704e01099b58b3c2fc4efe5e321fcd9",
+#     strip_prefix = "rules_k8s-0.3.1",
+#     urls = ["https://github.com/bazelbuild/rules_k8s/releases/download/v0.3.1/rules_k8s-v0.3.1.tar.gz"],
+# )
+
+load("@com_adobe_rules_gitops//skylib:k8s.bzl", "kubeconfig")
+
+kubeconfig(
+    name = "k8s_test",
+    cluster = "kind-kind",
+    # server = "https://127.0.0.1:62816",
+    # symlink = True,
+)

examples/e2e-test.sh

@@ -15,38 +15,55 @@ set -o errexit
 set -o nounset
 set -o xtrace
 
+KUBECTL_OPTS=""
+if [[ "$-" = *"x"* ]]; then
+    # -x is set
+    # enable kubectl logging
+    KUBECTL_OPTS="--logtostderr=true -v=5"
+fi
+
 bindir=$(cd `dirname "$0"` && pwd)
 repo_path=$bindir
 cd $repo_path
 
-#verify interactive workflow
+# verify interactive workflow
 MYNAMESPACE=$USER
 
 # kubectl config use-context kind-kind
 
-kubectl create namespace $MYNAMESPACE || true
-kubectl create namespace hwteam || true
+kubectl $KUBECTL_OPTS create namespace $MYNAMESPACE || true
+kubectl $KUBECTL_OPTS create namespace hwteam || true
 
 bazel run //helloworld:mynamespace.apply
-kubectl -n $MYNAMESPACE wait --timeout=60s --for=condition=Available deployment/helloworld
+kubectl $KUBECTL_OPTS -n $MYNAMESPACE wait --timeout=60s --for=condition=Available \
+    deployment/helloworld
 
 bazel run //helloworld:mynamespace.delete
 
-#TODO: verity it is deleted
-#kubectl -n $MYNAMESPACE wait --timeout=30s --for=delete deployment/helloworld
+# verify it is deleted
+kubectl -n $MYNAMESPACE wait --timeout=30s --for=delete \
+    deployment/helloworld \
+    || true
 
+# the result of .gitops operation goes into /cloud directory and should be submitted back to the repo
 rm -rf cloud
 
 bazel run //helloworld:canary.gitops
 bazel run //helloworld:release.gitops
 bazel run //helloworld:gitops_custom_path.gitops
-#the result of .gitops operation goes into /cloud directory and should be submitted back to the repo
 
-#apply everything generated
-kubectl apply -f cloud -R
+# apply everything generated
+kubectl $KUBECTL_OPTS apply -f cloud -R
+
+# apply gitops_custom_path gen
+kubectl $KUBECTL_OPTS apply -f custom_cloud -R
 
-#apply gitops_custom_path gen
-kubectl apply -f custom_cloud -R
+# wait for readiness
+kubectl $KUBECTL_OPTS -n hwteam wait --timeout=60s --for=condition=Available \
+    deployment/helloworld \
+    deployment/helloworld-canary \
+    deployment/helloworld-gitops-custom-path \
+    || false
 
-#wait for readiness
-kubectl -n hwteam wait --timeout=60s --for=condition=Available deployment/helloworld deployment/helloworld-canary deployment/helloworld-gitops-custom-path
+# delete
+kubectl $KUBECTL_OPTS delete namespace hwteam --now=true --ignore-not-found=true

examples/helloworld/BUILD

@@ -12,7 +12,7 @@ licenses(["notice"])  # Apache 2.0
 
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
-load("@com_adobe_rules_gitops//gitops:defs.bzl", "k8s_deploy")
+load("@com_adobe_rules_gitops//gitops:defs.bzl", "k8s_deploy", "k8s_test_setup")
 
 go_library(
     name = "go_default_library",
@@ -138,3 +138,27 @@ sh_test(
         "@bazel_tools//tools/bash/runfiles",
     ],
 )
+
+k8s_test_setup(
+    name = "service_it.setup",
+    kubeconfig = "@k8s_test//:kubeconfig",
+    objects = [
+        ":mynamespace",
+    ],
+)
+
+go_test(
+    name = "helloworld_integration_test",
+    srcs = ["helloworld_integration_test.go"],
+    args = [
+        "-setup",
+        "$(location :service_it.setup)",
+    ],
+    data = [
+        ":service_it.setup",
+    ],
+    rundir = ".",
+    deps = [
+        "@com_adobe_rules_gitops//testing/it_sidecar/client:go_default_library",
+    ],
+)

examples/helloworld/helloworld_integration_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/adobe/rules_gitops/testing/it_sidecar/client"
+)
+
+var setup client.K8STestSetup
+
+func TestMain(m *testing.M) {
+	setup = client.K8STestSetup{PortForwardServices: map[string]int{"helloworld": 8080}}
+	setup.TestMain(m)
+}
+
+func TestSuccessfulReceival(t *testing.T) {
+	localPort := setup.GetServiceLocalPort("helloworld")
+	fmt.Printf("helloworld server is available at localport %d\n", localPort)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d", localPort))
+	if err != nil {
+		t.Fatalf("request error %s", err)
+	}
+	if resp.StatusCode != 200 {
+		t.Fatalf("Unexpected status code %d, expectted 200", resp.StatusCode)
+	}
+	body, _ := ioutil.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "Hello World") {
+		t.Error("Unexpected content returned:", string(body))
+	}
+}

skylib/k8s.bzl

@@ -340,10 +340,27 @@ def _kubeconfig_impl(repository_ctx):
         kubecert_cert = certs.get_child("kubecert.cert")
         kubecert_key = certs.get_child("kubecert.key")
 
+        # detect server endpoint
+        # Note: this is an ugly hack
+        if not server:
+            exec_result = repository_ctx.execute(["kubectl", "cluster-info", "--cluster=%s" % repository_ctx.attr.cluster])
+            if exec_result.return_code != 0:
+                fail("Error detecting %s cluster server endpoint" % repository_ctx.attr.cluster)
+            cluster_info = exec_result.stdout.splitlines()
+            server_start_index = cluster_info[0].find("https://")
+            if server_start_index >= 0:
+                # remove traling terminal escape sequence
+                server_part = cluster_info[0][server_start_index:]
+                server = ""
+                for i in range(len(server_part)):
+                    c = server_part[i]
+                    if not (c.isalnum() or c in "/:."):
+                        break
+                    server += c
+
     # config set-cluster {cluster} \
     #     --certificate-authority=... \
     #     --server=https://dev3.k8s.tubemogul.info:443 \
-    #     --embed-certs",
     _kubectl_config(repository_ctx, [
         "set-cluster",
         repository_ctx.attr.cluster,
@@ -353,7 +370,7 @@ def _kubeconfig_impl(repository_ctx):
         ca_crt,
     ])
 
-    # config set-credentials {user} --token=...",
+    # config set-credentials {user} --token=...
     if token:
         _kubectl_config(repository_ctx, [
             "set-credentials",
@@ -362,7 +379,7 @@ def _kubeconfig_impl(repository_ctx):
             token,
         ])
 
-    # config set-credentials {user} --client-certificate=... --embed-certs",
+    # config set-credentials {user} --client-certificate=...
     if kubecert_cert and kubecert_cert.exists:
         _kubectl_config(repository_ctx, [
             "set-credentials",
@@ -371,7 +388,7 @@ def _kubeconfig_impl(repository_ctx):
             kubecert_cert.realpath,
         ])
 
-    # config set-credentials {user} --client-key=... --embed-certs",
+    # config set-credentials {user} --client-key=...
     if kubecert_key and kubecert_key.exists:
         _kubectl_config(repository_ctx, [
             "set-credentials",
@@ -430,7 +447,7 @@ def _k8s_cmd_impl(ctx):
         command_file = ctx.actions.declare_file(ctx.label.name + ".command")
         _stamp(ctx, ctx.attr.command, command_file)
         command_arg = "source %s" % _runfiles(ctx, command_file)
-        files += [command_file]
+        files.append(command_file)
 
     ctx.actions.expand_template(
         template = ctx.file._template,
@@ -492,7 +509,7 @@ def _k8s_test_namespace_impl(ctx):
         output = namespace_create,
         is_executable = True,
     )
-    files += [namespace_create]
+    files.append(namespace_create)
 
     return [DefaultInfo(
         executable = namespace_create,
@@ -533,16 +550,16 @@ def _k8s_test_setup_impl(ctx):
     for obj in ctx.attr.objects:
         if obj.files_to_run.executable:
             # add object' targets and excutables to runfiles
-            files += [obj.files_to_run.executable]
+            files.append(obj.files_to_run.executable)
             transitive.append(obj.default_runfiles.files)
 
             # add object' execution command
-            commands += [_runfiles(ctx, obj.files_to_run.executable) + " | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -"]
+            commands.append(_runfiles(ctx, obj.files_to_run.executable) + " | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -")
         else:
             files += obj.files.to_list()
             commands += [ctx.executable._template_engine.short_path + " --template=" + filename.short_path + " --variable=NAMESPACE=${NAMESPACE} | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -" for filename in obj.files.to_list()]
 
-    files += [ctx.executable._template_engine]
+    files.append(ctx.executable._template_engine)
 
     # create namespace script
     ctx.actions.expand_template(