From 172a34943c1be5f65cc206d247b2ebe36d7c9329 Mon Sep 17 00:00:00 2001 From: Pablo Chacin Date: Tue, 12 Sep 2023 22:07:26 +0200 Subject: [PATCH 1/3] Refactor k3s utils for reuse Signed-off-by: Pablo Chacin --- pkg/kubernetes/integration_test.go | 52 +++------------------------ pkg/testutils/k3sutils/k3sutils.go | 57 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 48 deletions(-) create mode 100644 pkg/testutils/k3sutils/k3sutils.go diff --git a/pkg/kubernetes/integration_test.go b/pkg/kubernetes/integration_test.go index d79e09e3..9fdfaa16 100644 --- a/pkg/kubernetes/integration_test.go +++ b/pkg/kubernetes/integration_test.go @@ -6,12 +6,13 @@ package kubernetes import ( "context" "fmt" - "regexp" "testing" "time" "github.com/grafana/xk6-disruptor/pkg/kubernetes/helpers" "github.com/grafana/xk6-disruptor/pkg/testutils/e2e/fixtures" + "github.com/grafana/xk6-disruptor/pkg/testutils/k3sutils" + "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/k3s" @@ -36,51 +37,6 @@ func createRandomTestNamespace(k8s Kubernetes) (string, error) { return ns.Name, nil } -// regexMatcher process lines from logs and notifies when a match is found -type regexMatcher struct { - exp *regexp.Regexp - found chan (bool) -} - -// Accept implements the LogConsumer interface -func (a *regexMatcher) Accept(log testcontainers.Log) { - if a.exp.MatchString(string(log.Content)) { - a.found <- true - } -} - -// waitForRegex waits until a match for the given regex is found in the log produced by the container -func waitForRegex(ctx context.Context, container testcontainers.Container, exp string, timeout time.Duration) error { - regexp, err := regexp.Compile(exp) - if err != nil { - return err - } - - found := make(chan bool) - container.FollowOutput(®exMatcher{ - exp: regexp, - found: found, - }) - - err = container.StartLogProducer(ctx) - if err != nil { - return err - } - defer func() { - _ = container.StopLogProducer() - }() - - expired := time.After(timeout) - select { - case <-found: - return nil - case <-expired: - return fmt.Errorf("timeout waiting for a match") - case <-ctx.Done(): - return ctx.Err() - } -} - func Test_Kubernetes(t *testing.T) { t.Parallel() @@ -95,7 +51,7 @@ func Test_Kubernetes(t *testing.T) { // see this issue for more details: // https://github.com/testcontainers/testcontainers-go/issues/1547 timeout := time.Second * 30 - err = waitForRegex(ctx, container, ".*Node controller sync successful.*", timeout) + err = k3sutils.WaitForRegex(ctx, container, ".*Node controller sync successful.*", timeout) if err != nil { t.Fatalf("failed waiting for cluster ready: %s", err) } @@ -321,7 +277,7 @@ func Test_UnsupportedKubernetesVersion(t *testing.T) { // see this issue for more details: // https://github.com/testcontainers/testcontainers-go/issues/1547 timeout := time.Second * 30 - err = waitForRegex(ctx, container, ".*Node controller sync successful.*", timeout) + err = k3sutils.WaitForRegex(ctx, container, ".*Node controller sync successful.*", timeout) if err != nil { t.Fatalf("failed waiting for cluster ready: %s", err) } diff --git a/pkg/testutils/k3sutils/k3sutils.go b/pkg/testutils/k3sutils/k3sutils.go new file mode 100644 index 00000000..8fad52cf --- /dev/null +++ b/pkg/testutils/k3sutils/k3sutils.go @@ -0,0 +1,57 @@ +// Package k3sutils implements helper functions for tests using TestContainer's k3s module +package k3sutils + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/testcontainers/testcontainers-go" +) + +// regexMatcher process lines from logs and notifies when a match is found +type regexMatcher struct { + exp *regexp.Regexp + found chan (bool) +} + +// Accept implements the LogConsumer interface +func (a *regexMatcher) Accept(log testcontainers.Log) { + if a.exp.MatchString(string(log.Content)) { + a.found <- true + } +} + +// WaitForRegex waits until a match for the given regex is found in the log produced by the container +// This utility function is a workaround for https://github.com/testcontainers/testcontainers-go/issues/1541 +func WaitForRegex(ctx context.Context, container testcontainers.Container, exp string, timeout time.Duration) error { + regexp, err := regexp.Compile(exp) + if err != nil { + return err + } + + found := make(chan bool) + container.FollowOutput(®exMatcher{ + exp: regexp, + found: found, + }) + + err = container.StartLogProducer(ctx) + if err != nil { + return err + } + defer func() { + _ = container.StopLogProducer() + }() + + expired := time.After(timeout) + select { + case <-found: + return nil + case <-expired: + return fmt.Errorf("timeout waiting for a match") + case <-ctx.Done(): + return ctx.Err() + } +} From 510e41d93afc4afaec90b8bf726c956bfe99a337 Mon Sep 17 00:00:00 2001 From: Pablo Chacin Date: Tue, 12 Sep 2023 22:08:53 +0200 Subject: [PATCH 2/3] Export NewFromConfig function Signed-off-by: Pablo Chacin --- pkg/kubernetes/integration_test.go | 4 ++-- pkg/kubernetes/kubernetes.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kubernetes/integration_test.go b/pkg/kubernetes/integration_test.go index 9fdfaa16..ad0f9973 100644 --- a/pkg/kubernetes/integration_test.go +++ b/pkg/kubernetes/integration_test.go @@ -73,7 +73,7 @@ func Test_Kubernetes(t *testing.T) { t.Fatalf("failed to create rest client for kubernetes : %s", err) } - k8s, err := newFromConfig(restcfg) + k8s, err := NewFromConfig(restcfg) if err != nil { t.Fatalf("error creating kubernetes client: %v", err) } @@ -299,7 +299,7 @@ func Test_UnsupportedKubernetesVersion(t *testing.T) { t.Fatalf("failed to create rest client for kubernetes : %s", err) } - _, err = newFromConfig(restcfg) + _, err = NewFromConfig(restcfg) if err == nil { t.Errorf("should had failed creating kubernetes client") return diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index ed620cae..cb97fdd1 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -31,8 +31,8 @@ type k8s struct { kubernetes.Interface } -// newFromConfig returns a Kubernetes instance configured with the provided kubeconfig. -func newFromConfig(config *rest.Config) (Kubernetes, error) { +// NewFromConfig returns a Kubernetes instance configured with the provided kubeconfig. +func NewFromConfig(config *rest.Config) (Kubernetes, error) { // As per the discussion in [1] client side rate limiting is no longer required. // Setting a large limit // [1] https://github.com/kubernetes/kubernetes/issues/111880 @@ -62,7 +62,7 @@ func NewFromKubeconfig(kubeconfig string) (Kubernetes, error) { return nil, err } - return newFromConfig(config) + return NewFromConfig(config) } // New returns a Kubernetes instance or an error when no config is eligible to be used. @@ -73,7 +73,7 @@ func NewFromKubeconfig(kubeconfig string) (Kubernetes, error) { func New() (Kubernetes, error) { k8sConfig, err := rest.InClusterConfig() if err == nil { - return newFromConfig(k8sConfig) + return NewFromConfig(k8sConfig) } if !errors.Is(err, rest.ErrNotInCluster) { From 2248fff40bf6cba08522b6e93374987230061be7 Mon Sep 17 00:00:00 2001 From: Pablo Chacin Date: Tue, 12 Sep 2023 23:08:57 +0200 Subject: [PATCH 3/3] Implement kubectl integration tests Signed-off-by: Pablo Chacin --- Makefile | 6 +- .../testutils/e2e/kubectl/integration_test.go | 58 +++++++++++++------ 2 files changed, 46 insertions(+), 18 deletions(-) rename e2e/kubectl/kubectl_e2e_test.go => pkg/testutils/e2e/kubectl/integration_test.go (58%) diff --git a/Makefile b/Makefile index fa36fef4..9d2bd5ac 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,11 @@ integration-agent: agent-image integration-kubernetes: go test -tags integration ./pkg/kubernetes/... -integration: integration-agent integration-kubernetes +integration-kubectl: + go test -tags integration ./pkg/testutils/e2e/kubectl/ + +integration: integration-agent integration-kubernetes integration-kubectl + # Running with -buildvcs=false works around the issue of `go list all` failing when git, which runs as root inside # the container, refuses to operate on the disruptor source tree as it is not owned by the same user (root). diff --git a/e2e/kubectl/kubectl_e2e_test.go b/pkg/testutils/e2e/kubectl/integration_test.go similarity index 58% rename from e2e/kubectl/kubectl_e2e_test.go rename to pkg/testutils/e2e/kubectl/integration_test.go index ae10498c..7cf55f73 100644 --- a/e2e/kubectl/kubectl_e2e_test.go +++ b/pkg/testutils/e2e/kubectl/integration_test.go @@ -1,7 +1,7 @@ -//go:build e2e -// +build e2e +//go:build integration +// +build integration -package e2e +package kubectl import ( "bytes" @@ -12,35 +12,59 @@ import ( "time" "github.com/grafana/xk6-disruptor/pkg/kubernetes" - "github.com/grafana/xk6-disruptor/pkg/testutils/e2e/cluster" "github.com/grafana/xk6-disruptor/pkg/testutils/e2e/deploy" - "github.com/grafana/xk6-disruptor/pkg/testutils/e2e/kubectl" "github.com/grafana/xk6-disruptor/pkg/testutils/e2e/kubernetes/namespace" + "github.com/grafana/xk6-disruptor/pkg/testutils/k3sutils" "github.com/grafana/xk6-disruptor/pkg/testutils/kubernetes/builders" + + + "github.com/testcontainers/testcontainers-go/modules/k3s" + + "k8s.io/client-go/tools/clientcmd" ) func Test_Kubectl(t *testing.T) { t.Parallel() - cluster, err := cluster.BuildE2eCluster( - cluster.DefaultE2eClusterConfig(), - ) + ctx := context.Background() + + container, err := k3s.RunContainer(ctx) if err != nil { - t.Errorf("failed to create cluster: %v", err) - return + t.Fatal(err) } + + // wait for the api server to complete initialization. + // see this issue for more details: + // https://github.com/testcontainers/testcontainers-go/issues/1547 + timeout := time.Second * 30 + err = k3sutils.WaitForRegex(ctx, container, ".*Node controller sync successful.*", timeout) + if err != nil { + t.Fatalf("failed waiting for cluster ready: %s", err) + } + + // Clean up the container after the test is complete t.Cleanup(func() { - _ = cluster.Cleanup() + if err = container.Terminate(ctx); err != nil { + t.Fatalf("failed to terminate container: %s", err) + } }) - k8s, err := kubernetes.NewFromKubeconfig(cluster.Kubeconfig()) + kubeConfigYaml, err := container.GetKubeConfig(ctx) + if err != nil { + t.Fatalf("failed to get kube-config : %s", err) + } + + restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml) + if err != nil { + t.Fatalf("failed to create rest client for kubernetes : %s", err) + } + + k8s, err := kubernetes.NewFromConfig(restcfg) if err != nil { - t.Errorf("error creating kubernetes client: %v", err) - return + t.Fatalf("error creating kubernetes client: %v", err) } - // Test Wait Pod Running - t.Run("Test local random port", func(t *testing.T) { + t.Run("Test port forwarding", func(t *testing.T) { namespace, err := namespace.CreateTestNamespace(context.TODO(), t, k8s.Client()) if err != nil { t.Errorf("failed to create test namespace: %v", err) @@ -63,7 +87,7 @@ func Test_Kubectl(t *testing.T) { return } - client, err := kubectl.NewFromKubeconfig(context.TODO(), cluster.Kubeconfig()) + client, err := NewForConfig(context.TODO(), restcfg) if err != nil { t.Errorf("failed to create kubectl client: %v", err) return