From 40dc8e676fb8a85ba84261b35b16131f3e145861 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Wed, 18 Jan 2023 17:29:19 +1100 Subject: [PATCH] Add DSE CDC test. --- .github/workflows/kindIntegTest.yml | 105 ++++++++-------- .../cdc_successful_test_c.go} | 2 +- .../cdc_successful_test_dse.go | 118 ++++++++++++++++++ tests/testdata/test-cdc/dse-datacenter.yaml | 23 ++++ 4 files changed, 195 insertions(+), 53 deletions(-) rename tests/{cdc_successful/cdc_successful_test.go => cdc_successful_c/cdc_successful_test_c.go} (99%) create mode 100644 tests/cdc_successful_dse/cdc_successful_test_dse.go create mode 100644 tests/testdata/test-cdc/dse-datacenter.yaml diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index feeeac6d..8e7e2bdf 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -5,7 +5,7 @@ on: - master - 1.10.x pull_request: - branches: [ master ] + branches: [master] jobs: build_docker_images: name: Build Docker Images @@ -53,9 +53,9 @@ jobs: strategy: matrix: integration_test: - - upgrade_operator # Test is not setup to run against 4.0 - - additional_seeds #TODO: Fails against C* 4.0, fix in https://github.com/k8ssandra/cass-operator/issues/459 - - scale_down_unbalanced_racks #TODO: Fails against C* 4.0 and DSE 6.8, fix in https://github.com/k8ssandra/cass-operator/issues/459 + - upgrade_operator # Test is not setup to run against 4.0 + - additional_seeds #TODO: Fails against C* 4.0, fix in https://github.com/k8ssandra/cass-operator/issues/459 + - scale_down_unbalanced_racks #TODO: Fails against C* 4.0 and DSE 6.8, fix in https://github.com/k8ssandra/cass-operator/issues/459 runs-on: ubuntu-latest env: CGO_ENABLED: 0 @@ -82,51 +82,52 @@ jobs: strategy: matrix: version: - - "4.0.7" + - "4.0.7" integration_test: - # Single worker tests: - - additional_serviceoptions - - additional_volumes - # - delete_node_terminated_container # This does not test any operator behavior - - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job - # - smoke_test_dse # Converted to test_all_the_things, see below job - # - terminate - # - timeout_prestop_termination - # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now - - webhook_validation - # Three worker tests: - - canary_upgrade - # - config_change_condition # config_change takes care of testing the same - - cdc_successful # OSS only - # - delete_node_lost_readiness # DSE specific behavior - - host_network - - internode-encryption-generated - #- no_infinite_reconcile # smoke_test_* should take care of this - - node_replace - - nodeport_service - - rolling_restart - # - stop_resume - - superuser-secret-generated - - superuser-secret-provided - - test_bad_config_and_fix - - test_mtls_mgmt_api - # More than 3 workers tests: - - add_racks - #- additional_seeds #TODO: Fails against C* 4.0, fix in https://github.com/k8ssandra/cass-operator/issues/459 - - cluster_wide_install - - config_change - - config_secret - # - multi_cluster_management - #- oss_test_all_the_things # This is now the smoke test, see kind_smoke_tests job - - scale_down - # - scale_down_not_enough_space # Not enough stable test - #- scale_down_unbalanced_racks #TODO: Fails against C* 4.0 and DSE, fix in https://github.com/k8ssandra/cass-operator/issues/459 - - scale_up - - scale_up_stop_resume - - seed_selection - - config_fql # OSS only - - decommission_dc + # Single worker tests: + - additional_serviceoptions + - additional_volumes + # - delete_node_terminated_container # This does not test any operator behavior + - podspec_simple + # - smoke_test_oss # Converted to test_all_the_things, see below job + # - smoke_test_dse # Converted to test_all_the_things, see below job + # - terminate + # - timeout_prestop_termination + # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now + - webhook_validation + # Three worker tests: + - canary_upgrade + # - config_change_condition # config_change takes care of testing the same + - cdc_successful_c # OSS only + - cdc_successful_dse # OSS only + # - delete_node_lost_readiness # DSE specific behavior + - host_network + - internode-encryption-generated + #- no_infinite_reconcile # smoke_test_* should take care of this + - node_replace + - nodeport_service + - rolling_restart + # - stop_resume + - superuser-secret-generated + - superuser-secret-provided + - test_bad_config_and_fix + - test_mtls_mgmt_api + # More than 3 workers tests: + - add_racks + #- additional_seeds #TODO: Fails against C* 4.0, fix in https://github.com/k8ssandra/cass-operator/issues/459 + - cluster_wide_install + - config_change + - config_secret + # - multi_cluster_management + #- oss_test_all_the_things # This is now the smoke test, see kind_smoke_tests job + - scale_down + # - scale_down_not_enough_space # Not enough stable test + #- scale_down_unbalanced_racks #TODO: Fails against C* 4.0 and DSE, fix in https://github.com/k8ssandra/cass-operator/issues/459 + - scale_up + - scale_up_stop_resume + - seed_selection + - config_fql # OSS only + - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA # let other tests continue to run # even if one fails @@ -157,11 +158,11 @@ jobs: strategy: matrix: version: - - "3.11.14" - - "4.0.7" - - "6.8.29" + - "3.11.14" + - "4.0.7" + - "6.8.29" integration_test: - - test_all_the_things + - test_all_the_things include: - version: 6.8.29 serverImage: datastax/dse-mgmtapi-6_8:6.8.29-jdk8 # DSE 6.8.29 diff --git a/tests/cdc_successful/cdc_successful_test.go b/tests/cdc_successful_c/cdc_successful_test_c.go similarity index 99% rename from tests/cdc_successful/cdc_successful_test.go rename to tests/cdc_successful_c/cdc_successful_test_c.go index 8076f6c1..468a76e5 100644 --- a/tests/cdc_successful/cdc_successful_test.go +++ b/tests/cdc_successful_c/cdc_successful_test_c.go @@ -1,7 +1,7 @@ // Copyright DataStax, Inc. // Please see the included license file for details. -package cdc_successful +package cdc_successful_c import ( "fmt" diff --git a/tests/cdc_successful_dse/cdc_successful_test_dse.go b/tests/cdc_successful_dse/cdc_successful_test_dse.go new file mode 100644 index 00000000..0579563c --- /dev/null +++ b/tests/cdc_successful_dse/cdc_successful_test_dse.go @@ -0,0 +1,118 @@ +// Copyright DataStax, Inc. +// Please see the included license file for details. + +package cdc_successful_dse + +import ( + "fmt" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8ssandra/cass-operator/tests/kustomize" + ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo" + "github.com/k8ssandra/cass-operator/tests/util/kubectl" + shutil "github.com/k8ssandra/cass-operator/tests/util/sh" +) + +var ( + testName = "DSE CDC flows work" + namespace = "test-cdc" + dcName = "dc1" + dcYaml = "../testdata/test-cdc/dse-datacenter.yaml" + pulsarValues = "../testdata/test-cdc/dev-values.yaml" + testUtilsDeployment = "../testdata/test-cdc/testutils-deployment.yaml" + ns = ginkgo_util.NewWrapper(testName, namespace) +) + +func TestLifecycle(t *testing.T) { + AfterSuite(func() { + logPath := fmt.Sprintf("%s/aftersuite", ns.LogDir) + err := kubectl.DumpAllLogs(logPath).ExecV() + if err != nil { + t.Logf("Failed to dump all the logs: %v", err) + } + + fmt.Printf("\n\tPost-run logs dumped at: %s\n\n", logPath) + ns.Terminate() + err = kustomize.Undeploy(namespace) + if err != nil { + t.Logf("Failed to undeploy cass-operator: %v", err) + } + kubectl.Delete("ns", "pulsar").OutputPanic() + }) + + RegisterFailHandler(Fail) + RunSpecs(t, testName) +} + +var _ = Describe(testName, func() { + Context("when in a new cluster with CDC enabled", func() { + Specify("CDC feeds will become available for read", func() { + + By("creating a namespace for the cass-operator") + err := kubectl.CreateNamespace(namespace).ExecV() + Expect(err).ToNot(HaveOccurred()) + + By("deploy cass-operator with kustomize") + err = kustomize.Deploy(namespace) + Expect(err).ToNot(HaveOccurred()) + ns.WaitForOperatorReady() + + step := "creating a DC" + testFile, err := ginkgo_util.CreateTestFile(dcYaml) + Expect(err).ToNot(HaveOccurred()) + + k := kubectl.ApplyFiles(testFile) + ns.ExecAndLog(step, k) + + By("Deploying Pulsar") + err = shutil.RunV("helm", "repo", "add", "datastax-pulsar", "https://datastax.github.io/pulsar-helm-chart") + Expect(err).ShouldNot(HaveOccurred()) + + err = shutil.RunV("helm", "repo", "update") + Expect(err).ShouldNot(HaveOccurred()) + + err = shutil.RunV("helm", "install", "--create-namespace", "-n", "pulsar", "-f", pulsarValues, "pulsar", "datastax-pulsar/pulsar") + Expect(err).ShouldNot(HaveOccurred()) + + By("Waiting for all components to be ready") + readyGetter := kubectl.Get("pods"). + WithFlag("selector", "app=cdc-testutil"). + WithFlag("selector", "component=proxy"). + WithFlag("namespace", "pulsar"). + FormatOutput("jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}") + err = kubectl.WaitForOutputContains(readyGetter, "True", 1800) + Expect(err).ShouldNot(HaveOccurred()) + + ns.WaitForDatacenterReadyWithTimeouts(dcName, 1200, 1200) + + step = "Creating a testutils deployment" + k = kubectl.ApplyFiles(testUtilsDeployment) + ns.ExecAndLog(step, k) + + step = "Confirming testutils ready" + readyGetter = kubectl.Get("pods"). + WithFlag("selector", "app=cdc-testutil"). + FormatOutput("jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}") + ns.WaitForOutputContainsAndLog(step, readyGetter, "True", 1800) + + step = "Running testutils applications" + podGetter := kubectl.Get("pods"). + WithFlag("selector", "app=cdc-testutil"). + WithFlag("namespace", namespace). + FormatOutput("jsonpath='{.items[0].metadata.name}'") + testUtilsPod := podGetter.OutputPanic() + testCommand := kubectl. + ExecOnPod( + strings.ReplaceAll(testUtilsPod, "'", ""), + "--", + "bash", "-c", + "/opt/bin/pulsar-cdc-testutil --cass-contact-points test-cluster-dc1-all-pods-service.test-cdc.svc.cluster.local:9042 --pulsar-url pulsar://pulsar-proxy.pulsar.svc.cluster.local:6650 --pulsar-admin-url http://pulsar-proxy.pulsar.svc.cluster.local:8080 --pulsar-cass-contact-point test-cluster-dc1-all-pods-service.test-cdc.svc.cluster.local"). + InNamespace(namespace) + ns.WaitForOutputContainsAndLog(step, testCommand, "SUCCESS", 1200) + }) + }) +}) diff --git a/tests/testdata/test-cdc/dse-datacenter.yaml b/tests/testdata/test-cdc/dse-datacenter.yaml new file mode 100644 index 00000000..334b8b6a --- /dev/null +++ b/tests/testdata/test-cdc/dse-datacenter.yaml @@ -0,0 +1,23 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc1 +spec: + clusterName: test-cluster + # The number of server nodes. + size: 1 + cdc: + pulsarServiceUrl: pulsar://pulsar-proxy.pulsar.svc.cluster.local:6650 + topicPrefix: persistent://public/default/events- + cdcWorkingDir: /var/lib/cassandra/cdc + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # Which server version to use. Required. + serverVersion: "6.8.26" + serverType: "dse"