From 0f8b9a25bdb5eed67d12e67cb878afee3e611362 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 5 Sep 2023 15:29:42 -0400 Subject: [PATCH] Add integration tests for TLS Solr (#611) - Fixed small bug. - Add tests for Secret TLS & CSI Driver TLS. - Multiple configurations tested, including verifyPeerName, wantAuth, needAuth, etc. For now, tests will only work with 8.11 --- controllers/solrcloud_controller_tls_test.go | 4 +- controllers/util/solr_tls_util.go | 6 +- go.mod | 4 +- go.sum | 7 +- tests/e2e/resource_utils_test.go | 5 +- tests/e2e/solrcloud_tls_test.go | 579 +++++++++++++++++++ tests/e2e/suite_test.go | 75 ++- tests/e2e/test_utils_test.go | 231 +++++--- tests/scripts/manage_e2e_tests.sh | 11 +- 9 files changed, 810 insertions(+), 112 deletions(-) create mode 100644 tests/e2e/solrcloud_tls_test.go diff --git a/controllers/solrcloud_controller_tls_test.go b/controllers/solrcloud_controller_tls_test.go index 34cbd5ca..fe55021a 100644 --- a/controllers/solrcloud_controller_tls_test.go +++ b/controllers/solrcloud_controller_tls_test.go @@ -827,7 +827,7 @@ func expectMountedTLSDirConfigOnPodTemplate(podTemplate *corev1.PodTemplateSpec, "-Djavax.net.ssl.trustStorePassword=$(cat " + expectedTruststorePasswordFile + ")" tlsJavaSysProps = "-Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE" } else { - expectedKeystorePassword := solrCloud.Spec.SolrTLS.MountedTLSDir.KeystorePassword + expectedKeystorePassword := "${SOLR_SSL_KEY_STORE_PASSWORD}" if solrCloud.Spec.SolrTLS.MountedTLSDir.KeystorePasswordFile != "" { expectedKeystorePassword = "$(cat " + solrCloud.Spec.SolrTLS.MountedTLSDir.Path + "/" + solrCloud.Spec.SolrTLS.MountedTLSDir.KeystorePasswordFile + ")" } @@ -835,7 +835,7 @@ func expectMountedTLSDirConfigOnPodTemplate(podTemplate *corev1.PodTemplateSpec, if solrCloud.Spec.SolrTLS.MountedTLSDir.TruststorePasswordFile != "" { expectedTruststorePassword = "$(cat " + solrCloud.Spec.SolrTLS.MountedTLSDir.Path + "/" + solrCloud.Spec.SolrTLS.MountedTLSDir.TruststorePasswordFile + ")" } else if solrCloud.Spec.SolrTLS.MountedTLSDir.TruststorePassword != "" { - expectedTruststorePassword = solrCloud.Spec.SolrTLS.MountedTLSDir.TruststorePassword + expectedTruststorePassword = "${SOLR_SSL_TRUST_STORE_PASSWORD}" } tlsJavaToolOpts = "-Djavax.net.ssl.keyStorePassword=" + expectedKeystorePassword + " " + diff --git a/controllers/util/solr_tls_util.go b/controllers/util/solr_tls_util.go index 1866581f..7316cb59 100644 --- a/controllers/util/solr_tls_util.go +++ b/controllers/util/solr_tls_util.go @@ -717,21 +717,23 @@ func secureProbeTLSJavaToolOpts(solrCloud *solr.SolrCloud) (tlsJavaToolOpts stri if solrCloud.Spec.SolrTLS != nil { // prefer the mounted client cert for probes if provided tlsDir := solrCloud.Spec.SolrTLS.MountedTLSDir + clientPrefix := "" if solrCloud.Spec.SolrClientTLS != nil && solrCloud.Spec.SolrClientTLS.MountedTLSDir != nil { tlsDir = solrCloud.Spec.SolrClientTLS.MountedTLSDir + clientPrefix = "CLIENT_" } if tlsDir != nil { // The keystore passwords are in a file, then we need to cat the file(s) into JAVA_TOOL_OPTIONS keyStorePassword := "$(cat " + mountedTLSKeystorePasswordPath(tlsDir) + ")" if tlsDir.KeystorePasswordFile == "" && tlsDir.KeystorePassword != "" { - keyStorePassword = "${SOLR_SSL_CLIENT_KEY_STORE_PASSWORD}" + keyStorePassword = "${SOLR_SSL_" + clientPrefix + "KEY_STORE_PASSWORD}" } tlsJavaToolOpts += " -Djavax.net.ssl.keyStorePassword=" + keyStorePassword trustStorePassword := keyStorePassword if tlsDir.TruststorePasswordFile != "" { trustStorePassword = "$(cat " + mountedTLSTruststorePasswordPath(tlsDir) + ")" } else if tlsDir.TruststorePassword != "" { - trustStorePassword = tlsDir.TruststorePassword + trustStorePassword = "${SOLR_SSL_" + clientPrefix + "TRUST_STORE_PASSWORD}" } tlsJavaToolOpts += " -Djavax.net.ssl.trustStorePassword=" + trustStorePassword } diff --git a/go.mod b/go.mod index 6193a4c0..b7d8c67b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/apache/solr-operator go 1.20 require ( + github.com/cert-manager/cert-manager v1.12.4 github.com/fsnotify/fsnotify v1.6.0 github.com/go-logr/logr v1.2.4 github.com/onsi/ginkgo/v2 v2.12.0 @@ -115,7 +116,6 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v1.1.0 // indirect - go.opencensus.io v0.24.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect @@ -140,9 +140,11 @@ require ( k8s.io/cli-runtime v0.26.0 // indirect k8s.io/component-base v0.27.2 // indirect k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-aggregator v0.27.2 // indirect k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 // indirect k8s.io/kubectl v0.26.0 // indirect oras.land/oras-go v1.2.2 // indirect + sigs.k8s.io/gateway-api v0.7.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.12.1 // indirect sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect diff --git a/go.sum b/go.sum index adc3c52c..2de97cce 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cert-manager/cert-manager v1.12.4 h1:HI38vtBYTG8b2JHDF65+Dbbd09kZps6bglIAlijoj1g= +github.com/cert-manager/cert-manager v1.12.4/go.mod h1:/RYHUvK9cxuU5dbRyhb7g6am9jCcZc8huF3AnADE+nA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -561,7 +563,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= @@ -1004,6 +1005,8 @@ k8s.io/component-base v0.27.2 h1:neju+7s/r5O4x4/txeUONNTS9r1HsPbyoPBAtHsDCpo= k8s.io/component-base v0.27.2/go.mod h1:5UPk7EjfgrfgRIuDBFtsEFAe4DAvP3U+M8RTzoSJkpo= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.27.2 h1:jfHoPip+qN/fn3OcrYs8/xMuVYvkJHKo0H0DYciqdns= +k8s.io/kube-aggregator v0.27.2/go.mod h1:mwrTt4ESjQ7A6847biwohgZWn8P/KzSFHegEScbSGY4= k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 h1:azYPdzztXxPSa8wb+hksEKayiz0o+PPisO/d+QhWnoo= k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5/go.mod h1:kzo02I3kQ4BTtEfVLaPbjvCkX97YqGve33wzlb3fofQ= k8s.io/kubectl v0.26.0 h1:xmrzoKR9CyNdzxBmXV7jW9Ln8WMrwRK6hGbbf69o4T0= @@ -1017,6 +1020,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/gateway-api v0.7.0 h1:/mG8yyJNBifqvuVLW5gwlI4CQs0NR/5q4BKUlf1bVdY= +sigs.k8s.io/gateway-api v0.7.0/go.mod h1:Xv0+ZMxX0lu1nSSDIIPEfbVztgNZ+3cfiYrJsa2Ooso= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= diff --git a/tests/e2e/resource_utils_test.go b/tests/e2e/resource_utils_test.go index 75bdb83c..a41770bb 100644 --- a/tests/e2e/resource_utils_test.go +++ b/tests/e2e/resource_utils_test.go @@ -18,6 +18,7 @@ package e2e import ( + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" @@ -89,7 +90,7 @@ func expectSolrCloudWithChecks(ctx context.Context, solrCloud *solrv1beta1.SolrC if additionalChecks != nil { additionalChecks(g, foundSolrCloud) } - }).WithContext(ctx).Should(Succeed()) + }).WithTimeout(time.Minute * 4).WithContext(ctx).Should(Succeed()) return foundSolrCloud } @@ -765,6 +766,8 @@ func cleanupTest(ctx context.Context, parentResource client.Object) { &solrv1beta1.SolrCloud{}, &solrv1beta1.SolrBackup{}, &solrv1beta1.SolrPrometheusExporter{}, &zkApi.ZookeeperCluster{}, + &certmanagerv1.Certificate{}, &certmanagerv1.Issuer{}, + // All dependent Kubernetes types, in order of dependence (deployment then replicaSet then pod) &corev1.ConfigMap{}, &netv1.Ingress{}, &corev1.PersistentVolumeClaim{}, &corev1.PersistentVolume{}, diff --git a/tests/e2e/solrcloud_tls_test.go b/tests/e2e/solrcloud_tls_test.go new file mode 100644 index 00000000..c814c25f --- /dev/null +++ b/tests/e2e/solrcloud_tls_test.go @@ -0,0 +1,579 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "context" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/utils/pointer" +) + +const ( + solrIssuerName = "solr-issuer" + + secretTlsPasswordKey = "password" + + clientAuthPasswordSecret = "client-auth-password" + clientAuthSecret = "client-auth" +) + +var _ = FDescribe("E2E - SolrCloud - TLS - Secrets", func() { + var ( + solrCloud *solrv1beta1.SolrCloud + + solrCollection = "e2e" + ) + + /* + Create a single SolrCloud that has TLS Enabled + */ + BeforeEach(func(ctx context.Context) { + installSolrIssuer(ctx, testNamespace()) + }) + + /* + Start the SolrCloud and ensure that it is running + */ + JustBeforeEach(func(ctx context.Context) { + By("creating the SolrCloud") + Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed()) + + DeferCleanup(func(ctx context.Context) { + cleanupTest(ctx, solrCloud) + }) + + By("waiting for the SolrCloud to come up healthy") + solrCloud = expectSolrCloudToBeReady(ctx, solrCloud) + + By("creating a Solr Collection to query metrics for") + createAndQueryCollection(ctx, solrCloud, solrCollection, 1, 2) + }) + + FContext("No Client TLS", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, false) + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + FContext("No Client TLS - Just a Keystore", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, false) + + solrCloud.Spec.SolrTLS.TrustStoreSecret = nil + solrCloud.Spec.SolrTLS.TrustStorePasswordSecret = nil + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + FContext("No Client TLS - VerifyClientHostname", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, false) + + solrCloud.Spec.SolrTLS.VerifyClientHostname = true + + solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + FContext("With Client TLS - VerifyClientHostname", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, true) + + solrCloud.Spec.SolrTLS.VerifyClientHostname = true + + solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + FContext("With Client TLS - CheckPeerName", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, true) + + solrCloud.Spec.SolrTLS.CheckPeerName = true + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func(ctx context.Context) { + By("Checking that using the wrong peer name fails") + response, err := callSolrApiInPod( + ctx, + solrCloud, + "get", + "/solr/admin/info/system", + nil, + "localhost", + ) + Expect(err).To(HaveOccurred(), "Error should have occurred while calling Solr API - Bad hostname for TLS") + Expect(response).To(ContainSubstring("doesn't match any of the subject alternative names"), "Wrong error when calling Solr - Bad hostname for TLS expected") + }) + }) + + FContext("With Client TLS - Client Auth Need", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, true) + + solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Need + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + FContext("With Client TLS - Client Auth Want", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithSecretTLS(ctx, 2, true) + + solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Want + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) +}) + +var _ = FDescribe("E2E - SolrCloud - TLS - Mounted Dir", func() { + var ( + solrCloud *solrv1beta1.SolrCloud + + solrCollection = "e2e" + ) + + /* + Create a single SolrCloud that has TLS Enabled + */ + BeforeEach(func(ctx context.Context) { + installSolrIssuer(ctx, testNamespace()) + }) + + /* + Start the SolrCloud and ensure that it is running + */ + JustBeforeEach(func(ctx context.Context) { + By("creating the SolrCloud") + Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed()) + + DeferCleanup(func(ctx context.Context) { + cleanupTest(ctx, solrCloud) + }) + + By("waiting for the SolrCloud to come up healthy") + solrCloud = expectSolrCloudToBeReady(ctx, solrCloud) + + By("creating a Solr Collection to query metrics for") + createAndQueryCollection(ctx, solrCloud, solrCollection, 1, 2) + }) + + FContext("ClientAuth - Want", func() { + + BeforeEach(func(ctx context.Context) { + solrCloud = generateBaseSolrCloudWithCSITLS(1, false, false) + + solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Want + + //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + }) + + FIt("Can run", func() {}) + }) + + //FContext("ClientAuth - Need", func() { + // + // BeforeEach(func(ctx context.Context) { + // solrCloud = generateBaseSolrCloudWithCSITLS(1, false, true) + // + // solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Need + // + // //solrCloud.Spec.SolrOpts = "-Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" + // }) + // + // FIt("Can run", func() {}) + //}) +}) + +func generateBaseSolrCloudWithSecretTLS(ctx context.Context, replicas int, includeClientTLS bool) (solrCloud *solrv1beta1.SolrCloud) { + solrCloud = generateBaseSolrCloud(replicas) + + solrCertSecret, tlsPasswordSecret, clientCertSecret, clientTlsPasswordSecret := generateSolrCert(ctx, solrCloud, includeClientTLS) + + solrCloud.Spec.SolrTLS = &solrv1beta1.SolrTLSOptions{ + PKCS12Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: solrCertSecret, + }, + Key: "keystore.p12", + }, + KeyStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tlsPasswordSecret, + }, + Key: secretTlsPasswordKey, + }, + TrustStoreSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: solrCertSecret, + }, + Key: "truststore.p12", + }, + TrustStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tlsPasswordSecret, + }, + Key: secretTlsPasswordKey, + }, + } + + if includeClientTLS { + solrCloud.Spec.SolrClientTLS = &solrv1beta1.SolrTLSOptions{ + PKCS12Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientCertSecret, + }, + Key: "keystore.p12", + }, + KeyStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientTlsPasswordSecret, + }, + Key: secretTlsPasswordKey, + }, + TrustStoreSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientCertSecret, + }, + Key: "truststore.p12", + }, + TrustStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientTlsPasswordSecret, + }, + Key: secretTlsPasswordKey, + }, + } + } + return +} + +func generateBaseSolrCloudWithCSITLS(replicas int, csiClientTLS bool, secretClientTLS bool) (solrCloud *solrv1beta1.SolrCloud) { + solrCloud = generateBaseSolrCloud(replicas) + solrCloud.Spec.CustomSolrKubeOptions.PodOptions.Volumes = []solrv1beta1.AdditionalVolume{ + { + Name: "server-tls", + Source: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "csi.cert-manager.io", + ReadOnly: pointer.Bool(true), + VolumeAttributes: map[string]string{ + "csi.cert-manager.io/issuer-name": solrIssuerName, + "csi.cert-manager.io/common-name": "${POD_NAME}." + solrCloud.Name + "-solrcloud-headless.${POD_NAMESPACE}", + "csi.cert-manager.io/dns-names": "${POD_NAME}." + solrCloud.Name + "-solrcloud-headless.${POD_NAMESPACE}.svc.cluster.local," + + solrCloud.Name + "-solrcloud-common.${POD_NAMESPACE}," + + solrCloud.Name + "-solrcloud-common.${POD_NAMESPACE}.svc.cluster.local," + + "${POD_NAME}," + + "${POD_NAME}.${POD_NAMESPACE}," + + "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local", + "csi.cert-manager.io/key-usages": "server auth,digital signature", + "csi.cert-manager.io/pkcs12-enable": "true", + "csi.cert-manager.io/pkcs12-password": "pass", + "csi.cert-manager.io/fs-group": "8983", + }, + }, + }, + DefaultContainerMount: &corev1.VolumeMount{ + ReadOnly: true, + MountPath: "/opt/server-tls", + }, + }, + } + + solrCloud.Spec.SolrTLS = &solrv1beta1.SolrTLSOptions{ + MountedTLSDir: &solrv1beta1.MountedTLSDirectory{ + Path: "/opt/server-tls", + KeystoreFile: "keystore.p12", + KeystorePassword: "pass", + }, + } + + if csiClientTLS { + solrCloud.Spec.CustomSolrKubeOptions.PodOptions.Volumes = append( + solrCloud.Spec.CustomSolrKubeOptions.PodOptions.Volumes, + solrv1beta1.AdditionalVolume{ + Name: "client-tls", + Source: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "csi.cert-manager.io", + ReadOnly: pointer.Bool(true), + VolumeAttributes: map[string]string{ + "csi.cert-manager.io/issuer-name": solrIssuerName, + "csi.cert-manager.io/common-name": "${POD_NAME}." + solrCloud.Name + "-solrcloud-headless.${POD_NAMESPACE}", + "csi.cert-manager.io/dns-names": "${POD_NAME}." + solrCloud.Name + "-solrcloud-headless.${POD_NAMESPACE}.svc.cluster.local," + + "${POD_NAME}," + + "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local", + "csi.cert-manager.io/key-usages": "client auth,digital signature", + "csi.cert-manager.io/pkcs12-enable": "true", + "csi.cert-manager.io/pkcs12-password": "pass", + "csi.cert-manager.io/fs-group": "8983", + }, + }, + }, + DefaultContainerMount: &corev1.VolumeMount{ + ReadOnly: true, + MountPath: "/opt/client-tls", + }, + }) + + solrCloud.Spec.SolrClientTLS = &solrv1beta1.SolrTLSOptions{ + MountedTLSDir: &solrv1beta1.MountedTLSDirectory{ + Path: "/opt/client-tls", + KeystoreFile: "keystore.p12", + KeystorePassword: "pass", + }, + } + } else if secretClientTLS { + // TODO: It is not currently supported to mix secret and mountedDir TLS. + // This will not work until that support is added. + solrCloud.Spec.SolrClientTLS = &solrv1beta1.SolrTLSOptions{ + PKCS12Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientAuthSecret, + }, + Key: "keystore.p12", + }, + TrustStoreSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientAuthSecret, + }, + Key: "truststore.p12", + }, + TrustStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientAuthPasswordSecret, + }, + Key: "password", + }, + } + } + return +} + +func installBootstrapIssuer(ctx context.Context) { + bootstrapIssuer := &certmanagerv1.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bootstrap-issuer", + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + SelfSigned: &certmanagerv1.SelfSignedIssuer{}, + }, + }, + } + Expect(k8sClient.Create(ctx, bootstrapIssuer)).To(Succeed(), "Failed to install SelfSigned ClusterIssuer for bootstrapping CA") + DeferCleanup(func(ctx context.Context) { + Expect(k8sClient.Delete(ctx, bootstrapIssuer)).To(Succeed(), "Failed to delete SelfSigned bootstrapping ClusterIssuer") + }) +} + +func installSolrIssuer(ctx context.Context, namespace string) { + secretName := "solr-ca-key-pair" + clusterCA := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "solr-ca", + Namespace: namespace, + }, + Spec: certmanagerv1.CertificateSpec{ + IsCA: true, + CommonName: "solr-ca", + SecretName: secretName, + PrivateKey: &certmanagerv1.CertificatePrivateKey{ + RotationPolicy: certmanagerv1.RotationPolicyNever, + Algorithm: "RSA", + }, + IssuerRef: certmanagermetav1.ObjectReference{ + Name: "bootstrap-issuer", + Kind: "ClusterIssuer", + Group: "cert-manager.io", + }, + }, + } + Expect(k8sClient.Create(ctx, clusterCA)).To(Succeed(), "Failed to install Solr CA for tests") + + namespaceIssuer := &certmanagerv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrIssuerName, + Namespace: namespace, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + CA: &certmanagerv1.CAIssuer{ + SecretName: secretName, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, namespaceIssuer)).To(Succeed(), "Failed to install CA Issuer for issuing test certs in namespace "+namespace) + + expectSecret(ctx, clusterCA, secretName) +} + +func generateSolrCert(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, includeClientTLS bool) (certSecretName string, tlsPasswordSecretName string, clientTLSCertSecretName string, clientTLSPasswordSecretName string) { + // First create a secret to use as a password for the keystore/truststore + tlsPasswordSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.Name + "-keystore-password", + Namespace: solrCloud.Namespace, + }, + StringData: map[string]string{ + secretTlsPasswordKey: rand.String(10), + }, + Type: corev1.SecretTypeOpaque, + } + Expect(k8sClient.Create(ctx, tlsPasswordSecret)).To(Succeed(), "Failed to create secret for tls password in namespace "+solrCloud.Namespace) + + expectSecret(ctx, solrCloud, tlsPasswordSecret.Name) + tlsPasswordSecretName = tlsPasswordSecret.Name + + allDNSNames := make([]string, *solrCloud.Spec.Replicas*2+1) + for _, pod := range solrCloud.GetAllSolrPodNames() { + allDNSNames = append(allDNSNames, pod, solrCloud.InternalNodeUrl(pod, false)) + } + + certSecretName = solrCloud.Name + "-secret-auth" + + solrCert := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.Name + "-secret-auth", + Namespace: solrCloud.Namespace, + }, + Spec: certmanagerv1.CertificateSpec{ + CommonName: solrCloud.InternalCommonUrl(false), + DNSNames: allDNSNames, + SecretName: certSecretName, + Keystores: &certmanagerv1.CertificateKeystores{ + PKCS12: &certmanagerv1.PKCS12Keystore{ + Create: true, + PasswordSecretRef: certmanagermetav1.SecretKeySelector{ + LocalObjectReference: certmanagermetav1.LocalObjectReference{ + Name: tlsPasswordSecret.Name, + }, + Key: secretTlsPasswordKey, + }, + }, + }, + IssuerRef: certmanagermetav1.ObjectReference{ + Name: solrIssuerName, + Kind: "Issuer", + Group: "cert-manager.io", + }, + IsCA: false, + Usages: []certmanagerv1.KeyUsage{certmanagerv1.UsageServerAuth, certmanagerv1.UsageDigitalSignature}, + PrivateKey: &certmanagerv1.CertificatePrivateKey{ + RotationPolicy: certmanagerv1.RotationPolicyNever, + Algorithm: "RSA", + }, + }, + } + Expect(k8sClient.Create(ctx, solrCert)).To(Succeed(), "Failed to install Solr secret cert for tests") + + expectSecret(ctx, solrCert, certSecretName) + + if includeClientTLS { + // First create a secret to use as a password for the keystore/truststore + clientTlsPasswordSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.Name + "-client-tls-password", + Namespace: solrCloud.Namespace, + }, + StringData: map[string]string{ + secretTlsPasswordKey: rand.String(10), + }, + Type: corev1.SecretTypeOpaque, + } + Expect(k8sClient.Create(ctx, clientTlsPasswordSecret)).To(Succeed(), "Failed to create secret for client tls password in namespace "+solrCloud.Namespace) + + expectSecret(ctx, solrCloud, clientTlsPasswordSecret.Name) + clientTLSPasswordSecretName = clientTlsPasswordSecret.Name + + clientTLSCertSecretName = solrCloud.Name + "-client-tls-secret-auth" + + solrClientCert := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.Name + "-client-secret-auth", + Namespace: solrCloud.Namespace, + }, + Spec: certmanagerv1.CertificateSpec{ + CommonName: solrCloud.InternalCommonUrl(false), + DNSNames: allDNSNames, + SecretName: clientTLSCertSecretName, + Keystores: &certmanagerv1.CertificateKeystores{ + PKCS12: &certmanagerv1.PKCS12Keystore{ + Create: true, + PasswordSecretRef: certmanagermetav1.SecretKeySelector{ + LocalObjectReference: certmanagermetav1.LocalObjectReference{ + Name: clientTlsPasswordSecret.Name, + }, + Key: secretTlsPasswordKey, + }, + }, + }, + IssuerRef: certmanagermetav1.ObjectReference{ + Name: solrIssuerName, + Kind: "Issuer", + Group: "cert-manager.io", + }, + IsCA: false, + Usages: []certmanagerv1.KeyUsage{certmanagerv1.UsageClientAuth, certmanagerv1.UsageDigitalSignature}, + PrivateKey: &certmanagerv1.CertificatePrivateKey{ + RotationPolicy: certmanagerv1.RotationPolicyNever, + Algorithm: "RSA", + }, + }, + } + Expect(k8sClient.Create(ctx, solrClientCert)).To(Succeed(), "Failed to install Solr clientTLS secret cert for tests") + + expectSecret(ctx, solrClientCert, clientTLSCertSecretName) + } + + return +} diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 90bbf946..d6060a39 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -21,9 +21,11 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/version" + certManagerApi "github.com/cert-manager/cert-manager/pkg/api" "github.com/go-logr/logr" "github.com/onsi/ginkgo/v2/types" zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" @@ -104,6 +106,7 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { k8sConfig, err = config.GetConfig() Expect(err).NotTo(HaveOccurred(), "Could not load in default kubernetes config") Expect(zkApi.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(certManagerApi.AddToScheme(scheme.Scheme)).To(Succeed()) k8sClient, err = client.New(k8sConfig, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred(), "Could not create controllerRuntime Kubernetes client") @@ -113,6 +116,10 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { By("creating a shared zookeeper cluster") zookeeper := runSharedZookeeperCluster(ctx) + // Set up a shared Bootstrap issuer for creating CAs for Solr + By("creating a boostrap cert issuer") + installBootstrapIssuer(ctx) + // Run this once before all tests, not per-test-process By("starting the test solr operator") solrOperatorRelease := runSolrOperator(ctx) @@ -144,6 +151,7 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { By("setting up the k8s clients") Expect(solrv1beta1.AddToScheme(scheme.Scheme)).To(Succeed()) Expect(zkApi.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(certManagerApi.AddToScheme(scheme.Scheme)).To(Succeed()) k8sClient, err = client.New(k8sConfig, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred(), "Could not create controllerRuntime Kubernetes client") @@ -204,7 +212,7 @@ func outputDirForTest(testText string) string { return outputDir + "/" + strings.ReplaceAll(strings.ReplaceAll(testName, " ", "-"), " ", "") } -var _ = JustAfterEach(func() { +var _ = JustAfterEach(func(ctx context.Context) { testOutputDir := outputDirForTest(CurrentSpecReport().FullText()) // We count "ran" as "passed" or "failed" @@ -214,14 +222,16 @@ var _ = JustAfterEach(func() { // Always save the logs of the Solr Operator for the test startTime := CurrentSpecReport().StartTime writePodLogsToFile( + ctx, testOutputDir+"solr-operator.log", - getSolrOperatorPodName(solrOperatorReleaseNamespace), + getSolrOperatorPodName(ctx, solrOperatorReleaseNamespace), solrOperatorReleaseNamespace, &startTime, fmt.Sprintf("%q: %q", "namespace", testNamespace()), ) // Always save the logs of the Solr Operator for the test - writeAllSolrLogsToFiles( + writeAllSolrInfoToFiles( + ctx, testOutputDir, testNamespace(), ) @@ -255,7 +265,7 @@ var _ = ReportAfterEach(func(report SpecReport) { } }) -func getSolrOperatorPodName(namespace string) string { +func getSolrOperatorPodName(ctx context.Context, namespace string) string { labelSelector := labels.SelectorFromSet(map[string]string{"control-plane": "solr-operator"}) listOps := &client.ListOptions{ Namespace: namespace, @@ -264,13 +274,13 @@ func getSolrOperatorPodName(namespace string) string { } foundPods := &corev1.PodList{} - Expect(k8sClient.List(context.TODO(), foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod") + Expect(k8sClient.List(ctx, foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod") Expect(foundPods).ToNot(BeNil(), "No Solr Operator pods could be found") Expect(foundPods.Items).ToNot(BeEmpty(), "No Solr Operator pods could be found") return foundPods.Items[0].Name } -func writeAllSolrLogsToFiles(directory string, namespace string) { +func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace string) { req, err := labels.NewRequirement("technology", selection.In, []string{solrv1beta1.SolrTechnologyLabel, solrv1beta1.SolrPrometheusExporterTechnologyLabel}) Expect(err).ToNot(HaveOccurred()) @@ -281,20 +291,55 @@ func writeAllSolrLogsToFiles(directory string, namespace string) { } foundPods := &corev1.PodList{} - Expect(k8sClient.List(context.TODO(), foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod") + Expect(k8sClient.List(ctx, foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod") Expect(foundPods).ToNot(BeNil(), "No Solr pods could be found") for _, pod := range foundPods.Items { - writePodLogsToFile( - directory+pod.Name+".log", - pod.Name, - namespace, - nil, - "", + writeAllPodInfoToFiles( + ctx, + directory+pod.Name, + &pod, ) } } -func writePodLogsToFile(filename string, podName string, podNamespace string, startTimeRaw *time.Time, filterLinesWithString string) { +// writeAllPodInfoToFile writes the following each to a separate file with the given base name & directory. +// - Pod Spec/Status +// - Pod Events +// - Pod logs +func writeAllPodInfoToFiles(ctx context.Context, baseFilename string, pod *corev1.Pod) { + // Write pod to a file + statusFile, err := os.Create(baseFilename + ".status.json") + defer statusFile.Close() + Expect(err).ToNot(HaveOccurred(), "Could not open file to save pod status: %s", baseFilename+".status.json") + jsonBytes, marshErr := json.MarshalIndent(pod, "", "\t") + Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize pod json") + _, writeErr := statusFile.Write(jsonBytes) + Expect(writeErr).ToNot(HaveOccurred(), "Could not write pod json to file") + + // Write events for pod to a file + eventsFile, err := os.Create(baseFilename + ".events.json") + defer eventsFile.Close() + Expect(err).ToNot(HaveOccurred(), "Could not open file to save status: %s", baseFilename+".events.yaml") + + eventList, err := rawK8sClient.CoreV1().Events(pod.Namespace).Search(scheme.Scheme, pod) + Expect(err).ToNot(HaveOccurred(), "Could not find events for pod: %s", pod.Name) + jsonBytes, marshErr = json.MarshalIndent(eventList, "", "\t") + Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize events json") + _, writeErr = eventsFile.Write(jsonBytes) + Expect(writeErr).ToNot(HaveOccurred(), "Could not write events json to file") + + // Write pod logs to a file + writePodLogsToFile( + ctx, + baseFilename+".log", + pod.Name, + pod.Namespace, + nil, + "", + ) +} + +func writePodLogsToFile(ctx context.Context, filename string, podName string, podNamespace string, startTimeRaw *time.Time, filterLinesWithString string) { logFile, err := os.Create(filename) defer logFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save logs: %s", filename) @@ -306,7 +351,7 @@ func writePodLogsToFile(filename string, podName string, podNamespace string, st } req := rawK8sClient.CoreV1().Pods(podNamespace).GetLogs(podName, &podLogOpts) - podLogs, logsErr := req.Stream(context.Background()) + podLogs, logsErr := req.Stream(ctx) defer podLogs.Close() Expect(logsErr).ToNot(HaveOccurred(), "Could not open stream to fetch pod logs. namespace: %s, pod: %s", podNamespace, podName) diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go index 85e08a0a..ca5b01c7 100644 --- a/tests/e2e/test_utils_test.go +++ b/tests/e2e/test_utils_test.go @@ -44,6 +44,7 @@ import ( "k8s.io/utils/pointer" "os" "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" "strings" "time" ) @@ -211,55 +212,52 @@ func createAndQueryCollection(ctx context.Context, solrCloud *solrv1beta1.SolrCl func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, shards int, replicasPerShard int, g Gomega, additionalOffset int, nodes ...int) { asyncId := fmt.Sprintf("create-collection-%s-%d-%d", collection, shards, replicasPerShard) - var nodeSet []string - for _, node := range nodes { - nodeSet = append(nodeSet, util.SolrNodeName(solrCloud, solrCloud.GetSolrPodName(node))) + createParams := map[string]string{ + "action": "CREATE", + "name": collection, + "replicationFactor": strconv.Itoa(replicasPerShard), + "numShards": strconv.Itoa(shards), + "maxShardsPerNode": "10", + "async": asyncId, + "wt": "json", } - createNodeSet := "" - if len(nodeSet) > 0 { - createNodeSet = "&createNodeSet=" + strings.Join(nodeSet, ",") + + if len(nodes) > 0 { + var nodeSet []string + for _, node := range nodes { + nodeSet = append(nodeSet, util.SolrNodeName(solrCloud, solrCloud.GetSolrPodName(node))) + } + createParams["createNodeSet"] = strings.Join(nodeSet, ",") } additionalOffset += 1 g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf( - "http://localhost:%d/solr/admin/collections?action=CREATE&name=%s&replicationFactor=%d&numShards=%d%s&async=%s&maxShardsPerNode=10", - solrCloud.Spec.SolrAddressability.PodPort, - collection, - replicasPerShard, - shards, - createNodeSet, - asyncId), - }, + solrCloud, + "get", + "/solr/admin/collections", + createParams, ) innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while starting async command to create Solr Collection") innerG.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while starting async command to create Solr Collection") - }, time.Second*5).WithContext(ctx).Should(Succeed(), "Collection creation command start was not successful") + }).Within(time.Second*10).WithContext(ctx).Should(Succeed(), "Collection creation command start was not successful") // Only wait 5 seconds when trying to create the asyncCommand g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf( - "http://localhost:%d/solr/admin/collections?action=REQUESTSTATUS&requestid=%s", - solrCloud.Spec.SolrAddressability.PodPort, - asyncId), + solrCloud, + "get", + "/solr/admin/collections", + map[string]string{ + "action": "REQUESTSTATUS", + "requestid": asyncId, + "wt": "json", }, ) innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while checking if Solr Collection creation command was successful") - if strings.Contains(response, "\"state\":\"failed\"") || strings.Contains(response, "\"state\":\"notfound\"") { + if strings.Contains(response, "failed") || strings.Contains(response, "notfound") { StopTrying("A failure occurred while creating the Solr Collection"). Attach("Collection", collection). Attach("Shards", shards). @@ -269,25 +267,23 @@ func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1be } innerG.Expect(response).To(ContainSubstring("\"status\":0"), "A failure occurred while creating the Solr Collection") innerG.Expect(response).To(ContainSubstring("\"state\":\"completed\""), "Did not finish creating Solr Collection in time") - }).WithContext(ctx).Should(Succeed(), "Collection creation was not successful") + }).Within(time.Second*40).WithContext(ctx).Should(Succeed(), "Collection creation was not successful") g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf( - "http://localhost:%d/solr/admin/collections?action=DELETESTATUS&requestid=%s", - solrCloud.Spec.SolrAddressability.PodPort, - asyncId), + solrCloud, + "get", + "/solr/admin/collections", + map[string]string{ + "action": "DELETESTATUS", + "requestid": asyncId, + "wt": "json", }, ) innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while deleting Solr CollectionsAPI AsyncID") innerG.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while deleting Solr CollectionsAPI AsyncID") - }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not delete aysncId after collection creation") + }).Within(time.Second*10).WithContext(ctx).Should(Succeed(), "Could not delete aysncId after collection creation") // Only wait 5 seconds when trying to delete the async requestId queryCollectionWithGomega(ctx, solrCloud, collection, 0, g, additionalOffset) @@ -299,31 +295,31 @@ func queryCollection(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, coll func queryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, docCount int, g Gomega, additionalOffset ...int) { g.EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega) { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf("http://localhost:%d/solr/%s/select?rows=0", solrCloud.Spec.SolrAddressability.PodPort, collection), + solrCloud, + "get", + fmt.Sprintf("/solr/%s/select", collection), + map[string]string{ + "rows": "0", + "wt": "json", }, ) innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection") innerG.Expect(response).To(ContainSubstring("\"numFound\":%d", docCount), "Error occurred while querying Solr Collection '%s'", collection) - }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection: %v", fetchClusterStatus(ctx, solrCloud)) + }).Within(time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection: %v", fetchClusterStatus(ctx, solrCloud)) // Only wait 5 seconds for the collection to be query-able } func fetchClusterStatus(ctx context.Context, solrCloud *solrv1beta1.SolrCloud) string { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf("http://localhost:%d/solr/admin/collections?action=CLUSTERSTATUS", solrCloud.Spec.SolrAddressability.PodPort), + solrCloud, + "get", + "/solr/admin/collections", + map[string]string{ + "action": "CLUSTERSTATUS", + "wt": "json", }, ) Expect(err).ToNot(HaveOccurred(), "Could not fetch clusterStatus for cloud") @@ -333,19 +329,21 @@ func fetchClusterStatus(ctx context.Context, solrCloud *solrv1beta1.SolrCloud) s func queryCollectionWithNoReplicaAvailable(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, additionalOffset ...int) { EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega) { - response, err := runExecForContainer( + response, _ := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - fmt.Sprintf("http://localhost:%d/solr/%s/select", solrCloud.Spec.SolrAddressability.PodPort, collection), + solrCloud, + "get", + fmt.Sprintf("/solr/%s/select", collection), + map[string]string{ + "rows": "0", + "wt": "json", }, ) - innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection") - innerG.Expect(response).To(ContainSubstring("Error trying to proxy request for url"), "Wrong occurred while querying Solr Collection '%s', expected a proxy forwarding error", collection) - }, time.Second*5).WithContext(ctx).Should(Succeed(), "Collection query did not fail in the correct way") + innerG.Expect(response).To( + // "Exception in thread "main" is for 8.11, which does not handle the exception correctly + Or(ContainSubstring("Error trying to proxy request for url"), ContainSubstring("Exception in thread \"main\" java.lang.NullPointerException")), + "Wrong occurred while querying Solr Collection '%s', expected a proxy forwarding error", collection) + }).Within(time.Second*5).WithContext(ctx).Should(Succeed(), "Collection query did not fail in the correct way") } func getPrometheusExporterPod(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter) (podName string) { @@ -414,25 +412,24 @@ func checkBackupWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud g.Expect(solrCloud.Spec.BackupRepositories).To(Not(BeEmpty()), "Solr BackupRepository list cannot be empty in backup test") } for _, collection := range solrBackup.Spec.Collections { - curlCommand := fmt.Sprintf( - "http://localhost:%d/solr/admin/collections?action=LISTBACKUP&name=%s&repository=%s&collection=%s&location=%s", - solrCloud.Spec.SolrAddressability.PodPort, - util.FullCollectionBackupName(collection, solrBackup.Name), - repositoryName, - collection, - util.BackupLocationPath(repository, solrBackup.Spec.Location)) + backupParams := map[string]string{ + "action": "LISTBACKUP", + "name": util.FullCollectionBackupName(collection, solrBackup.Name), + "repository": repositoryName, + "collection": collection, + "location": util.BackupLocationPath(repository, solrBackup.Spec.Location), + "wt": "json", + } + g.Eventually(func(innerG Gomega) { - response, err := runExecForContainer( + response, err := callSolrApiInPod( ctx, - util.SolrNodeContainer, - solrCloud.GetRandomSolrPodName(), - solrCloud.Namespace, - []string{ - "curl", - curlCommand, - }, + solrCloud, + "get", + "/solr/admin/collections", + backupParams, ) - innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while fetching backup '%s' for collection '%s': %s", solrBackup.Name, collection, curlCommand) + innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while fetching backup '%s' for collection '%s': %s", solrBackup.Name, collection, backupParams) backupListResponse := &solr_api.SolrBackupListResponse{} innerG.Expect(json.Unmarshal([]byte(response), &backupListResponse)).To(Succeed(), "Could not parse json from Solr BackupList API") @@ -443,6 +440,49 @@ func checkBackupWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud } } +type ExecError struct { + Command string + + Err error + + ErrorOutput string + + ResponseOutput string +} + +func (r *ExecError) Error() string { + return fmt.Sprintf("Error from Pod Exec: %v\n\nError output from Pod Exec: %sResponse output from Pod Exec: %s", r.Err, r.ErrorOutput, r.ResponseOutput) +} + +func callSolrApiInPod(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, httpMethod string, apiPath string, queryParams map[string]string, hostnameOptional ...string) (response string, err error) { + hostname := "${POD_HOSTNAME}" + if len(hostnameOptional) > 0 { + hostname = hostnameOptional[0] + } + var queryParamsSlice []string + for param, val := range queryParams { + queryParamsSlice = append(queryParamsSlice, param+"="+val) + } + queryParamsString := strings.Join(queryParamsSlice, "&") + if len(queryParamsString) > 0 { + queryParamsString = "?" + queryParamsString + } + + command := []string{ + "solr", + "api", + "-" + strings.ToLower(httpMethod), + fmt.Sprintf( + "\"%s://%s:%d%s%s\"", + solrCloud.UrlScheme(false), + hostname, + solrCloud.Spec.SolrAddressability.PodPort, + apiPath, + queryParamsString), + } + return runExecForContainer(ctx, util.SolrNodeContainer, solrCloud.GetRandomSolrPodName(), solrCloud.Namespace, command) +} + func runExecForContainer(ctx context.Context, container string, podName string, namespace string, command []string) (response string, err error) { req := rawK8sClient.CoreV1().RESTClient().Post(). Resource("pods"). @@ -456,7 +496,7 @@ func runExecForContainer(ctx context.Context, container string, podName string, parameterCodec := runtime.NewParameterCodec(scheme) req.VersionedParams(&corev1.PodExecOptions{ - Command: command, + Command: []string{"sh", "-c", strings.Join(command, " ")}, Container: container, Stdin: false, Stdout: true, @@ -477,11 +517,24 @@ func runExecForContainer(ctx context.Context, container string, podName string, Tty: false, }) + responseOutput := stdout.String() + errOutput := stderr.String() + if err != nil { - return "", fmt.Errorf("error in Stream: %v", err) + err = &ExecError{ + Command: strings.Join(command, " "), + Err: err, + ResponseOutput: responseOutput, + ErrorOutput: errOutput, + } + } + if len(responseOutput) == 0 { + response = errOutput + } else { + response = responseOutput } - return stdout.String(), err + return response, err } func generateBaseSolrCloud(replicas int) *solrv1beta1.SolrCloud { diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh index 27ca4f28..d6a3b5e1 100755 --- a/tests/scripts/manage_e2e_tests.sh +++ b/tests/scripts/manage_e2e_tests.sh @@ -35,7 +35,7 @@ Available actions are: run-tests, create-cluster, destroy-cluster, kubeconfig -h Display this help and exit -i Solr Operator docker image to use (Optional, defaults to apache/solr-operator:) -k Kubernetes Version to test with (full tag, e.g. v1.24.16) (Optional, defaults to a compatible version) - -s Full solr image, or image tag (for the official Solr image), to test with (e.g. apache/solr-nightly:9.0.0, 8.11). (Optional, defaults to a compatible version) + -s Full solr image, or image tag (for the official Solr image), to test with (e.g. apache/solr-nightly:9.4.0, 8.11). (Optional, defaults to a compatible version) -a Load additional local images into the test Kubernetes cluster. Provide option multiple times for multiple images. (Optional) EOF } @@ -96,6 +96,9 @@ export RAW_GINKGO export REUSE_KIND_CLUSTER_IF_EXISTS="${REUSE_KIND_CLUSTER_IF_EXISTS:-true}" # This is used for all start_cluster calls export LEAVE_KIND_CLUSTER_ON_SUCCESS="${LEAVE_KIND_CLUSTER_ON_SUCCESS:-false}" # This is only used when using run_tests or run_with_cluster +export CERT_MANAGER_VERSION=1.12.3 +export CERT_MANAGER_CSI_DRIVER_VERSION=0.5.0 + function add_image_to_kind_repo_if_local() { IMAGE="$1" PULL_IF_NOT_LOCAL="$2" @@ -179,6 +182,12 @@ function setup_cluster() { kubectl create -f "${REPO_DIR}/config/crd/bases/" 2>/dev/null || kubectl replace -f "${REPO_DIR}/config/crd/bases/" kubectl create -f "${REPO_DIR}/config/dependencies/" 2>/dev/null || kubectl replace -f "${REPO_DIR}/config/dependencies/" echo "" + + printf "Installing Cert Manager\n" + helm repo add cert-manager https://charts.jetstack.io --force-update + helm upgrade -i -n cert-manager --create-namespace cert-manager cert-manager/cert-manager --version "${CERT_MANAGER_VERSION}" --set installCRDs=true + helm upgrade -i -n cert-manager cert-manager-csi-driver cert-manager/cert-manager-csi-driver --version "${CERT_MANAGER_CSI_DRIVER_VERSION}" + echo "" } case "$ACTION" in