chore(cStorVolume_bdd): add negative test cases for cStorVolumeProvis…

…ioning (openebs-archive#1530)

1. This PR provides the capability to get debug build for the cstor-pool-mgmt.
2. Debug build has an injection server running on port 8080 on all the cstor-pool-mgmgt
   which can be used to inject the errors for CR's and ZFS operations for negative path testing.

**Perform the following steps to run the negative tests on your local machine:**
1. Navigate to tests/cstor/volume using cd.
2. In your command line run the following command to execute negative BDD
   CMD: ginkgo -v  -focus="\[Cstor Volume Provisioning Negative\]" -- -kubeconfig=/home/sai/.kube/config -cstor-maxpools=3 -cstor-replicas=3

Signed-off-by: mittachaitu <[email protected]>

GNUmakefile

@@ -108,6 +108,7 @@ VOLUME_MGMT=cstor-volume-mgmt
 EXPORTER=maya-exporter
 CSPC_OPERATOR=cspc-operator
 CSPC_OPERATOR_DEBUG=cspc-operator-debug
+CSP_OPERATOR_DEBUG=cstor-pool-mgmt-debug
 
 
 # Specify the date o build
@@ -311,6 +312,18 @@ pool-mgmt-image: cstor-pool-mgmt
 	@cd buildscripts/cstor-pool-mgmt && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:${IMAGE_TAG} --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache
 	@rm buildscripts/cstor-pool-mgmt/${POOL_MGMT}
 
+#Use this to build debug image of cstor-pool-mgmt
+.PHONY: pool-mgmt-debug-image
+pool-mgmt-debug-image:
+	@echo "----------------------------"
+	@echo -n "--> cstor-pool-mgmt debug image "
+	@echo "${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject"
+	@echo "----------------------------"
+	@PNAME=${CSP_OPERATOR_DEBUG} CTLNAME=${POOL_MGMT} BUILD_TAG="-tags=debug" sh -c "'$(PWD)/buildscripts/build.sh'"
+	@cp bin/${CSP_OPERATOR_DEBUG}/${POOL_MGMT} buildscripts/${CSP_OPERATOR_DEBUG}/
+	@cd buildscripts/${CSP_OPERATOR_DEBUG} && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache
+	@rm buildscripts/${CSP_OPERATOR_DEBUG}/${POOL_MGMT}
+
 #Use this to build cspi-mgmt
 .PHONY: cspi-mgmt
 cspi-mgmt:

buildscripts/cstor-pool-mgmt-debug/Dockerfile

@@ -0,0 +1,30 @@
+#
+# This Dockerfile builds a recent cstor-pool-mgmt-debug using the latest binary from
+# cstor-pool-mgmt  releases.
+#
+
+#openebs/cstor-base is the image that contains cstor related binaries and
+#libraries - zpool, zfs, zrepl
+#FROM openebs/cstor-base:ci
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+COPY cstor-pool-mgmt /usr/local/bin/
+COPY entrypoint.sh /usr/local/bin/
+
+RUN printf '#!/bin/bash\nif [ $# -lt 1 ]; then\n\techo "argument missing"\n\texit 1\nfi\neval "$*"\n' >> /usr/local/bin/execute.sh
+
+RUN chmod +x /usr/local/bin/execute.sh
+RUN apt install netcat -y
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ARG BUILD_DATE
+LABEL org.label-schema.name="cstor-pool-mgmt"
+LABEL org.label-schema.description="OpenEBS"
+LABEL org.label-schema.url="http://www.openebs.io/"
+LABEL org.label-schema.vcs-url="https://github.com/openebs/maya"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+
+ENTRYPOINT entrypoint.sh
+EXPOSE 7676

buildscripts/cstor-pool-mgmt-debug/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -ex
+
+rm /usr/local/bin/zrepl
+exec /usr/local/bin/cstor-pool-mgmt start
+exec service ssh start
+exec service rsyslog start

cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/openebs/maya/cmd/cstor-pool-mgmt/controller/common"
+	"github.com/openebs/maya/pkg/debug"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
@@ -34,6 +35,10 @@ func (c *CStorPoolController) Run(threadiness int, stopCh <-chan struct{}) error
 	defer runtime.HandleCrash()
 	defer c.workqueue.ShutDown()
 
+	// Start server only in debug build
+	debug.LogBuildDetails()
+	debug.StartInjectionServer()
+
 	// Start the informer factories to begin populating the informer caches
 	klog.Info("Starting CStorPool controller")
 

cmd/cstor-pool-mgmt/controller/replica-controller/handler.go

@@ -27,6 +27,7 @@ import (
 	"github.com/openebs/maya/cmd/cstor-pool-mgmt/volumereplica"
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
 	clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned"
+	"github.com/openebs/maya/pkg/debug"
 	merrors "github.com/openebs/maya/pkg/errors/v1alpha1"
 	pkg_errors "github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -146,6 +147,9 @@ func (c *CStorVolumeReplicaController) syncHandler(
 
 	// need to update cvr before returning this error
 	if err != nil {
+		if debug.EI.IsCVRUpdateErrorInjected() {
+			return merrors.Errorf("CVR update error via injection")
+		}
 		_, err1 := c.clientset.
 			OpenebsV1alpha1().
 			CStorVolumeReplicas(cvrGot.Namespace).

cmd/cstor-pool-mgmt/volumereplica/volumereplica.go

@@ -27,6 +27,7 @@ import (
 	"encoding/json"
 
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	"github.com/openebs/maya/pkg/debug"
 	"github.com/openebs/maya/pkg/hash"
 	"github.com/openebs/maya/pkg/util"
 	zfs "github.com/openebs/maya/pkg/zfs/cmd/v1alpha1"
@@ -183,6 +184,10 @@ func CreateVolumeReplica(cStorVolumeReplica *apis.CStorVolumeReplica, fullVolNam
 	isClone := cStorVolumeReplica.Labels[string(apis.CloneEnableKEY)] == "true"
 	snapName := ""
 
+	if debug.EI.IsZFSCreateErrorInjected() {
+		return errors.New("ZFS create error via injection")
+	}
+
 	if isClone {
 		srcVolume := cStorVolumeReplica.Annotations[string(apis.SourceVolumeKey)]
 		snapName = cStorVolumeReplica.Annotations[string(apis.SnapshotNameKey)]
@@ -413,6 +418,10 @@ func buildVolumeRestoreCommand(poolName, fullVolName, restoreSrc string) []strin
 
 // GetVolumes returns the slice of volumes.
 func GetVolumes() ([]string, error) {
+	if debug.EI.IsZFSGetErrorInjected() {
+		return []string{}, errors.New("ZFS get error via injection")
+	}
+
 	volStrCmd := []string{"get", "-Hp", "name", "-o", "name"}
 	volnameByte, err := RunnerVar.RunStdoutPipe(VolumeReplicaOperator, volStrCmd...)
 	if err != nil || string(volnameByte) == "" {
@@ -431,6 +440,10 @@ func GetVolumes() ([]string, error) {
 
 // DeleteVolume deletes the specified volume.
 func DeleteVolume(fullVolName string) error {
+	if debug.EI.IsZFSDeleteErrorInjected() {
+		return errors.New("ZFS delete error via injection")
+	}
+
 	deleteVolStr := []string{"destroy", "-R", fullVolName}
 	stdoutStderr, err := RunnerVar.RunCombinedOutput(VolumeReplicaOperator, deleteVolStr...)
 	if err != nil {

pkg/debug/debug.go

@@ -22,10 +22,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"k8s.io/klog"
 	"log"
 	"net/http"
 	"time"
+
+	"k8s.io/klog"
 )
 
 // LogBuildDetails logs the build details when the cspc-operator starts.
@@ -287,3 +288,49 @@ func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool {
 	}
 	return false
 }
+
+// IsZFSGetErrorInjected returns true if error is injected for ZFS get command
+func (ei *ErrorInjection) IsZFSGetErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectGetError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsZFSDeleteErrorInjected returns true if error is injected for ZFS delete command
+func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectDeleteError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsZFSCreateErrorInjected returns true if error is injected for ZFS create command
+func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectCreateError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsCVRCreateErrorInjected returns true if error is injected for CVR create
+// command
+func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool {
+	return false
+}
+
+// IsCVRDeleteErrorInjected returns true if error is injected for CVR delete
+// command
+func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRGetErrorInjected returns true if error is injected for CVR get command
+func (ei *ErrorInjection) IsCVRGetErrorInjected() bool {
+	return false
+}
+
+// IsCVRUpdateErrorInjected returns true if error is injected for CVR update command
+func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool {
+	return false
+}

pkg/debug/release.go

@@ -128,3 +128,45 @@ func (ei *ErrorInjection) IsDeploymentUpdateErrorInjected() bool {
 func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool {
 	return false
 }
+
+// IsZFSGetErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSGetErrorInjected() bool {
+	return false
+}
+
+// IsZFSCreateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool {
+	return false
+}
+
+// IsZFSDeleteErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRCreateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool {
+	return false
+}
+
+// IsCVRDeleteErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRGetErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRGetErrorInjected() bool {
+	return false
+}
+
+// IsCVRUpdateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool {
+	return false
+}