chore(cStorVolume_bdd): add negative test cases for cStorVolumeProvis…

…ioning (openebs-archive#1530)

1. This PR provides the capability to get debug build for the cstor-pool-mgmt.
2. Debug build has an injection server running on port 8080 on all the cstor-pool-mgmgt
   which can be used to inject the errors for CR's and ZFS operations for negative path testing.

**Perform the following steps to run the negative tests on your local machine:**
1. Navigate to tests/cstor/volume using cd.
2. In your command line run the following command to execute negative BDD
   CMD: ginkgo -v  -focus="\[Cstor Volume Provisioning Negative\]" -- -kubeconfig=/home/sai/.kube/config -cstor-maxpools=3 -cstor-replicas=3

Signed-off-by: mittachaitu <[email protected]>

GNUmakefile

@@ -108,6 +108,7 @@ VOLUME_MGMT=cstor-volume-mgmt
 EXPORTER=maya-exporter
 CSPC_OPERATOR=cspc-operator
 CSPC_OPERATOR_DEBUG=cspc-operator-debug
+CSP_OPERATOR_DEBUG=cstor-pool-mgmt-debug
 
 
 # Specify the date o build
@@ -311,6 +312,18 @@ pool-mgmt-image: cstor-pool-mgmt
 	@cd buildscripts/cstor-pool-mgmt && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:${IMAGE_TAG} --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache
 	@rm buildscripts/cstor-pool-mgmt/${POOL_MGMT}
 
+#Use this to build debug image of cstor-pool-mgmt
+.PHONY: pool-mgmt-debug-image
+pool-mgmt-debug-image:
+	@echo "----------------------------"
+	@echo -n "--> cstor-pool-mgmt debug image "
+	@echo "${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject"
+	@echo "----------------------------"
+	@PNAME=${CSP_OPERATOR_DEBUG} CTLNAME=${POOL_MGMT} BUILD_TAG="-tags=debug" sh -c "'$(PWD)/buildscripts/build.sh'"
+	@cp bin/${CSP_OPERATOR_DEBUG}/${POOL_MGMT} buildscripts/${CSP_OPERATOR_DEBUG}/
+	@cd buildscripts/${CSP_OPERATOR_DEBUG} && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache
+	@rm buildscripts/${CSP_OPERATOR_DEBUG}/${POOL_MGMT}
+
 #Use this to build cspi-mgmt
 .PHONY: cspi-mgmt
 cspi-mgmt:

buildscripts/cstor-pool-mgmt-debug/Dockerfile

@@ -0,0 +1,30 @@
+#
+# This Dockerfile builds a recent cstor-pool-mgmt-debug using the latest binary from
+# cstor-pool-mgmt  releases.
+#
+
+#openebs/cstor-base is the image that contains cstor related binaries and
+#libraries - zpool, zfs, zrepl
+#FROM openebs/cstor-base:ci
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+COPY cstor-pool-mgmt /usr/local/bin/
+COPY entrypoint.sh /usr/local/bin/
+
+RUN printf '#!/bin/bash\nif [ $# -lt 1 ]; then\n\techo "argument missing"\n\texit 1\nfi\neval "$*"\n' >> /usr/local/bin/execute.sh
+
+RUN chmod +x /usr/local/bin/execute.sh
+RUN apt install netcat -y
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ARG BUILD_DATE
+LABEL org.label-schema.name="cstor-pool-mgmt"
+LABEL org.label-schema.description="OpenEBS"
+LABEL org.label-schema.url="http://www.openebs.io/"
+LABEL org.label-schema.vcs-url="https://github.com/openebs/maya"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+
+ENTRYPOINT entrypoint.sh
+EXPOSE 7676

buildscripts/cstor-pool-mgmt-debug/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -ex
+
+rm /usr/local/bin/zrepl
+exec /usr/local/bin/cstor-pool-mgmt start
+exec service ssh start
+exec service rsyslog start

cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/openebs/maya/cmd/cstor-pool-mgmt/controller/common"
+	"github.com/openebs/maya/pkg/debug"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
@@ -34,6 +35,10 @@ func (c *CStorPoolController) Run(threadiness int, stopCh <-chan struct{}) error
 	defer runtime.HandleCrash()
 	defer c.workqueue.ShutDown()
 
+	// Start server only in debug build
+	debug.LogBuildDetails()
+	debug.StartInjectionServer()
+
 	// Start the informer factories to begin populating the informer caches
 	klog.Info("Starting CStorPool controller")
 

cmd/cstor-pool-mgmt/controller/replica-controller/handler.go

@@ -27,6 +27,7 @@ import (
 	"github.com/openebs/maya/cmd/cstor-pool-mgmt/volumereplica"
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
 	clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned"
+	"github.com/openebs/maya/pkg/debug"
 	merrors "github.com/openebs/maya/pkg/errors/v1alpha1"
 	pkg_errors "github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -146,6 +147,9 @@ func (c *CStorVolumeReplicaController) syncHandler(
 
 	// need to update cvr before returning this error
 	if err != nil {
+		if debug.EI.IsCVRUpdateErrorInjected() {
+			return merrors.Errorf("CVR update error via injection")
+		}
 		_, err1 := c.clientset.
 			OpenebsV1alpha1().
 			CStorVolumeReplicas(cvrGot.Namespace).

cmd/cstor-pool-mgmt/volumereplica/volumereplica.go

@@ -27,6 +27,7 @@ import (
 	"encoding/json"
 
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	"github.com/openebs/maya/pkg/debug"
 	"github.com/openebs/maya/pkg/hash"
 	"github.com/openebs/maya/pkg/util"
 	zfs "github.com/openebs/maya/pkg/zfs/cmd/v1alpha1"
@@ -183,6 +184,10 @@ func CreateVolumeReplica(cStorVolumeReplica *apis.CStorVolumeReplica, fullVolNam
 	isClone := cStorVolumeReplica.Labels[string(apis.CloneEnableKEY)] == "true"
 	snapName := ""
 
+	if debug.EI.IsZFSCreateErrorInjected() {
+		return errors.New("ZFS create error via injection")
+	}
+
 	if isClone {
 		srcVolume := cStorVolumeReplica.Annotations[string(apis.SourceVolumeKey)]
 		snapName = cStorVolumeReplica.Annotations[string(apis.SnapshotNameKey)]
@@ -413,6 +418,10 @@ func buildVolumeRestoreCommand(poolName, fullVolName, restoreSrc string) []strin
 
 // GetVolumes returns the slice of volumes.
 func GetVolumes() ([]string, error) {
+	if debug.EI.IsZFSGetErrorInjected() {
+		return []string{}, errors.New("ZFS get error via injection")
+	}
+
 	volStrCmd := []string{"get", "-Hp", "name", "-o", "name"}
 	volnameByte, err := RunnerVar.RunStdoutPipe(VolumeReplicaOperator, volStrCmd...)
 	if err != nil || string(volnameByte) == "" {
@@ -431,6 +440,10 @@ func GetVolumes() ([]string, error) {
 
 // DeleteVolume deletes the specified volume.
 func DeleteVolume(fullVolName string) error {
+	if debug.EI.IsZFSDeleteErrorInjected() {
+		return errors.New("ZFS delete error via injection")
+	}
+
 	deleteVolStr := []string{"destroy", "-R", fullVolName}
 	stdoutStderr, err := RunnerVar.RunCombinedOutput(VolumeReplicaOperator, deleteVolStr...)
 	if err != nil {

cmd/provisioner-localpv/app/config.go

@@ -227,3 +227,12 @@ func GetNodeHostname(n *v1.Node) string {
 	}
 	return hostname
 }
+
+// GetTaints extracts the Taints from the Spec on the node
+// If Taints are empty, it just returns empty structure of corev1.Taints
+func GetTaints(n *v1.Node) []v1.Taint {
+	var taints []v1.Taint
+	taints = n.Spec.Taints
+	return taints
+
+}

cmd/provisioner-localpv/app/helper_hostpath.go

@@ -38,6 +38,7 @@ import (
 type podConfig struct {
 	pOpts                         *HelperPodOptions
 	parentDir, volumeDir, podName string
+	toleration                    []corev1.Toleration
 }
 
 var (
@@ -66,6 +67,8 @@ type HelperPodOptions struct {
 
 	// serviceAccountName is the service account with which the pod should be launched
 	serviceAccountName string
+
+	selectedNodeTaints []corev1.Taint
 }
 
 // validate checks that the required fields to launch
@@ -93,6 +96,24 @@ func (p *Provisioner) createInitPod(pOpts *HelperPodOptions) error {
 		return err
 	}
 
+	// Convert the taints into pod tolerations
+	// to be passed to pod builder to set appropriate toleration
+	var tolerations []corev1.Toleration
+	nodeTaints := pOpts.selectedNodeTaints
+	if nodeTaints != nil {
+		for i, _ := range nodeTaints {
+			tolerations[i].Key = nodeTaints[i].Key
+			tolerations[i].Effect = nodeTaints[i].Effect
+			if len(nodeTaints[i].Value) == 0 {
+				tolerations[i].Operator = corev1.TolerationOpExists
+			} else {
+				tolerations[i].Value = nodeTaints[i].Value
+				tolerations[i].Operator = corev1.TolerationOpEqual
+			}
+
+		}
+	}
+	config.toleration = tolerations
 	// Initialize HostPath builder and validate that
 	// volume directory is not directly under root.
 	// Extract the base path and the volume unique path.
@@ -157,11 +178,12 @@ func (p *Provisioner) launchPod(config podConfig) (*corev1.Pod, error) {
 	// Helper pods need to create and delete directories on the host.
 	privileged := true
 
-	helperPod, _ := pod.NewBuilder().
+	helperPod, err := pod.NewBuilder().
 		WithName(config.podName + "-" + config.pOpts.name).
 		WithRestartPolicy(corev1.RestartPolicyNever).
 		WithNodeSelectorHostnameNew(config.pOpts.nodeHostname).
 		WithServiceAccountName(config.pOpts.serviceAccountName).
+		WithTolerations(config.toleration).
 		WithContainerBuilder(
 			container.NewBuilder().
 				WithName("local-path-" + config.podName).
@@ -183,6 +205,10 @@ func (p *Provisioner) launchPod(config podConfig) (*corev1.Pod, error) {
 		).
 		Build()
 
+	if err != nil {
+		klog.Infof("Unable to build Pod Error : %v", err)
+	}
+	klog.Infof("Creating Pod : %v", helperPod)
 	//Launch the helper pod.
 	hPod, err := p.kubeClient.CoreV1().Pods(p.namespace).Create(helperPod)
 	return hPod, err

cmd/provisioner-localpv/app/provisioner_hostpath.go

@@ -33,6 +33,7 @@ import (
 func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, error) {
 	pvc := opts.PVC
 	nodeHostname := GetNodeHostname(opts.SelectedNode)
+	taints := GetTaints(opts.SelectedNode)
 	name := opts.PVName
 	stgType := volumeConfig.GetStorageType()
 	saName := getOpenEBSServiceAccountName()
@@ -59,6 +60,7 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeC
 		path:               path,
 		nodeHostname:       nodeHostname,
 		serviceAccountName: saName,
+		selectedNodeTaints: taints,
 	}
 
 	iErr := p.createInitPod(podOpts)

pkg/debug/debug.go

@@ -22,10 +22,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"k8s.io/klog"
 	"log"
 	"net/http"
 	"time"
+
+	"k8s.io/klog"
 )
 
 // LogBuildDetails logs the build details when the cspc-operator starts.
@@ -287,3 +288,49 @@ func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool {
 	}
 	return false
 }
+
+// IsZFSGetErrorInjected returns true if error is injected for ZFS get command
+func (ei *ErrorInjection) IsZFSGetErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectGetError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsZFSDeleteErrorInjected returns true if error is injected for ZFS delete command
+func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectDeleteError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsZFSCreateErrorInjected returns true if error is injected for ZFS create command
+func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool {
+	if ei.ZFSError.CRUDErrorInjection.InjectCreateError == Inject {
+		return true
+	}
+	return false
+}
+
+// IsCVRCreateErrorInjected returns true if error is injected for CVR create
+// command
+func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool {
+	return false
+}
+
+// IsCVRDeleteErrorInjected returns true if error is injected for CVR delete
+// command
+func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRGetErrorInjected returns true if error is injected for CVR get command
+func (ei *ErrorInjection) IsCVRGetErrorInjected() bool {
+	return false
+}
+
+// IsCVRUpdateErrorInjected returns true if error is injected for CVR update command
+func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool {
+	return false
+}

pkg/debug/release.go

@@ -128,3 +128,45 @@ func (ei *ErrorInjection) IsDeploymentUpdateErrorInjected() bool {
 func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool {
 	return false
 }
+
+// IsZFSGetErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSGetErrorInjected() bool {
+	return false
+}
+
+// IsZFSCreateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool {
+	return false
+}
+
+// IsZFSDeleteErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRCreateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool {
+	return false
+}
+
+// IsCVRDeleteErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool {
+	return false
+}
+
+// IsCVRGetErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRGetErrorInjected() bool {
+	return false
+}
+
+// IsCVRUpdateErrorInjected is production alternative for the same function that
+// exists in debug build
+func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool {
+	return false
+}