CI: Enable upstream virtualization operator E2E tests. (#1556)

* Add option for upstream virtualization operator. Use this to tweak the subscription settings and some expected namespaces. Signed-off-by: Matthew Arnold <[email protected]> * Do not fail test on missing images namespace. Signed-off-by: Matthew Arnold <[email protected]> * Avoid "cannot configure to watch own namespace". Signed-off-by: Matthew Arnold <[email protected]> * Add DataVolume clone permissions for cirros-test. Signed-off-by: Matthew Arnold <[email protected]> * Remove permissions for kubevirt-os-images. Going to try to keep everything in openshift-virtualization-os-images namespace instead. Installing this CRD as-is fails because kubevirt-os-images does not exist yet. Signed-off-by: Matthew Arnold <[email protected]> * Force upstream test run. Signed-off-by: Matthew Arnold <[email protected]> * Copy fedora data source to os-images namespace. The community KubeVirt operator puts these in "kubevirt-os-images", but to keep fixed YAML templates the test needs them in "openshift-virtualization-os-images". This adds a way to copy an existing data source to a new namespace. Signed-off-by: Matthew Arnold <[email protected]> * Turn off upstream-only CI. Undo temporary change to force community operator in tests. Signed-off-by: Matthew Arnold <[email protected]> * Don't fail if test storage classes already exist. Signed-off-by: Matthew Arnold <[email protected]> * Explicitly set Fedora volume mode for test. Trying to work around image clone failure in CI. Signed-off-by: Matthew Arnold <[email protected]> * Revert "Explicitly set Fedora volume mode for test." This reverts commit a663e29. Signed-off-by: Matthew Arnold <[email protected]> --------- Signed-off-by: Matthew Arnold <[email protected]>
openshift · Oct 22, 2024 · 20d8c61 · 20d8c61
1 parent 1d6ed3c
commit 20d8c61
Show file tree

Hide file tree

Showing 7 changed files with 172 additions and 56 deletions.
diff --git a/Makefile b/Makefile
@@ -32,6 +32,7 @@ OC_CLI = $(shell which oc)
 TEST_VIRT ?= false
 KVM_EMULATION ?= true
 TEST_UPGRADE ?= false
+HCO_UPSTREAM ?= false
 
 ifdef CLI_DIR
 	OC_CLI = ${CLI_DIR}/oc
@@ -524,6 +525,7 @@ test-e2e: test-e2e-setup install-ginkgo
 	-artifact_dir=$(ARTIFACT_DIR) \
 	-oc_cli=$(OC_CLI) \
 	-kvm_emulation=$(KVM_EMULATION) \
+	-hco_upstream=$(HCO_UPSTREAM) \
 	--ginkgo.vv \
 	--ginkgo.no-color=$(OPENSHIFT_CI) \
 	--ginkgo.label-filter="$(TEST_FILTER)" \

diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go
@@ -49,7 +49,8 @@ var (
 	knownFlake          bool
 	accumulatedTestLogs []string
 
-	kvmEmulation bool
+	kvmEmulation   bool
+	useUpstreamHco bool
 )
 
 func init() {
@@ -65,6 +66,7 @@ func init() {
 	flag.StringVar(&oc_cli, "oc_cli", "oc", "OC CLI Client")
 	flag.Int64Var(&flakeAttempts, "flakeAttempts", 3, "Customize the number of flake retries (3)")
 	flag.BoolVar(&kvmEmulation, "kvm_emulation", true, "Enable or disable KVM emulation for virtualization testing")
+	flag.BoolVar(&useUpstreamHco, "hco_upstream", false, "Force use of upstream virtualization operator")
 
 	// helps with launching debug sessions from IDE
 	if os.Getenv("E2E_USE_ENV_FLAGS") == "true" {
@@ -110,6 +112,13 @@ func init() {
 				log.Println("Error parsing KVM_EMULATION, it will be enabled by default: ", err)
 			}
 		}
+		if envValue := os.Getenv("HCO_UPSTREAM"); envValue != "" {
+			if parsedValue, err := strconv.ParseBool(envValue); err == nil {
+				useUpstreamHco = parsedValue
+			} else {
+				log.Println("Error parsing HCO_UPSTREAM, it will be disabled by default: ", err)
+			}
+		}
 	}
 
 }

diff --git a/tests/e2e/lib/subscription_helpers.go b/tests/e2e/lib/subscription_helpers.go
@@ -35,6 +35,9 @@ func getOperatorSubscription(c client.Client, namespace, label string) (*Subscri
 
 func (v *VirtOperator) getOperatorSubscription() (*Subscription, error) {
 	label := "operators.coreos.com/kubevirt-hyperconverged.openshift-cnv"
+	if v.Upstream {
+		label = "operators.coreos.com/community-kubevirt-hyperconverged.kubevirt-hyperconverged"
+	}
 	return getOperatorSubscription(v.Client, v.Namespace, label)
 }
 

diff --git a/tests/e2e/lib/virt_helpers.go b/tests/e2e/lib/virt_helpers.go
@@ -59,13 +59,19 @@ type VirtOperator struct {
 	Namespace string
 	Csv       string
 	Version   *version.Version
+	Upstream  bool
 }
 
 // GetVirtOperator fills out a new VirtOperator
-func GetVirtOperator(c client.Client, clientset *kubernetes.Clientset, dynamicClient dynamic.Interface) (*VirtOperator, error) {
+func GetVirtOperator(c client.Client, clientset *kubernetes.Clientset, dynamicClient dynamic.Interface, upstream bool) (*VirtOperator, error) {
 	namespace := "openshift-cnv"
+	manifest := "kubevirt-hyperconverged"
+	if upstream {
+		namespace = "kubevirt-hyperconverged"
+		manifest = "community-kubevirt-hyperconverged"
+	}
 
-	csv, operatorVersion, err := getCsvFromPackageManifest(dynamicClient, "kubevirt-hyperconverged")
+	csv, operatorVersion, err := getCsvFromPackageManifest(dynamicClient, manifest)
 	if err != nil {
 		log.Printf("Failed to get CSV from package manifest")
 		return nil, err
@@ -78,6 +84,7 @@ func GetVirtOperator(c client.Client, clientset *kubernetes.Clientset, dynamicCl
 		Namespace: namespace,
 		Csv:       csv,
 		Version:   operatorVersion,
+		Upstream:  upstream,
 	}
 
 	return v, nil
@@ -86,16 +93,23 @@ func GetVirtOperator(c client.Client, clientset *kubernetes.Clientset, dynamicCl
 // Helper to create an operator group object, common to installOperatorGroup
 // and removeOperatorGroup.
 func (v *VirtOperator) makeOperatorGroup() *operatorsv1.OperatorGroup {
+	// Community operator fails with "cannot configure to watch own namespace",
+	// need to remove target namespaces.
+	spec := operatorsv1.OperatorGroupSpec{}
+	if !v.Upstream {
+		spec = operatorsv1.OperatorGroupSpec{
+			TargetNamespaces: []string{
+				v.Namespace,
+			},
+		}
+	}
+
 	return &operatorsv1.OperatorGroup{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "kubevirt-hyperconverged-group",
 			Namespace: v.Namespace,
 		},
-		Spec: operatorsv1.OperatorGroupSpec{
-			TargetNamespaces: []string{
-				v.Namespace,
-			},
-		},
+		Spec: spec,
 	}
 }
 
@@ -174,9 +188,9 @@ func getCsvFromPackageManifest(dynamicClient dynamic.Interface, name string) (st
 }
 
 // Checks the existence of the operator's target namespace
-func (v *VirtOperator) checkNamespace() bool {
+func (v *VirtOperator) checkNamespace(ns string) bool {
 	// First check that the namespace exists
-	exists, _ := DoesNamespaceExist(v.Clientset, v.Namespace)
+	exists, _ := DoesNamespaceExist(v.Clientset, ns)
 	return exists
 }
 
@@ -236,7 +250,7 @@ func (v *VirtOperator) checkHco() bool {
 
 // Check if KVM emulation is enabled.
 func (v *VirtOperator) checkEmulation() bool {
-	hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{})
+	hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{})
 	if err != nil {
 		return false
 	}
@@ -260,11 +274,12 @@ func (v *VirtOperator) checkEmulation() bool {
 	return false
 }
 
-// Creates the target virtualization namespace, likely openshift-cnv or kubevirt-hyperconverged
-func (v *VirtOperator) installNamespace() error {
-	err := v.Client.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: v.Namespace}})
+// Creates the target namespace, likely openshift-cnv or kubevirt-hyperconverged,
+// but also used for openshift-virtualization-os-images if not already present.
+func (v *VirtOperator) installNamespace(ns string) error {
+	err := v.Client.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
 	if err != nil {
-		log.Printf("Failed to create namespace %s: %v", v.Namespace, err)
+		log.Printf("Failed to create namespace %s: %v", ns, err)
 		return err
 	}
 	return nil
@@ -285,19 +300,30 @@ func (v *VirtOperator) installOperatorGroup() error {
 
 // Creates the subscription, which triggers creation of the ClusterServiceVersion.
 func (v *VirtOperator) installSubscription() error {
-	subscription := &operatorsv1alpha1.Subscription{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "hco-operatorhub",
-			Namespace: v.Namespace,
-		},
-		Spec: &operatorsv1alpha1.SubscriptionSpec{
-			CatalogSource:          "redhat-operators",
+	spec := &operatorsv1alpha1.SubscriptionSpec{
+		CatalogSource:          "redhat-operators",
+		CatalogSourceNamespace: "openshift-marketplace",
+		Package:                "kubevirt-hyperconverged",
+		Channel:                "stable",
+		StartingCSV:            v.Csv,
+		InstallPlanApproval:    operatorsv1alpha1.ApprovalAutomatic,
+	}
+	if v.Upstream {
+		spec = &operatorsv1alpha1.SubscriptionSpec{
+			CatalogSource:          "community-operators",
 			CatalogSourceNamespace: "openshift-marketplace",
-			Package:                "kubevirt-hyperconverged",
+			Package:                "community-kubevirt-hyperconverged",
 			Channel:                "stable",
 			StartingCSV:            v.Csv,
 			InstallPlanApproval:    operatorsv1alpha1.ApprovalAutomatic,
+		}
+	}
+	subscription := &operatorsv1alpha1.Subscription{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "hco-operatorhub",
+			Namespace: v.Namespace,
 		},
+		Spec: spec,
 	}
 	err := v.Client.Create(context.Background(), subscription)
 	if err != nil {
@@ -332,7 +358,7 @@ func (v *VirtOperator) installHco() error {
 }
 
 func (v *VirtOperator) configureEmulation() error {
-	hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{})
+	hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
@@ -353,7 +379,7 @@ func (v *VirtOperator) configureEmulation() error {
 		return err
 	}
 
-	_, err = v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Update(context.Background(), hco, metav1.UpdateOptions{})
+	_, err = v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Update(context.Background(), hco, metav1.UpdateOptions{})
 	if err != nil {
 		return err
 	}
@@ -362,19 +388,19 @@ func (v *VirtOperator) configureEmulation() error {
 }
 
 // Creates target namespace if needed, and waits for it to exist
-func (v *VirtOperator) ensureNamespace(timeout time.Duration) error {
-	if !v.checkNamespace() {
-		if err := v.installNamespace(); err != nil {
+func (v *VirtOperator) EnsureNamespace(ns string, timeout time.Duration) error {
+	if !v.checkNamespace(ns) {
+		if err := v.installNamespace(ns); err != nil {
 			return err
 		}
 		err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
-			return v.checkNamespace(), nil
+			return v.checkNamespace(ns), nil
 		})
 		if err != nil {
-			return fmt.Errorf("timed out waiting to create namespace %s: %w", v.Namespace, err)
+			return fmt.Errorf("timed out waiting to create namespace %s: %w", ns, err)
 		}
 	} else {
-		log.Printf("Namespace %s already present, no action required", v.Namespace)
+		log.Printf("Namespace %s already present, no action required", ns)
 	}
 
 	return nil
@@ -449,10 +475,10 @@ func (v *VirtOperator) ensureHco(timeout time.Duration) error {
 }
 
 // Deletes the virtualization operator namespace (likely openshift-cnv).
-func (v *VirtOperator) removeNamespace() error {
-	err := v.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: v.Namespace}})
+func (v *VirtOperator) removeNamespace(ns string) error {
+	err := v.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
 	if err != nil {
-		log.Printf("Failed to delete namespace %s: %v", v.Namespace, err)
+		log.Printf("Failed to delete namespace %s: %v", ns, err)
 		return err
 	}
 	return nil
@@ -494,21 +520,21 @@ func (v *VirtOperator) removeHco() error {
 }
 
 // Makes sure the virtualization operator's namespace is removed.
-func (v *VirtOperator) ensureNamespaceRemoved(timeout time.Duration) error {
-	if !v.checkNamespace() {
-		log.Printf("Namespace %s already removed, no action required", v.Namespace)
+func (v *VirtOperator) ensureNamespaceRemoved(ns string, timeout time.Duration) error {
+	if !v.checkNamespace(ns) {
+		log.Printf("Namespace %s already removed, no action required", ns)
 		return nil
 	}
 
-	if err := v.removeNamespace(); err != nil {
+	if err := v.removeNamespace(ns); err != nil {
 		return err
 	}
 
 	err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) {
-		return !v.checkNamespace(), nil
+		return !v.checkNamespace(ns), nil
 	})
 	if err != nil {
-		return fmt.Errorf("timed out waiting to delete namespace %s: %w", v.Namespace, err)
+		return fmt.Errorf("timed out waiting to delete namespace %s: %w", ns, err)
 	}
 
 	return nil
@@ -696,7 +722,7 @@ func (v *VirtOperator) EnsureEmulation(timeout time.Duration) error {
 // IsVirtInstalled returns whether or not the OpenShift Virtualization operator
 // is installed and ready, by checking for a HyperConverged operator resource.
 func (v *VirtOperator) IsVirtInstalled() bool {
-	if !v.checkNamespace() {
+	if !v.checkNamespace(v.Namespace) {
 		return false
 	}
 
@@ -712,7 +738,7 @@ func (v *VirtOperator) EnsureVirtInstallation() error {
 	}
 
 	log.Printf("Creating virtualization namespace %s", v.Namespace)
-	if err := v.ensureNamespace(10 * time.Second); err != nil {
+	if err := v.EnsureNamespace(v.Namespace, 10*time.Second); err != nil {
 		return err
 	}
 	log.Printf("Created namespace %s", v.Namespace)
@@ -771,7 +797,7 @@ func (v *VirtOperator) EnsureVirtRemoval() error {
 	log.Println("Deleted operator group")
 
 	log.Printf("Deleting virtualization namespace %s", v.Namespace)
-	if err := v.ensureNamespaceRemoved(3 * time.Minute); err != nil {
+	if err := v.ensureNamespaceRemoved(v.Namespace, 3*time.Minute); err != nil {
 		return err
 	}
 	log.Printf("Deleting namespace %s", v.Namespace)

diff --git a/tests/e2e/lib/virt_storage_helpers.go b/tests/e2e/lib/virt_storage_helpers.go
@@ -173,26 +173,30 @@ func (v *VirtOperator) RemoveDataSource(namespace, name string) error {
 // Create a DataSource from an existing PVC, with the same name and namespace.
 // This way, the PVC can be specified as a sourceRef in the VM spec.
 func (v *VirtOperator) CreateDataSourceFromPvc(namespace, name string) error {
+	return v.CreateTargetDataSourceFromPvc(namespace, namespace, name, name)
+}
+
+func (v *VirtOperator) CreateTargetDataSourceFromPvc(sourceNamespace, destinationNamespace, sourcePvcName, destinationDataSourceName string) error {
 	unstructuredDataSource := unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"apiVersion": "cdi.kubevirt.io/v1beta1",
 			"kind":       "DataSource",
 			"metadata": map[string]interface{}{
-				"name":      name,
-				"namespace": namespace,
+				"name":      destinationDataSourceName,
+				"namespace": destinationNamespace,
 			},
 			"spec": map[string]interface{}{
 				"source": map[string]interface{}{
 					"pvc": map[string]interface{}{
-						"name":      name,
-						"namespace": namespace,
+						"name":      sourcePvcName,
+						"namespace": sourceNamespace,
 					},
 				},
 			},
 		},
 	}
 
-	_, err := v.Dynamic.Resource(dataSourceGVR).Namespace(namespace).Create(context.Background(), &unstructuredDataSource, metav1.CreateOptions{})
+	_, err := v.Dynamic.Resource(dataSourceGVR).Namespace(destinationNamespace).Create(context.Background(), &unstructuredDataSource, metav1.CreateOptions{})
 	if err != nil {
 		if apierrors.IsAlreadyExists(err) {
 			return nil
@@ -207,6 +211,36 @@ func (v *VirtOperator) CreateDataSourceFromPvc(namespace, name string) error {
 	return nil
 }
 
+// Find the given DataSource, and return the PVC it points to
+func (v *VirtOperator) GetDataSourcePvc(ns, name string) (string, string, error) {
+	unstructuredDataSource, err := v.Dynamic.Resource(dataSourceGVR).Namespace(ns).Get(context.Background(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Printf("Error getting DataSource %s: %v", name, err)
+		return "", "", err
+	}
+
+	pvcName, ok, err := unstructured.NestedString(unstructuredDataSource.UnstructuredContent(), "status", "source", "pvc", "name")
+	if err != nil {
+		log.Printf("Error getting PVC from DataSource: %v", err)
+		return "", "", err
+	}
+	if !ok {
+		return "", "", errors.New("failed to get PVC from " + name + " DataSource")
+	}
+
+	pvcNamespace, ok, err := unstructured.NestedString(unstructuredDataSource.UnstructuredContent(), "status", "source", "pvc", "namespace")
+	if err != nil {
+		log.Printf("Error getting PVC namespace from DataSource: %v", err)
+		return "", "", err
+	}
+	if !ok {
+		return "", "", errors.New("failed to get PVC namespace from " + name + " DataSource")
+	}
+
+	return pvcNamespace, pvcName, nil
+
+}
+
 // Find the default storage class
 func (v *VirtOperator) GetDefaultStorageClass() (*storagev1.StorageClass, error) {
 	storageClasses, err := v.Clientset.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{})
@@ -241,6 +275,9 @@ func (v *VirtOperator) CreateImmediateModeStorageClass(name string) error {
 	immediateStorageClass.Annotations["storageclass.kubernetes.io/is-default-class"] = "false"
 
 	_, err = v.Clientset.StorageV1().StorageClasses().Create(context.Background(), immediateStorageClass, metav1.CreateOptions{})
+	if apierrors.IsAlreadyExists(err) {
+		return nil
+	}
 	return err
 }