Pre-installation testing framework

Signed-off-by: Kanha gupta <[email protected]>

.github/workflows/kind.yml

@@ -772,13 +772,16 @@ jobs:
       - name: Create Kind Cluster
         run: |
           kind create cluster --config ci/kind/config-3nodes.yml
+      - name: Build antctl binary
+        run: |
+          make antctl-linux
+      - name: Run Pre-installation checks
+        run: |
+          ./bin/antctl-linux check cluster
       - name: Load Docker images and deploy Antrea
         run: |
           kind load docker-image antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-agent-ubuntu-coverage:latest
           kubectl apply -f build/yamls/antrea.yml
-      - name: Build antctl binary
-        run: |
-          make antctl-linux
       - name: Run antctl command
         run: |
           ./bin/antctl-linux check installation

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.6.0
+	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/memberlist v0.5.1
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0
 	github.com/k8snetworkplumbingwg/sriov-cni v2.1.0+incompatible

go.sum

@@ -413,6 +413,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=

pkg/antctl/antctl.go

@@ -19,6 +19,7 @@ import (
 
 	agentapis "antrea.io/antrea/pkg/agent/apis"
 	fallbackversion "antrea.io/antrea/pkg/antctl/fallback/version"
+	checkcluster "antrea.io/antrea/pkg/antctl/raw/check/cluster"
 	checkinstallation "antrea.io/antrea/pkg/antctl/raw/check/installation"
 	"antrea.io/antrea/pkg/antctl/raw/featuregates"
 	"antrea.io/antrea/pkg/antctl/raw/multicluster"
@@ -640,6 +641,12 @@ $ antctl get podmulticaststats pod -n namespace`,
 			supportController: false,
 			commandGroup:      check,
 		},
+		{
+			cobraCommand:      checkcluster.Command(),
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      check,
+		},
 		{
 			cobraCommand:      supportbundle.Command,
 			supportAgent:      true,

pkg/antctl/raw/check/cluster/command.go

@@ -0,0 +1,209 @@
+// Copyright 2024 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/utils/ptr"
+
+	"antrea.io/antrea/pkg/antctl/raw/check"
+)
+
+func Command() *cobra.Command {
+	o := newOptions()
+	command := &cobra.Command{
+		Use:   "cluster",
+		Short: "Runs pre installation checks",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return Run(o)
+		},
+	}
+	command.Flags().StringVarP(&o.antreaNamespace, "Namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running")
+	return command
+}
+
+type options struct {
+	antreaNamespace string
+}
+
+func newOptions() *options {
+	return &options{
+		antreaNamespace: "kube-system",
+	}
+}
+
+const (
+	antreaNamespace = "kube-system"
+	deploymentName  = "cluster-check"
+	podReadyTimeout = 1 * time.Minute
+)
+
+type Test interface {
+	Run(ctx context.Context, testContext *testContext) error
+}
+
+var testsRegistry = make(map[string]Test)
+
+func RegisterTest(name string, test Test) {
+	testsRegistry[name] = test
+}
+
+type testContext struct {
+	client          kubernetes.Interface
+	config          *rest.Config
+	clusterName     string
+	antreaNamespace string
+}
+
+func Run(o *options) error {
+	client, config, clusterName, err := check.NewClient()
+	if err != nil {
+		return fmt.Errorf("unable to create Kubernetes client: %s", err)
+	}
+	ctx := context.Background()
+	testContext := NewTestContext(client, config, clusterName, o)
+	if err := testContext.setup(ctx); err != nil {
+		return err
+	}
+	for name, test := range testsRegistry {
+		testContext.Header("Running test: %s", name)
+		if err := test.Run(ctx, testContext); err != nil {
+			testContext.Header("Test %s failed: %s", name, err)
+		} else {
+			testContext.Header("Test %s passed", name)
+		}
+	}
+	testContext.Log("Test finished")
+	testContext.teardown(ctx, deploymentName, antreaNamespace)
+	return nil
+}
+
+func (t *testContext) setup(ctx context.Context) error {
+	deployment := check.NewDeployment(check.DeploymentParameters{
+		Name:        deploymentName,
+		Image:       "antrea/antrea-agent-ubuntu:latest",
+		Replicas:    1,
+		Command:     []string{"sleep", "infinity"},
+		Labels:      map[string]string{"app": "cluster-check"},
+		HostNetwork: true,
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "cni-conf", MountPath: "/etc/cni/net.d"},
+			{Name: "lib-modules", MountPath: "/lib/modules"},
+		},
+		Tolerations: []corev1.Toleration{
+			{
+				Key:      "node-role.kubernetes.io/control-plane",
+				Operator: "Exists",
+				Effect:   "NoSchedule",
+			},
+			{
+				Key:      "node-role.kubernetes.io/master",
+				Operator: "Exists",
+				Effect:   "NoSchedule",
+			},
+			{
+				Key:      "node.kubernetes.io/not-ready",
+				Operator: "Exists",
+				Effect:   "NoSchedule",
+			},
+		},
+		Volumes: []corev1.Volume{
+			{
+				Name: "cni-conf",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/etc/cni/net.d",
+					},
+				},
+			},
+			{
+				Name: "lib-modules",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/lib/modules",
+						Type: ptr.To(corev1.HostPathType("Directory")),
+					},
+				},
+			},
+		},
+	})
+
+	t.Log("Creating Deployment")
+	_, err := t.client.AppsV1().Deployments(antreaNamespace).Create(ctx, deployment, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to create Deployment: %w", err)
+	}
+
+	t.Log("Waiting for Deployment to become ready")
+	check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.antreaNamespace, t.clusterName, deploymentName)
+	if err != nil {
+		return fmt.Errorf("error while waiting for Deployment to become ready: %w", err)
+	}
+	return nil
+}
+
+func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext {
+	return &testContext{
+		client:          client,
+		config:          config,
+		clusterName:     clusterName,
+		antreaNamespace: o.antreaNamespace,
+	}
+}
+
+func (t *testContext) teardown(ctx context.Context, deploymentName, namespace string) error {
+	err := t.client.AppsV1().Deployments(namespace).Delete(ctx, deploymentName, metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+	t.Log("Waiting for the deletion of Deployment %s in Namespace %s...", deploymentName, namespace)
+	err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
+		_, err := t.client.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for Deployment %s to be deleted in Namespace %s: %w", deploymentName, namespace, err)
+	}
+	t.Log("Deployment %s successfully deleted from Namespace %s", deploymentName, namespace)
+	return nil
+}
+
+func (t *testContext) Log(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...)
+}
+
+func (t *testContext) Header(format string, a ...interface{}) {
+	t.Log("-------------------------------------------------------------------------------------------")
+	t.Log(format, a...)
+	t.Log("-------------------------------------------------------------------------------------------")
+}

pkg/antctl/raw/check/cluster/test_checkCNIExistence.go

@@ -0,0 +1,60 @@
+// Copyright 2024 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"antrea.io/antrea/pkg/antctl/raw/check"
+)
+
+type checkCNIExistence struct{}
+
+func init() {
+	RegisterTest("Check if another CNI is Present", &checkCNIExistence{})
+}
+
+func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) error {
+	pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "name=cluster-check"})
+	if err != nil {
+		return fmt.Errorf("failed to list Pods: %v", err)
+	}
+	testContext.Log("Checking CNI configurations in Pod: %s", pods.Items[0].Name)
+	command := []string{"ls", "/etc/cni/net.d"}
+	output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", command)
+	if err != nil {
+		testContext.Log("Failed to execute command in pod: %s, error: %v", pods.Items[0].Name, err)
+	}
+	outputStr := strings.TrimSpace(output)
+	if outputStr == "" {
+		testContext.Log("No files present in /etc/cni/net.d in pod: %s", pods.Items[0].Name)
+	} else {
+		files := strings.Split(outputStr, "\n")
+		sort.Strings(files)
+		if len(files) > 0 && files[0] < "10-antrea.conflist" {
+			testContext.Log("Warning: Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0])
+		} else if len(files) > 0 && files[0] != "10-antrea.conflist" {
+			testContext.Log("Warning: Another CNI configuration file found: %s.", files[0])
+		} else {
+			testContext.Log("Antrea's CNI configuration file already present: %s", files[0])
+		}
+	}
+	return nil
+}

pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go

@@ -0,0 +1,52 @@
+// Copyright 2024 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type checkControlPlaneAvailability struct{}
+
+func init() {
+	RegisterTest("Check Control Plane Availability", &checkControlPlaneAvailability{})
+}
+
+func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error {
+	controlPlaneLabel := "node-role.kubernetes.io/control-plane"
+	masterNodeLabel := "node-role.kubernetes.io/master"
+	controlPlaneNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: controlPlaneLabel})
+	if err != nil {
+		return fmt.Errorf("failed to list control plane Nodes: %w", err)
+	}
+	masterNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: masterNodeLabel})
+	if err != nil {
+		return fmt.Errorf("failed to list master Nodes: %w", err)
+	}
+	if len(controlPlaneNode.Items) == 0 && len(masterNode.Items) == 0 {
+		testContext.Log("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.")
+	} else {
+		for _, node := range controlPlaneNode.Items {
+			testContext.Log("Control plane Node %s found", node.Name)
+		}
+		for _, node := range masterNode.Items {
+			testContext.Log("Master Node %s found", node.Name)
+		}
+	}
+	return nil
+}