Enable Calico on ARM64 and add configureable flags for Calico installation

scripts/lib/integration.sh

@@ -36,17 +36,30 @@ function run_calico_test() {
   echo "Starting Helm installing Tigera operator and running Calico STAR tests"
   pushd ./test
   VPC_ID=$(eksctl get cluster $CLUSTER_NAME -oyaml | grep vpc | cut -d ":" -f 2 | awk '{$1=$1};1')
-  # we can automatically use latest version in Calico repo, or use the known highest version (currently v3.22.0)
+
   calico_version=$CALICO_VERSION
-  if [[ $RUN_LATEST_CALICO_VERSION == true ]]; then
-    version_tag=$(curl -i https://api.github.com/repos/projectcalico/calico/releases/latest | grep "tag_name") || true
-    if [[ -n $version_tag ]]; then
-      calico_version=$(echo $version_tag | cut -d ":" -f 2 | cut -d '"' -f 2 )
-    else
-      echo "Getting Calico latest version failed, will fall back to default/set version $calico_version instead"
+  if [[ $1 == "true" ]]; then
+    # we can automatically use latest version in Calico repo, or use the known highest version (currently v3.23.0)
+    if [[ $RUN_LATEST_CALICO_VERSION == true ]]; then
+      version_tag=$(curl -i https://api.github.com/repos/projectcalico/calico/releases/latest | grep "tag_name") || true
+      if [[ -n $version_tag ]]; then
+        calico_version=$(echo $version_tag | cut -d ":" -f 2 | cut -d '"' -f 2 )
+      else
+        echo "Getting Calico latest version failed, will fall back to default/set version $calico_version instead"
+      fi
+    else echo "Using default Calico version"
     fi
+    echo "Using Calico version $calico_version to test"
+  else
+    version=$(kubectl describe ds -n calico-system calico-node | grep "calico/node:" | cut -d ':' -f3)
+    echo "Calico has been installed in testing cluster, keep using the version $version"
   fi
-  echo "Using Calico version $calico_version to test"
-  ginkgo -v e2e/calico -- --cluster-kubeconfig=$KUBECONFIG --cluster-name=$CLUSTER_NAME --aws-region=$AWS_DEFAULT_REGION --aws-vpc-id=$VPC_ID --calico-version=$calico_version
+
+  echo "Testing amd64"
+  instance_type="amd64"
+  ginkgo -v e2e/calico -- --cluster-kubeconfig=$KUBECONFIG --cluster-name=$CLUSTER_NAME --aws-region=$AWS_DEFAULT_REGION --aws-vpc-id=$VPC_ID --calico-version=$calico_version --instance-type=$instance_type --install-calico=$1
+  echo "Testing arm64"
+  instance_type="arm64"
+  ginkgo -v e2e/calico -- --cluster-kubeconfig=$KUBECONFIG --cluster-name=$CLUSTER_NAME --aws-region=$AWS_DEFAULT_REGION --aws-vpc-id=$VPC_ID --calico-version=$calico_version --instance-type=$instance_type --install-calico=false
   popd
 }

scripts/run-cni-release-tests.sh

@@ -51,7 +51,7 @@ function run_integration_test() {
 
 function run_calico_tests(){
   # get version from run-integration-tests.sh
-  : "${CALICO_VERSION:=3.22.0}"
+  : "${CALICO_VERSION:=v3.23.0}"
   echo "Running calico tests, version $CALICO_VERSION"
   START=$SECONDS
   TEST_RESULT=success

scripts/run-integration-tests.sh

@@ -31,7 +31,7 @@ ARCH=$(go env GOARCH)
 : "${RUNNING_PERFORMANCE:=false}"
 : "${RUN_CALICO_TEST:=false}"
 : "${RUN_LATEST_CALICO_VERSION:=false}"
-: "${CALICO_VERSION:=3.22.0}"
+: "${CALICO_VERSION:=v3.23.0}"
 : "${RUN_CALICO_TEST_WITH_PD:=true}"
 
 
@@ -272,16 +272,20 @@ if [[ $TEST_PASS -eq 0 ]]; then
 fi
 
 if [[ $RUN_CALICO_TEST == true ]]; then
-  run_calico_test
+  # need to install Calico
+  run_calico_test "true"
   if [[ "$RUN_CALICO_TEST_WITH_PD" == true ]]; then
       # if we run prefix delegation tests as well, we need update CNI env and terminate all nodes to restore iptables rules for following tests
       echo "Run Calico tests with Prefix Delegation enabled"
       $KUBECTL_PATH set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true
+      # we shouldn't rely on other tests to set this required ENV
+      $KUBECTL_PATH set env ds aws-node -n kube-system WARM_PREFIX_TARGET=1
       ids=( $(aws ec2 describe-instances --filters Name=vpc-id,Values=$VPC_ID --query 'Reservations[*].Instances[*].InstanceId' --output text) )
       aws ec2 terminate-instances --instance-ids $ids
       echo "Waiting 15 minutes for new nodes being ready"
       sleep 900
-      run_calico_test
+      # no longer need to install Calico again for PD
+      run_calico_test "false"
   fi
 
   emit_cloudwatch_metric "calico_test_status" "1"

test/e2e/calico/calico_suite_test.go

@@ -3,6 +3,7 @@ package calico
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
@@ -13,22 +14,21 @@ import (
 )
 
 var (
-	f                *framework.Framework
-	err              error
-	uiNamespace      = "management-ui"
-	clientNamespace  = "client"
-	starsNamespace   = "stars"
-	uiLabel          = map[string]string{"role": "management-ui"}
-	clientLabel      = map[string]string{"role": "client"}
-	feLabel          = map[string]string{"role": "frontend"}
-	beLabel          = map[string]string{"role": "backend"}
-	nodeArchKey      = "kubernetes.io/arch"
-	nodeArchARMValue = "arm64"
-	nodeArchAMDValue = "amd64"
-	uiPod            v1.Pod
-	clientPod        v1.Pod
-	fePod            v1.Pod
-	bePod            v1.Pod
+	f               *framework.Framework
+	err             error
+	uiNamespace     = "management-ui"
+	clientNamespace = "client"
+	starsNamespace  = "stars"
+	uiLabel         = map[string]string{"role": "management-ui"}
+	clientLabel     = map[string]string{"role": "client"}
+	feLabel         = map[string]string{"role": "frontend"}
+	beLabel         = map[string]string{"role": "backend"}
+	nodeArchKey     = "kubernetes.io/arch"
+	starImage       = "calico/star-probe:multiarch"
+	uiPod           v1.Pod
+	clientPod       v1.Pod
+	fePod           v1.Pod
+	bePod           v1.Pod
 )
 
 func TestCalicoPoliciesWithVPCCNI(t *testing.T) {
@@ -38,15 +38,16 @@ func TestCalicoPoliciesWithVPCCNI(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	f = framework.New(framework.GlobalOptions)
-	By("installing Calico operator")
 
-	tigeraVersion := f.Options.CalicoVersion
-	err := f.InstallationManager.InstallTigeraOperator(tigeraVersion)
-	Expect(err).ToNot(HaveOccurred())
-
-	By("Patching ARM64 node unschedulable")
-	err = updateNodesSchedulability(nodeArchKey, nodeArchARMValue, true)
-	Expect(err).ToNot(HaveOccurred())
+	if f.Options.InstallCalico {
+		By("installing Calico operator")
+		tigeraVersion := f.Options.CalicoVersion
+		err := f.InstallationManager.InstallTigeraOperator(tigeraVersion)
+		// wait for Calico resources being provisioned and setup.
+		// we don't have control on Calico pods metadata thus we may not poll the pods for waiting wisely.
+		time.Sleep(utils.DefaultDeploymentReadyTimeout / 2)
+		Expect(err).ToNot(HaveOccurred())
+	}
 
 	By("installing Calico Start Policy Tests Resources")
 	err = f.K8sResourceManagers.NamespaceManager().CreateNamespaceWithLabels(uiNamespace, map[string]string{"role": "management-ui"})
@@ -58,7 +59,7 @@ var _ = BeforeSuite(func() {
 
 	uiContainer := manifest.NewBaseContainer().
 		Name("management-ui").
-		Image("calico/star-collect:v0.1.0").
+		Image(starImage).
 		ImagePullPolicy(v1.PullAlways).
 		Port(v1.ContainerPort{ContainerPort: 9001}).
 		Build()
@@ -68,15 +69,15 @@ var _ = BeforeSuite(func() {
 		Container(uiContainer).
 		Replicas(1).
 		PodLabel("role", "management-ui").
-		NodeSelector(nodeArchKey, nodeArchAMDValue).
+		NodeSelector(nodeArchKey, f.Options.InstanceType).
 		Labels(map[string]string{"role": "management-ui"}).
 		Build()
 	_, err = f.K8sResourceManagers.DeploymentManager().CreateAndWaitTillDeploymentIsReady(uiDeployment, utils.DefaultDeploymentReadyTimeout)
 	Expect(err).ToNot(HaveOccurred())
 
 	clientContainer := manifest.NewBaseContainer().
 		Name("client").
-		Image("calico/star-probe:v0.1.0").
+		Image(starImage).
 		ImagePullPolicy(v1.PullAlways).
 		Command([]string{"probe", "--urls=http://frontend.stars:80/status,http://backend.stars:6379/status"}).
 		Port(v1.ContainerPort{ContainerPort: 9000}).
@@ -87,15 +88,15 @@ var _ = BeforeSuite(func() {
 		Container(clientContainer).
 		Replicas(1).
 		PodLabel("role", "client").
-		NodeSelector(nodeArchKey, nodeArchAMDValue).
+		NodeSelector(nodeArchKey, f.Options.InstanceType).
 		Labels(map[string]string{"role": "client"}).
 		Build()
 	_, err = f.K8sResourceManagers.DeploymentManager().CreateAndWaitTillDeploymentIsReady(clientDeployment, utils.DefaultDeploymentReadyTimeout)
 	Expect(err).ToNot(HaveOccurred())
 
 	feContainer := manifest.NewBaseContainer().
 		Name("frontend").
-		Image("calico/star-probe:v0.1.0").
+		Image(starImage).
 		ImagePullPolicy(v1.PullAlways).
 		Command([]string{
 			"probe",
@@ -110,15 +111,15 @@ var _ = BeforeSuite(func() {
 		Container(feContainer).
 		Replicas(1).
 		PodLabel("role", "frontend").
-		NodeSelector(nodeArchKey, nodeArchAMDValue).
+		NodeSelector(nodeArchKey, f.Options.InstanceType).
 		Labels(map[string]string{"role": "frontend"}).
 		Build()
 	_, err = f.K8sResourceManagers.DeploymentManager().CreateAndWaitTillDeploymentIsReady(feDeployment, utils.DefaultDeploymentReadyTimeout)
 	Expect(err).ToNot(HaveOccurred())
 
 	beContainer := manifest.NewBaseContainer().
 		Name("backend").
-		Image("calico/star-probe:v0.1.0").
+		Image(starImage).
 		ImagePullPolicy(v1.PullAlways).
 		Command([]string{
 			"probe",
@@ -133,7 +134,7 @@ var _ = BeforeSuite(func() {
 		Container(beContainer).
 		Replicas(1).
 		PodLabel("role", "backend").
-		NodeSelector(nodeArchKey, nodeArchAMDValue).
+		NodeSelector(nodeArchKey, f.Options.InstanceType).
 		Labels(map[string]string{"role": "backend"}).
 		Build()
 	_, err = f.K8sResourceManagers.DeploymentManager().CreateAndWaitTillDeploymentIsReady(beDeployment, utils.DefaultDeploymentReadyTimeout)
@@ -215,11 +216,10 @@ var _ = AfterSuite(func() {
 	f.K8sResourceManagers.NetworkPolicyManager().DeleteNetworkPolicy(&networkPolicyAllowFE)
 	f.K8sResourceManagers.NetworkPolicyManager().DeleteNetworkPolicy(&networkPolicyAllowClient)
 
-	By("Helm Uninstall Calico Installation")
-	f.InstallationManager.UninstallTigeraOperator()
-
-	By("Restore ARM64 Nodes Schedulability")
-	updateNodesSchedulability(nodeArchKey, nodeArchARMValue, false)
+	// TODO: disable Calico uninstallation for now. We can add this back after a number of successful test runs.
+	////we are using dynamic cluster to run the test. Not uninstalling Calico is fine.
+	//By("Helm Uninstall Calico Installation")
+	//f.InstallationManager.UninstallTigeraOperator()
 })
 
 func installNetcatToolInContainer(name string, namespace string) error {

test/framework/options.go

@@ -41,6 +41,7 @@ type Options struct {
 	TargetAddon      string
 	InitialManifest  string
 	TargetManifest   string
+	InstallCalico    bool
 }
 
 func (options *Options) BindFlags() {
@@ -55,9 +56,10 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
 	flag.StringVar(&options.InitialManifest, "initial-manifest-file", "", "Initial CNI manifest, can be local file path or remote Url")
 	flag.StringVar(&options.TargetManifest, "target-manifest-file", "", "Target CNI manifest, can be local file path or remote Url")
-	flag.StringVar(&options.CalicoVersion, "calico-version", "3.22.0", "calico version to be tested")
+	flag.StringVar(&options.CalicoVersion, "calico-version", "v3.23.0", "calico version to be tested")
 	flag.StringVar(&options.ContainerRuntime, "container-runtime", "", "Optionally can specify it as 'containerd' for the test nodes")
 	flag.StringVar(&options.InstanceType, "instance-type", "amd64", "Optionally specify instance type as arm64 for the test nodes")
+	flag.BoolVar(&options.InstallCalico, "install-calico", true, "Install Calico operator before running e2e tests")
 }
 
 func (options *Options) Validate() error {