Merge pull request #1445 from saschagrunert/e2e-tests

Update e2e test suite

.github/workflows/crio.yml

@@ -1,4 +1,4 @@
-name: critest CRI-O
+name: CRI-O
 on:
   push:
     tags:
@@ -7,14 +7,13 @@ on:
       - master
   pull_request:
 jobs:
-  #
-  # Run CRI tests against CRI-O
-  #
-  linux-build-and-critest-cri-o:
+  test:
     strategy:
       matrix:
-        version: [main]
-    name: ${{matrix.version}} / linux amd64
+        suite:
+          - e2e
+          - critest
+    name: ${{matrix.suite}}
     runs-on: ubuntu-22.04
     steps:
       - name: Install go
@@ -50,21 +49,13 @@ jobs:
           sudo sysctl -w net.ipv4.ip_forward=1
           sudo iptables -t nat -I POSTROUTING -s 127.0.0.0/8 ! -d 127.0.0.0/8 -j MASQUERADE
 
-          # enable criu support
-          sudo apt-get update
-          sudo apt-get install -y criu
-
       - name: Install ginkgo
         run: |
           go install github.com/onsi/ginkgo/v2/ginkgo@latest
           ginkgo version
           sudo cp $(command -v ginkgo) /usr/local/bin
 
-      - name: Setup GCloud
-        uses: google-github-actions/setup-gcloud@v2
-
       - name: Install CRI-O latest main
-        if: ${{matrix.version == 'main'}}
         run: |
           curl https://raw.githubusercontent.com/cri-o/packaging/main/get | sudo bash
 
@@ -88,13 +79,21 @@ jobs:
         working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools
 
       - name: Run critest
+        if: ${{matrix.suite == 'critest'}}
         run: |
           sudo -E PATH=$PATH critest \
             --runtime-endpoint=unix:///var/run/crio/crio.sock \
             --ginkgo.flakeAttempts=3 \
             --parallel=$(nproc)
           sudo journalctl -u crio > cri-o.log
 
+      - name: Run crictl e2e tests
+        if: ${{matrix.suite == 'e2e'}}
+        run: |
+          sudo -E PATH=$PATH make test-e2e \
+            TESTFLAGS="-crictl-runtime-endpoint=unix://var/run/crio/crio.sock"
+        working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools
+
       - name: Upload CRI-O logs
         uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -33,3 +33,4 @@ Session.vim
 
 /build
 release-notes.md
+test/e2e/e2e.test

Makefile

@@ -130,12 +130,14 @@ release-notes:
 
 # needs to run as root to work
 test-e2e: $(GINKGO)
-	$(GINKGO) $(TESTFLAGS) \
-		-r -p \
-		--randomizeAllSpecs \
-		--randomizeSuites \
-		--slowSpecThreshold 60 \
-		test
+	$(GINKGO) \
+		-r \
+		--randomize-all \
+		--randomize-suites \
+		--slow-spec-threshold 60s \
+		test \
+		-- \
+		$(TESTFLAGS)
 
 test-crictl: $(GINKGO)
 	# Run go test for templates_test.go and util_test.go

test/e2e/events_test.go

@@ -20,7 +20,6 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gbytes"
-	. "github.com/onsi/gomega/gexec"
 )
 
 // The actual test suite
@@ -40,20 +39,8 @@ var _ = t.Describe("events options validation", func() {
 
 // The actual test suite
 var _ = t.Describe("events", func() {
-	var (
-		endpoint, testDir string
-		crio              *Session
-	)
-	BeforeEach(func() {
-		endpoint, testDir, crio = t.StartCrio()
-	})
-
-	AfterEach(func() {
-		t.StopCrio(testDir, crio)
-	})
-
 	It("should succeed", func() {
-		session := t.CrictlWithEndpointNoWait(endpoint, "events")
+		session := t.CrictlNoWait("events")
 		defer session.Terminate()
 		Expect(session.Out).ToNot(Say("unknown method GetContainerEvents")) // no errors
 	})

test/e2e/help_test.go

@@ -18,7 +18,6 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega/gexec"
 )
 
 // The actual test suite
@@ -46,40 +45,27 @@ var _ = t.Describe("help", func() {
 
 // The actual test suite
 var _ = t.Describe("help subcommand", func() {
-
-	var (
-		endpoint, testDir string
-		crio              *Session
-	)
-	BeforeEach(func() {
-		endpoint, testDir, crio = t.StartCrio()
-	})
-
-	AfterEach(func() {
-		t.StopCrio(testDir, crio)
-	})
-
 	It("should show help running rm without params", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint, "rm", "crictl rm command")
+		t.CrictlExpectSuccess("rm", "crictl rm command")
 	})
 
 	It("should show help running rmi without params", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint, "rmi", "crictl rmi command")
+		t.CrictlExpectSuccess("rmi", "crictl rmi command")
 	})
 
 	It("should show help running rmp without params", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint, "rmp", "crictl rmp command")
+		t.CrictlExpectSuccess("rmp", "crictl rmp command")
 	})
 
 	It("should not show help running rm -a", func() {
-		t.CrictlExpect(endpoint, "rm -a", 0, "", "No containers to remove")
+		t.CrictlExpect("rm -a", 0, "", "No containers to remove")
 	})
 
 	It("should not show help running rmi -a", func() {
-		t.CrictlExpect(endpoint, "rmi -a", 0, "", "No images to remove")
+		t.CrictlExpect("rmi -a", 0, "", "No images to remove")
 	})
 
 	It("should not show help running rmp -a", func() {
-		t.CrictlExpect(endpoint, "rmp -a", 0, "", "No pods to remove")
+		t.CrictlExpect("rmp -a", 0, "", "No pods to remove")
 	})
 })

test/e2e/info_test.go

@@ -18,26 +18,12 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega/gexec"
 )
 
 // The actual test suite
 var _ = t.Describe("info", func() {
-
-	var (
-		endpoint, testDir string
-		crio              *Session
-	)
-	BeforeEach(func() {
-		endpoint, testDir, crio = t.StartCrio()
-	})
-
-	AfterEach(func() {
-		t.StopCrio(testDir, crio)
-	})
-
 	It("should succeed", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint, "info", "NetworkReady")
+		t.CrictlExpectSuccess("info", "NetworkReady")
 	})
 
 	It("should fail with additional argument", func() {

test/e2e/pull_test.go

@@ -18,42 +18,35 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
 )
 
 // The actual test suite
 var _ = t.Describe("pull", func() {
-	var (
-		endpoint, testDir string
-		crio              *Session
-	)
-	BeforeEach(func() {
-		endpoint, testDir, crio = t.StartCrio()
-	})
-
-	AfterEach(func() {
-		t.StopCrio(testDir, crio)
-	})
-
 	const (
 		imageSuccessText = "Image is up to date"
 		registry         = "gcr.io/k8s-staging-cri-tools/"
 	)
 
+	AfterEach(func() {
+		Expect(t.Crictl("rmi -a")).To(Exit(0))
+	})
+
 	It("should succeed without tag or digest", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint,
+		t.CrictlExpectSuccess(
 			"pull "+registry+"test-image-1",
 			imageSuccessText)
 	})
 
 	It("should succeed with tag", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint,
+		t.CrictlExpectSuccess(
 			"pull "+registry+"test-image-1:latest",
 			imageSuccessText)
 	})
 
 	It("should succeed with digest", func() {
-		t.CrictlExpectSuccessWithEndpoint(endpoint,
+		t.CrictlExpectSuccess(
 			"pull "+registry+"test-image-digest"+
 				"@sha256:9700f9a2f5bf2c45f2f605a0bd3bce7cf37420ec9d3ed50ac2758413308766bf",
 			imageSuccessText)
@@ -64,7 +57,6 @@ var _ = t.Describe("pull", func() {
 	})
 
 	It("should fail on not existing image", func() {
-		t.CrictlExpectFailureWithEndpoint(endpoint, "pull localhost/wrong",
-			"", "pulling image")
+		t.CrictlExpectFailure("pull localhost/wrong", "", "pulling image")
 	})
 })

test/e2e/suite_test.go

@@ -32,14 +32,9 @@ func TestE2E(t *testing.T) {
 
 var t *TestFramework
 
-var _ = SynchronizedBeforeSuite(func() []byte {
-	// Setup only once
-	dir := SetupCrio()
-	return []byte(dir)
-
-}, func(dir []byte) {
+var _ = BeforeSuite(func() {
 	t = NewTestFramework()
-	t.Setup(string(dir))
+	t.Setup()
 })
 
 var _ = AfterSuite(func() {

test/e2e/version_test.go

@@ -23,13 +23,6 @@ import (
 // The actual test suite
 var _ = t.Describe("version", func() {
 	It("should succeed", func() {
-		// Given
-		endpoint, testDir, crio := t.StartCrio()
-
-		// When
-		t.CrictlExpectSuccessWithEndpoint(endpoint, "version", "RuntimeName:  cri-o")
-
-		// Then
-		t.StopCrio(testDir, crio)
+		t.CrictlExpectSuccess("version", "RuntimeName: ")
 	})
 })