tests: add smoke test for nydusd hot upgrade

.github/workflows/smoke.yml

@@ -286,7 +286,7 @@ jobs:
         export PERFORMANCE_TEST_MODE=${{ matrix.mode }}
         sudo -E make smoke-performance
 
-  failover-test:
+  takeover-test:
     runs-on: ubuntu-latest
     needs: [contrib-build, nydus-build]
     steps:
@@ -304,7 +304,8 @@ jobs:
         path: contrib/nydusify/cmd
     - name: Prepare Nydus Container Environment
       run: |
-        sudo bash misc/failover/prepare.sh
-    - name: Failover Test
+        sudo bash misc/takeover/prepare.sh
+    - name: Takeover Test
       run: |
-        sudo -E make smoke-failover
+        export NEW_NYDUSD_BINARY_PATH=target/release/nydusd
+        sudo -E make smoke-takeover

Makefile

@@ -133,8 +133,8 @@ smoke-performance:
 smoke-benchmark:
 	make -C smoke test-benchmark
 
-smoke-failover:
-	make -C smoke test-failover
+smoke-takeover:
+	make -C smoke test-takeover
 
 smoke: release smoke-only
 

misc/failover/prepare.sh → misc/takeover/prepare.sh

@@ -18,6 +18,6 @@ sudo tar -xzvf cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz -C /opt/cni/bin
 sudo install -D misc/performance/containerd_config.toml /etc/containerd/config.toml
 sudo systemctl restart containerd
 sudo install -D misc/performance/nydusd_config.json /etc/nydus/nydusd-config.fusedev.json
-sudo install -D misc/failover/snapshotter_config.toml /etc/nydus/config.toml
+sudo install -D misc/takeover/snapshotter_config.toml /etc/nydus/config.toml
 sudo install -D misc/performance/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service
 sudo systemctl start nydus-snapshotter

smoke/Makefile

@@ -48,6 +48,7 @@ test-compatibility: build
 
 # SNAPSHOTTER_SYSTEM_SOCK=/run/containerd-nydus/system.sock
 # SNAPSHOTTER=nydus
-# FAILOVER_TEST_IMAGE=wordpress
-test-failover: build 
-	FAILOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestFailover
+# TAKEOVER_TEST_IMAGE=wordpress
+# NEW_NYDUSD_BINARY_PATH=target/release/nydusd
+test-takeover: build 
+	TAKEOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestTakeover

smoke/tests/takeover_test.go

@@ -0,0 +1,163 @@
+// Copyright 2023 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tests
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/dragonflyoss/nydus/smoke/tests/tool"
+	"github.com/dragonflyoss/nydus/smoke/tests/tool/test"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd.
+// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md.
+
+var (
+	snapshotter           string
+	takeoverTestImage     string
+	snapshotterSystemSock string
+)
+
+type TakeoverTestSuit struct {
+	t              *testing.T
+	ctx            *tool.Context
+	testImage      string
+	snapshotterCli *tool.SnapshotterClient
+}
+
+func NewTakeoverTestSuit(t *testing.T) *TakeoverTestSuit {
+	snapshotterCli := tool.NewSnapshotterClient(snapshotterSystemSock)
+	ctx := tool.DefaultContext(t)
+
+	// prepare and convert image
+	sourceImage := tool.PrepareImage(t, takeoverTestImage)
+	imageName := fmt.Sprintf("%s:nydus", sourceImage)
+	tool.ConvertImage(t, ctx, sourceImage, imageName)
+
+	return &TakeoverTestSuit{
+		t:              t,
+		ctx:            ctx,
+		testImage:      imageName,
+		snapshotterCli: snapshotterCli,
+	}
+}
+
+func (f *TakeoverTestSuit) clear() {
+	tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s image rm %s", snapshotter, f.testImage))
+}
+
+func (f *TakeoverTestSuit) rmContainer(conatinerName string) {
+	tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s rm -f %s", snapshotter, conatinerName))
+}
+
+func (f *TakeoverTestSuit) TestFailover(t *testing.T) {
+	imageName := f.testImage
+
+	containerName := uuid.NewString()
+	tool.RunContainerSimple(t, imageName, snapshotter, containerName, false)
+	defer f.rmContainer(containerName)
+
+	daemons, err := f.snapshotterCli.GetNydusDaemonInfos()
+	require.NoError(t, err, "get nydus daemon infos")
+
+	// kill the nydus daemons
+	for _, daemon := range daemons {
+		killCmd := fmt.Sprintf("kill -9 %d", daemon.Pid)
+		tool.Run(t, killCmd)
+	}
+
+	// wait for the nydus daemons recover
+	time.Sleep(5 * time.Second)
+
+	// check the container by requesting its wait url
+	runArgs := tool.GetRunArgs(t, imageName)
+	resp, err := http.Get(runArgs.WaitURL)
+	require.NoError(t, err, "access to the wait url of the recoverd container")
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		t.Fatalf("Failed to access the wait url of the recoverd container")
+	}
+}
+
+func (f *TakeoverTestSuit) TestHotUpgrade(t *testing.T) {
+	imageName := f.testImage
+
+	containerName := uuid.NewString()
+	tool.RunContainerSimple(t, imageName, snapshotter, containerName, false)
+	defer f.rmContainer(containerName)
+
+	// hot upgrade nydusd
+	newNydusdPath := os.Getenv("NEW_NYDUSD_BINARY_PATH")
+	if newNydusdPath == "" {
+		newNydusdPath = "target/release/nydusd"
+	}
+	nydusdPath, err := filepath.Abs("../" + newNydusdPath)
+	require.NoErrorf(t, err, "get the abs path of new nydusd path (%s)", newNydusdPath)
+	err = os.Chmod(nydusdPath, 0755)
+	require.NoErrorf(t, err, "chmod nydusd binary file (%s)", nydusdPath)
+
+	upgradeReq := &tool.UpgradeRequest{
+		NydusdPath: nydusdPath,
+		Version:    getNydusdVersion(nydusdPath),
+		Policy:     "rolling",
+	}
+	err = f.snapshotterCli.Upgrade(upgradeReq)
+	require.NoError(t, err, "call the snapshotter to upgrade nydus daemons")
+
+	// wait for the nydus daemons recover
+	time.Sleep(5 * time.Second)
+
+	// check the container by requesting its wait url
+	runArgs := tool.GetRunArgs(t, imageName)
+	resp, err := http.Get(runArgs.WaitURL)
+	require.NoError(t, err, "access to the wait url of the recoverd container")
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		t.Fatalf("Failed to access the wait url of the recoverd container")
+	}
+}
+
+func getNydusdVersion(nydusdPath string) string {
+	versionOutput := tool.RunWithOutput(fmt.Sprintf("%s --version", nydusdPath))
+	lines := strings.Split(versionOutput, "\n")
+	version := ""
+	for _, line := range lines {
+		if strings.HasPrefix(line, "Version:") {
+			fields := strings.Fields(line)
+			version = strings.TrimSpace(fields[1])
+		}
+	}
+	return version
+}
+
+func TestTakeover(t *testing.T) {
+	if v, ok := os.LookupEnv("TAKEOVER_TEST"); !ok || v != "true" {
+		t.Skip("skipping takeover test")
+	}
+	snapshotter = os.Getenv("SNAPSHOTTER")
+	if snapshotter == "" {
+		snapshotter = defaultSnapshotter
+	}
+	takeoverTestImage = os.Getenv("TAKEOVER_TEST_IMAGE")
+	if takeoverTestImage == "" {
+		takeoverTestImage = "wordpress"
+	}
+	snapshotterSystemSock = os.Getenv("SNAPSHOTTER_SYSTEM_SOCK")
+	if snapshotterSystemSock == "" {
+		snapshotterSystemSock = defaultSnapshotterSystemSock
+	}
+	suite := NewTakeoverTestSuit(t)
+	defer suite.clear()
+
+	test.Run(t, suite, test.Sync)
+}

smoke/tests/tool/snapshotter.go

@@ -5,13 +5,16 @@
 package tool
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 // SnapshotterClient commnicates with nydus-snapshotter via
@@ -41,6 +44,12 @@ type rafsInstanceInfo struct {
 	ImageID     string `json:"image_id"`
 }
 
+type UpgradeRequest struct {
+	NydusdPath string `json:"nydusd_path"`
+	Version    string `json:"version"`
+	Policy     string `json:"policy"`
+}
+
 func NewSnapshotterClient(sock string) *SnapshotterClient {
 	transport := &http.Transport{
 		MaxIdleConns:          10,
@@ -64,14 +73,42 @@ func NewSnapshotterClient(sock string) *SnapshotterClient {
 	}
 }
 
-func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotter, error) {
-	resp, err := cli.client.Get(fmt.Sprintf("http://unix%s", "/api/v1/daemons"))
+func (cli *SnapshotterClient) request(method, urlSuffix string, body any) (respBody []byte, err error) {
+	var reqBody io.Reader
+	if body != nil {
+		reqJSON, err := json.Marshal(body)
+		if err != nil {
+			return nil, errors.Wrap(err, "marshal request body")
+		}
+
+		reqBody = bytes.NewBuffer(reqJSON)
+	}
+
+	url := fmt.Sprintf("http://unix%s", urlSuffix)
+	req, err := http.NewRequest(method, url, reqBody)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "build request")
+	}
+	resp, err := cli.client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "do request")
 	}
 	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp.Body)
+	respBody, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "read response body")
+	}
+
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("faild  to do request(%v), got status %s, and resp %s", req, resp.Status, respBody)
+	}
+
+	return
+}
+
+func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotter, error) {
+	body, err := cli.request("GET", "/api/v1/daemons", nil)
 	if err != nil {
 		return nil, err
 	}
@@ -83,3 +120,8 @@ func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotte
 
 	return infos, nil
 }
+
+func (cli *SnapshotterClient) Upgrade(req *UpgradeRequest) error {
+	_, err := cli.request("PUT", "/api/v1/daemons/upgrade", req)
+	return err
+}