From fb1a8fc3f80bada5a12386522b9275083d6bb658 Mon Sep 17 00:00:00 2001 From: loheagn Date: Wed, 27 Dec 2023 13:42:39 +0000 Subject: [PATCH] tests: add smoke test for hot upgrade Signed-off-by: loheagn --- .github/workflows/smoke.yml | 9 +- Makefile | 4 +- misc/{failover => takeover}/prepare.sh | 2 +- .../snapshotter_config.toml | 0 smoke/Makefile | 7 +- smoke/tests/failover_test.go | 80 --------- smoke/tests/takeover_test.go | 163 ++++++++++++++++++ smoke/tests/tool/snapshotter.go | 42 +++++ 8 files changed, 217 insertions(+), 90 deletions(-) rename misc/{failover => takeover}/prepare.sh (95%) rename misc/{failover => takeover}/snapshotter_config.toml (100%) delete mode 100644 smoke/tests/failover_test.go create mode 100644 smoke/tests/takeover_test.go diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 6ddfd0fd4e4..d4c6b9f96cf 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -286,7 +286,7 @@ jobs: export PERFORMANCE_TEST_MODE=${{ matrix.mode }} sudo -E make smoke-performance - failover-test: + takeover-test: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] steps: @@ -304,7 +304,8 @@ jobs: path: contrib/nydusify/cmd - name: Prepare Nydus Container Environment run: | - sudo bash misc/failover/prepare.sh - - name: Failover Test + sudo bash misc/takeover/prepare.sh + - name: Takeover Test run: | - sudo -E make smoke-failover + export NEW_NYDUSD_BINARY_PATH=target/release/nydusd + sudo -E make smoke-takeover diff --git a/Makefile b/Makefile index 22a9ceebf4b..3f601b221c7 100644 --- a/Makefile +++ b/Makefile @@ -133,8 +133,8 @@ smoke-performance: smoke-benchmark: make -C smoke test-benchmark -smoke-failover: - make -C smoke test-failover +smoke-takeover: + make -C smoke test-takeover smoke: release smoke-only diff --git a/misc/failover/prepare.sh b/misc/takeover/prepare.sh similarity index 95% rename from misc/failover/prepare.sh rename to misc/takeover/prepare.sh index cf8f5081024..92782426c34 100755 --- a/misc/failover/prepare.sh +++ b/misc/takeover/prepare.sh @@ -18,6 +18,6 @@ sudo tar -xzvf cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz -C /opt/cni/bin sudo install -D misc/performance/containerd_config.toml /etc/containerd/config.toml sudo systemctl restart containerd sudo install -D misc/performance/nydusd_config.json /etc/nydus/nydusd-config.fusedev.json -sudo install -D misc/failover/snapshotter_config.toml /etc/nydus/config.toml +sudo install -D misc/takeover/snapshotter_config.toml /etc/nydus/config.toml sudo install -D misc/performance/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service sudo systemctl start nydus-snapshotter diff --git a/misc/failover/snapshotter_config.toml b/misc/takeover/snapshotter_config.toml similarity index 100% rename from misc/failover/snapshotter_config.toml rename to misc/takeover/snapshotter_config.toml diff --git a/smoke/Makefile b/smoke/Makefile index 697c486fbd3..05562e6f769 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -48,6 +48,7 @@ test-compatibility: build # SNAPSHOTTER_SYSTEM_SOCK=/run/containerd-nydus/system.sock # SNAPSHOTTER=nydus -# FAILOVER_TEST_IMAGE=wordpress -test-failover: build - FAILOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestFailover +# TAKEOVER_TEST_IMAGE=wordpress +# NEW_NYDUSD_BINARY_PATH=target/release/nydusd +test-takeover: build + TAKEOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestTakeover diff --git a/smoke/tests/failover_test.go b/smoke/tests/failover_test.go deleted file mode 100644 index ad2c8db85fb..00000000000 --- a/smoke/tests/failover_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2023 Nydus Developers. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -package tests - -import ( - "fmt" - "net/http" - "os" - "testing" - "time" - - "github.com/dragonflyoss/nydus/smoke/tests/tool" - "github.com/dragonflyoss/nydus/smoke/tests/tool/test" - "github.com/google/uuid" -) - -// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd. -// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. - -type FailoverTestSuit struct { - t *testing.T -} - -func (f *FailoverTestSuit) TestFailover(t *testing.T) { - // prepare the basic constants - snapshotter := os.Getenv("SNAPSHOTTER") - if snapshotter == "" { - snapshotter = defaultSnapshotter - } - sourceImage := os.Getenv("FAILOVER_TEST_IMAGE") - if sourceImage == "" { - sourceImage = "wordpress" - } - snapshotterSystemSock := os.Getenv("SNAPSHOTTER_SYSTEM_SOCK") - if snapshotterSystemSock == "" { - snapshotterSystemSock = defaultSnapshotterSystemSock - } - - ctx := tool.DefaultContext(t) - - // prepare and convert image - sourceImage = tool.PrepareImage(t, sourceImage) - imageName := fmt.Sprintf("%s:nydus", sourceImage) - tool.ConvertImage(t, ctx, sourceImage, imageName) - - containerName := uuid.NewString() - tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) - defer tool.ClearContainer(t, imageName, snapshotter, containerName) - - snapshotterCli := tool.NewSnapshotterClient(snapshotterSystemSock) - daemons, err := snapshotterCli.GetNydusDaemonInfos() - if err != nil { - t.Fatalf("Failed to get nydus daemon infos: %s", err.Error()) - } - - // kill the nydus daemons - for _, daemon := range daemons { - killCmd := fmt.Sprintf("kill -9 %d", daemon.Pid) - tool.Run(t, killCmd) - } - - // wait for the nydus daemons recover - time.Sleep(5 * time.Second) - - // check the container by requesting its wait url - runArgs := tool.GetRunArgs(t, imageName) - resp, err := http.Get(runArgs.WaitURL) - if err != nil || !(resp.StatusCode >= 200 && resp.StatusCode < 300) { - t.Fatal("Failed to access the wait url of the recoverd container") - } -} - -func TestFailover(t *testing.T) { - if v, ok := os.LookupEnv("FAILOVER_TEST"); !ok || v != "true" { - t.Skip("skipping failover test") - } - test.Run(t, &FailoverTestSuit{t: t}) -} diff --git a/smoke/tests/takeover_test.go b/smoke/tests/takeover_test.go new file mode 100644 index 00000000000..2535176168c --- /dev/null +++ b/smoke/tests/takeover_test.go @@ -0,0 +1,163 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dragonflyoss/nydus/smoke/tests/tool" + "github.com/dragonflyoss/nydus/smoke/tests/tool/test" + "github.com/google/uuid" +) + +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd. +// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. + +var ( + snapshotter string + takeoverTestImage string + snapshotterSystemSock string +) + +type TakeoverTestSuit struct { + t *testing.T + ctx *tool.Context + testImage string + snapshotterCli *tool.SnapshotterClient +} + +func NewTakeoverTestSuit(t *testing.T) *TakeoverTestSuit { + snapshotterCli := tool.NewSnapshotterClient(snapshotterSystemSock) + ctx := tool.DefaultContext(t) + + // prepare and convert image + sourceImage := tool.PrepareImage(t, takeoverTestImage) + imageName := fmt.Sprintf("%s:nydus", sourceImage) + tool.ConvertImage(t, ctx, sourceImage, imageName) + + return &TakeoverTestSuit{ + t: t, + ctx: ctx, + testImage: imageName, + snapshotterCli: snapshotterCli, + } +} + +func (f *TakeoverTestSuit) clear() { + tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s image rm %s", snapshotter, f.testImage)) +} + +func (f *TakeoverTestSuit) rmContainer(conatinerName string) { + tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s rm -f %s", snapshotter, conatinerName)) +} + +func (f *TakeoverTestSuit) TestFailover(t *testing.T) { + imageName := f.testImage + + containerName := uuid.NewString() + tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) + defer f.rmContainer(containerName) + + daemons, err := f.snapshotterCli.GetNydusDaemonInfos() + if err != nil { + t.Fatalf("Failed to get nydus daemon infos: %s", err.Error()) + } + + // kill the nydus daemons + for _, daemon := range daemons { + killCmd := fmt.Sprintf("kill -9 %d", daemon.Pid) + tool.Run(t, killCmd) + } + + // wait for the nydus daemons recover + time.Sleep(5 * time.Second) + + // check the container by requesting its wait url + runArgs := tool.GetRunArgs(t, imageName) + resp, err := http.Get(runArgs.WaitURL) + if err != nil || !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + t.Fatal("Failed to access the wait url of the recoverd container") + } +} + +func (f *TakeoverTestSuit) TestHotUpgrade(t *testing.T) { + imageName := f.testImage + + containerName := uuid.NewString() + tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) + defer f.rmContainer(containerName) + + // hot upgrade nydusd + newNydusdPath := os.Getenv("NEW_NYDUSD_BINARY_PATH") + if newNydusdPath == "" { + newNydusdPath = "target/release/nydusd" + } + nydusdPath, err := filepath.Abs("../" + newNydusdPath) + if err != nil { + t.Fatalf("Failed to get the abs path of new nydusd path(%s)", newNydusdPath) + } + if err := os.Chmod(nydusdPath, 0755); err != nil { + t.Fatalf("Failed to chmod nydusd binary file (%s): %s", nydusdPath, err.Error()) + } + upgradeReq := &tool.UpgradeRequest{ + NydusdPath: nydusdPath, + Version: getNydusdVersion(nydusdPath), + Policy: "rolling", + } + if err := f.snapshotterCli.Upgrade(upgradeReq); err != nil { + t.Fatalf("failed to upgrade: %s", err.Error()) + } + + // wait for the nydus daemons recover + time.Sleep(5 * time.Second) + + // check the container by requesting its wait url + runArgs := tool.GetRunArgs(t, imageName) + resp, err := http.Get(runArgs.WaitURL) + if err != nil || resp.StatusCode/100 != 2 { + t.Fatal("Failed to access the wait url of the container after hot upgraded") + } +} + +func getNydusdVersion(nydusdPath string) string { + versionOutput := tool.RunWithOutput(fmt.Sprintf("%s --version", nydusdPath)) + lines := strings.Split(versionOutput, "\n") + version := "" + for _, line := range lines { + if strings.HasPrefix(line, "Version:") { + fields := strings.Fields(line) + version = strings.TrimSpace(fields[1]) + } + } + return version +} + +func TestTakeover(t *testing.T) { + if v, ok := os.LookupEnv("TAKEOVER_TEST"); !ok || v != "true" { + t.Skip("skipping takeover test") + } + snapshotter = os.Getenv("SNAPSHOTTER") + if snapshotter == "" { + snapshotter = defaultSnapshotter + } + takeoverTestImage = os.Getenv("TAKEOVER_TEST_IMAGE") + if takeoverTestImage == "" { + takeoverTestImage = "wordpress" + } + snapshotterSystemSock = os.Getenv("SNAPSHOTTER_SYSTEM_SOCK") + if snapshotterSystemSock == "" { + snapshotterSystemSock = defaultSnapshotterSystemSock + } + suite := NewTakeoverTestSuit(t) + defer suite.clear() + + test.Run(t, suite, test.Sync) +} diff --git a/smoke/tests/tool/snapshotter.go b/smoke/tests/tool/snapshotter.go index 5f500115df6..e90bf62515d 100644 --- a/smoke/tests/tool/snapshotter.go +++ b/smoke/tests/tool/snapshotter.go @@ -5,6 +5,7 @@ package tool import ( + "bytes" "context" "encoding/json" "fmt" @@ -12,6 +13,8 @@ import ( "net" "net/http" "time" + + "github.com/pkg/errors" ) // SnapshotterClient commnicates with nydus-snapshotter via @@ -41,6 +44,12 @@ type rafsInstanceInfo struct { ImageID string `json:"image_id"` } +type UpgradeRequest struct { + NydusdPath string `json:"nydusd_path"` + Version string `json:"version"` + Policy string `json:"policy"` +} + func NewSnapshotterClient(sock string) *SnapshotterClient { transport := &http.Transport{ MaxIdleConns: 10, @@ -64,6 +73,34 @@ func NewSnapshotterClient(sock string) *SnapshotterClient { } } +func (cli *SnapshotterClient) request(method, urlSuffix string, body any) (respBody []byte, err error) { + var reqJSON []byte + if body != nil { + reqJSON, err = json.Marshal(body) + if err != nil { + return nil, errors.Wrap(err, "marshal request body") + } + } else { + reqJSON = make([]byte, 0) + } + + url := fmt.Sprintf("http://unix%s", urlSuffix) + req, err := http.NewRequest(method, url, bytes.NewBuffer(reqJSON)) + if err != nil { + return nil, errors.Wrap(err, "build request") + } + resp, err := cli.client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "do request") + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("faild to do request(%v), got %s", req, resp.Status) + } + defer resp.Body.Close() + + return io.ReadAll(resp.Body) +} + func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotter, error) { resp, err := cli.client.Get(fmt.Sprintf("http://unix%s", "/api/v1/daemons")) if err != nil { @@ -83,3 +120,8 @@ func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotte return infos, nil } + +func (cli *SnapshotterClient) Upgrade(req *UpgradeRequest) error { + _, err := cli.request("PUT", "/api/v1/daemons/upgrade", req) + return err +}