diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index fbb1fc29c39..bcf46015a64 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -286,7 +286,7 @@ jobs: export PERFORMANCE_TEST_MODE=${{ matrix.mode }} sudo -E make smoke-performance - failover-test: + takeover-test: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] steps: @@ -304,7 +304,8 @@ jobs: path: contrib/nydusify/cmd - name: Prepare Nydus Container Environment run: | - sudo bash misc/failover/prepare.sh - - name: Failover Test + sudo bash misc/takeover/prepare.sh + - name: Takeover Test run: | - sudo -E make smoke-failover + export NEW_NYDUSD_BINARY_PATH=target/release/nydusd + sudo -E make smoke-takeover diff --git a/Makefile b/Makefile index 22a9ceebf4b..3f601b221c7 100644 --- a/Makefile +++ b/Makefile @@ -133,8 +133,8 @@ smoke-performance: smoke-benchmark: make -C smoke test-benchmark -smoke-failover: - make -C smoke test-failover +smoke-takeover: + make -C smoke test-takeover smoke: release smoke-only diff --git a/misc/failover/prepare.sh b/misc/takeover/prepare.sh similarity index 95% rename from misc/failover/prepare.sh rename to misc/takeover/prepare.sh index cf8f5081024..92782426c34 100755 --- a/misc/failover/prepare.sh +++ b/misc/takeover/prepare.sh @@ -18,6 +18,6 @@ sudo tar -xzvf cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz -C /opt/cni/bin sudo install -D misc/performance/containerd_config.toml /etc/containerd/config.toml sudo systemctl restart containerd sudo install -D misc/performance/nydusd_config.json /etc/nydus/nydusd-config.fusedev.json -sudo install -D misc/failover/snapshotter_config.toml /etc/nydus/config.toml +sudo install -D misc/takeover/snapshotter_config.toml /etc/nydus/config.toml sudo install -D misc/performance/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service sudo systemctl start nydus-snapshotter diff --git a/misc/failover/snapshotter_config.toml b/misc/takeover/snapshotter_config.toml similarity index 100% rename from misc/failover/snapshotter_config.toml rename to misc/takeover/snapshotter_config.toml diff --git a/smoke/Makefile b/smoke/Makefile index 697c486fbd3..05562e6f769 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -48,6 +48,7 @@ test-compatibility: build # SNAPSHOTTER_SYSTEM_SOCK=/run/containerd-nydus/system.sock # SNAPSHOTTER=nydus -# FAILOVER_TEST_IMAGE=wordpress -test-failover: build - FAILOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestFailover +# TAKEOVER_TEST_IMAGE=wordpress +# NEW_NYDUSD_BINARY_PATH=target/release/nydusd +test-takeover: build + TAKEOVER_TEST=true sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestTakeover diff --git a/smoke/tests/failover_test.go b/smoke/tests/failover_test.go deleted file mode 100644 index ad2c8db85fb..00000000000 --- a/smoke/tests/failover_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2023 Nydus Developers. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -package tests - -import ( - "fmt" - "net/http" - "os" - "testing" - "time" - - "github.com/dragonflyoss/nydus/smoke/tests/tool" - "github.com/dragonflyoss/nydus/smoke/tests/tool/test" - "github.com/google/uuid" -) - -// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd. -// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. - -type FailoverTestSuit struct { - t *testing.T -} - -func (f *FailoverTestSuit) TestFailover(t *testing.T) { - // prepare the basic constants - snapshotter := os.Getenv("SNAPSHOTTER") - if snapshotter == "" { - snapshotter = defaultSnapshotter - } - sourceImage := os.Getenv("FAILOVER_TEST_IMAGE") - if sourceImage == "" { - sourceImage = "wordpress" - } - snapshotterSystemSock := os.Getenv("SNAPSHOTTER_SYSTEM_SOCK") - if snapshotterSystemSock == "" { - snapshotterSystemSock = defaultSnapshotterSystemSock - } - - ctx := tool.DefaultContext(t) - - // prepare and convert image - sourceImage = tool.PrepareImage(t, sourceImage) - imageName := fmt.Sprintf("%s:nydus", sourceImage) - tool.ConvertImage(t, ctx, sourceImage, imageName) - - containerName := uuid.NewString() - tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) - defer tool.ClearContainer(t, imageName, snapshotter, containerName) - - snapshotterCli := tool.NewSnapshotterClient(snapshotterSystemSock) - daemons, err := snapshotterCli.GetNydusDaemonInfos() - if err != nil { - t.Fatalf("Failed to get nydus daemon infos: %s", err.Error()) - } - - // kill the nydus daemons - for _, daemon := range daemons { - killCmd := fmt.Sprintf("kill -9 %d", daemon.Pid) - tool.Run(t, killCmd) - } - - // wait for the nydus daemons recover - time.Sleep(5 * time.Second) - - // check the container by requesting its wait url - runArgs := tool.GetRunArgs(t, imageName) - resp, err := http.Get(runArgs.WaitURL) - if err != nil || !(resp.StatusCode >= 200 && resp.StatusCode < 300) { - t.Fatal("Failed to access the wait url of the recoverd container") - } -} - -func TestFailover(t *testing.T) { - if v, ok := os.LookupEnv("FAILOVER_TEST"); !ok || v != "true" { - t.Skip("skipping failover test") - } - test.Run(t, &FailoverTestSuit{t: t}) -} diff --git a/smoke/tests/takeover_test.go b/smoke/tests/takeover_test.go new file mode 100644 index 00000000000..f556be0b930 --- /dev/null +++ b/smoke/tests/takeover_test.go @@ -0,0 +1,163 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dragonflyoss/nydus/smoke/tests/tool" + "github.com/dragonflyoss/nydus/smoke/tests/tool/test" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd. +// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. + +var ( + snapshotter string + takeoverTestImage string + snapshotterSystemSock string +) + +type TakeoverTestSuit struct { + t *testing.T + ctx *tool.Context + testImage string + snapshotterCli *tool.SnapshotterClient +} + +func NewTakeoverTestSuit(t *testing.T) *TakeoverTestSuit { + snapshotterCli := tool.NewSnapshotterClient(snapshotterSystemSock) + ctx := tool.DefaultContext(t) + + // prepare and convert image + sourceImage := tool.PrepareImage(t, takeoverTestImage) + imageName := fmt.Sprintf("%s:nydus", sourceImage) + tool.ConvertImage(t, ctx, sourceImage, imageName) + + return &TakeoverTestSuit{ + t: t, + ctx: ctx, + testImage: imageName, + snapshotterCli: snapshotterCli, + } +} + +func (f *TakeoverTestSuit) clear() { + tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s image rm %s", snapshotter, f.testImage)) +} + +func (f *TakeoverTestSuit) rmContainer(conatinerName string) { + tool.RunWithoutOutput(f.t, fmt.Sprintf("sudo nerdctl --snapshotter %s rm -f %s", snapshotter, conatinerName)) +} + +func (f *TakeoverTestSuit) TestFailover(t *testing.T) { + imageName := f.testImage + + containerName := uuid.NewString() + tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) + defer f.rmContainer(containerName) + + daemons, err := f.snapshotterCli.GetNydusDaemonInfos() + require.NoError(t, err, "get nydus daemon infos") + + // kill the nydus daemons + for _, daemon := range daemons { + killCmd := fmt.Sprintf("kill -9 %d", daemon.Pid) + tool.Run(t, killCmd) + } + + // wait for the nydus daemons recover + time.Sleep(5 * time.Second) + + // check the container by requesting its wait url + runArgs := tool.GetRunArgs(t, imageName) + resp, err := http.Get(runArgs.WaitURL) + require.NoError(t, err, "access to the wait url of the recoverd container") + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + t.Fatalf("Failed to access the wait url of the recoverd container") + } +} + +func (f *TakeoverTestSuit) TestHotUpgrade(t *testing.T) { + imageName := f.testImage + + containerName := uuid.NewString() + tool.RunContainerSimple(t, imageName, snapshotter, containerName, false) + defer f.rmContainer(containerName) + + // hot upgrade nydusd + newNydusdPath := os.Getenv("NEW_NYDUSD_BINARY_PATH") + if newNydusdPath == "" { + newNydusdPath = "target/release/nydusd" + } + nydusdPath, err := filepath.Abs("../" + newNydusdPath) + require.NoErrorf(t, err, "get the abs path of new nydusd path (%s)", newNydusdPath) + err = os.Chmod(nydusdPath, 0755) + require.NoErrorf(t, err, "chmod nydusd binary file (%s)", nydusdPath) + + upgradeReq := &tool.UpgradeRequest{ + NydusdPath: nydusdPath, + Version: getNydusdVersion(nydusdPath), + Policy: "rolling", + } + err = f.snapshotterCli.Upgrade(upgradeReq) + require.NoError(t, err, "call the snapshotter to upgrade nydus daemons") + + // wait for the nydus daemons recover + time.Sleep(5 * time.Second) + + // check the container by requesting its wait url + runArgs := tool.GetRunArgs(t, imageName) + resp, err := http.Get(runArgs.WaitURL) + require.NoError(t, err, "access to the wait url of the recoverd container") + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + t.Fatalf("Failed to access the wait url of the recoverd container") + } +} + +func getNydusdVersion(nydusdPath string) string { + versionOutput := tool.RunWithOutput(fmt.Sprintf("%s --version", nydusdPath)) + lines := strings.Split(versionOutput, "\n") + version := "" + for _, line := range lines { + if strings.HasPrefix(line, "Version:") { + fields := strings.Fields(line) + version = strings.TrimSpace(fields[1]) + } + } + return version +} + +func TestTakeover(t *testing.T) { + if v, ok := os.LookupEnv("TAKEOVER_TEST"); !ok || v != "true" { + t.Skip("skipping takeover test") + } + snapshotter = os.Getenv("SNAPSHOTTER") + if snapshotter == "" { + snapshotter = defaultSnapshotter + } + takeoverTestImage = os.Getenv("TAKEOVER_TEST_IMAGE") + if takeoverTestImage == "" { + takeoverTestImage = "wordpress" + } + snapshotterSystemSock = os.Getenv("SNAPSHOTTER_SYSTEM_SOCK") + if snapshotterSystemSock == "" { + snapshotterSystemSock = defaultSnapshotterSystemSock + } + suite := NewTakeoverTestSuit(t) + defer suite.clear() + + test.Run(t, suite, test.Sync) +} diff --git a/smoke/tests/tool/snapshotter.go b/smoke/tests/tool/snapshotter.go index 5f500115df6..c301e46c2a8 100644 --- a/smoke/tests/tool/snapshotter.go +++ b/smoke/tests/tool/snapshotter.go @@ -5,6 +5,7 @@ package tool import ( + "bytes" "context" "encoding/json" "fmt" @@ -12,6 +13,8 @@ import ( "net" "net/http" "time" + + "github.com/pkg/errors" ) // SnapshotterClient commnicates with nydus-snapshotter via @@ -41,6 +44,12 @@ type rafsInstanceInfo struct { ImageID string `json:"image_id"` } +type UpgradeRequest struct { + NydusdPath string `json:"nydusd_path"` + Version string `json:"version"` + Policy string `json:"policy"` +} + func NewSnapshotterClient(sock string) *SnapshotterClient { transport := &http.Transport{ MaxIdleConns: 10, @@ -64,14 +73,42 @@ func NewSnapshotterClient(sock string) *SnapshotterClient { } } -func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotter, error) { - resp, err := cli.client.Get(fmt.Sprintf("http://unix%s", "/api/v1/daemons")) +func (cli *SnapshotterClient) request(method, urlSuffix string, body any) (respBody []byte, err error) { + var reqBody io.Reader + if body != nil { + reqJSON, err := json.Marshal(body) + if err != nil { + return nil, errors.Wrap(err, "marshal request body") + } + + reqBody = bytes.NewBuffer(reqJSON) + } + + url := fmt.Sprintf("http://unix%s", urlSuffix) + req, err := http.NewRequest(method, url, reqBody) if err != nil { - return nil, err + return nil, errors.Wrap(err, "build request") + } + resp, err := cli.client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "do request") } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + respBody, err = io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "read response body") + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("faild to do request(%v), got status %s, and resp %s", req, resp.Status, respBody) + } + + return +} + +func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotter, error) { + body, err := cli.request("GET", "/api/v1/daemons", nil) if err != nil { return nil, err } @@ -83,3 +120,8 @@ func (cli *SnapshotterClient) GetNydusDaemonInfos() ([]*DaemonInfoFromSnapshotte return infos, nil } + +func (cli *SnapshotterClient) Upgrade(req *UpgradeRequest) error { + _, err := cli.request("PUT", "/api/v1/daemons/upgrade", req) + return err +}