ci: switch from jenkins to github actions

Jenkins has become troublesome to maintain and makes it harder for users to see and test output without adding new users. We could probably spend some time improving that, but the current Jenkins is also setup on systems managed by the sponsoring company so it makes sense to move away from that at least. In order to run the CI tests we need a relatively powerful CI system so we are making use of the ubuntu-latest-16-cores runners. The tests seem to run slower than on Jenkins. Part of it may be explained because the dependencies are not pre-cache. Signed-off-by: Tiago Castro <[email protected]>
openebs · Dec 26, 2024 · 82dc1bb · 82dc1bb
1 parent 17488f4
commit 82dc1bb
Show file tree

Hide file tree

Showing 33 changed files with 362 additions and 102 deletions.
diff --git a/.github/workflows/bdd.yml b/.github/workflows/bdd.yml
@@ -0,0 +1,61 @@
+name: BDD CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  bdd-tests:
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+      - name: Build binaries
+        run: nix-shell --run "cargo build --bins --features=io-engine-testing"
+      - name: Setup Test Pre-Requisites
+        run: |
+          sudo sysctl -w vm.nr_hugepages=2560
+          sudo apt-get install linux-modules-extra-$(uname -r)
+          for module in nvme_tcp nbd nvme_rdma; do
+            sudo modprobe $module
+          done
+          # for the coredump check
+          sudo apt-get install gdb
+      - name: Setup VENV
+        run: nix-shell --run "./test/python/setup.sh"
+      - name: Run BDD Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/pytest-tests.sh"
+      - name: Test Report
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: test/python/reports/**/xunit-report.xml
+          summary: true
+          display-options: a
+          fail-on-empty: true
+          title: Test results
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/pytest-tests.sh --clean-all-exit"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+# debugging
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   timeout-minutes: 240
+      #   uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml
@@ -0,0 +1,19 @@
+name: ImageBuild CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  image-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Test building the release images
+        run: ./scripts/release.sh --skip-publish --build-bins
diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
@@ -0,0 +1,29 @@
+name: Image Push
+on:
+  push:
+    branches:
+      - develop
+      - 'release/**'
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+**'
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  image-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Test building the release images
+        run: ./scripts/release.sh
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -0,0 +1,32 @@
+name: Unit/Int CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  code-linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+      - name: Lint rust code
+        run: |
+          nix-shell --run "FMT_OPTS=--check ./scripts/rust-style.sh"
+          nix-shell --run "./scripts/rust-linter.sh"
+      - name: Lint JS code
+        run: nix-shell --run "./scripts/js-check.sh"
+      - name: Lint Nix code
+        run: nix-shell --run "nixpkgs-fmt --check ."
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
@@ -0,0 +1,15 @@
+name: Nightly CI
+on:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    uses: ./.github/workflows/pr-ci.yml
+  nightly-ci:
+    if: ${{ success() }}
+    needs:
+      - ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
@@ -0,0 +1,27 @@
+name: Bors CI
+on:
+  push:
+    branches:
+      - staging
+      - trying
+
+jobs:
+  lint-ci:
+    uses: ./.github/workflows/lint.yml
+  int-ci:
+    uses: ./.github/workflows/unit-int.yml
+  bdd-ci:
+    uses: ./.github/workflows/bdd.yml
+  image-ci:
+    uses: ./.github/workflows/image-pr.yml
+  bors-ci:
+    if: ${{ success() }}
+    needs:
+      - lint-ci
+      - int-ci
+      - bdd-ci
+      - image-ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0
diff --git a/.github/workflows/unit-int.yml b/.github/workflows/unit-int.yml
@@ -0,0 +1,75 @@
+name: Integration CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  int-tests:
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ startsWith(github.ref_name, 'release/') || github.ref_name == 'develop' }}
+      - name: Build binaries
+        run: nix-shell --run "cargo build --bins --features=io-engine-testing"
+      - name: Setup Test Pre-Requisites
+        run: |
+          sudo sysctl -w vm.nr_hugepages=3584
+          sudo apt-get install linux-modules-extra-$(uname -r)
+          for module in nvme_tcp nbd nvme_rdma; do
+            sudo modprobe $module
+          done
+          sudo apt-get install gdb
+      - name: Run Rust Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/cargo-test.sh"
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/clean-cargo-tests.sh"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+      - name: Run JS Grpc Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/grpc-test.sh"
+          mkdir js-reports
+          for file in *-xunit-report.xml; do
+            echo "<testsuites>" > "js-reports/$file"
+            cat $file >> "js-reports/$file"
+            echo "</testsuites>" >> "js-reports/$file"
+          done
+      - name: Test Report
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: 'js-reports/*-xunit-report.xml'
+          summary: true
+          display-options: a
+          fail-on-empty: true
+          title: Test results
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/clean-cargo-tests.sh"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+# debugging
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   timeout-minutes: 240
+      #   uses: mxschmitt/action-tmate@v3
diff --git a/io-engine-tests/src/lib.rs b/io-engine-tests/src/lib.rs
@@ -342,7 +342,7 @@ pub fn mount_and_get_md5(device: &str) -> Result<String, String> {
 pub fn fio_run_verify(device: &str) -> Result<String, String> {
     let (exit, stdout, stderr) = run_script::run(
         r"
-        fio --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
+        $FIO --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
         --runtime=5 --bs=4k --verify=crc32 --group_reporting=1 --output-format=terse \
         --verify_fatal=1 --verify_async=2 --filename=$1
     ",
@@ -511,11 +511,11 @@ pub async fn wait_for_rebuild(dst_uri: String, state: RebuildState, timeout: Dur
 pub fn fio_verify_size(device: &str, size: u64) -> i32 {
     let (exit, stdout, stderr) = run_script::run(
         r"
-        fio --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
+        $FIO --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
         --direct=1 --ioengine=libaio --rw=randwrite --verify=crc32 \
         --verify_fatal=1 --name=write_verify --filename=$1 --size=$2
 
-        fio --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
+        $FIO --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
         --direct=1 --ioengine=libaio --verify=crc32 --verify_only \
         --verify_fatal=1 --name=verify --filename=$1
     ",

diff --git a/io-engine/.cargo/runner.sh b/io-engine/.cargo/runner.sh
@@ -4,7 +4,7 @@
 ARGS="${@}"
 
 if [[ $EUID -ne 0 ]]; then
-  MAYBE_SUDO='sudo -E'
+  MAYBE_SUDO='sudo -E --preserve-env=PATH'
 else
   MAYBE_SUDO=''
 fi

diff --git a/io-engine/tests/ftl_mount_fs.rs b/io-engine/tests/ftl_mount_fs.rs
@@ -97,7 +97,7 @@ async fn ftl_mount_fs_multiple() {
 pub fn csal_fio_run_verify(device: &str) -> Result<String, String> {
     let (exit, stdout, stderr) = run_script::run(
         r#"
-        fio --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
+        $FIO --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
         --runtime=10 --bs=64k --verify=crc32 --group_reporting=1  \
         --verify_fatal=1 --verify_async=2 --filename=$1
         "#,

diff --git a/libnvme-rs/src/nvme_uri.rs b/libnvme-rs/src/nvme_uri.rs
@@ -347,6 +347,12 @@ impl NvmeTarget {
     }
 }
 
+impl Drop for NvmeTarget {
+    fn drop(&mut self) {
+        self.disconnect().ok();
+    }
+}
+
 #[test]
 fn nvme_parse_uri() {
     let target = NvmeTarget::try_from("nvmf://1.2.3.4:1234/testnqn.what-ever.foo").unwrap();

diff --git a/scripts/grpc-test.sh b/scripts/grpc-test.sh
@@ -16,7 +16,7 @@ set -euxo pipefail
 export PATH="$PATH:${HOME}/.cargo/bin"
 export npm_config_jobs=$(nproc)
 
-cargo build --all
+cargo build --bins --features=io-engine-testing
 cd "$(dirname "$0")/../test/grpc"
 npm install --legacy-peer-deps
 

diff --git a/scripts/nvmx-test.sh b/scripts/nvmx-test.sh
diff --git a/test/python/cross-grpc-version/nexus/docker-compose.yml b/test/python/cross-grpc-version/nexus/docker-compose.yml
@@ -8,15 +8,15 @@ services:
     container_name: "ms0"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.2
+      - MY_POD_IP=10.1.0.2
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.2
+        ipv4_address: 10.1.0.2
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -36,15 +36,15 @@ services:
     container_name: "ms1"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.3
+      - MY_POD_IP=10.1.0.3
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.3
+        ipv4_address: 10.1.0.3
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -66,4 +66,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: "10.0.0.0/16"
+        - subnet: "10.1.0.0/16"
diff --git a/test/python/cross-grpc-version/pool/docker-compose.yml b/test/python/cross-grpc-version/pool/docker-compose.yml
@@ -8,15 +8,15 @@ services:
     container_name: "ms0"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.2
+      - MY_POD_IP=10.1.0.2
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock --env-context=--iova-mode=pa
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.2
+        ipv4_address: 10.1.0.2
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -38,4 +38,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: "10.0.0.0/16"
+        - subnet: "10.1.0.0/16"