Try #1793:

.github/workflows/bdd.yml

@@ -0,0 +1,61 @@
+name: BDD CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  bdd-tests:
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+      - name: Build binaries
+        run: nix-shell --run "cargo build --bins --features=io-engine-testing"
+      - name: Setup Test Pre-Requisites
+        run: |
+          sudo sysctl -w vm.nr_hugepages=2560
+          sudo apt-get install linux-modules-extra-$(uname -r)
+          for module in nvme_tcp nbd nvme_rdma; do
+            sudo modprobe $module
+          done
+          # for the coredump check
+          sudo apt-get install gdb
+      - name: Setup VENV
+        run: nix-shell --run "./test/python/setup.sh"
+      - name: Run BDD Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/pytest-tests.sh"
+      - name: Test Report
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: test/python/reports/**/xunit-report.xml
+          summary: true
+          display-options: a
+          fail-on-empty: true
+          title: Test results
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/pytest-tests.sh --clean-all-exit"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+# debugging
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   timeout-minutes: 240
+      #   uses: mxschmitt/action-tmate@v3

.github/workflows/image-pr.yml

@@ -0,0 +1,19 @@
+name: ImageBuild CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  image-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Test building the release images
+        run: ./scripts/release.sh --skip-publish --build-bins

.github/workflows/image.yml

@@ -0,0 +1,29 @@
+name: Image Push
+on:
+  push:
+    branches:
+      - develop
+      - 'release/**'
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+**'
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  image-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Test building the release images
+        run: ./scripts/release.sh

.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: Unit/Int CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  code-linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+      - name: Lint rust code
+        run: |
+          nix-shell --run "FMT_OPTS=--check ./scripts/rust-style.sh"
+          nix-shell --run "./scripts/rust-linter.sh"
+      - name: Lint JS code
+        run: nix-shell --run "./scripts/js-check.sh"
+      - name: Lint Nix code
+        run: nix-shell --run "nixpkgs-fmt --check ."

.github/workflows/nightly-ci.yml

@@ -0,0 +1,15 @@
+name: Nightly CI
+on:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    uses: ./.github/workflows/pr-ci.yml
+  nightly-ci:
+    if: ${{ success() }}
+    needs:
+      - ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0

.github/workflows/pr-ci.yml

@@ -0,0 +1,27 @@
+name: Bors CI
+on:
+  push:
+    branches:
+      - staging
+      - trying
+
+jobs:
+  lint-ci:
+    uses: ./.github/workflows/lint.yml
+  int-ci:
+    uses: ./.github/workflows/unit-int.yml
+  bdd-ci:
+    uses: ./.github/workflows/bdd.yml
+  image-ci:
+    uses: ./.github/workflows/image-pr.yml
+  bors-ci:
+    if: ${{ success() }}
+    needs:
+      - lint-ci
+      - int-ci
+      - bdd-ci
+      - image-ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0

.github/workflows/unit-int.yml

@@ -0,0 +1,75 @@
+name: Integration CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  int-tests:
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ startsWith(github.ref_name, 'release/') || github.ref_name == 'develop' }}
+      - name: Build binaries
+        run: nix-shell --run "cargo build --bins --features=io-engine-testing"
+      - name: Setup Test Pre-Requisites
+        run: |
+          sudo sysctl -w vm.nr_hugepages=3584
+          sudo apt-get install linux-modules-extra-$(uname -r)
+          for module in nvme_tcp nbd nvme_rdma; do
+            sudo modprobe $module
+          done
+          sudo apt-get install gdb
+      - name: Run Rust Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/cargo-test.sh"
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/clean-cargo-tests.sh"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+      - name: Run JS Grpc Tests
+        run: |
+          echo "TEST_START_DATE=$(date +"%Y-%m-%d %H:%M:%S")" >> $GITHUB_ENV
+          nix-shell --run "./scripts/grpc-test.sh"
+          mkdir js-reports
+          for file in *-xunit-report.xml; do
+            echo "<testsuites>" > "js-reports/$file"
+            cat $file >> "js-reports/$file"
+            echo "</testsuites>" >> "js-reports/$file"
+          done
+      - name: Test Report
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: 'js-reports/*-xunit-report.xml'
+          summary: true
+          display-options: a
+          fail-on-empty: true
+          title: Test results
+      - name: Cleanup
+        if: always()
+        run: nix-shell --run "./scripts/clean-cargo-tests.sh"
+      - name: Check Coredumps
+        run: sudo ./scripts/check-coredumps.sh --since "${TEST_START_DATE}"
+# debugging
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   timeout-minutes: 240
+      #   uses: mxschmitt/action-tmate@v3

io-engine-tests/src/lib.rs

@@ -342,7 +342,7 @@ pub fn mount_and_get_md5(device: &str) -> Result<String, String> {
 pub fn fio_run_verify(device: &str) -> Result<String, String> {
     let (exit, stdout, stderr) = run_script::run(
         r"
-        fio --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
+        $FIO --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
         --runtime=5 --bs=4k --verify=crc32 --group_reporting=1 --output-format=terse \
         --verify_fatal=1 --verify_async=2 --filename=$1
     ",
@@ -511,11 +511,11 @@ pub async fn wait_for_rebuild(dst_uri: String, state: RebuildState, timeout: Dur
 pub fn fio_verify_size(device: &str, size: u64) -> i32 {
     let (exit, stdout, stderr) = run_script::run(
         r"
-        fio --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
+        $FIO --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
         --direct=1 --ioengine=libaio --rw=randwrite --verify=crc32 \
         --verify_fatal=1 --name=write_verify --filename=$1 --size=$2
 
-        fio --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
+        $FIO --thread=1 --numjobs=1 --iodepth=16 --bs=512 \
         --direct=1 --ioengine=libaio --verify=crc32 --verify_only \
         --verify_fatal=1 --name=verify --filename=$1
     ",

io-engine/.cargo/runner.sh

@@ -4,7 +4,7 @@
 ARGS="${@}"
 
 if [[ $EUID -ne 0 ]]; then
-  MAYBE_SUDO='sudo -E'
+  MAYBE_SUDO='sudo -E --preserve-env=PATH'
 else
   MAYBE_SUDO=''
 fi

io-engine/tests/ftl_mount_fs.rs

@@ -97,7 +97,7 @@ async fn ftl_mount_fs_multiple() {
 pub fn csal_fio_run_verify(device: &str) -> Result<String, String> {
     let (exit, stdout, stderr) = run_script::run(
         r#"
-        fio --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
+        $FIO --name=randrw --rw=randrw --ioengine=libaio --direct=1 --time_based=1 \
         --runtime=10 --bs=64k --verify=crc32 --group_reporting=1  \
         --verify_fatal=1 --verify_async=2 --filename=$1
         "#,

libnvme-rs/src/nvme_uri.rs

@@ -347,6 +347,12 @@ impl NvmeTarget {
     }
 }
 
+impl Drop for NvmeTarget {
+    fn drop(&mut self) {
+        self.disconnect().ok();
+    }
+}
+
 #[test]
 fn nvme_parse_uri() {
     let target = NvmeTarget::try_from("nvmf://1.2.3.4:1234/testnqn.what-ever.foo").unwrap();

scripts/grpc-test.sh

@@ -16,7 +16,7 @@ set -euxo pipefail
 export PATH="$PATH:${HOME}/.cargo/bin"
 export npm_config_jobs=$(nproc)
 
-cargo build --all
+cargo build --bins --features=io-engine-testing
 cd "$(dirname "$0")/../test/grpc"
 npm install --legacy-peer-deps
 

test/python/cross-grpc-version/nexus/docker-compose.yml

@@ -8,15 +8,15 @@ services:
     container_name: "ms0"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.2
+      - MY_POD_IP=10.1.0.2
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.2
+        ipv4_address: 10.1.0.2
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -36,15 +36,15 @@ services:
     container_name: "ms1"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.3
+      - MY_POD_IP=10.1.0.3
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.3
+        ipv4_address: 10.1.0.3
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -66,4 +66,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: "10.0.0.0/16"
+        - subnet: "10.1.0.0/16"

test/python/cross-grpc-version/pool/docker-compose.yml

@@ -8,15 +8,15 @@ services:
     container_name: "ms0"
     image: rust:latest
     environment:
-      - MY_POD_IP=10.0.0.2
+      - MY_POD_IP=10.1.0.2
       - NEXUS_NVMF_ANA_ENABLE=1
       - NEXUS_NVMF_RESV_ENABLE=1
       - PATH=${LLVM_SYMBOLIZER_DIR:-}
       - ASAN_OPTIONS=detect_leaks=0
     command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock --env-context=--iova-mode=pa
     networks:
       mayastor_net:
-        ipv4_address: 10.0.0.2
+        ipv4_address: 10.1.0.2
     cap_add:
       # NUMA related
       - SYS_ADMIN
@@ -38,4 +38,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: "10.0.0.0/16"
+        - subnet: "10.1.0.0/16"