Using dws-test-driver for DWS state progression integration tests

Signed-off-by: Nathan Lee <[email protected]>

.github/workflows/build.yml

@@ -29,6 +29,7 @@ jobs:
   publish-unit-test:
     uses: ./.github/workflows/publish-unit-test.yml
     needs: unit-test
+    if: always()
   integration-test:
     uses: ./.github/workflows/integration-test.yml
     needs: unit-test

.github/workflows/integration-test.yml

@@ -66,8 +66,7 @@ jobs:
           cache-from: type=gha,scope=slurm-${{ env.BRANCH }}
           cache-to: type=gha,mode=max,scope=slurm-${{ env.BRANCH }}
 
-      # Pre-build slurm image with docker cache. Expect 3 minutes for a full
-      # DWS build.
+      # Pre-build dws image with docker cache. Expect 3 minutes for a full
       - name: Build DWS
         uses: docker/build-push-action@v3
         with:
@@ -78,6 +77,17 @@ jobs:
           cache-from: type=gha,scope=dws-${{ env.BRANCH }}
           cache-to: type=gha,mode=max,scope=dws-${{ env.BRANCH }}
 
+      # Pre-build dws-test-driver image with docker cache. Expect 2 minutes for a full
+      - name: Build dws-test-driver
+        uses: docker/build-push-action@v3
+        with:
+          context: testsuite/submodules/dws-test-driver
+          push: false
+          load: true
+          tags: local/dws-test-driver-operator:test
+          cache-from: type=gha,scope=dws-test-driver-${{ env.BRANCH }}
+          cache-to: type=gha,mode=max,scope=dws-test-driver-${{ env.BRANCH }}
+
       - name: Integration Test
         run: cd testsuite/integration && make setup test reports
 

.gitmodules

@@ -6,3 +6,6 @@
 	path = testsuite/submodules/slurm-docker-cluster
 	url = [email protected]:DataWorkflowServices/slurm-docker-cluster.git
         branch = master
+[submodule "testsuite/submodules/dws-test-driver"]
+	path = testsuite/submodules/dws-test-driver
+	url = [email protected]:DataWorkflowServices/dws-test-driver.git

src/burst_buffer/burst_buffer.lua

@@ -163,7 +163,7 @@ function DWS:initialize(wlmID, jobID, userID, groupID, dw_directives, labels)
 	else
 		dwd_block = "dwDirectives:\n"
 		for k, v in ipairs(dw_directives) do
-			dwd_block = dwd_block .. "  - " .. dw_directives[k] .. "\n"
+			dwd_block = dwd_block .. "  - \"" .. dw_directives[k] .. "\"\n"
 		end
 		yaml = string.gsub(yaml, "DWDIRECTIVES", dwd_block)
 	end
@@ -181,6 +181,7 @@ function DWS:save(fname)
 		local msg = "unable to open " .. fname
 		return false, msg
 	end
+	slurm.log_info(self.yaml)
 	f:write(self.yaml)
 	local rc = {f:close()}
 	-- Success or failure is a boolean in rc[1].

testsuite/integration/Dockerfile

@@ -14,8 +14,6 @@ RUN yum update -y && \
     install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \
     pip install -r requirements.txt
 
-COPY src /tests
-
 RUN mkdir /reports
 
 FROM testbase AS test
@@ -27,5 +25,5 @@ CMD [ \
     "--junitxml=/reports/integration.junit.xml", \
     "--html=/reports/integration.html", \
     "--gherkin-terminal-reporter", \
-    "-v" \
+    "-v", "." \
 ]

testsuite/integration/Makefile

@@ -39,13 +39,23 @@ setup-dws:
 	@{\
 		set -e ; \
 		cd ../submodules/dws ; \
-		docker buildx build -t local/dws-operator:test --cache-from="/tmp/.buildx-cache" --load . ; \
+		docker buildx build -t local/dws-operator:test --load . ; \
 		IMAGE_TAG_BASE=local/dws-operator VERSION=test KIND_CLUSTER=dws make kind-push deploy ; \
 		kubectl wait deployment --timeout=60s -n dws-operator-system dws-operator-controller-manager --for condition=Available=True ; \
 	}
 
+.PHONY: setup-dws-test-driver
+setup-dws-test-driver:
+	@{\
+		set -e ; \
+		cd ../submodules/dws-test-driver ; \
+		docker buildx build -t local/dws-test-driver-operator:test --load . ; \
+		IMAGE_TAG_BASE=local/dws-test-driver-operator VERSION=test KIND_CLUSTER=dws make kind-push deploy ; \
+		kubectl wait deployment --timeout=60s -n dws-operator-system dws-test-driver-controller-manager --for condition=Available=True ; \
+	}
+
 .PHONY: setup
-setup: setup-kind setup-dws
+setup: setup-kind setup-dws setup-dws-test-driver
 
 .PHONY: test
 test:

testsuite/integration/docker-compose.yml

@@ -35,6 +35,7 @@ services:
         target: /jobs
       - ./kubeconfig:/root/.kube/config
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./src/:/tests
 
 networks:
   default:

testsuite/integration/kind/kind.sh

@@ -58,6 +58,9 @@ install_dependencies () {
   # Make sure the current context is set to dws
   kubectl config use-context kind-dws
 
+  # Create the slurm namespace. This will be the default location of dws-slurm-bb-plugin workflows
+  kubectl create namespace slurm
+
   # Install the cert-manager for the DWS webhook.
   kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.7.0/cert-manager.yaml
 
@@ -70,8 +73,9 @@ prep_kubeconfig () {
   set -e
   cp ~/.kube/config kubeconfig
   yq -i e '(.clusters | map(select(.name=="kind-dws")))[0].cluster.server |= "https://dws-control-plane:6443"' kubeconfig
-  yq -i e '.current-context |= "kind-dws"' kubeconfig
   chmod a+r kubeconfig
+  KUBECONFIG=kubeconfig kubectl config use-context kind-dws
+  KUBECONFIG=kubeconfig kubectl config set-context --current --namespace=slurm
 }
 
 teardown () {

testsuite/integration/requirements.txt

@@ -2,4 +2,5 @@ pytest == 7
 pytest-bdd == 6
 pytest-html == 3
 docker == 6
-kubernetes >= 25.3
+kubernetes >= 25.3
+tenacity == 8

testsuite/integration/slurm/jobs/integration-test-bb.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+#
+# Copyright 2022 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#SBATCH --output=/jobs/slurm-%j.out
+#DW Proposal action=complete
+#DW DataIn action=wait
+/bin/hostname
+srun -l /bin/hostname
+srun -l /bin/pwd

testsuite/integration/src/features/test_dws_states.feature

@@ -0,0 +1,117 @@
+#
+# Copyright 2022 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Feature: Data Workflow Services State Progression
+    Verify that the DWS-Slurm Burst Buffer Plugin progresses through Data
+    Workflow Services states
+
+    Scenario: The DWS-BB Plugin progresses through DWS states
+        Given a job script:
+            #!/bin/bash
+
+            #DW Proposal action=wait
+            #DW Setup action=wait
+            #DW DataIn action=wait
+            #DW PreRun action=wait
+            #DW PostRun action=wait
+            #DW DataOut action=wait
+            #DW Teardown action=wait
+            /bin/hostname
+
+        When the job is run
+        And a Workflow is created for the job
+        #Then the job's temporary Workflow is not found
+        Then the Workflow and job progress to the Proposal state
+        And the Workflow and job progress to the Setup state
+        And the Workflow and job progress to the DataIn state
+        And the Workflow and job progress to the PreRun state
+        And the Workflow and job progress to the PostRun state
+        And the Workflow and job progress to the DataOut state
+        And the Workflow and job progress to the Teardown state
+        And the job completes successfully
+
+    @todo
+    Scenario: The DWS-BB Plugin can handle DWS driver errors
+        Given a job script:
+            #!/bin/bash
+
+            #DW <state> action=error message=TEST_ERROR
+            #DW Teardown action=wait
+            /bin/hostname
+
+        When the job is run
+        And a Workflow is created for the job
+        Then the Workflow and job progress to the Teardown state
+        And the job shows an error with message "TEST ERROR"
+
+        Examples:
+            # *** HEADER ***
+            | state    |
+            # *** VALUES ***
+            | Proposal |
+            | Setup    |
+            | DataIn   |
+            | PreRun   |
+            | PostRun  |
+            | DataOut  |
+
+    @todo
+    Scenario: The DWS-BB Plugin can handle a DWS driver error during Teardown
+        Given a job script:
+            #!/bin/bash
+
+            #DW Teardown action=error message=TEST_ERROR
+            /bin/hostname
+
+        When the job is run
+        And a Workflow is created for the job
+        Then the job shows an error with message "TEST ERROR"
+
+    @todo
+    Scenario: The DWS-BB Plugin can cancel jobs
+        Given a job script:
+            #!/bin/bash
+
+            #DW <state> action=wait
+            #DW Teardown action=wait
+            /bin/hostname
+
+        When the job is run
+        And a Workflow is created for the job
+        And the Workflow and job progress to the <state> state
+        And the job is canceled with the hurry flag set to <hurry_flag>
+        Then the Workflow and job progress to the Teardown state
+        And the Workflow's hurry flag is set to <hurry_flag>
+
+        Examples:
+            # *** HEADER ***
+            | state    | hurry_flag |
+            # *** VALUES ***
+            | Proposal | false      |
+            | Setup    | false      |
+            | DataIn   | false      |
+            | PreRun   | false      |
+            | PostRun  | false      |
+            | DataOut  | false      |
+            | Proposal | true       |
+            | Setup    | true       |
+            | DataIn   | true       |
+            | PreRun   | true       |
+            | PostRun  | true       |
+            | DataOut  | true       |

testsuite/integration/src/features/test_environment.feature

@@ -24,19 +24,18 @@ Feature: Integration test environment
         When kubernetes cluster nodes are queried
         Then one or more kubernetes nodes are available
 
-    # Scenario: Using DataWorkflowServices
-    #     When the DataWorkflowServices deployment is queried
-    #     Then the DataWorkflowServices deployment is found
+    Scenario: The DataWorkflowServices deployment exists
+        When the DataWorkflowServices deployment is queried
+        Then the DataWorkflowServices deployment is found
 
-    # Scenario: Using Slurm
-    #     Given a simple job script
-    #         """
-    #         /bin/hostname
-    #         srun -l /bin/hostname
-    #         srun -l /bin/pwd
-    #         """
-    #     When the job is run
-    #     Then the job completes successfully
+    Scenario: Slurm is usable
+        Given a job script:
+            #!/bin/bash
+            /bin/hostname
+            srun -l /bin/hostname
+            srun -l /bin/pwd
+        When the job is run
+        Then the job completes successfully
 
     Scenario: Kubernetes and slurm are connected
         Given the kubernetes cluster kube-system UID

testsuite/integration/src/tests/conftest.py

@@ -0,0 +1,79 @@
+#
+# Copyright 2022 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import pytest
+import secrets
+
+from .slurmctld import Slurmctld
+from kubernetes import client, config
+from pytest_bdd import (
+    given,
+    scenarios,
+    parsers,
+    then,
+    when,
+)
+
+@pytest.fixture
+def k8s():
+    config.load_kube_config()
+    return client
+
+@pytest.fixture
+def slurmctld():
+    return Slurmctld()
+
+def pytest_bdd_apply_tag(tag, function):
+    if tag == 'todo':
+        marker = pytest.mark.skip(reason="Not implemented yet")
+        marker(function)
+        return True
+    else:
+        # Fall back to the default behavior of pytest-bdd
+        return None
+
+@given(parsers.parse('a job script:\n{script}'), target_fixture="script_path")
+def _(script):
+    """a simple job script: <script>"""
+    path = "/jobs/" + secrets.token_hex(5) + "-job.sh"
+    with open(path, "w") as file:
+        file.write(script)
+
+    yield path
+
+    os.remove(path)
+
+@when('the job is run', target_fixture="jobId")
+def _(slurmctld, script_path):
+    """the job is run."""
+    jobId,outputFilePath = slurmctld.submit_job(script_path)
+    print("submitted job: " + str(jobId))
+
+    yield jobId
+
+    # remove the slurm output from the jobs folder
+    slurmctld.remove_job_output(outputFilePath)
+
+@then('the job completes successfully')
+def _(slurmctld, jobId):
+    """the job completes successfully."""
+    state = slurmctld.get_final_job_state(jobId)
+    assert state=="COMPLETED", "Unexpected Job State: " + state
+