From 725f8b770e4a686f532c74be9f90dc643706d233 Mon Sep 17 00:00:00 2001 From: Nathan Lee <113044550+nathandotleeathpe@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:21:54 -0600 Subject: [PATCH] Issue/11 - Create integration test stages and cleanup workflows (#22) Signed-off-by: Nathan Lee --- .github/workflows/build.yml | 37 +++++ .github/workflows/deploy.yml | 10 -- .github/workflows/integration-test.yml | 94 ++++++++++++ .../workflows/publish-integration-test.yml | 95 ++++++++++++ .github/workflows/publish-test.yml | 76 ---------- .github/workflows/publish-unit-test.yml | 138 ++++++++++++++++++ .github/workflows/pull_request.yml | 43 ++++++ .github/workflows/{test.yml => unit-test.yml} | 52 +++++-- .gitignore | 4 + Makefile | 2 + testsuite/integration/Dockerfile | 31 ++++ testsuite/integration/Makefile | 67 +++++++++ testsuite/integration/docker-compose.yml | 42 ++++++ testsuite/integration/kind/kind.sh | 81 ++++++++++ testsuite/integration/requirements.txt | 5 + .../integration/slurm/docker-compose.yml | 24 ++- testsuite/integration/slurm/jobs/test-bb.sh | 22 ++- .../src/features/test_environment.feature | 44 ++++++ testsuite/integration/src/pytest.ini | 21 +++ .../integration/src/tests/test_environment.py | 74 ++++++++++ testsuite/submodules/dws | 2 +- testsuite/submodules/slurm-docker-cluster | 2 +- testsuite/unit/bin/validate | 19 +++ testsuite/unit/luacov.lua | 19 +++ testsuite/unit/output.lua | 19 +++ tools/README.md | 138 ------------------ tools/kind.sh | 63 -------- 27 files changed, 924 insertions(+), 300 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/integration-test.yml create mode 100644 .github/workflows/publish-integration-test.yml delete mode 100644 .github/workflows/publish-test.yml create mode 100644 .github/workflows/publish-unit-test.yml create mode 100644 .github/workflows/pull_request.yml rename .github/workflows/{test.yml => unit-test.yml} (50%) create mode 100644 testsuite/integration/Dockerfile create mode 100644 testsuite/integration/Makefile create mode 100644 testsuite/integration/docker-compose.yml create mode 100755 testsuite/integration/kind/kind.sh create mode 100644 testsuite/integration/requirements.txt create mode 100644 testsuite/integration/src/features/test_environment.feature create mode 100644 testsuite/integration/src/pytest.ini create mode 100644 testsuite/integration/src/tests/test_environment.py delete mode 100644 tools/README.md delete mode 100755 tools/kind.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..f65c9c1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Build +on: + push: + +jobs: + unit-test: + uses: ./.github/workflows/unit-test.yml + publish-unit-test: + uses: ./.github/workflows/publish-unit-test.yml + needs: unit-test + integration-test: + uses: ./.github/workflows/integration-test.yml + needs: unit-test + publish-integration-test: + uses: ./.github/workflows/publish-integration-test.yml + needs: integration-test \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 7ee02bb..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json -name: Deploy -on: - push: - branches: - - "main" - -jobs: - unit-test: - uses: ./.github/workflows/test.yml \ No newline at end of file diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..5dfba16 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,94 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Integration Test +on: + workflow_call: + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + steps: + + # Publish event file if debug is enabled + - name: Publish Event File + uses: actions/upload-artifact@v3 + if: ${{ runner.debug }} + with: + name: integration-test-event-file + path: ${{ github.event_path }} + + # Needed to maintain branch scope for separate docker caches + - name: Get Branch + run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV + + - uses: actions/checkout@v2 + with: + submodules: recursive + + # Requireed for docker caching + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Pre-build slurm image with docker cache. This will also generate an + # inline cache used by the docker build in the "Integration Test" job. + # Docker caches are need to be image and branch scoped. The first build + # on new branches will need to build the image from scratch. Expect 10 + # minutes for a full slurm build. + - name: Build Slurm + uses: docker/build-push-action@v3 + with: + context: testsuite/submodules/slurm-docker-cluster + push: false + load: true + tags: slurm-bb:test + build-args: | + SLURM_TAG=slurm-22-05-4-1 + cache-from: type=gha,scope=slurm-${{ env.BRANCH }} + cache-to: type=gha,mode=max,scope=slurm-${{ env.BRANCH }} + + # Pre-build slurm image with docker cache. Expect 3 minutes for a full + # DWS build. + - name: Build DWS + uses: docker/build-push-action@v3 + with: + context: testsuite/submodules/dws + push: false + load: true + tags: local/dws-operator:test + cache-from: type=gha,scope=dws-${{ env.BRANCH }} + cache-to: type=gha,mode=max,scope=dws-${{ env.BRANCH }} + + - name: Integration Test + run: cd testsuite/integration && make setup test reports + + - name: Publish Test Results + uses: actions/upload-artifact@v3 + with: + name: integration-test-results + path: testsuite/integration/reports + + # The "Integration Test" step should never fail, otherwise the test + # reports will not be available. This step will check the integration + # test container to see if a failure occured. + - name: Check Integration Test + run: test $(docker inspect integration-test --format="{{.State.ExitCode}}") -eq 0 \ No newline at end of file diff --git a/.github/workflows/publish-integration-test.yml b/.github/workflows/publish-integration-test.yml new file mode 100644 index 0000000..80fe616 --- /dev/null +++ b/.github/workflows/publish-integration-test.yml @@ -0,0 +1,95 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See "publish-unit-test.yml" for information about actions in this workflow +name: Publish Unit Test +on: + workflow_call: + workflow_run: + workflows: [Pull Request] + types: + - completed + +jobs: + publish: + name: Publish + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Publish Event File + uses: actions/upload-artifact@v3 + if: ${{ runner.debug }} + with: + name: publish-integration-event-file + path: ${{ github.event_path }} + + - uses: actions/download-artifact@v3 + with: + name: integration-test-results + + - name: Publish Integration Test Results + uses: scacap/action-surefire-report@v1 + with: + report_paths: "**/*.junit.xml" + check_name: "Integration Test Report" + + publish_checkrun: + name: Publish Checkrun + if: ${{ github.event_name == 'workflow_run' }} + runs-on: ubuntu-latest + steps: + - name: Publish Event File + uses: actions/upload-artifact@v3 + if: ${{ runner.debug }} + with: + name: publish-integration-event-file + path: ${{ github.event_path }} + + - name: 'Download artifacts' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifactTestResults = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "integration-test-results" + })[0]; + let downloadTestResults = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifactTestResults.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/unit-test-results.zip`, Buffer.from(downloadTestResults.data)); + + - name: Unzip artifacts + run: unzip integration-test-results.zip + + - name: Publish Integration Test Results + uses: scacap/action-surefire-report@v1 + with: + report_paths: "**/*.junit.xml" + check_name: "Unit Test Report" + commit: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/publish-test.yml b/.github/workflows/publish-test.yml deleted file mode 100644 index 8d76197..0000000 --- a/.github/workflows/publish-test.yml +++ /dev/null @@ -1,76 +0,0 @@ -# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json -name: Publish Unit Test -on: - workflow_run: - workflows: [Unit Test] - types: - - completed - -jobs: - test: - name: Publish - runs-on: ubuntu-latest - steps: - - name: 'Download artifacts' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifactTestResults = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "unit-test-results" - })[0]; - let matchArtifactEventFile = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "event-file" - })[0]; - let downloadTestResults = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifactTestResults.id, - archive_format: 'zip', - }); - let downloadEventFile = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifactEventFile.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/unit-test-results.zip`, Buffer.from(downloadTestResults.data)); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/event.json.zip`, Buffer.from(downloadEventFile.data)); - - - - name: Unzip artifacts - run: unzip unit-test-results.zip && unzip event.json.zip - - - name: Publish Unit Test Results - uses: scacap/action-surefire-report@v1 - if: ( github.event.workflow_run.event == 'push' ) || ( github.event.repository.full_name != github.event.workflow_run.head_repository.full_name ) - with: - report_paths: "**/*.junit.xml" - check_name: "Unit Test Report" - commit: ${{ github.event.workflow_run.head_sha }} - - - name: Get PR Number - if: github.event.workflow_run.event == 'pull_request' - run: | - NUM="$(jq -r ".number" event.json)" - echo "TEST_PR_NUMBER=$NUM" >> $GITHUB_ENV - - - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@v2 - if: github.event.workflow_run.event == 'pull_request' - with: - path: code-coverage-results.md - number: ${{ env.TEST_PR_NUMBER }} - - - name: Publish Event File - uses: actions/upload-artifact@v2 - with: - name: event-file - path: ${{ github.event_path }} - - diff --git a/.github/workflows/publish-unit-test.yml b/.github/workflows/publish-unit-test.yml new file mode 100644 index 0000000..193ef7f --- /dev/null +++ b/.github/workflows/publish-unit-test.yml @@ -0,0 +1,138 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Publish Unit Test +on: + workflow_call: + workflow_run: + workflows: [Pull Request] + types: + - completed + +jobs: + # During a push event, the user will have the required permissions to create + # a checkrun. Since all workflows are included in the same run, the + # download-artifact action can be used to retrieve the test reports. + publish: + name: Publish + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Publish Event File + uses: actions/upload-artifact@v3 + if: ${{ runner.debug }} + with: + name: publish-unit-event-file + path: ${{ github.event_path }} + + - uses: actions/download-artifact@v3 + with: + name: unit-test-results + + - name: Publish Unit Test Results + uses: scacap/action-surefire-report@v1 + with: + report_paths: "**/*.junit.xml" + check_name: "Unit Test Report" + + # During a workflow_run event, pull requests coming from a forked repository + # will not have the required permissions to create a checkrun or post the + # code coverage summary comment. Github actions will trigger this workflow + # from the main branch and run the workflow on behalf of the pull request. + publish_checkrun: + name: Publish Checkrun + if: ${{ github.event_name == 'workflow_run' }} + runs-on: ubuntu-latest + steps: + - name: Publish Event File + uses: actions/upload-artifact@v2 + if: ${{ runner.debug }} + with: + name: publish-unit-event-file + path: ${{ github.event_path }} + + # Since this workflow is invoked by a "workflow_run" trigger, the + # "download-artifact" action will not find the test results. The + # "github-script" action provides a handy javascript environment + # with access to a github actions client and basic packages. This + # is used to query for the triggering workflow and download the + # test results and event file. + - name: 'Download artifacts' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifactTestResults = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "unit-test-results" + })[0]; + let matchArtifactEventFile = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "unit-test-event-file" + })[0]; + let downloadTestResults = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifactTestResults.id, + archive_format: 'zip', + }); + let downloadEventFile = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifactEventFile.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/unit-test-results.zip`, Buffer.from(downloadTestResults.data)); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/event.json.zip`, Buffer.from(downloadEventFile.data)); + + - name: Unzip artifacts + run: unzip unit-test-results.zip && unzip event.json.zip + + # This action creates a checkrun that publishes the junit test report in + # the first workflow run in the commit. This should only be the + # "Pull Request" workflow, but could end up being the "Build" workflow + # if the PR is made from a branch within the same repository + - name: Publish Unit Test Results + uses: scacap/action-surefire-report@v1 + with: + report_paths: "**/*.junit.xml" + check_name: "Unit Test Report" + commit: ${{ github.event.workflow_run.head_sha }} + + # Get the PR number to post the code coverage summary comment to the + # correct PR. The triggering workflow's event file is the only way to + # retrieve the PR number in this context. The PR number is saved to the + # "GITHUB_ENV" environment variable so it can be used below. + - name: Get PR Number + if: github.event.workflow_run.event == 'pull_request' + run: | + NUM="$(jq -r ".number" event.json)" + echo "TEST_PR_NUMBER=$NUM" >> $GITHUB_ENV + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event.workflow_run.event == 'pull_request' + with: + path: code-coverage-results.md + number: ${{ env.TEST_PR_NUMBER }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..3a82324 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,43 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# PRs from forked repos will not have the permissions to invoke check runs +# (i.e. publish test reports) or publish code coverage summary comments. +# Instead, the "Publish Integration Test" and "Publish Unit Test" workflows +# on the main branch will be activated using the "workflow_run" trigger. These +# workflows will publish the test reports and code coverage summary comment on +# behalf of the pull request run. +# +# NOTE: This workflow is designed to run on PRs from forked repositories and is +# not guaranteed to run perfectly in PRs from branches in the same repository. +# In particular, test reports from check runs will be written to the wrong +# workflow summary, since both the "Build" and "Pull Request" workflows will +# run in the same repository. +name: Pull Request +on: + pull_request: + +jobs: + unit-test: + uses: ./.github/workflows/unit-test.yml + integration-test: + uses: ./.github/workflows/integration-test.yml + needs: unit-test \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/unit-test.yml similarity index 50% rename from .github/workflows/test.yml rename to .github/workflows/unit-test.yml index 28ac237..3e76092 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/unit-test.yml @@ -1,14 +1,27 @@ # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Unit Test on: workflow_call: - pull_request: - branches: - - "main" - push: - branches: - - "**" - - "!main" jobs: test: @@ -19,15 +32,20 @@ jobs: with: submodules: recursive + # Required to use the "--output" option in the docker build - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + # The event file needs to be published during PRs so the + # "Publish Unit Test" workflow can look up the PR number. - name: Publish Event File - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 + if: ( github.event_name == 'pull_request' ) || runner.debug with: - name: event-file + name: unit-test-event-file path: ${{ github.event_path }} + # Run the tests and publish the junit report and raw coverage results - name: Unit Tests run: docker buildx build --target testresults @@ -35,12 +53,15 @@ jobs: --output test_artifacts . + # Upload junit report. If a test error occurs (e.g. bad Lua syntax) then + # the raw code coverage results won't be published. - name: Publish Test Results uses: actions/upload-artifact@v3 with: name: unit-test-results path: test_artifacts + # Process the raw code coverage results - name: Unit Test Artifacts run: docker buildx build --target testartifacts @@ -48,6 +69,9 @@ jobs: --output test_artifacts . + # Build the code coverage summary from the coverage results. These + # results still need to be uploaded, so the job shouldn't be allowed to + # fail yet - name: Analyze Coverage Results uses: irongut/CodeCoverageSummary@v1.3.0 with: @@ -61,24 +85,34 @@ jobs: output: both thresholds: '80 85' + # Add the coverage summary to the workflow summary. - name: Adding markdown run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY + # Include the coverage summary when uploading test artifacts. This makes + # the summary available when the "Publish Unit Test" workflow creates the + # code coverage summary PR comment - name: Move Coverage Report run: mv code-coverage-results.md test_artifacts/ + # Re-upload the test artifacts. This will overwrite the old artifacts, + # but the only delta should be the processed coverage results - name: Publish Coverage Results uses: actions/upload-artifact@v3 with: name: unit-test-results path: test_artifacts + # This stage of the docke build has a separate check for unit test + # failures, after the test artifacts have been collected - name: Fail on Test Failures run: docker buildx build --target test -t test . + # Re-run the coverage tool, this time failing the build if the coverage + # is too low. - name: Fail on Low Coverage uses: irongut/CodeCoverageSummary@v1.3.0 with: diff --git a/.gitignore b/.gitignore index 917c5d2..748588c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ tools/kind-config.yaml # Slurm Output testsuite/integration/jobs/slurm*.out + +# Kind files +testsuite/integration/kubeconfig +testsuite/integration/kind-config.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index 5519d2c..1d331bc 100644 --- a/Makefile +++ b/Makefile @@ -35,3 +35,5 @@ test-mocks: test-realk8s: MOCK_SLURM=yes REAL_K8S=yes busted $(TAG) $(OUTPUT_HANDLER) testsuite/unit/src/burst_buffer/dws-test.lua +integration-test: $(find testsuite/integration/src -type f) testsuite/integration/Dockerfile + cd testsuite/integration && make setup test clean diff --git a/testsuite/integration/Dockerfile b/testsuite/integration/Dockerfile new file mode 100644 index 0000000..3040c48 --- /dev/null +++ b/testsuite/integration/Dockerfile @@ -0,0 +1,31 @@ +FROM rockylinux:9 AS testbase + +COPY requirements.txt /requirements.txt + +RUN yum update -y && \ + yum in -y \ + dnf-plugins-core \ + python3 \ + python3-pip && \ + dnf check-update && \ + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ + dnf install -y docker-ce-cli && \ + curl -LO https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl && \ + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \ + pip install -r requirements.txt + +COPY src /tests + +RUN mkdir /reports + +FROM testbase AS test + +WORKDIR /tests + +CMD [ \ + "pytest", \ + "--junitxml=/reports/integration.junit.xml", \ + "--html=/reports/integration.html", \ + "--gherkin-terminal-reporter", \ + "-v" \ +] \ No newline at end of file diff --git a/testsuite/integration/Makefile b/testsuite/integration/Makefile new file mode 100644 index 0000000..a07b15a --- /dev/null +++ b/testsuite/integration/Makefile @@ -0,0 +1,67 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +SKIP_KIND_GENERATION?="" + +.PHONY: setup-kind +setup-kind: + @{ \ + set -e ; \ + source kind/kind.sh ; \ + test ! -z "$(SKIP_KIND_GENERATION)" && echo "Skipping cluster generation" || generate_cluster ; \ + install_dependencies ; \ + prep_kubeconfig ; \ + } + cd slurm && docker compose build && docker compose up --wait + docker network connect slurm_default dws-control-plane + +.PHONY: setup-dws +setup-dws: + @{\ + set -e ; \ + cd ../submodules/dws ; \ + docker buildx build -t local/dws-operator:test --cache-from="/tmp/.buildx-cache" --load . ; \ + IMAGE_TAG_BASE=local/dws-operator VERSION=test KIND_CLUSTER=dws make kind-push deploy ; \ + kubectl wait deployment --timeout=60s -n dws-operator-system dws-operator-controller-manager --for condition=Available=True ; \ + } + +.PHONY: setup +setup: setup-kind setup-dws + +.PHONY: test +test: + docker compose build + docker compose up + +.PHONY: reports +reports: + mkdir reports + docker cp -a integration-test:/reports/ . + +.PHONY: clean +clean: + docker compose down || echo "Integration test container cleanup failed" + docker network disconnect slurm_default dws-control-plane || echo "Docker network cleanup failed" + cd slurm && docker compose down || echo "Slurm cleanup failed" + source kind/kind.sh && teardown || echo "Kind cleanup failed" + +all: setup test \ No newline at end of file diff --git a/testsuite/integration/docker-compose.yml b/testsuite/integration/docker-compose.yml new file mode 100644 index 0000000..f270e12 --- /dev/null +++ b/testsuite/integration/docker-compose.yml @@ -0,0 +1,42 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "2.2" + +services: + integration-test: + image: local/integration-test:${TAG:-test} + build: + context: . + cache_from: + - "/tmp/.buildx-cache" + container_name: integration-test + privileged: true + hostname: integration-test + volumes: + - type: bind + source: ./slurm/jobs + target: /jobs + - ./kubeconfig:/root/.kube/config + - /var/run/docker.sock:/var/run/docker.sock + +networks: + default: + name: slurm_default + external: true \ No newline at end of file diff --git a/testsuite/integration/kind/kind.sh b/testsuite/integration/kind/kind.sh new file mode 100755 index 0000000..87e0681 --- /dev/null +++ b/testsuite/integration/kind/kind.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +generate_cluster () { + set -e + + CONFIG=$(dirname $0)/kind-config.yaml + # Only write the config if it's not present. + if ! [[ -f $CONFIG ]] + then + # System Local Controllers (SLC) + SLCCONFIG=$(cat << EOF + + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: cray.wlm.manager=true +EOF + ) + + cat > $CONFIG <= 25.3 \ No newline at end of file diff --git a/testsuite/integration/slurm/docker-compose.yml b/testsuite/integration/slurm/docker-compose.yml index 0eec2c6..d25be32 100644 --- a/testsuite/integration/slurm/docker-compose.yml +++ b/testsuite/integration/slurm/docker-compose.yml @@ -1,3 +1,22 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + version: "2.2" services: @@ -19,6 +38,8 @@ services: context: ../../submodules/slurm-docker-cluster args: SLURM_TAG: ${SLURM_TAG:-slurm-22-05-4-1} + cache_from: + - "/tmp/.buildx-cache" command: ["slurmdbd"] container_name: slurmdbd hostname: slurmdbd @@ -42,8 +63,9 @@ services: target: /jobs - var_log_slurm:/var/log/slurm - type: bind - source: ../../../src/burst_buffer/burst_buffer.lua.example + source: ../../../src/burst_buffer/burst_buffer.lua target: /etc/slurm/burst_buffer.lua + - ../kubeconfig:/home/slurm/.kube/config expose: - "6817" depends_on: diff --git a/testsuite/integration/slurm/jobs/test-bb.sh b/testsuite/integration/slurm/jobs/test-bb.sh index d14e216..acf7e9e 100644 --- a/testsuite/integration/slurm/jobs/test-bb.sh +++ b/testsuite/integration/slurm/jobs/test-bb.sh @@ -1,6 +1,26 @@ #!/bin/sh + +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + #SBATCH --output=/jobs/slurm-%j.out -#BB_LUA pool=pool1 capacity=1K +#BB_LUA /bin/hostname srun -l /bin/hostname srun -l /bin/pwd diff --git a/testsuite/integration/src/features/test_environment.feature b/testsuite/integration/src/features/test_environment.feature new file mode 100644 index 0000000..bb00c3a --- /dev/null +++ b/testsuite/integration/src/features/test_environment.feature @@ -0,0 +1,44 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Feature: Integration test environment + Verify the integration test environment has been setup correctly + + Scenario: Kubernetes is available + When kubernetes cluster nodes are queried + Then one or more kubernetes nodes are available + + # Scenario: Using DataWorkflowServices + # When the DataWorkflowServices deployment is queried + # Then the DataWorkflowServices deployment is found + + # Scenario: Using Slurm + # Given a simple job script + # """ + # /bin/hostname + # srun -l /bin/hostname + # srun -l /bin/pwd + # """ + # When the job is run + # Then the job completes successfully + + Scenario: Kubernetes and slurm are connected + Given the kubernetes cluster kube-system UID + When the kube-system UID is queried from slurmctld + Then the UIDs match and the cluster is the same diff --git a/testsuite/integration/src/pytest.ini b/testsuite/integration/src/pytest.ini new file mode 100644 index 0000000..1899dfb --- /dev/null +++ b/testsuite/integration/src/pytest.ini @@ -0,0 +1,21 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[pytest] +bdd_features_base_dir = features \ No newline at end of file diff --git a/testsuite/integration/src/tests/test_environment.py b/testsuite/integration/src/tests/test_environment.py new file mode 100644 index 0000000..382e8fb --- /dev/null +++ b/testsuite/integration/src/tests/test_environment.py @@ -0,0 +1,74 @@ +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Integration test environment feature tests.""" + +import docker +import json +from kubernetes import client, config +import pytest +from pytest_bdd import ( + given, + scenarios, + then, + when, +) + +scenarios("test_environment.feature") + +@pytest.fixture +def k8s(): + config.load_kube_config() + v1 = client.CoreV1Api() + return v1 + +@pytest.fixture +def slurmctld(): + return docker.from_env().containers.get("slurmctld") + +@given('the kubernetes cluster kube-system UID', target_fixture="kube_system_uid") +def _(k8s): + """the kubernetes cluster kube-system UID.""" + ns_list = k8s.list_namespace(field_selector="metadata.name==kube-system") + assert len(ns_list.items) == 1, "kube-system namespace not found" + return ns_list.items[0].metadata.uid + +@when('kubernetes cluster nodes are queried', target_fixture="k8s_nodes") +def _(k8s): + """kubernetes cluster nodes are queried.""" + return k8s.list_node().items + +@when('the kube-system UID is queried from slurmctld', target_fixture="kube_system_uid_from_slurmctld") +def _(slurmctld): + """the kube-system UID is queried from slurmctld.""" + rc,out = slurmctld.exec_run(["kubectl", "get", "namespace","-o=json", "kube-system"], + user="slurm") + assert rc==0, "non-zero return code: \n" + str(out) + return json.loads(str(out, 'utf-8'))["metadata"]["uid"] + + +@then('one or more kubernetes nodes are available') +def _(k8s_nodes): + """one or more kubernetes nodes are available.""" + assert len(k8s_nodes) > 0 + +@then('the UIDs match and the cluster is the same') +def _(kube_system_uid, kube_system_uid_from_slurmctld): + """the UIDs match and the cluster is the same.""" + assert kube_system_uid == kube_system_uid_from_slurmctld \ No newline at end of file diff --git a/testsuite/submodules/dws b/testsuite/submodules/dws index abf44bb..ebb254e 160000 --- a/testsuite/submodules/dws +++ b/testsuite/submodules/dws @@ -1 +1 @@ -Subproject commit abf44bba3b47f43be7546e0fae010fb6b8bb0aba +Subproject commit ebb254e1c9d446ab9adf0271a8a44287487f286d diff --git a/testsuite/submodules/slurm-docker-cluster b/testsuite/submodules/slurm-docker-cluster index 3d26a21..53c4b87 160000 --- a/testsuite/submodules/slurm-docker-cluster +++ b/testsuite/submodules/slurm-docker-cluster @@ -1 +1 @@ -Subproject commit 3d26a213b78467ad0c688268bbe744de10755d43 +Subproject commit 53c4b8714bb5e3c3f8a9f67a12c36b6f4415cc63 diff --git a/testsuite/unit/bin/validate b/testsuite/unit/bin/validate index 298de45..2c03777 100755 --- a/testsuite/unit/bin/validate +++ b/testsuite/unit/bin/validate @@ -1,5 +1,24 @@ #! /usr/bin/python3 +# +# Copyright 2022 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import sys import yaml import io diff --git a/testsuite/unit/luacov.lua b/testsuite/unit/luacov.lua index a703264..80be1a8 100644 --- a/testsuite/unit/luacov.lua +++ b/testsuite/unit/luacov.lua @@ -1,3 +1,22 @@ +-- +-- Copyright 2022 Hewlett Packard Enterprise Development LP +-- Other additional copyright holders may be indicated within. +-- +-- The entirety of this work is licensed under the Apache License, +-- Version 2.0 (the "License"); you may not use this file except +-- in compliance with the License. +-- +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + -- This file will be copied to a ".luacov" file before testing with busted. -- -- Refer to the following link for more information on configuraing Luacov. diff --git a/testsuite/unit/output.lua b/testsuite/unit/output.lua index f7541a5..a3c8006 100644 --- a/testsuite/unit/output.lua +++ b/testsuite/unit/output.lua @@ -1,3 +1,22 @@ +-- +-- Copyright 2022 Hewlett Packard Enterprise Development LP +-- Other additional copyright holders may be indicated within. +-- +-- The entirety of this work is licensed under the Apache License, +-- Version 2.0 (the "License"); you may not use this file except +-- in compliance with the License. +-- +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + -- The path to this script is provided to Busted using the "-o/--output" flag, -- allowing busted to output execution results in the junit and terminal -- format. diff --git a/tools/README.md b/tools/README.md deleted file mode 100644 index 568fb35..0000000 --- a/tools/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Using KIND to create a Kubernetes environment for DWS - -See https://kind.sigs.k8s.io for KIND docs. - -## Install KIND - -```bash -$ brew install kind -``` - -## Start a cluster - -This will create a simple one-node k8s cluster that has a node label for DWS and has a cert-manager installed for the DWS webhook. - -```bash -$ tools/kind.sh -``` - -### Delete a cluster - -When your testing with DWS is complete you may delete the cluster. - -```bash -$ kind delete cluster -``` - -## Build DWS - -Build a container image for DWS. - -```bash -$ git clone git@github.com:HewlettPackard/dws.git -$ cd dws -$ make docker-build -``` - -## Deploy DWS in K8s - -Push the DWS container into the KIND environment. This relies on the node label we applied with the `tools/kind.sh` tool above. - -```bash -$ make kind-push -``` - -Install the DWS CRDs, service accounts, roles, rolebindings, deployment, and other necessary pieces. - -```bash -$ make deploy -``` - -Wait for DWS to be ready. - -```bash -$ kubectl get pods -n dws-operator-system -w -``` - -# The DWS API - -## State: Proposal - -For proposal state, the Lua script will create the initial workflow resource, with desiredState=proposal. The Lua script would add the BB_LUA lines to the dwDirectives array, and fill in the wlmID, jobID, userID, and groupID prior to submitting this resource. For now we're using an empty dwDirectives array because we don't have a mock conduit driver and ruleset. - -Create a basic Workflow resource. - -```bash -$ cat << END > silver_workflow_proposal.yaml -apiVersion: dws.cray.hpe.com/v1alpha1 -kind: Workflow -metadata: - name: silver -spec: - desiredState: "proposal" - dwDirectives: [] - wlmID: "23423-sdf0239-324lkj" - jobID: 100 - userID: 100 - groupID: 100 -END -``` - -```bash -$ kubectl apply -f silver_workflow_proposal.yaml -workflow.dws.cray.hpe.com/silver created -``` - -The Lua script must wait and confirm that the workflow completes the proposal state. We're using an empty dwDirectives array, so DWS will automatically transition this to a completed state. If we had a dwDirectives array, with BB_LUA lines, then we'd expect a mock conduit driver and ruleset to transition this to a completed state for us. - -```bash -$ kubectl get workflow silver -NAME STATE READY STATUS AGE -silver proposal true Completed 39s - -$ kubectl get workflow silver -o jsonpath='{.status.status}' -Completed -``` - -To get current state so it can be compared to desiredState: - -```bash -$ kubectl get workflow silver -o jsonpath='desiredState={.spec.desiredState}{"\n"}currentState={.status.state}{"\n"}status={.status.status}{"\n"}' -desiredState=proposal -currentState=proposal -status=Completed -``` - -## State: Setup - -For setup state, the Lua script will first confirm that proposal state has completed, then it will patch the spec to changed the desiredState to setup. - -```bash -$ kubectl patch workflow silver --type=merge -p '{"spec":{"desiredState": "setup"}}' -workflow.dws.cray.hpe.com/silver patched -``` - -The Lua script must wait and confirm that the workflow completes the setup state. - -```bash -$ kubectl get workflow silver -NAME STATE READY STATUS AGE -silver setup true Completed 23m - -$ kubectl get workflow silver -o jsonpath='{.status.status}' -Completed -``` - -## State: others - -Same as above. - -## State: Teardown - -For teardown state there is an option to set the "hurry" flag. - -```bash -$ kubectl patch workflow silver --type=merge -p '{"spec":{"desiredState": "teardown","hurry":true}}' -workflow.dws.cray.hpe.com/silver patched -``` - diff --git a/tools/kind.sh b/tools/kind.sh deleted file mode 100755 index f8bfeba..0000000 --- a/tools/kind.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright 2022 Hewlett Packard Enterprise Development LP -# Other additional copyright holders may be indicated within. -# -# The entirety of this work is licensed under the Apache License, -# Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -CONFIG=$(dirname $0)/kind-config.yaml -# Only write the config if it's not present. -if ! [[ -f $CONFIG ]] -then - # System Local Controllers (SLC) - SLCCONFIG=$(cat << EOF - - kubeadmConfigPatches: - - | - kind: JoinConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: cray.wlm.manager=true -EOF -) - - cat > $CONFIG <