Merge pull request cyclus#1637 from cyclus/downstream-testing

CI Workflow Refactoring

.github/workflows/build_test.yml

@@ -1,27 +1,32 @@
-name: Build/Test for PR and collaborator push
+name: Build/Test Cyclus
 
 on:
   # allows us to run workflows manually
   workflow_dispatch:
   pull_request:
     paths-ignore:
-      - '.github/workflows/build_test_publish.yml'
+      - '.github/workflows/publish_latest.yml'
+      - '.github/workflows/publish_release.yml'
       - '.github/workflows/changelog_test.yml'
-      - 'docker/**'
+      - '.github/workflows/code_coverage.yml'
       - 'doc/**'
       - 'CHANGELOG.rst'
   push:
     paths-ignore:
-      - '.github/workflows/build_test_publish.yml'
+      - '.github/workflows/publish_latest.yml'
+      - '.github/workflows/publish_release.yml'
       - '.github/workflows/changelog_test.yml'
-      - 'docker/**'
+      - '.github/workflows/code_coverage.yml'
       - 'doc/**'
       - 'CHANGELOG.rst'
 
 jobs:
-  build-and-test:
+  build-test-cyclus:
     runs-on: ubuntu-latest
-
+    permissions:
+      contents: read
+      packages: write
+      pull-requests: write
     strategy:
       fail-fast: false
       matrix:
@@ -30,77 +35,95 @@ jobs:
           22.04,
         ]
         pkg_mgr : [
-            apt,
-            conda,
+          apt,
+          conda,
         ]
-
-    container:
-      image: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/${{ matrix.pkg_mgr }}-deps
 
     steps:
-      - name: Checkout and Build Cyclus
-        uses: actions/checkout@v3
-
-      - name: Building Cyclus
-        run: |
-          mkdir -p `python -m site --user-site`
-          python install.py -j 2 --build-type=Release --core-version 999999.999999 --allow-milps --code_coverage
-          echo "PATH=${HOME}/.local/bin:$PATH" >> "$GITHUB_ENV"
-          echo "LD_LIBRARY_PATH=${HOME}/.local/lib:${HOME}/.local/lib/cyclus:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install lcov for apt
-        if: matrix.pkg_mgr == 'apt'
-        run: |
-          apt update --fix-missing && apt install -y lcov curl
-          echo "GCOV=/usr/bin/gcov" >> "$GITHUB_ENV"
-      
-      - name: Install lcov for conda
-        if: matrix.pkg_mgr == 'conda'
-        run: |
-            mamba install -c conda-forge -y lcov curl
-            echo "GCOV=/opt/conda/bin/x86_64-conda-linux-gnu-gcov" >> "$GITHUB_ENV"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Initial Coverage Report
-        run: |
-          lcov -c --rc lcov_branch_coverage=1 --initial -d ${PWD}/build/src -o ${HOME}/initial_coverage.info --gcov-tool ${{ env.GCOV }}
+      - name: Checkout Cyclus
+        uses: actions/checkout@v4
 
-      - name: Cyclus Unit Tests
-        run: |
-          cyclus_unit_tests
+      - name: Build and Test Cyclus
+        id: build-cyclus
+        uses: docker/build-push-action@v5
+        with:
+          file: docker/Dockerfile
+          cache-from: type=registry,ref=ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cyclus:ci-layer-cache
+          cache-to: type=registry,ref=ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cyclus:ci-layer-cache,mode=max
+          tags: ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cyclus:ci-image-cache
+          push: true
+          build-args: |
+            pkg_mgr=${{ matrix.pkg_mgr }}
+            ubuntu_version=${{ matrix.ubuntu_versions }}
 
-      - name: Cyclus Python Tests
-        run: |
-          cd tests && python -m pytest
+      - name: Checkout Cycamore
+        if: ${{ github.event_name == 'pull_request' && steps.build-cyclus.outcome == 'success' }}
+        uses: actions/checkout@v4
+        with:
+          repository: cyclus/cycamore
+          path: ${{ github.workspace }}/cycamore
 
-      - name: Unit Test Coverage Report
-        run: |
-          lcov -c --rc lcov_branch_coverage=1 -d ${PWD}/build/src -o ${HOME}/test_coverage.info --gcov-tool ${{ env.GCOV }}
-          lcov --add-tracefile ${HOME}/initial_coverage.info --add-tracefile  ${HOME}/test_coverage.info --rc lcov_branch_coverage=1 -o ${HOME}/total_coverage.info
-          lcov --remove ${HOME}/total_coverage.info -o ${HOME}/src_coverage.info --rc lcov_branch_coverage=1 "/usr/**" "/opt/**"
-          mkdir -p ${HOME}/cyclus_coverage/html
-          genhtml ${HOME}/src_coverage.info --output-directory ${HOME}/cyclus_coverage/html --branch-coverage
-          echo "artifactPath=${HOME}/cyclus_coverage/html/" >> $GITHUB_ENV
-     
-      - name: Upload Coverage HTML Artifacts
-        uses: actions/upload-pages-artifact@v2
+      - name: Build and Test Cycamore
+        id: build-cycamore
+        if: ${{ github.event_name == 'pull_request' && steps.build-cyclus.outcome == 'success' }}
+        continue-on-error: true
+        uses: docker/build-push-action@v5
         with:
-          name: cyclus-coverage-report-${{ matrix.ubuntu_versions }}-${{ matrix.pkg_mgr }}
-          path: ${{ env.artifactPath }}
+          context: ${{ github.workspace }}/cycamore
+          file: ${{ github.workspace }}/cycamore/docker/Dockerfile
+          cache-from: type=registry,ref=ghcr.io/cyclus/cycamore_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cycamore:ci-layer-cache
+          cache-to: type=registry,ref=ghcr.io/cyclus/cycamore_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cycamore:ci-layer-cache,mode=max
+          tags: ghcr.io/cyclus/cycamore_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cycamore:ci-image-cache
+          push: true
+          build-args: |
+            pkg_mgr=${{ matrix.pkg_mgr }}
+            ubuntu_version=${{ matrix.ubuntu_versions }}
+            cyclus_tag=ci-image-cache@${{ steps.build-cyclus.outputs.digest }}
 
-      - name: Coveralls
-        uses: coverallsapp/github-action@v2
+      - name: Checkout Cymetric
+        if: ${{ github.event_name == 'pull_request' && steps.build-cycamore.outcome == 'success' }}
+        uses: actions/checkout@v4
         with:
-          flag-name: ${{ join(matrix.*, '-') }}
-          parallel: true
-          file: ${HOME}/src_coverage.info
-          format: lcov
-          compare-ref: main
+          repository: cyclus/cymetric
+          path: ${{ github.workspace }}/cymetric
 
-  finish-coveralls:
-    needs: build-and-test
-    runs-on: ubuntu-latest
-    steps:
-    - name: Coveralls Finished
-      uses: coverallsapp/github-action@v2
-      with:
-        parallel-finished: true
+      - name: Build and Test Cymetric
+        if: ${{ github.event_name == 'pull_request' && steps.build-cycamore.outcome == 'success' }}
+        id: build-cymetric
+        continue-on-error: true
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ github.workspace }}/cymetric
+          file: ${{ github.workspace }}/cymetric/docker/Dockerfile
+          cache-from: type=registry,ref=ghcr.io/cyclus/cymetric_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cymetric:ci-layer-cache
+          cache-to: type=registry,ref=ghcr.io/cyclus/cymetric_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cymetric:ci-layer-cache,mode=max
+          tags: ghcr.io/cyclus/cymetric_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cymetric:ci-image-cache
+          push: true
+          build-args: |
+            pkg_mgr=${{ matrix.pkg_mgr }}
+            ubuntu_version=${{ matrix.ubuntu_versions }}
+            cycamore_tag=ci-image-cache@${{ steps.build-cycamore.outputs.digest }}
+  
+      - name: PR Comment
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          comment_tag: ${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}
+          message: |
+            ## Downstream build statuses using cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}
+            - Cycamore: ${{steps.build-cycamore.outcome == 'success' && '*Success* :white_check_mark:' || 
+                steps.build-cycamore.outcome == 'failure' && '**Failure** :x:' || 
+                '**Skipped due to upstream failure** :warning:'}}
+            - Cymetric: ${{steps.build-cymetric.outcome == 'success' && '*Success* :white_check_mark:' || 
+                steps.build-cymetric.outcome == 'failure' && '**Failure** :x:' || 
+                '**Skipped due to upstream failure** :warning:'}}

.github/workflows/changelog_test.yml

@@ -22,7 +22,7 @@ jobs:
           git --version
 
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - run: |
           git config --global --add safe.directory ${GITHUB_WORKSPACE}

.github/workflows/code_coverage.yml

@@ -0,0 +1,71 @@
+name: Code Coverage
+
+on:
+  # allows us to run workflows manually
+  workflow_dispatch:
+  pull_request:
+    paths-ignore:
+      - '.github/workflows/build_test.yml'
+      - '.github/workflows/publish_latest.yml'
+      - '.github/workflows/publish_release.yml'
+      - '.github/workflows/changelog_test.yml'
+      - 'doc/**'
+      - 'CHANGELOG.rst'
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu_versions : [
+          22.04,
+        ]
+        pkg_mgr : [
+          apt,
+        ]
+
+    name: Build, Test, Report Coverage
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Cyclus
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build, Test, and Report Coverage
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=registry,ref=ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cyclus-coverage:ci-layer-cache
+          cache-to: type=registry,ref=ghcr.io/cyclus/cyclus_${{ matrix.ubuntu_versions }}_${{ matrix.pkg_mgr }}/cyclus-coverage:ci-layer-cache,mode=max
+          file: docker/Dockerfile
+          target: coverage-report
+          outputs: type=local,dest=.
+          build-args: |
+            pkg_mgr=${{ matrix.pkg_mgr }}
+            ubuntu_version=${{ matrix.ubuntu_versions }}
+
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          file: ${{ github.workspace }}/total_coverage.info
+          format: lcov
+          compare-ref: main
+
+      - name: Upload Coverage HTML Artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          name: cyclus-coverage-report-${{ matrix.ubuntu_versions }}-${{ matrix.pkg_mgr }}
+          path: ${{ github.workspace }}/html
+