Update dockerfiles to Debian base image (IntelLabs#120)

cwlacewe · Jul 14, 2023 · 45de5d9 · 45de5d9
1 parent 0794e15
commit 45de5d9
Show file tree

Hide file tree

Showing 16 changed files with 287 additions and 386 deletions.
diff --git a/.github/requirements.txt b/.github/requirements.txt
@@ -1,16 +1,24 @@
-certifi==2019.11.28
-chardet==3.0.4
-coverage==7.2.3
-Cython==0.29.34
+blinker==1.6.2
+click==8.1.5
 dbus-python==1.2.16
+Flask==2.3.2
 grpcio==1.40.0
 grpcio-tools==1.40.0
-idna==2.8
-numpy==1.24.2
+importlib-metadata==6.8.0
+imutils==0.5.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+numpy==1.25.1
+opencv-python==4.5.5.64
 protobuf==3.20.3
-PyGObject==3.36.0
-python-apt==2.0.1+ubuntu0.20.4.1
-requests==2.22.0
-requests-unixsocket==0.2.0
-six==1.14.0
-urllib3==1.25.8
+pycurl==7.43.0.6
+PyGObject==3.38.0
+python-apt==2.2.1
+pyzmq==25.1.0
+scipy==1.11.1
+six==1.16.0
+sk-video==1.1.10
+Werkzeug==2.3.6
+zipp==3.16.1
+zmq==0.0.0
diff --git a/.github/workflows/auto-formatter.sh b/.github/workflows/auto-formatter.sh
@@ -35,4 +35,4 @@ find "${REPO_DIR}" -type f -not -path "${REPO_DIR}/src/pmgd/*" \
 
 # Run Linter on Python Code
 check_package python 'black>=23.1.0'
-black ${REPO_DIR}/
+black ${REPO_DIR}/ --exclude="client/python/vdms/queryMessage_pb2.py"
diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml
@@ -64,12 +64,8 @@ jobs:
           fetch-depth: 0
 
       - if: matrix.coverage_type  == 'Source'
-        name: Format C++ Code (clang-format)
-        run: find "${PWD}" -type f -not -path "${PWD}/src/pmgd/*" -not -path "${PWD}/build/*" -regex '.*\.\(cc\|cpp\|h\|hpp\)' | xargs clang-format -i || true
-
-      - if: matrix.coverage_type  == 'Source'
-        name: Format Python Code (black code)
-        uses: DataDog/[email protected]
+        name: Format C++ Code (clang-format) and Python (black code)
+        run: ./.github/workflows/auto-formatter.sh
 
       - if: matrix.coverage_type  == 'Source'
         name: Check for modified files
@@ -79,15 +75,13 @@ jobs:
           echo "modify_source=$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT
           echo "added_modified=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} HEAD -- . ':!.github' ':!docker'| xargs)" >> $GITHUB_OUTPUT
 
-      - name: Build and Run Docker Container
+      - name: Build Docker Container
         run: |
           set -x
 
-          docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true
-          docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true
+          docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") | xargs docker rm || true
 
-          docker build --rm -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} .
-          # docker run --rm -d -v ${PWD}:/local_repo --name ${{ matrix.container_name }} ${{ matrix.container_tag }}
+          docker build --rm --build-arg="BUILD_COVERAGE=on" --build-arg="BUILD_COVERITY=on" -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} .
 
       - if: matrix.coverage_type  == 'Source' &&  steps.git_check.outputs.added_modified
         uses: ./.github/actions/coverity-incremental-scan
@@ -105,11 +99,10 @@ jobs:
         run: |
           set -x
           mkdir -p coverage
-          echo "${{ matrix.container_name }}"
+
           docker run --rm -d -v ${PWD}:/local_repo --name ${{ matrix.container_name }} \
             --env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-            --env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-            ${{ matrix.container_tag }}
+            --env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} ${{ matrix.container_tag }}
 
           docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh && cd / && ./run_coverage_py.sh"
 
@@ -148,7 +141,8 @@ jobs:
         run: |
           rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_REPOSITORY} || true
           rm -rf /tmp/tmp-* ${{ env.ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/*  || true
-          docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker stop || true
+
+          docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") | xargs docker rm || true
           docker rmi $(docker images | grep '<none>' | awk '{print $3}') || true
 
   compare_coverage:
@@ -209,10 +203,7 @@ jobs:
           token: ${{ secrets.FACELESS_TOKEN || github.token }}
 
       - if: needs.coverage_job.outputs.modify_source == 'true'
-        run: find "${PWD}" -type f -not -path "${PWD}/src/pmgd/*" -not -path "${PWD}/build/*" -regex '.*\.\(cc\|cpp\|h\|hpp\)' | xargs clang-format -i || true
-
-      - if: needs.coverage_job.outputs.modify_source == 'true'
-        uses: DataDog/[email protected]
+        run: ./.github/workflows/auto-formatter.sh
 
       # Update Code and Push (Should be last steps of workflow since it changes commit)
       - if: needs.coverage_job.outputs.modify_source == 'true'

diff --git a/.github/workflows/sdl_req.yml b/.github/workflows/sdl_req.yml
@@ -1,19 +1,19 @@
-# Uses docker/check-in/Dockerfile.base
-# Dockerfile.base -> Same as docker/base/Dockerfile but builds VDMS with local changes instead of external repo
+# Uses docker/check-in/Dockerfile without coverage or coverity
+# Same as docker/base/Dockerfile but builds VDMS with local changes instead of external repo
 name: SDL Requirements using Docker Image
 
-# Controls when the action will run. Triggers the workflow on push or pull request
+# Controls when the action will run. Triggers the workflow on push or pull request (for testing)
 # events but only for the master and develop branch
+on:
+  push:
+    branches:
+      - develop
 # on:
 #   pull_request:
 #     types: [ opened, edited, synchronize, reopened ]
 #     branches:
 #       - develop
 #       - master
-on:
-  push:
-    branches:
-      - develop
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -23,20 +23,42 @@ concurrency:
 env:
   ARTIFACT_DIR: SDL_artifacts
   DOCKER_ARTIFACT_DIR: Docker_artifacts
-  CHECKIN_DOCKERFILE: docker/check-in/Dockerfile.base
-  SNYK_TOKEN: ${{ secrets.SNYK_TOKEN}}
-  SNYK_API: ${{ secrets.SNYK_API}}
+  CHECKIN_DOCKERFILE: docker/check-in/Dockerfile
   # CHECKOUT_REF: ${{ github.event.pull_request.head.sha }}
   FACELESS_USERNAME: ${{ secrets.FACELESS_NAME}}
-  COVERITY_DOCKERFILE: docker/check-in/Dockerfile.coverity
   FACELESS_AUTHKEY: ${{ secrets.FACELESS_AUTHKEY}}
   COVERITYSTREAM: ${{ secrets.COVERITYSTREAM}}
   COVERITYSERVER: ${{ secrets.COVERITYSERVER }}
 
 jobs:
+  delete:
+      name: Remove old artifacts
+      runs-on:
+        group: intellabs-vdms-runners
+        labels: vdms-check-in
+      steps:
+        - uses: actions/github-script@v6
+          id: artifact
+          with:
+            # Delete all artifacts
+            script: |
+              const res = await github.rest.actions.listArtifactsForRepo({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+              })
+
+              res.data.artifacts
+                .forEach(({ id }) => {
+                  github.rest.actions.deleteArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: id,
+                  })
+                })
+
   # RUN HADOLINT & BANDIT; NO DOCKER BUILD NEEDED
-  # Check format of Dockerfile we will release (docker/base/Dockerfile)
   Hadolint:
+    # Check format of Dockerfile we will release (docker/base/Dockerfile)
     name: Haskell Dockerfile Linter
     needs: delete
     runs-on:
@@ -104,31 +126,8 @@ jobs:
           rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true
           rm -rf /tmp/tmp-* ${GITHUB_WORKSPACE}/*  || true
 
-  # BUILD LATEST CODE AS DOCKER IMAGE; USED WITH SNYK, CIS, & BDBA JOBS
-  delete:
-      name: Remove old artifacts
-      runs-on:
-        group: intellabs-vdms-runners
-        labels: vdms-check-in
-      steps:
-      - uses: actions/github-script@v6
-        id: artifact
-        with:
-          # Delete all artifacts
-          script: |
-            const res = await github.rest.actions.listArtifactsForRepo({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-            })
-
-            res.data.artifacts
-              .forEach(({ id }) => {
-                github.rest.actions.deleteArtifact({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  artifact_id: id,
-                })
-              })
+  # BUILD LATEST CODE AS DOCKER IMAGE
+  # USED WITH TRIVY, CIS, & BDBA JOBS
   BuildLatest:
     # This job builds docker container for later use
     name: Build Latest Docker
@@ -145,7 +144,8 @@ jobs:
       - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }}
       - name: Build Docker Container
         run: |
-          docker build --rm -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:latest .
+          docker build --rm --build-arg="BUILD_COVERAGE=off" --build-arg="BUILD_COVERITY=off" \
+            -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:latest .
           docker save -o ${{ env.DOCKER_ARTIFACT_DIR }}/vdms_latest.tar vdms:latest
       - name: Upload Docker Image Artifact
         if: success()
@@ -168,8 +168,6 @@ jobs:
       labels: vdms-check-in
     name: BDBA
     needs: BuildLatest
-    # container:
-    #   image: python:3.8-slim
     steps:
       - name: Download Docker Image
         uses: actions/download-artifact@v3
@@ -241,6 +239,7 @@ jobs:
       - name: Get Docker Image SBOM
         run: |
           docker sbom --format spdx-tag-value --output ${{ env.ARTIFACT_DIR }}/sbom_docker_CT36.txt vdms:latest
+
           python3 ${GITHUB_WORKSPACE}/docker/check-in/spdx2csv.py -i ${{ env.ARTIFACT_DIR }}/sbom_docker_CT36.txt \
             -o ${{ env.ARTIFACT_DIR }}/vdms_sbom_docker_CT36.csv
 
@@ -262,7 +261,6 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          docker stop snyk_py && docker rm snyk_py || true
           rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true
           rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/*  || true
 
@@ -324,7 +322,7 @@ jobs:
         # cf. https://github.com/actions/upload-artifact/issues/256
         if: always()
         run: |
-          docker stop vdms_test-CIS && docker rm vdms_test-CIS
+          docker stop vdms_test-CIS && docker rm vdms_test-CIS || true
           docker rmi $(docker images | grep '<none>' | awk '{print $3}') || true
           rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true
           rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/*  || true
@@ -343,16 +341,9 @@ jobs:
           # ref: ${{ env.CHECKOUT_REF }}
       - name: Build Docker Container with Coverity
         run: |
-          cp ${{ env.CHECKIN_DOCKERFILE}} ${{ env.COVERITY_DOCKERFILE}}
-          sed -i -e 's|CMD \["/start.sh"]|RUN mkdir /coverity \&\& cd /coverity \&\& \\|g' ${{ env.COVERITY_DOCKERFILE}}
-          echo "    curl -L -o cov-analysis-linux64-2023.3.0.sh  https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/cov-analysis-linux64-2023.3.0.sh && chmod +x cov-analysis-linux64-2023.3.0.sh && \\"  >> ${{ env.COVERITY_DOCKERFILE}}
-          echo "    curl -L -o license.dat https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/license.dat && \\"  >> ${{ env.COVERITY_DOCKERFILE}}
-          echo "    ./cov-analysis-linux64-2023.3.0.sh -q --installation.dir=/opt/coverity/analysis/ \\
-            --license.agreement=agree --license.region=0 --license.type.choice=0 \\
-            --license.cov.path=/coverity/license.dat --component.sdk=false --component.skip.documentation=true"  >> ${{ env.COVERITY_DOCKERFILE}}
-          echo "ENV PATH /opt/coverity/analysis/bin:$PATH" >> ${{ env.COVERITY_DOCKERFILE}}
-          echo 'CMD ["/start.sh"]' >> ${{ env.COVERITY_DOCKERFILE}}
-          docker build --rm -f ${{ env.COVERITY_DOCKERFILE}} -t vdms:coverity .
+          docker build --rm --build-arg="BUILD_COVERAGE=off" --build-arg="BUILD_COVERITY=on" \
+            -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:coverity .
+
       - name: Run Coverity with GCC
         env:
             DOCKER_PROXY_RUN_ARGS: "--env HTTPS_PROXY=$HTTPS_PROXY \
@@ -384,7 +375,7 @@ jobs:
         # cf. https://github.com/actions/upload-artifact/issues/256
         if: always()
         run: |
-          docker stop vdms_test-Coverity || true
+          docker stop vdms_test-Coverity && docker rm vdms_test-Coverity || true
           docker rmi $(docker images | grep '<none>' | awk '{print $3}') || true
           rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true
           rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/*  || true