Merge pull request #338 from helxplatform/develop

Sync to master
helxplatform · Jan 25, 2024 · 1f75acd · 1f75acd
2 parents 114c915 + 073aefa
commit 1f75acd
Show file tree

Hide file tree

Showing 46 changed files with 2,155 additions and 1,332 deletions.
diff --git a/.github/workflows/build-push-dev-image.yml b/.github/workflows/build-push-dev-image.yml
@@ -48,20 +48,20 @@ jobs:
     # https://github.com/marketplace/actions/build-and-push-docker-images
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -72,7 +72,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true

diff --git a/.github/workflows/build-push-release.yml b/.github/workflows/build-push-release.yml
@@ -18,7 +18,7 @@ on:
       - .dockerignore
       - .githooks
     tags-ignore:
-      - 'v[0-9]+.[0-9]+.*'
+      - '*'
 jobs:
   build-push-release:
     runs-on: ubuntu-latest
@@ -63,20 +63,20 @@ jobs:
     # step
     # https://github.com/marketplace/actions/build-and-push-docker-images
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -86,7 +86,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         push: true
         # Push to renci-registry and dockerhub here.

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -1,23 +1,27 @@
 # Workflow responsible for core acceptance testing.
 # Tests Currently Run:
 #     - flake8-linter
-#     - image-build-test
-#    
-# This workflow only validates images can build
-# but does not push images to any repository.
-#
+#     - PYTest
+#     - Bandit
+# For PR Vulnerability Scanning a separate workflow will run.
 # The build-push-dev-image and build-push-release workflows 
 # handle the develop and release image storage respectively.
 #
 #
 
 name: Code-Checks
 on:
-  push:
-    branches-ignore:
-      - master
-      - main
+  # push:
+  #   branches-ignore:
+  #     - master
+  #     - main
+  #     - develop
+  pull_request:
+    branches:
       - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
     paths-ignore:
       - README.md
       - .old_cicd/*
@@ -27,13 +31,6 @@ on:
       - .gitignore
       - .dockerignore
       - .githooks
-  pull_request:
-    branches:
-      - develop
-      - master
-      - main 
-    types: [ opened, synchronize ]
-
 
 jobs:
 ############################## flake8-linter ##############################
@@ -45,7 +42,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     # Currently actions/setup-python supports caching
     # but the cache is not as robust as cache action.
@@ -69,35 +66,6 @@ jobs:
       # flake8 --ignore=E,W --exit-zero . 
       continue-on-error: true
 
-############################## test-image-build ##############################
-  test-image-build:
-    # needs: flake8-linter
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-      with:
-        driver-opts: |
-          network=host
-
-    - name: Login to DockerHub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-        logout: true
-
-    # Notes on Cache: 
-    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
-    - name: Build Container
-      uses: docker/build-push-action@v4
-      with:
-        context: .
-        push: false
-        cache-from: type=registry,ref=${{ github.repository }}:buildcache
-        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
 ################################### PYTEST ###################################
   pytest:
     runs-on: ubuntu-latest
@@ -106,7 +74,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install Requirements
       run: |
@@ -116,8 +84,7 @@ jobs:
 
     - name: Test with pytest
       run: |
-        pytest --doctest-modules src
-        coverage run -m pytest tests/unit
+        make test
 
 ############################ Bandit ################################
   bandit:
@@ -127,7 +94,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install Requirements
       run: |
@@ -138,4 +105,48 @@ jobs:
     # Only report high security issues
     - name: Test with Bandit
       run: |
-        bandit -r src -n3 -lll
+        bandit -r src -n3 -lll
+
+############################## test-image-build ##############################
+  test-image-build:
+    runs-on: ubuntu-latest
+    # if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set short git commit SHA
+      id: vars
+      run: |
+        echo "short_sha=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_OUTPUT
+    # https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
+
+    - name: Confirm git commit SHA output
+      run: echo ${{ steps.vars.outputs.short_sha }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        logout: true
+
+    - name: Parse Github Reference Name
+      id: branch
+      run: |
+        REF=${{ github.ref_name }}
+        echo "GHR=${REF%/*}" >> $GITHUB_OUTPUT
+        
+    # Notes on Cache: 
+    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+    - name: Build Container
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: true
+        tags: |
+          ${{ github.repository }}:test_${{ steps.branch.outputs.GHR }}
+        cache-from: type=registry,ref=${{ github.repository }}:buildcache
+        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
diff --git a/.github/workflows/trivy-pr-scan.yml b/.github/workflows/trivy-pr-scan.yml
@@ -0,0 +1,68 @@
+
+name: trivy-pr-scan 
+on:
+  pull_request:
+    branches:
+      - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
+    paths-ignore:
+    - README.md
+    - .old_cicd/*
+    - .github/*
+    - .github/workflows/*
+    - LICENSE
+    - .gitignore
+    - .dockerignore
+    - .githooks
+
+jobs:
+ trivy-pr-scan:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: |
+          network=host
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        logout: true
+
+    # Notes on Cache: 
+    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+    - name: Build Container
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: false
+        load: true
+        tags: ${{ github.repository }}:vuln-test
+        cache-from: type=registry,ref=${{ github.repository }}:buildcache
+        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
+
+    # We will not be concerned with Medium and Low vulnerabilities
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: '${{ github.repository }}:vuln-test'
+        format: 'sarif'
+        severity: 'CRITICAL,HIGH'
+        ignore-unfixed: true
+        output: 'trivy-results.sarif'
+        exit-code: '1'
+    # Scan results should be viewable in GitHub Security Dashboard
+    # We still fail the job if results are found, so below will always run
+    # unless manually canceled.
+    - name: Upload Trivy scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v2
+      if: '!cancelled()'
+      with:
+        sarif_file: 'trivy-results.sarif'
diff --git a/Dockerfile b/Dockerfile
@@ -3,19 +3,23 @@
 # A container for the core semantic-search capability.
 #
 ######################################################
-FROM python:3.10.10-slim
+FROM python:3.12.1-alpine3.19
+
 
 # Install required packages
-RUN apt-get update && \
-    apt-get install -y curl make vim && \
-    rm -rf /var/cache/apt/*
+RUN apk update && \
+    apk add g++ make
+
+#upgrade openssl \
+RUN apk  add openssl=3.1.4-r4
 
+RUN pip install --upgrade pip
 # Create a non-root user.
 ENV USER dug
 ENV HOME /home/$USER
 ENV UID 1000
 
-RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER
+RUN adduser -D --home $HOME  --uid $UID $USER
 
 USER $USER
 WORKDIR $HOME
@@ -31,4 +35,4 @@ RUN make install
 RUN make install.dug
 
 # Run it
-ENTRYPOINT dug
+ENTRYPOINT dug
diff --git a/Makefile b/Makefile
@@ -40,8 +40,6 @@ install.dug:
 
 #test: Run all tests
 test:
-	# ${PYTHON} -m flake8 src
-	${PYTHON} -m pytest --doctest-modules src
 	coverage run -m pytest tests
 
 coverage:

diff --git a/README.md b/README.md
@@ -57,13 +57,13 @@ dug crawl tests/integration/data/test_variables_v1.0.csv -p "TOPMedTag"
 
 After crawling, you can search:
 ```shell
-dug search -q "heart attack" -t "concepts"
-dug search -q "heart attack" -t "variables" -k "concept=MONDO:0005068"
+dug search -q "vein" -t "concepts"
+dug search -q "vein" -t "variables" -k "concept=UBERON:0001638"
 ```
 
 You can also query Dug's REST API:
 ```shell
-query="`echo '{"index" : "concepts_index", "query" : "heart attack"}'`"
+query="`echo '{"index" : "concepts_index", "query" : "vein"}'`"
 
 curl --data "$query" \
      --header "Content-Type: application/json" \
@@ -290,3 +290,8 @@ TOPMed phenotypic concept data is [here](https://github.com/helxplatform/dug/tre
 ## Release
 
 To release, commit the change and select feature.
+
+#### Fail on Vulnerability Detection
+
+During PR's several vulnerability scanners are run. If there are vulnerabilities detected, the pr checks will fail and a report will be sent to Github Security Dashboard for viewing. Please ensure the vulnerability is mitigated prior to continuing the merge to protected branches.
+