refactor the integration of blackduck and self-hosted runner

.github/workflows/blossom-ci.yml

@@ -13,20 +13,100 @@
 # limitations under the License.
 
 # A workflow to trigger blossom-CI on self-hosted runner
-name: blossom-ci trigger
-
+name: Blossom-CI
 on:
   issue_comment:
     types: [created]
-    
+
 jobs:
-  build:
+  authorization:
+    name: Authorization
+    # trigger on pre-defined text
     if: github.event.comment.body == 'build'
     runs-on: [self-hosted, linux, blossom]
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          VERSION: '1'
+
+  vulnerability-scan-job:
+    name: Vulnerability scan job
+    needs: [authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get pull request data
+        id: pull_request_data
+        uses: octokit/[email protected]
+        with:
+          route: GET /repos/:repository/pulls/:issue_id
+          repository: ${{ github.repository }}
+          issue_id: ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Set blackduck project version
+        id: blackduck-project-version
+        run: echo "${{ fromJson(steps.pull_request_data.outputs.data).head.ref }}-${{ github.run_id }}"
+
+      - name: Update status
+        uses: octokit/[email protected]
+        with:
+          route: POST /repos/:repository/statuses/:sha
+          repository: ${{ github.repository }}
+          sha: ${{ fromJson(steps.pull_request_data.outputs.data).head.sha }}
+          target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          description: "vulnerability scan running"
+          state: "pending"
+          context: "blossom-ci"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ fromJson(steps.pull_request_data.outputs.data).head.ref }}
+          lfs: 'true'
+
+      - name: Setup java
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Maven install
+        run: mvn clean install -DskipTests -T 1.5C
+
+      - name: Add mask
+        run: echo "::add-mask::${{ secrets.BLACKDUCK_URL }}"
+
+      - name: Run synopsys detect
+        id: scan_result
+        uses: blackducksoftware/[email protected]
+        env:
+          PROJECTS: ${{ env.projects }}
+        with:
+          args: >
+            --blackduck.url="https://${{ secrets.BLACKDUCK_URL }}"
+            --blackduck.api.token="${{ secrets.BLACKDUCK_API_TOKEN }}"
+            --detect.force.success=false
+            --detect.parallel.processors=0
+            --detect.project.name="${{ github.repository }}"
+            --detect.project.version.name="${{ github.run_id }}"
+
+  vulnerability-check-trigger:
+    name: Vulnerability check & start ci job
+    needs: [vulnerability-scan-job]
+    runs-on: [self-hosted, linux, blossom]
     steps:
-    - name: trigger blossom ci
-      run: blossom-ci
-      env:
-        CI_SERVER: ${{ secrets.CI_SERVER }}
-        REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check for new issue in vulnerability scan & start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'SCAN-CHECK-CI-JOB-START'
+          VERSION: '1'
+          BLACKDUCK_TOKEN: "${{ secrets.BLACKDUCK_API_TOKEN }}"
+          BLACKDUCK_URL: "${{ secrets.BLACKDUCK_URL }}"
+          BLACKDUCK_PROJECT_VERSION: "${{ github.run_id }}"
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/signoff-check.yml

@@ -0,0 +1,34 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to check if PR got sign-off
+name: signoff check
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  signoff-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: sigoff-check job
+        uses: ./.github/workflows/signoff-check
+        env:
+          OWNER: NVIDIA
+          REPO_NAME: spark-rapids
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PULL_NUMBER: ${{ github.event.number }}

.github/workflows/signoff-check/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:alpine
+
+WORKDIR /
+COPY signoff-check .
+RUN pip install PyGithub && chmod +x /signoff-check
+
+# require envs: OWNER,REPO_NAME,GITHUB_TOKEN,PULL_NUMBER
+ENTRYPOINT ["/signoff-check"]

.github/workflows/signoff-check/action.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'signoff check action'
+description: 'check if PR got signed off'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/workflows/signoff-check/signoff-check

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A signoff check
+
+The tool checks if any commit got signoff in a pull request.
+
+NOTE: this script is for github actions only, you should not use it anywhere else.
+"""
+import os
+import re
+import sys
+from argparse import ArgumentParser
+
+from github import Github
+
+SIGNOFF_REGEX = re.compile('Signed-off-by:')
+
+
+def signoff(token: str, owner: str, repo_name: str, pull_number: int):
+    gh = Github(token, per_page=100, user_agent='signoff-check', verify=True)
+    pr = gh.get_repo(f"{owner}/{repo_name}").get_pull(pull_number)
+    for c in pr.get_commits():
+        if SIGNOFF_REGEX.search(c.commit.message):
+            print('Found signoff.\n')
+            print(f"Commit sha:\n{c.commit.sha}")
+            print(f"Commit message:\n{c.commit.message}")
+            return True
+    return False
+
+
+def main(token: str, owner: str, repo_name: str, pull_number: int):
+    try:
+        if not signoff(token, owner, repo_name, pull_number):
+            raise Exception('No commits w/ signoff')
+    except Exception as e:  # pylint: disable=broad-except
+        print(e)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description="signoff check")
+    parser.add_argument("--owner", help="repo owner", default='')
+    parser.add_argument("--repo_name", help="repo name", default='')
+    parser.add_argument("--token", help="github token, will use GITHUB_TOKEN if empty", default='')
+    parser.add_argument("--pull_number", help="pull request number", type=int)
+    args = parser.parse_args()
+
+    GITHUB_TOKEN = args.token if args.token else os.environ.get('GITHUB_TOKEN')
+    assert GITHUB_TOKEN, 'env GITHUB_TOKEN should not be empty'
+    OWNER = args.owner if args.owner else os.environ.get('OWNER')
+    assert OWNER, 'env OWNER should not be empty'
+    REPO_NAME = args.repo_name if args.repo_name else os.environ.get('REPO_NAME')
+    assert REPO_NAME, 'env REPO_NAME should not be empty'
+    PULL_NUMBER = args.pull_number if args.pull_number else int(os.environ.get('PULL_NUMBER'))
+    assert PULL_NUMBER, 'env PULL_NUMBER should not be empty'
+
+    main(token=GITHUB_TOKEN, owner=OWNER, repo_name=REPO_NAME, pull_number=PULL_NUMBER)

jenkins/Dockerfile-blossom.ubuntu16

@@ -35,6 +35,6 @@ RUN add-apt-repository ppa:deadsnakes/ppa && \
     openjdk-8-jdk python3.6 python3-pip tzdata git
 
 RUN ln -s /usr/bin/python3.6 /usr/bin/python
-RUN python -m pip install pytest sre_yield requests
+RUN python -m pip install pytest sre_yield requests pandas pyarrow
 
 RUN apt install -y inetutils-ping expect

jenkins/Jenkinsfile-blossom.premerge

@@ -84,8 +84,8 @@ spec:
             values:
             - Tesla_V100S_PCIE_32GB
             - Tesla_V100_PCIE_32GB
-            - Tesla_T4
             - TITAN_RTX
+            - Tesla_T4
 """
 
 def githubHelper // blossom github helper
@@ -103,9 +103,9 @@ pipeline {
 
     options {
         ansiColor('xterm')
-        buildDiscarder(logRotator(numToKeepStr: '30'))
+        buildDiscarder(logRotator(numToKeepStr: '50'))
         skipDefaultCheckout true
-        timeout(time: 120, unit: 'MINUTES')
+        timeout(time: 180, unit: 'MINUTES')
     }
 
     environment {
@@ -120,7 +120,7 @@ pipeline {
     }
 
     stages {
-        stage("init githubHelper") {
+        stage("Init githubHelper") {
             steps {
                 script {
                     githubHelper = GithubHelper.getInstance("${GITHUB_TOKEN}", githubData)
@@ -134,7 +134,7 @@ pipeline {
                     }
                 }
             }
-        }
+        } // end of Init githubHelper
 
         stage('Build docker image') {
             when {