diff --git a/.github/jobs/build_sonarqube_image.sh b/.github/jobs/build_sonarqube_image.sh new file mode 100755 index 0000000000..9838d3b760 --- /dev/null +++ b/.github/jobs/build_sonarqube_image.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=met-sonarqube-gha + +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile.sonarqube + +CMD_LOGFILE=${GITHUB_WORKSPACE}/sonarqube_build.log + +# +# Define the $SONAR_REFERENCE_BRANCH as the +# - Target of any requests +# - Manual setting for workflow dispatch +# - Source branch for any pushes (e.g. develop) +# +if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then + export SONAR_REFERENCE_BRANCH=${GITHUB_BASE_REF} +elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then + export SONAR_REFERENCE_BRANCH=${WD_REFERENCE_BRANCH} +else + export SONAR_REFERENCE_BRANCH=${SOURCE_BRANCH} +fi + +echo SONAR_REFERENCE_BRANCH=${SONAR_REFERENCE_BRANCH} + +time_command docker build -t ${DOCKERHUB_TAG} \ + --build-arg MET_BASE_REPO \ + --build-arg MET_BASE_TAG \ + --build-arg SOURCE_BRANCH \ + --build-arg SONAR_SCANNER_VERSION \ + --build-arg SONAR_HOST_URL \ + --build-arg SONAR_TOKEN \ + --build-arg SONAR_REFERENCE_BRANCH \ + -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE} +if [ $? != 0 ]; then + cat ${CMD_LOGFILE} + exit 1 +fi + +# Copy the .scannerwork directory from the image +id=$(docker create ${DOCKERHUB_TAG}) +time_command docker cp $id:/met/.scannerwork /tmp/met_scannerwork +docker rm -v $id + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d3a97c7a05..0ed1004546 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -22,6 +22,9 @@ If **yes**, describe the new output and/or changes to the existing output:
- [ ] Will this PR result in changes to existing METplus Use Cases? **[Yes or No]**
If **yes**, create a new **Update Truth** [METplus issue](https://github.com/dtcenter/METplus/issues/new/choose) to describe them. +- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**
+If **yes**, please describe: + - [ ] Please complete this pull request review by **[Fill in date]**.
## Pull Request Checklist ## diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 0000000000..f7e7ddf4c1 --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,102 @@ +name: SonarQube Scan + +# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches + +on: + + # Trigger analysis for pushes to develop and main_vX.Y branches + push: + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/labels/**' + - '**/README.md' + - '**/LICENSE.md' + + # Trigger analysis for pull requests to develop and main_vX.Y branches + pull_request: + types: [opened, synchronize, reopened] + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/labels/**' + - '**/README.md' + - '**/LICENSE.md' + + workflow_dispatch: + inputs: + reference_branch: + description: 'Reference Branch' + default: develop + type: string + +jobs: + build: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v4 + with: + # Disable shallow clones for better analysis + fetch-depth: 0 + + - name: Create output directories + run: mkdir -p ${RUNNER_WORKSPACE}/logs + + - name: Get branch name + id: get_branch_name + run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT + + - name: SonarQube Scan in Docker + run: .github/jobs/build_sonarqube_image.sh + env: + MET_BASE_REPO: met-base + MET_BASE_TAG: v3.2 + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} + SONAR_SCANNER_VERSION: 5.0.1.3006 + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Python Quality Gate check + id: sonarqube-python-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + with: + scanMetadataReportFile: /tmp/met_scannerwork/python-report-task.txt + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube CXX Quality Gate check + id: sonarqube-cxx-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + with: + scanMetadataReportFile: /tmp/met_scannerwork/cxx-report-task.txt + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: Copy log files into logs directory + if: always() + run: cp ${GITHUB_WORKSPACE}/*.log ${RUNNER_WORKSPACE}/logs/ + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs_sonarqube + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + diff --git a/internal/scripts/docker/Dockerfile.sonarqube b/internal/scripts/docker/Dockerfile.sonarqube new file mode 100644 index 0000000000..156fa14bf7 --- /dev/null +++ b/internal/scripts/docker/Dockerfile.sonarqube @@ -0,0 +1,95 @@ +ARG MET_BASE_REPO=met-base +ARG MET_BASE_TAG=v3.2 + +FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} +MAINTAINER John Halley Gotway + +# +# This Dockerfile checks out MET from GitHub and runs the +# SonarQube static code analysis on the specified branch or tag. +# https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/ +# +ARG SONAR_SCANNER_VERSION=5.0.1.3006 +ARG SONAR_HOST_URL +ARG SONAR_TOKEN +ARG SOURCE_BRANCH +ARG SONAR_REFERENCE_BRANCH + +# +# SONAR_HOST_URL is required. +# +RUN if [ "x${SONAR_HOST_URL}" = "x" ]; then \ + echo "ERROR: SONAR_HOST_URL undefined! Rebuild with \"--build-arg SONAR_HOST_URL={url}\""; \ + exit 1; \ + fi + +# +# SONAR_TOKEN is required. +# +RUN if [ "x${SONAR_TOKEN}" = "x" ]; then \ + echo "ERROR: SONAR_TOKEN undefined! Rebuild with \"--build-arg SONAR_TOKEN={token}\""; \ + exit 1; \ + fi + +# +# SOURCE_BRANCH is the branch name of the MET source code. +# +RUN if [ "x${SOURCE_BRANCH}" = "x" ]; then \ + echo "ERROR: SOURCE_BRANCH undefined! Rebuild with \"--build-arg SOURCE_BRANCH={branch name}\""; \ + exit 1; \ + else \ + echo "Build Argument SOURCE_BRANCH=${SOURCE_BRANCH}"; \ + fi + +# +# SONAR_REFERENCE_BRANCH defines to the version against which this scan should be compared. +# +RUN if [ "x${SONAR_REFERENCE_BRANCH}" = "x" ]; then \ + echo "ERROR: SONAR_REFERENCE_BRANCH undefined! Rebuild with \"--build-arg SONAR_REFERENCE_BRANCH={branch name}\""; \ + exit 1; \ + else \ + echo "Build Argument SONAR_REFERENCE_BRANCH=${SONAR_REFERENCE_BRANCH}"; \ + fi + +ENV MET_GIT_NAME ${SOURCE_BRANCH} +ENV MET_REPO_DIR /met/MET-${MET_GIT_NAME} +ENV MET_GIT_URL https://github.com/dtcenter/MET + +# +# Download and install the Sonar software. +# +RUN echo "Installing SonarQube into $HOME/.sonar" \ + && mkdir -p $HOME/.sonar \ + && curl -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip \ + && unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ \ + && echo export PATH="$HOME/.sonar/sonar-scanner-${SONAR_SCANNER_VERSION}-linux/bin:\$PATH" >> $HOME/.bashrc \ + && curl -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip ${SONAR_HOST_URL}/static/cpp/build-wrapper-linux-x86.zip \ + && unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ \ + && echo export PATH="$HOME/.sonar/build-wrapper-linux-x86:\$PATH" >> $HOME/.bashrc + +# +# Update the OS, as needed. +# +RUN apt update + +# +# Set the working directory. +# +WORKDIR /met + +# +# Copy MET Download and install MET. +# +RUN echo "Copying MET into ${MET_REPO_DIR}" \ + && mkdir -p ${MET_REPO_DIR} + +COPY . ${MET_REPO_DIR} + +RUN if [ ! -e "${MET_REPO_DIR}/configure.ac" ]; then \ + echo "ERROR: docker build must be run from the MET directory: `ls`"; \ + exit 1; \ + fi + +RUN cd ${MET_REPO_DIR} \ + && internal/scripts/docker/build_met_sonarqube.sh + diff --git a/internal/scripts/docker/build_met_sonarqube.sh b/internal/scripts/docker/build_met_sonarqube.sh new file mode 100755 index 0000000000..f7d491ea3b --- /dev/null +++ b/internal/scripts/docker/build_met_sonarqube.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Run SonarQube Source Code Analyzer within a Docker container +#======================================================================= +# +# This build_met_sonarqube.sh script must be run from the top-level +# directory of the MET repository to be analyzed. It runs SonarQube to +# scan both the Python and C/C++ MET source code. +# +# Usage: internal/scripts/docker/build_met_sonarqube.sh +# +# Required Enviornment Variables: +# SONAR_HOST_URL +# SONAR_TOKEN +# MET_GIT_NAME +# SONAR_REFERENCE_BRANCH +# +#======================================================================= + +# Check that this is being run from the top-level MET directory +if [ ! -e internal/scripts/docker/build_met_sonarqube.sh ]; then + echo "ERROR: ${0} -> must be run from the top-level MET directory" + exit 1 +fi + +echo "Running script to scan MET with SonarQube in Docker" + +# Source the docker build environment +source ~/.bashrc +source internal/scripts/environment/development.docker +source .github/jobs/bash_functions.sh + +# Check required environment variables +if [ -z ${SONAR_HOST_URL+x} ]; then + echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!" + exit 1 +fi +if [ -z ${SONAR_TOKEN+x} ]; then + echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!" + exit 1 +fi +if [ -z ${MET_GIT_NAME+x} ]; then + echo "ERROR: ${0} -> \$MET_GIT_NAME not defined!" + exit 1 +fi +if [ -z ${SONAR_REFERENCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$SONAR_REFERENCE_BRANCH not defined!" + exit 1 +fi + +# Locate the wrapper +WRAPPER_NAME=build-wrapper-linux-x86-64 +SONAR_WRAPPER=$(which $WRAPPER_NAME 2> /dev/null) + +if [ ! -e $SONAR_WRAPPER ]; then + echo "ERROR: ${0} -> $WRAPPER_NAME not found in the path" + exit 1 +else + echo "SONAR_WRAPPER=$SONAR_WRAPPER" +fi + +# Locate the scanner +SCANNER_NAME=sonar-scanner +SONAR_SCANNER=$(which $SCANNER_NAME 2> /dev/null) + +if [ ! -e $SONAR_SCANNER ]; then + echo "ERROR: ${0} -> $SCANNER_NAME not found in the path" + exit 1 +else + echo "SONAR_SCANNER=$SONAR_SCANNER" +fi + +# Set output directory name +if [ -z ${SONARQUBE_OUT_DIR} ]; then + export SONARQUBE_OUT_DIR=bw-outputs +fi + +# Store the full path to the scripts directory +SONAR_PROPERTIES_DIR=internal/scripts/sonarqube +SONAR_PROPERTIES=sonar-project.properties + +# Copy sonar-project.properties for Python code +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_PROJECT_KEY|MET-GHA-Python|" \ + -e "s|SONAR_PROJECT_NAME|MET GHA Python|" \ + -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \ + $SONAR_PROPERTIES_DIR/python.sonar-project.properties > $SONAR_PROPERTIES + +# The source and reference branches must differ to define new code +if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +# Run SonarQube scan for Python code +time_command $SONAR_SCANNER + +# Copy the Python scan report-task.txt file +mkdir -p /met/.scannerwork +cp .scannerwork/report-task.txt /met/.scannerwork/python-report-task.txt + +# Copy sonar-project.properties for C/C++ code +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_PROJECT_KEY|MET-GHA-CXX|" \ + -e "s|SONAR_PROJECT_NAME|MET GHA CXX|" \ + -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \ + $SONAR_PROPERTIES_DIR/sonar-project.properties > $SONAR_PROPERTIES + +# The source and reference branches must differ to define new code +if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +# Run the configure script +time_command ./configure \ + BUFRLIB_NAME=${BUFRLIB_NAME} \ + GRIB2CLIB_NAME=${GRIB2CLIB_NAME} \ + --enable-grib2 \ + --enable-modis \ + --enable-mode_graphics \ + --enable-lidar2nc \ + --enable-python + CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" \ + LIBS="-ltirpc" + +# Run make clean +time_command make clean + +# Run SonarQube make +time_command $SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make + +# Run SonarQube scan for C/C++ code +time_command $SONAR_SCANNER + +# Copy the C/C++ scan report-task.txt file +mkdir -p /met/.scannerwork +cp .scannerwork/report-task.txt /met/.scannerwork/cxx-report-task.txt + +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES diff --git a/internal/scripts/sonarqube/python.sonar-project.properties b/internal/scripts/sonarqube/python.sonar-project.properties index 7beb252151..71b6b36b71 100644 --- a/internal/scripts/sonarqube/python.sonar-project.properties +++ b/internal/scripts/sonarqube/python.sonar-project.properties @@ -1,8 +1,9 @@ -sonar.projectKey=MET_python_NB -sonar.projectName=MET python Nightly Build +sonar.projectKey=SONAR_PROJECT_KEY +sonar.projectName=SONAR_PROJECT_NAME sonar.projectVersion=1.0 sonar.sources=scripts/python,data/wrappers +sonar.python.version=3.6.3 # The build-wrapper output dir @@ -10,8 +11,7 @@ sonar.sources=scripts/python,data/wrappers sonar.sourceEncoding=UTF-8 #----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=SONAR_SERVER_URL +sonar.host.url=SONAR_HOST_URL -sonar.token=SONAR_TOKEN_VALUE -sonar.branch.name=develop +sonar.token=SONAR_TOKEN +sonar.branch.name=SONAR_BRANCH_NAME diff --git a/internal/scripts/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh index bda39c1c7f..69f1b80e86 100755 --- a/internal/scripts/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -112,11 +112,11 @@ export MET_DEVELOPMENT=true # Run the configure script run_command "./configure --prefix=`pwd` \ - --enable-grib2 \ - --enable-modis \ - --enable-mode_graphics \ - --enable-lidar2nc \ - --enable-python" + --enable-grib2 \ + --enable-modis \ + --enable-mode_graphics \ + --enable-lidar2nc \ + --enable-python" # Set the build id #BUILD_ID="MET-${1}" @@ -125,19 +125,29 @@ SONAR_PROPERTIES=sonar-project.properties # Copy sonar-project.properties for Python code [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -[ -z "$SONAR_SERVER_URL" ] && SONAR_SERVER_URL="http://localhost:9000" -if [ -z "$SONAR_TOKEN_VALUE" ]; then - echo " == ERROR == SONAR_TOKEN_VALUE is not defined" +[ -z "$SONAR_HOST_URL" ] && SONAR_HOST_URL="http://localhost:9000" +if [ -z "$SONAR_TOKEN" ]; then + echo " == ERROR == SONAR_TOKEN is not defined" exit 1 else - sed -e "s|SONAR_TOKEN_VALUE|$SONAR_TOKEN_VALUE|" -e "s|SONAR_SERVER_URL|$SONAR_SERVER_URL|" $SCRIPT_DIR/python.sonar-project.properties > $SONAR_PROPERTIES + sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_PROJECT_KEY|MET_python_NB|" \ + -e "s|SONAR_PROJECT_NAME|MET python Nightly Build|" \ + -e "s|SONAR_BRANCH_NAME|develop|" \ + $SCRIPT_DIR/python.sonar-project.properties > $SONAR_PROPERTIES # Run SonarQube scan for Python code run_command "$SONAR_SCANNER" # Copy sonar-project.properties for C/C++ code [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES - sed -e "s|SONAR_TOKEN_VALUE|$SONAR_TOKEN_VALUE|" -e "s|SONAR_SERVER_URL|$SONAR_SERVER_URL|" $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES + sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_PROJECT_KEY|MET_develop_NB|" \ + -e "s|SONAR_PROJECT_NAME|MET Nightly Build|" \ + -e "s|SONAR_BRANCH_NAME|develop|" \ + $SCRIPT_DIR/sonar-project.properties > $SONAR_PROPERTIES # Run SonarQube clean run_command "make clean" diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 39e1e73745..c09f1d31b1 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -1,5 +1,5 @@ -sonar.projectKey=MET_develop_NB -sonar.projectName=MET Nightly Build +sonar.projectKey=SONAR_PROJECT_KEY +sonar.projectName=SONAR_PROJECT_NAME sonar.projectVersion=1.0 sonar.sources=src @@ -11,8 +11,7 @@ sonar.cfamily.build-wrapper-output=bw-outputs sonar.sourceEncoding=UTF-8 #----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=SONAR_SERVER_URL +sonar.host.url=SONAR_HOST_URL -sonar.token=SONAR_TOKEN_VALUE -#sonar.branch.name=develop +sonar.token=SONAR_TOKEN +sonar.branch.name=SONAR_BRANCH_NAME