From 6c1b48735f1d8ae229a3e63b4feca4e2e9010cfe Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Wed, 19 Jan 2022 15:28:29 -0500 Subject: [PATCH 1/5] Update issue templates --- .github/ISSUE_TEMPLATE/issue-template.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/issue-template.md diff --git a/.github/ISSUE_TEMPLATE/issue-template.md b/.github/ISSUE_TEMPLATE/issue-template.md new file mode 100644 index 0000000..ea91145 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue-template.md @@ -0,0 +1,18 @@ +--- +name: Issue template +about: Issue template for fecfile-imagegenerator +title: '' +labels: '' +assignees: '' + +--- + +### Business Reason ### + +As a [role], I will be able to [blank] so that I can [business reason] + +### Acceptance Criteria ### + +**If** [precedent] +**When** [action] +**Then** [result] From d4092b8e6d2e83e1317e00e8e5a55af5cfc29195 Mon Sep 17 00:00:00 2001 From: Al Crowley Date: Tue, 1 Feb 2022 16:19:42 -0500 Subject: [PATCH 2/5] Adding SonarCloud scanning * Modify test CI job to run sonar-scanner * Adding sonar-project.properties --- .circleci/config.yml | 124 +++++++++++++++++++++++++++++++++++++++ requirements-test.txt | 2 + sonar-project.properties | 19 ++++++ 3 files changed, 145 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 requirements-test.txt create mode 100644 sonar-project.properties diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..0fa6f90 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,124 @@ +# See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 + +# See: https://circleci.com/docs/2.0/orb-intro/ +orbs: + # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python + python: circleci/python@1.2 + +# See: https://circleci.com/docs/2.0/configuration-reference/#jobs +jobs: + test: + # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/ + # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python + docker: + - image: cimg/python:3.8 + + steps: + - checkout + + - run: + name: Create unified requirements so CircleCI can cache them + command: | + cat requirements.txt > requirements-all.txt + echo >> requirements-all.txt # blank in case new newline at end of requirements.txt + cat requirements-test.txt >> requirements-all.txt + + # default settings in python/install-packages has trouble resolving + # dependencies and takes a very long time + - run: + name: install Python packages + command: + pip install --use-deprecated=legacy-resolver -r requirements-all.txt + +# - python/install-packages: +# pkg-manager: pip +# pip-dependency-file: requirements-all.txt + + - run: + name: Run tests, save a coverage report, and save coverage percentage + command: | + pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term || EXIT_CODE=$? + if [ $EXIT_CODE == "0" ]; then + echo "Tests ran successfully" + elif [ $EXIT_CODE == "5" ]; then + echo "No tests found. Continuing." + else + echo "Unit test errors. pytest exited with $EXIT_CODE." + exit 1 + fi + + - store_artifacts: + path: htmlcov + + # Sonar cloud setup and scanning + - run: + name: Create sonar-scanner cache directory if it doesn't exist + command: mkdir -p /tmp/cache/scanner + - restore_cache: + keys: + - v1-sonarcloud-scanner-4.6.2.2472 + - run: + name: SonarCloud + command: | + set -e + VERSION=4.6.2.2472 + if [ -z "$SONAR_TOKEN" ]; then + echo "You must set SONAR_TOKEN environemnt variable" + exit 1 + fi + SCANNER_DIRECTORY=/tmp/cache/scanner + export SONAR_USER_HOME=$SCANNER_DIRECTORY/.sonar + OS="linux" + echo $SONAR_USER_HOME + if [[ ! -x "$SCANNER_DIRECTORY/sonar-scanner-$VERSION-$OS/bin/sonar-scanner" ]]; then + curl -Ol https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$VERSION-$OS.zip + unzip -qq -o sonar-scanner-cli-$VERSION-$OS.zip -d $SCANNER_DIRECTORY + fi + + chmod +x $SCANNER_DIRECTORY/sonar-scanner-$VERSION-$OS/bin/sonar-scanner + chmod +x $SCANNER_DIRECTORY/sonar-scanner-$VERSION-$OS/jre/bin/java + $SCANNER_DIRECTORY/sonar-scanner-$VERSION-$OS/bin/sonar-scanner + environment: + SONARQUBE_SCANNER_PARAMS: '{"sonar.host.url":"https://sonarcloud.io"}' + - save_cache: + key: v1-sonarcloud-scanner-4.6.2.2472 + paths: /tmp/cache/scanner + + dependency-check: + docker: + - image: cimg/python:3.8 + + steps: + - checkout + + - python/install-packages: + pkg-manager: pip + pip-dependency-file: requirements-test.txt + + - run: + name: Run depency check + command: | + export today=$(date "+%Y-%m-%d") + + # gather up the -i ignore IDs fro safety check + export ignores=$( + grep -vE "^\s*#" .safety.dependency.ignore | # print out any non-comment line + grep "[0-9]" | # filter out any line that doesn't have a number in it + awk -v "today=${today}" '{ if ($2 > today || $2 == "") print "-i", $1}' | # print any line with date after today + xargs echo # put all the output from previous command on one line + ) + export command="safety check -r requirements.txt --full-report $ignores" + + echo "----------------------------------------------------" + echo "If you need to modify the ignore list for the safety" + echo "check, edit .safety.dependency.ignore file" + echo "----------------------------------------------------" + eval $command + +# See: https://circleci.com/docs/2.0/configuration-reference/#workflows +workflows: + test: + jobs: + - test + - dependency-check \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..811ee3c --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==6.2.5 +pytest-cov==3.0.0 diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..58cfbb8 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,19 @@ +sonar.projectKey=fecgov_fecfile-image-generator +sonar.organization=fecgov + +# This is the name and version displayed in the SonarCloud UI. +sonar.projectName=fecfile-image-generator +#sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +sonar.sources=. +sonar.python.coverage.reportPaths=coverage.xml +sonar.python.version=3 + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 + + +sonar.host.url=https://sonarcloud.io + + From 76e9738f60067033dcc6d8566b53b774a37dc94b Mon Sep 17 00:00:00 2001 From: Al Crowley Date: Tue, 1 Feb 2022 16:23:38 -0500 Subject: [PATCH 3/5] * Adding safety to requirements-test.txt --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 811ee3c..2df6a92 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ pytest==6.2.5 pytest-cov==3.0.0 +safety==1.10.3 \ No newline at end of file From e60d24cfc13555920a8d0adc9e07983bc66f3bfb Mon Sep 17 00:00:00 2001 From: Al Crowley Date: Tue, 1 Feb 2022 16:27:56 -0500 Subject: [PATCH 4/5] * Adding .safety.dependency.ignore * Using python:3.8-node for CI container --- .circleci/config.yml | 2 +- .safety.dependency.ignore | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .safety.dependency.ignore diff --git a/.circleci/config.yml b/.circleci/config.yml index 0fa6f90..6c085b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/ # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python docker: - - image: cimg/python:3.8 + - image: cimg/python:3.8-node steps: - checkout diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore new file mode 100644 index 0000000..223ee2c --- /dev/null +++ b/.safety.dependency.ignore @@ -0,0 +1,12 @@ +# Any vulnerability ID numbers listed in this file will be ignored when +# running the safety dependency check. Each line should have the ID number +# and a date. The ID will be ignored by the CI pipeline check unitl the date +# in YYYY-MM-DD format listed for that line. +# If no date is listed, the exception will never expire. (NOT RECOMMENDED) +# +# test +# Example: +# 40104 2022-01-15 +# +40104 2022-01-25 # gunicorn +40105 2022-01-25 # gunicorn From 720cb207ed391439dc461651a5319c31326dbe8f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 2 Feb 2022 16:53:59 -0500 Subject: [PATCH 5/5] Updated requirements.txt to meet safety check fails --- .safety.dependency.ignore | 5 +++-- requirements.txt | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index 223ee2c..83438bd 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,5 +8,6 @@ # Example: # 40104 2022-01-15 # -40104 2022-01-25 # gunicorn -40105 2022-01-25 # gunicorn +38834 2022-03-01 # urllib3 +43975 2022-03-01 # urllib3 +27519 2022-03-01 # urllib3 diff --git a/requirements.txt b/requirements.txt index d6cf78f..9aa9272 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,18 +4,18 @@ chardet==3.0.4 Click==7.0 cx-Oracle==7.0.0 Flask==1.0.2 -Flask-Cors==3.0.6 +Flask-Cors==3.0.9 Flask-HTTPAuth==3.2.4 idna==2.7 itsdangerous==0.24 -Jinja2==2.10.1 +Jinja2==2.11.3 MarkupSafe==1.1.1 passlib==1.7.1 python-json-logger==0.1.9 requests==2.22.0 six==1.11.0 urllib3==1.25.3 -Werkzeug==0.15.3 +Werkzeug==2.0.2 WeasyPrint==47 ipdb==0.12.3 pypdftk==0.4 @@ -26,5 +26,5 @@ PyPDF2==1.26.0 pdfkit==0.6.1 beautifulsoup4==4.7.1 -gunicorn==19.9.0 +gunicorn==19.10.0 flake8==3.6.0