Modified deployment pipeline to deploy to cloud.gov, fixed some linting errors

.circleci/config.yml

@@ -111,9 +111,34 @@ jobs:
             echo "----------------------------------------------------"
             eval $command
 
+  deploy:
+
+    docker:
+      - image: cimg/python:3.7
+
+    steps:
+      - checkout
+
+      - python/install-packages:
+          pkg-manager: pip
+          pip-dependency-file: requirements.txt
+
+      - run:
+          name: Installs for deploy
+          command: |
+            mkdir -p $HOME/bin
+            export PATH=$HOME/bin:$PATH
+            curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.1.0" | tar xzv -C $HOME/bin
+      - deploy:
+          name: Deploy API
+          command: |
+            export PATH=$HOME/bin:$PATH
+            invoke deploy --branch $CIRCLE_BRANCH --login
+
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
 workflows:
   test:
     jobs:
       - test
-      - dependency-check
+      - dependency-check
+      - deploy

.flake8

@@ -1,4 +1,4 @@
 [flake8]
-max-line-lenght = 160
+max-line-length = 160
 ignore =
-    Z111  # not a real code, just placeholder for an example
+    W503  # line break before binary operator

.gitignore

@@ -58,11 +58,12 @@ testem.log
 *.bower-registry
 *.bower-tmp
 
-
-
 # System Files
 .DS_Store
 Thumbs.db
 .vscode/settings.json
 front-end/.vscode/launch.json
 *pycache*
+
+# cloud foundry
+.cfmeta

Procfile

@@ -0,0 +1 @@
+web: bin/run.sh

bin/run.sh

@@ -0,0 +1 @@
+gunicorn --bind 0.0.0.0:8080 wsgi:APP -w 5 -t 200

config.py

@@ -16,10 +16,10 @@
 ALLOWED_HOSTS = ["*"]
 
 
-FORM_TEMPLATES_LOCATION = 'templates/forms/{}.pdf'
-HTML_FORM_TEMPLATES_LOCATION = 'templates/forms/F99/{}.html'
-FORMS_LOCATION = 'templates/forms/{}'
-REQUEST_FILE_LOCATION = 'temp/json/{}.json'
+FORM_TEMPLATES_LOCATION = "templates/forms/{}.pdf"
+HTML_FORM_TEMPLATES_LOCATION = "templates/forms/F99/{}.html"
+FORMS_LOCATION = "templates/forms/{}"
+REQUEST_FILE_LOCATION = "temp/json/{}.json"
 # OUTPUT_FILE_LOCATION = 'output/pdf/{}.pdf'
 OUTPUT_DIR_LOCATION = "output/pdf/{}/"
 
@@ -33,16 +33,14 @@
 # AWS_SECRET_ACCESS_KEY = os.environ.get('SECRET_KEY', None)
 # AWS_HOST_NAME = 'us-east-1'
 # AWS_REGION = 'us-east-1'
-AWS_SES_AUTO_THROTTLE = (
-    0.5
-)  # (default; safety factor applied to rate limit, turn off automatic throttling, set this to None)
+AWS_SES_AUTO_THROTTLE = 0.5  # (default; safety factor applied to rate limit, turn off automatic throttling, set this to None)
 
 # AWS FECFile components bucket name
 
-AWS_FECFILE_OUTPUT_DIRECTORY = os.environ.get('OUTPUT_DIRECTORY', 'output')
+AWS_FECFILE_OUTPUT_DIRECTORY = os.environ.get("OUTPUT_DIRECTORY", "output")
 AWS_FECFILE_COMPONENTS_BUCKET_NAME = "fecfile-dev-components"
-AWS_FECFILE_TEMP_BUCKET_NAME = os.environ.get('TEMP_BUCKET', 'dev-efile-repo')
-AWS_FECFILE_PDF_BUCKET_NAME = os.environ.get('PERM_BUCKET', 'fecfile-pdf')
+AWS_FECFILE_TEMP_BUCKET_NAME = os.environ.get("TEMP_BUCKET", "dev-efile-repo")
+AWS_FECFILE_PDF_BUCKET_NAME = os.environ.get("PERM_BUCKET", "fecfile-pdf")
 
 
 # if False it will create unique file names for every uploaded file
@@ -56,28 +54,40 @@
     "%s.s3.amazonaws.com" % AWS_FECFILE_TEMP_BUCKET_NAME
 )
 
-S3_FILE_URL = "https://%s/%s/" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, AWS_FECFILE_OUTPUT_DIRECTORY)
+S3_FILE_URL = "https://%s/%s/" % (
+    AWS_S3_FECFILE_COMPONENTS_DOMAIN,
+    AWS_FECFILE_OUTPUT_DIRECTORY,
+)
 
 # the sub-directories of temp and output files
 # TEMP_FILES_LOCATION = 'temp'
 # OUTPUT_FILE_FOLDER = 'output'
 
 # TEMP_FILES_URL = "https://%s/%s/{}" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, TEMP_FILES_LOCATION)
-PRINT_OUTPUT_FILE_URL = "https://%s/%s" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, OUTPUT_DIR_LOCATION)
+PRINT_OUTPUT_FILE_URL = "https://%s/%s" % (
+    AWS_S3_FECFILE_COMPONENTS_DOMAIN,
+    OUTPUT_DIR_LOCATION,
+)
 
-FECFILE_UTIL_PRINT_API_URL = os.environ.get('FECFILE_UTIL_URL', 'https://dev-efile-api.efdev.fec.gov/printpdf')
+FECFILE_UTIL_PRINT_API_URL = os.environ.get(
+    "FECFILE_UTIL_URL", "https://dev-efile-api.efdev.fec.gov/printpdf"
+)
 FECFILE_UTIL_API_VERSION = "/v1/fecfileutil"
 
-NXG_FEC_PARSER_API_URL = os.environ.get('PARSER_URL', 'https://dev-efile-api.efdev.fec.gov/receiver')
+NXG_FEC_PARSER_API_URL = os.environ.get(
+    "PARSER_URL", "https://dev-efile-api.efdev.fec.gov/receiver"
+)
 # NXG_FEC_PARSER_API_URL = os.environ.get('PARSER_URL', 'http://localhost:8090')
 NXG_FEC_PARSER_API_VERSION = "/v1"
 
 
 # SQS Details
-DATA_RECEIVER_SQS_QUEUE = os.environ.get('DATA_RECEIVER_SQS_QUEUE', 'datareceiver-dev')
-VALIDATION_SQS_QUEUE = os.environ.get('VALIDATION_SQS_QUEUE', 'validator-dev')
-IMAGE_NUMBER_SQS_QUEUE = os.environ.get('IMAGE_NUMBER_SQS_QUEUE', 'imaging')
+DATA_RECEIVER_SQS_QUEUE = os.environ.get("DATA_RECEIVER_SQS_QUEUE", "datareceiver-dev")
+VALIDATION_SQS_QUEUE = os.environ.get("VALIDATION_SQS_QUEUE", "validator-dev")
+IMAGE_NUMBER_SQS_QUEUE = os.environ.get("IMAGE_NUMBER_SQS_QUEUE", "imaging")
 # IMAGE_NUMBER_SQS_QUEUE = os.environ.get('IMAGE_NUMBER_SQS_QUEUE', 'validator-dev')
-IMAGE_GENERATOR_SQS_QUEUE = os.environ.get('IMAGE_GENERATOR_SQS_QUEUE', 'imaging-generator')
+IMAGE_GENERATOR_SQS_QUEUE = os.environ.get(
+    "IMAGE_GENERATOR_SQS_QUEUE", "imaging-generator"
+)
 # Timeout for the message in queue, after the set time interval other process will see it
 MESSAGE_VISIBILITY_TIMEOUT = 30  # setting it to 0 for testing

manifest-dev.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-dev.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

manifest-prod.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-prod.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

manifest-stage.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-stage.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

requirements.txt

@@ -1,29 +1,24 @@
-BeautifulSoup4==4.7.1
+beautifulsoup4==4.7.1
+boto3==1.21.2
 certifi==2018.10.15
+cfenv==0.5.2
 chardet==3.0.4
-Click==7.0
-cx-Oracle==7.0.0
-Flask==2.0.2
+Flask-API==3.0.post1
 Flask-Cors==3.0.9
 Flask-HTTPAuth==3.2.4
+Flask==2.0.2
+GitPython==3.1.0
+gunicorn==19.10.0
 idna==2.7
-itsdangerous==0.24
-Jinja2==2.11.3
-MarkupSafe==1.1.1
+invoke==0.15.0
+ipdb==0.12.3
+itsdangerous==2.1.0
 passlib==1.7.1
+pdfkit==0.6.1
+PyPDF2==1.26.0
+pypdftk==0.4
 python-json-logger==0.1.9
-requests==2.22.0
+requests==2.27.1
 six==1.11.0
-urllib3==1.25.3
-Werkzeug==2.0.2
+urllib3==1.26.8
 WeasyPrint==47
-ipdb==0.12.3
-pypdftk==0.4
-boto3==1.14.60
-botocore==1.17.60
-Flask-API==3.0.post1
-PyPDF2==1.26.0
-pdfkit==0.6.1
-beautifulsoup4==4.7.1
-
-gunicorn==19.10.0

routes/src/common.py

@@ -49,8 +49,8 @@ def check_response_and_return_or_log(response, url):
                     return True, response_text.get("data")
             except Exception as e:
                 LOGGER.error(
-                    u'Exception "%s" raised when trying to read '
-                    u"response.text as json" % str(e)
+                    'Exception "%s" raised when trying to read '
+                    "response.text as json" % str(e)
                 )
                 # if response.text does not indicate failure assume
                 # success = True
@@ -61,13 +61,13 @@ def check_response_and_return_or_log(response, url):
 
     elif 400 <= response.status_code < 500:
         LOGGER.error(
-            u"%s Bouncer Service Error: %s for url: %s"
+            "%s Bouncer Service Error: %s for url: %s"
             % (response.status_code, response.text, url)
         )
 
     elif 500 <= response.status_code < 600:
         LOGGER.error(
-            u"%s Server Error: %s for url: %s"
+            "%s Server Error: %s for url: %s"
             % (response.status_code, response.text, url)
         )
 

routes/src/controllers.py

@@ -19,6 +19,12 @@
 CORS(app)
 
 
+@app.route("/", methods=["GET"])
+@app.route("/app-name", methods=["GET"])
+def index():
+    return "fecfile-image-generator"
+
+
 @app.route("/print", methods=["POST"])
 def print_pdf():
     """
@@ -359,7 +365,7 @@ def image_number_data(next_imaging=None):
             "submissionId": submission_id,
             "totalPages": total_pages,
         }
-        ## data_obj = json.dumps(data_obj)
+        # data_obj = json.dumps(data_obj)
         begin_image_number_object = requests.post(
             cfg.NXG_FEC_PARSER_API_URL
             + cfg.NXG_FEC_PARSER_API_VERSION
@@ -487,7 +493,7 @@ def image_generator_data(next_image_generator=None):
     json_file_name = next_image_generator["fileName"]
     begin_image_number = next_image_generator["beginImageNumber"]
     filing_timestamp = next_image_generator["receivedTime"]
-    rep_id = json_file_name[0 : json_file_name.index(".json")]
+    rep_id = json_file_name[0:json_file_name.index(".json")]
     # rep_id = '8'
     # print(rep_id)
 

routes/src/f3x/form3x.py

@@ -1,23 +1,24 @@
 import flask
 import boto3
-import re
 import os
 import os.path
 import pypdftk
 import shutil
 import sys
 import traceback
-import urllib.request
 
 from collections import OrderedDict
 from os import path
 from flask import json
 from flask import request, current_app
 from flask_api import status
-from PyPDF2 import PdfFileWriter, PdfFileReader, PdfFileMerger
-from PyPDF2.generic import BooleanObject, NameObject, IndirectObject
-from routes.src import tmoflask, utils, common, form
-from routes.src.utils import md5_for_text, md5_for_file, directory_files, merge, error, delete_directory
+from routes.src import common
+from routes.src.utils import (
+    md5_for_text,
+    md5_for_file,
+    error,
+    delete_directory,
+)
 from routes.src.f3x.helper import (
     calculate_page_count,
     calculate_sh3_page_count,
@@ -460,13 +461,18 @@ def print_pdftk(
                     extraArgs = {"ContentType": "application/pdf", "ACL": "public-read"}
 
                     if silent_print:
-                        response["pdf_url"] = current_app.config['S3_FILE_URL'] + rep_id + '.pdf'
+                        response["pdf_url"] = (
+                            current_app.config["S3_FILE_URL"] + rep_id + ".pdf"
+                        )
                         s3.upload_file(
-                            md5_directory + 'all_pages.pdf',
-                            current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'],
-                            current_app.config['AWS_FECFILE_OUTPUT_DIRECTORY'] + '/' +
-                            str(rep_id) + '.pdf',
-                            ExtraArgs=extraArgs)
+                            md5_directory + "all_pages.pdf",
+                            current_app.config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"],
+                            current_app.config["AWS_FECFILE_OUTPUT_DIRECTORY"]
+                            + "/"
+                            + str(rep_id)
+                            + ".pdf",
+                            ExtraArgs=extraArgs,
+                        )
 
                         s3.upload_file(
                             md5_directory + "all_pages.pdf",
@@ -1559,7 +1565,7 @@ def process_schedules_pages(
                     for rec in values:
                         if (
                             rec[0].get("designatingCommitteeName")
-                            and rec[0].get("coordinateExpenditure") is "Y"
+                            and rec[0].get("coordinateExpenditure") == "Y"
                         ):
                             cord_name = "designatingCommittee"
                         elif (
@@ -1569,7 +1575,7 @@ def process_schedules_pages(
                             cord_name = "designatingNamewithoutEXP"
                         elif (
                             rec[0].get("subordinateCommitteeName")
-                            and rec[0].get("coordinateExpenditure") is "N"
+                            and rec[0].get("coordinateExpenditure") == "N"
                         ):
                             cord_name = "subCommittee"
                         elif (
@@ -1580,7 +1586,7 @@ def process_schedules_pages(
 
                         elif (
                             rec[0].get("subordinateCommitteeName") == ""
-                            and rec[0].get("coordinateExpenditure") is "N"
+                            and rec[0].get("coordinateExpenditure") == "N"
                         ):
                             cord_name = "withsubCommittee"