Updates and build system in preparation for Sprint 0

.circleci/config.yml

@@ -0,0 +1,88 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# Orbs are reusable packages of CircleCI configuration that you may share across projects, enabling you to create encapsulated, parameterized commands, jobs, and executors that can be used across multiple projects.
+# See: https://circleci.com/docs/2.0/orb-intro/
+orbs:
+  # The python orb contains a set of prepackaged CircleCI configuration you can use repeatedly in your configuration files
+  # Orb commands and jobs help you with common scripting around a language/tool
+  # so you dont have to copy and paste it everywhere.
+  # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python
+  python: circleci/[email protected]
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  build-and-test-python: # This is the name of the job, feel free to change it to better match what you're trying to do!
+    # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/
+    # You can specify an image from Dockerhub or use one of the convenience images from CircleCI's Developer Hub
+    # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python
+    # The executor is the environment in which the steps below will be executed - below will use a python 3.8 container
+    # Change the version below to your required version of python
+    docker:
+      - image: cimg/python:3.7
+    # Checkout the code as the first step. This is a dedicated CircleCI step.
+    # The python orb's install-packages step will install the dependencies from a Pipfile via Pipenv by default.
+    # Here we're making sure we use just use the system-wide pip. By default it uses the project root's requirements.txt.
+    # Then run your tests!
+    # CircleCI will report the results back to your VCS provider.
+    steps:
+      - checkout
+
+      - python/install-packages:
+          pkg-manager: pip
+          app-dir: ~/project/django-backend/
+          pip-dependency-file: requirements.txt
+      - run:
+          name: Run tests
+          # This assumes pytest is installed via the install-package step above
+          command: python manage.py test
+          working_directory: ~/project/django-backend/
+  build-and-test-angular: # This is the name of the job, feel free to change it to better match what you're trying to do!
+    # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/
+    # You can specify an image from Dockerhub or use one of the convenience images from CircleCI's Developer Hub
+    # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python
+    # The executor is the environment in which the steps below will be executed - below will use a python 3.8 container
+    # Change the version below to your required version of python
+    docker:
+      #- image: cimg/node:10.23.2
+      - image: cimg/node:16.12.0
+    # Checkout the code as the first step. This is a dedicated CircleCI step.
+    # The python orb's install-packages step will install the dependencies from a Pipfile via Pipenv by default.
+    # Here we're making sure we use just use the system-wide pip. By default it uses the project root's requirements.txt.
+    # Then run your tests!
+    # CircleCI will report the results back to your VCS provider.
+    steps:
+      - checkout
+#      - run:
+#          name: npm install
+#          command: npm install
+#          working_directory: ~/project/front-end/
+
+      - run:
+          name: intall eslint eslint-config-google
+          command: npm install eslint  @typescript-eslint/parser
+          working_directory: ~/project/front-end/
+
+      - run:
+          command: ls ~/project/front-end/node_modules/ || echo not found
+
+      - run:
+          name: run linter version
+          command: ./node_modules/.bin/eslint --version
+          working_directory: ~/project/front-end/
+
+      - run:
+          name: run linter
+          command: ./node_modules/.bin/eslint "src/**"
+          working_directory: ~/project/front-end/
+
+# Invoke jobs via workflows
+# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
+workflows:
+  test: # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    jobs:
+      - build-and-test-python
+      - build-and-test-angular

README.md

@@ -58,9 +58,17 @@ when running docker-compose you will need to be in the root directory of the pro
 # Build the application
 ` docker-compose build `
 # Run the containers after being built
-`docker-compose up -d`
-Alternatively if you have made changes to the application and need to push the changes to a container
-` docker-compose up --build -d` will build the containers then start them.
+`docker-compose up --build`
+
+You should set the following environment variables in the shell where you are running 'docker-compose up'.
+Proper values for the development variables are shown here as an example
+```
+export FECFILE_DB_HOST=db
+export FECFILE_DB_USERNAME=postgres
+export FECFILE_DB_PASSWORD=postgres
+export FECFILE_DB_NAME=postgres
+```
+
 # Shut down the containers
 ` docker-compose down `
 # see all running containers

cf-deploy.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# magic StackOverflow one liner to get the dirname where the script lives
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+cd ${SCRIPT_DIR}/front-end
+
+#clean enverything up so we get a fresh build
+rm -rf dist node_modules
+
+#create the docker we will use to perform the build
+docker build -t fecfile-web-cf-build -f build-scripts/Dockerfile-cf-build .
+
+#build the app 
+docker run -v ${SCRIPT_DIR}/front-end/:/usr/src/app fecfile-web-cf-build sh -c 'cd /usr/src/app/ && npm install && node --max_old_space_size=4000 ./node_modules/\@angular/cli/bin/ng build'
+
+#copy all the nginx and cloud foundry files into the Angualr dist dir
+cp ${SCRIPT_DIR}/deploy-config/front-end-nginx-config/* ${SCRIPT_DIR}/front-end/dist
+cd ${SCRIPT_DIR}/front-end/dist
+cf push 

db/Dockerfile

@@ -0,0 +1,18 @@
+FROM postgres:10
+
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_NAME=postgres
+ENV POSTGRES_PASSWORD=postgres
+ARG ENCRYPTION_PASSWORD
+
+
+# Load scripts to run at container initialization. They are run in alphabetical order.
+COPY initdb.sql* /tmp/
+RUN if [ -f "/tmp/initdb.sql.gpg" ]; then gpg --batch --output /docker-entrypoint-initdb.d/1.sql --passphrase $ENCRYPTION_PASSWORD --decrypt /tmp/initdb.sql.gpg; fi
+RUN if [ -f "/tmp/initdb.sql" ]; then cp /tmp/initdb.sql /docker-entrypoint-initdb.d/1.sql; fi
+
+COPY fecfile-frontend-dev.sql.gz* /tmp/
+RUN if [ -f "/tmp/fecfile-frontend-dev.sql.gz.gpg" ]; then gpg --batch --output /docker-entrypoint-initdb.d/2.sql.gz --passphrase $ENCRYPTION_PASSWORD --decrypt /tmp/fecfile-frontend-dev.sql.gz.gpg; fi
+RUN if [ -f "/tmp/fecfile-frontend-dev.sql.gz" ]; then cp /tmp/fecfile-frontend-dev.sql.gz /docker-entrypoint-initdb.d/2.sql.gz; fi
+
+EXPOSE 5432

deploy-config/front-end-nginx-config/buildpack.yml

@@ -0,0 +1,3 @@
+---
+nginx:
+  version: mainline

deploy-config/front-end-nginx-config/manifest.yml

@@ -0,0 +1,6 @@
+applications:
+ - name: fecfile-online-frontend
+   instances: 1
+   memory: 256M
+   buildpack: paketo-buildpacks/nginx
+

deploy-config/front-end-nginx-config/mime.types

@@ -0,0 +1,78 @@
+types {
+  text/html html htm shtml;
+  text/css css;
+  text/xml xml;
+  image/gif gif;
+  image/jpeg jpeg jpg;
+  application/x-javascript js;
+  application/atom+xml atom;
+  application/rss+xml rss;
+  font/ttf ttf;
+  font/woff woff;
+  font/woff2 woff2;
+  text/mathml mml;
+  text/plain txt;
+  text/vnd.sun.j2me.app-descriptor jad;
+  text/vnd.wap.wml wml;
+  text/x-component htc;
+  text/cache-manifest manifest;
+  image/png png;
+  image/tiff tif tiff;
+  image/vnd.wap.wbmp wbmp;
+  image/x-icon ico;
+  image/x-jng jng;
+  image/x-ms-bmp bmp;
+  image/svg+xml svg svgz;
+  image/webp webp;
+  application/java-archive jar war ear;
+  application/mac-binhex40 hqx;
+  application/msword doc;
+  application/pdf pdf;
+  application/postscript ps eps ai;
+  application/rtf rtf;
+  application/vnd.ms-excel xls;
+  application/vnd.ms-powerpoint ppt;
+  application/vnd.wap.wmlc wmlc;
+  application/vnd.google-earth.kml+xml  kml;
+  application/vnd.google-earth.kmz kmz;
+  application/x-7z-compressed 7z;
+  application/x-cocoa cco;
+  application/x-java-archive-diff jardiff;
+  application/x-java-jnlp-file jnlp;
+  application/x-makeself run;
+  application/x-perl pl pm;
+  application/x-pilot prc pdb;
+  application/x-rar-compressed rar;
+  application/x-redhat-package-manager  rpm;
+  application/x-sea sea;
+  application/x-shockwave-flash swf;
+  application/x-stuffit sit;
+  application/x-tcl tcl tk;
+  application/x-x509-ca-cert der pem crt;
+  application/x-xpinstall xpi;
+  application/xhtml+xml xhtml;
+  application/zip zip;
+  application/octet-stream bin exe dll;
+  application/octet-stream deb;
+  application/octet-stream dmg;
+  application/octet-stream eot;
+  application/octet-stream iso img;
+  application/octet-stream msi msp msm;
+  application/json json;
+  audio/midi mid midi kar;
+  audio/mpeg mp3;
+  audio/ogg ogg;
+  audio/x-m4a m4a;
+  audio/x-realaudio ra;
+  video/3gpp 3gpp 3gp;
+  video/mp4 mp4;
+  video/mpeg mpeg mpg;
+  video/quicktime mov;
+  video/webm webm;
+  video/x-flv flv;
+  video/x-m4v m4v;
+  video/x-mng mng;
+  video/x-ms-asf asx asf;
+  video/x-ms-wmv wmv;
+  video/x-msvideo avi;
+}

deploy-config/front-end-nginx-config/nginx.conf

@@ -0,0 +1,24 @@
+worker_processes 1;
+daemon off;
+
+error_log stderr;
+events { worker_connections 1024; }
+
+http {
+  charset utf-8;
+  log_format cloudfoundry 'NginxLog "$request" $status $body_bytes_sent';
+  access_log /dev/stdout cloudfoundry;
+  default_type application/octet-stream;
+  include mime.types;
+  sendfile on;
+
+  tcp_nopush on;
+  keepalive_timeout 30;
+  port_in_redirect off; # Ensure that redirects don't include the internal container PORT - 8080
+
+  server {
+    listen {{port}};
+    root fec-eFilling;
+    index index.html index.htm Default.htm;
+  }
+}

django-backend/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.7
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 RUN mkdir /opt/nxg_fec
 WORKDIR /opt/nxg_fec
@@ -12,4 +12,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt
 user nxgu
 
 EXPOSE 8080
-ENTRYPOINT ["sh", "-c", "python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200"]
+ENTRYPOINT ["sh", "-c", "python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200 --reload"]

django-backend/Dockerfile-live

@@ -0,0 +1,15 @@
+FROM python:3.7
+ENV PYTHONUNBUFFERED=1
+
+RUN mkdir /opt/nxg_fec
+WORKDIR /opt/nxg_fec
+ADD requirements.txt /opt/nxg_fec/
+RUN pip3 install -r requirements.txt
+
+RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime
+
+RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt/nxg_fec
+user nxgu
+
+EXPOSE 8080
+ENTRYPOINT ["sh", "-c", "pip3 install -r requirements.txt && python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200 --reload"]

django-backend/fecfiler/authentication/login.py

@@ -136,7 +136,8 @@ def verify_login(request):
 
             username = user_list["username"]
             key = user_list["secret_key"]
-            unix_time = user_list["code_time"]
+            # unix_time = user_list["code_time"]
+            unix_time = "code_time" in user_list and user_list["code_time"]
             otp_class = TOTPVerification(username)
             token_val = otp_class.verify_token(key, unix_time)
 

django-backend/fecfiler/core/transactions_chk_csv_duplicates.py

@@ -17,11 +17,11 @@ def addapt_numpy_int64(numpy_int64):
 register_adapter(numpy.int64, addapt_numpy_int64)
 
 
-PG_HOST = os.getenv('DB_HOST')
-PG_PORT = os.getenv('DB_PORT')
-PG_DATABASE = os.getenv('DB_NAME')
-PG_USER = os.getenv('DB_USERNAME')
-PG_PASSWORD = os.getenv('DB_PASSWORD')
+PG_HOST = os.getenv('FECFILE_DB_HOST', 'localhost')
+PG_PORT = os.getenv('FECFILE_DB_PORT', 5432)
+PG_DATABASE = os.getenv('FECFILE_DB_NAME', 'postgres')
+PG_USER = os.getenv('FECFILE_DB_USERNAME', 'postgres')
+PG_PASSWORD = os.getenv('FECFILE_DB_PASSWORD', 'postgres')
 SQS_QUEUE_NAME = os.getenv('SQS_QUEUE_NAME') #
 
 

django-backend/fecfiler/core/transactions_validate_csv.py

@@ -22,11 +22,11 @@
 from django.db import connection
 
 # Postgres Database Settings - local
-PG_HOST = os.getenv('DB_HOST')
-PG_PORT = os.getenv('DB_PORT')
-PG_DATABASE = os.getenv('DB_NAME')
-PG_USER = os.getenv('DB_USERNAME')
-PG_PASSWORD = os.getenv('DB_PASSWORD')
+PG_HOST = os.getenv('FECFILE_DB_HOST', 'localhost')
+PG_PORT = os.getenv('FECFILE_DB_PORT', '5432')
+PG_DATABASE = os.getenv('FECFILE_DB_NAME', 'postgres')
+PG_USER = os.getenv('FECFILE_DB_USERNAME', 'postgres')
+PG_PASSWORD = os.getenv('FECFILE_DB_PASSWORD', 'postgres')
 SQS_QUEUE_NAME = os.getenv('SQS_QUEUE_NAME') #
 
 

django-backend/fecfiler/core/views.py

@@ -1960,7 +1960,7 @@ def reposit_f99_data(cmte_id, report_id):
             os.environ.get("BACKEND_DB_NAME"),
             os.environ.get("BACKEND_DB_USER"),
             os.environ.get("BACKEND_DB_HOST"),
-            os.environ.get("BACKEND_DB_PASSWORD"),
+            os.environ.get("BACKEND_FECFILE_DB_PASSWORD"),
         )
     )
     # conn.close()