14816 - Batch job to make limited restoration historical (#3165)

* Add GH CI/CD for the expired limited restoration job

* Add setup and requirements files

* Add Dockerfile

* Add Makefile

* Add devops configuration

* Add Kubernetes configuration

* Add logging configuration

* Add utils files

* Add main job implementation

.github/workflows/expired-limited-restoration-cd.yml

@@ -0,0 +1,103 @@
+name: Expired Limited Restoration Job CD
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "jobs/expired-limited-restoration/**"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment (dev/test/prod)"
+        required: true
+        default: "dev"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./jobs/expired-limited-restoration
+
+env:
+  APP_NAME: "expired-limited-restoration"
+  TAG_NAME: "dev"
+
+jobs:
+  expired-limited-restoration-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'push' && github.repository == 'bcgov/lear'
+    environment:
+      name: "dev"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*Future Effective Filings Job Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  expired-limited-restoration-cd-by-dispatch:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'workflow_dispatch' && github.repository == 'bcgov/lear'
+    environment:
+      name: "${{ github.event.inputs.environment }}"
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set env by input
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.environment }}" >> $GITHUB_ENV
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*Future Effective Filings Job Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/expired-limited-restoration-ci.yml

@@ -0,0 +1,83 @@
+name: Expired Limited Restoration Job CI
+
+on:
+  pull_request:
+    types: [assigned, synchronize]
+    paths:
+      - "jobs/expired-limited-restoration/**"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./jobs/expired-limited-restoration
+
+jobs:
+  setup-job:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/lear'
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: "true"
+
+  linting:
+    needs: setup-job
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          make setup
+      - name: Lint with pylint
+        id: pylint
+        run: |
+          make pylint
+      - name: Lint with flake8
+        id: flake8
+        run: |
+          make flake8
+
+  testing:
+    needs: setup-job
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          make setup
+      - name: Test with pytest
+        id: test
+        run: |
+          make test
+      #- name: Upload coverage to Codecov
+      #  uses: codecov/codecov-action@v3
+      #  with:
+      #    file: ./queue_services/entity-pay/coverage.xml
+      #    flags: entitypay
+      #    name: codecov-entity-pay
+      #    fail_ci_if_error: false
+
+  build-check:
+    needs: setup-job
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: build to check strictness
+        id: build
+        run: |
+          make build-nc

jobs/expired-limited-restoration/Dockerfile

@@ -0,0 +1,25 @@
+# platform=linux/amd64
+FROM python:3.8.5-buster
+USER root
+
+# Create working directory
+RUN mkdir /opt/app-root && chmod 755 /opt/app-root
+WORKDIR /opt/app-root
+
+# Install the requirements
+COPY ./requirements.txt .
+
+#RUN pip install --upgrade pip
+RUN pip install pip==20.1.1
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+USER 1001
+
+# Set Python path
+ENV PYTHONPATH=/opt/app-root/src
+
+EXPOSE 8080
+
+CMD [ "python", "/opt/app-root/file_expired_limited_restoration.py" ]

jobs/expired-limited-restoration/Makefile

@@ -0,0 +1,148 @@
+.PHONY: license
+.PHONY: setup
+.PHONY: ci cd
+.PHONY: run
+
+MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST)))
+CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH)))
+
+PROJECT_NAME:=expired-limited-restoration
+DOCKER_NAME:=expired-limited-restoration
+
+#################################################################################
+# COMMANDS -- Setup                                                             #
+#################################################################################
+setup: install install-dev ## Setup the project
+
+clean: clean-build clean-pyc clean-test ## Clean the project
+	rm -rf venv/
+
+clean-build: ## Clean build files
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -fr {} +
+
+clean-pyc: ## Clean cache files
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## clean test files
+	find . -name '.pytest_cache' -exec rm -fr {} +
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+
+build-req: clean ## Upgrade requirements
+	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	. venv/bin/activate ;\
+	pip install pip==20.1.1 ;\
+	pip install -Ur requirements/prod.txt ;\
+	pip freeze | sort > requirements.txt ;\
+	cat requirements/bcregistry-libraries.txt >> requirements.txt ;\
+	pip install -Ur requirements/bcregistry-libraries.txt
+
+install: clean ## Install python virtual environment
+	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	. venv/bin/activate ;\
+	pip install pip==20.1.1 ;\
+	pip install -Ur requirements.txt
+
+install-dev: ## Install local application
+	. venv/bin/activate ; \
+	pip install -Ur requirements/dev.txt; \
+	pip install -e .
+
+#################################################################################
+# COMMANDS - CI                                                                 #
+#################################################################################
+ci: lint flake8 test ## CI flow
+
+pylint: ## Linting with pylint
+	. venv/bin/activate && pylint --rcfile=setup.cfg file_expired_limited_restoration.py
+
+flake8: ## Linting with flake8
+	. venv/bin/activate && flake8 file_expired_limited_restoration.py
+
+lint: pylint flake8 ## run all lint type scripts
+
+test: ## Unit testing
+	. venv/bin/activate && pytest
+
+mac-cov: test ## Run the coverage report and display in a browser window (mac)
+	@open -a "Google Chrome" htmlcov/index.html
+
+#################################################################################
+# COMMANDS - CD
+# expects the terminal to be openshift login
+# expects export OPENSHIFT_DOCKER_REGISTRY=""
+# expects export OPENSHIFT_SA_NAME="$(oc whoami)"
+# expects export OPENSHIFT_SA_TOKEN="$(oc whoami -t)"
+# expects export OPENSHIFT_REPOSITORY=""
+# expects export TAG_NAME="dev/test/prod"
+# expects export OPS_REPOSITORY=""                                                        #
+#################################################################################
+cd: ## CD flow
+ifeq ($(TAG_NAME), test)
+# cd: update-env
+cd:
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME)
+else ifeq ($(TAG_NAME), prod)
+# cd: update-env
+cd:
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F)
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME)
+else
+TAG_NAME=dev
+# cd: build update-env tag
+cd: build tag
+endif
+
+build: ## Build the docker container
+	docker build . -t $(DOCKER_NAME) \
+		--build-arg VCS_REF=$(shell git rev-parse --short HEAD) \
+		--build-arg BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
+
+build-nc: ## Build the docker container without caching
+	docker build --no-cache -t $(DOCKER_NAME) .
+
+REGISTRY_IMAGE=$(OPENSHIFT_DOCKER_REGISTRY)/$(OPENSHIFT_REPOSITORY)-tools/$(DOCKER_NAME)
+push: #build ## Push the docker container to the registry & tag latest
+	@echo "$(OPENSHIFT_SA_TOKEN)" | docker login $(OPENSHIFT_DOCKER_REGISTRY) -u $(OPENSHIFT_SA_NAME) --password-stdin ;\
+    docker tag $(DOCKER_NAME) $(REGISTRY_IMAGE):latest ;\
+    docker push $(REGISTRY_IMAGE):latest
+
+# 1Password CLI1 will be deprecated on Oct 1, 2024
+# VAULTS=`cat devops/vaults.json`
+# update-env: ## Update env from 1pass
+# 	oc -n "$(OPS_REPOSITORY)-$(TAG_NAME)" exec "dc/vault-service-$(TAG_NAME)" -- ./scripts/1pass.sh \
+# 		-m "secret" \
+# 		-e "$(TAG_NAME)" \
+# 		-a "$(DOCKER_NAME)-$(TAG_NAME)" \
+# 		-n "$(OPENSHIFT_REPOSITORY)-$(TAG_NAME)" \
+# 		-v "$(VAULTS)" \
+# 		-r "false" \
+# 		-f "false"
+
+tag: push ## tag image
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):latest $(DOCKER_NAME):$(TAG_NAME)
+
+#################################################################################
+# COMMANDS - Local                                                              #
+#################################################################################
+
+run: ## Run the project in local
+	. venv/bin/activate && python file_expired_limited_restoration.py
+
+#################################################################################
+# Self Documenting Commands                                                     #
+#################################################################################
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'