From 33b490b08a2c38d14d251d0643ed162873948014 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 14:37:41 -0700 Subject: [PATCH 01/22] checkpoint-kai-1711402661 --- analytics/Makefile | 82 +++++++++++++++++---- analytics/pyproject.toml | 33 ++++----- analytics/src/analytics/cli.py | 7 +- analytics/src/analytics/metrics/burndown.py | 6 +- analytics/src/analytics/metrics/burnup.py | 20 +---- analytics/src/analytics/metrics/utils.py | 21 +++--- 6 files changed, 105 insertions(+), 64 deletions(-) diff --git a/analytics/Makefile b/analytics/Makefile index 6923a9e29..42943685e 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -1,5 +1,7 @@ -POETRY ?= poetry -GITHUB ?= gh +############# +# Constants # +############# + ORG ?= HHS REPO ?= simpler-grants-gov SPRINT_PROJECT ?= 13 @@ -9,49 +11,97 @@ SPRINT ?= @current UNIT ?= points ACTION ?= show-results MIN_TEST_COVERAGE ?= 80 +APP_NAME ?= grants-analytics + +# Required for CI to work properly +SHELL = /bin/bash -o pipefail + +# By default, all python/poetry commands will run inside of the docker container +# if you wish to run this natively, add PY_RUN_APPROACH=local to your environment vars +# You can set this by either running `export PY_RUN_APPROACH=local` in your shell or add +# it to your ~/.zshrc file (and run `source ~/.zshrc`) +ifeq "$(PY_RUN_APPROACH)" "local" +POETRY := poetry run +GITHUB := gh +else +POETRY := docker-compose run $(DOCKER_EXEC_ARGS) --rm $(APP_NAME) poetry run +GITHUB := docker-compose run $(DOCKER_EXEC_ARGS) --rm $(APP_NAME) gh +endif + +# Docker user configuration +# This logic is to avoid issues with permissions and mounting local volumes, +# which should be owned by the same UID for Linux distros. Mac OS can use root, +# but it is best practice to run things as with least permission where possible + +# Can be set by adding user= and/ or uid= after the make command +# If variables are not set explicitly: try looking up values from current +# environment, otherwise fixed defaults. +# uid= defaults to 0 if user= set (which makes sense if user=root, otherwise you +# probably want to set uid as well). +ifeq ($(user),) +RUN_USER ?= $(or $(strip $(USER)),nodummy) +RUN_UID ?= $(or $(strip $(shell id -u)),4000) +else +RUN_USER = $(user) +RUN_UID = $(or $(strip $(uid)),0) +endif + +export RUN_USER +export RUN_UID + +################## +# Build Commands # +################## check-prereqs: @echo "=> Checking for pre-requisites" - @if ! $(POETRY) --version; then echo "=> Poetry isn't installed"; fi - @if ! $(GITHUB) --version; then echo "=> GitHub CLI isn't installed"; fi + @if ! poetry --version; then echo "=> Poetry isn't installed"; fi + @if ! github --version; then echo "=> GitHub CLI isn't installed"; fi @echo "=> All pre-requisites satisfied" install: check-prereqs @echo "=> Installing python dependencies" - $(POETRY) install + poetry install -setup: install - $(GITHUB) auth login +login: + gh auth login + +build: + docker-compose build lint: @echo "=> Running code quality checks" @echo "=============================" - $(POETRY) run black src tests - $(POETRY) run ruff src tests --fix - $(POETRY) run pylint src tests - $(POETRY) run mypy src + $(POETRY) black src tests + $(POETRY) ruff src tests --fix + $(POETRY) pylint src tests + $(POETRY) mypy src @echo "=============================" @echo "=> All checks succeeded" unit-test: @echo "=> Running unit tests" @echo "=============================" - $(POETRY) run pytest --cov=src + $(POETRY) pytest --cov=src e2e-test: @echo "=> Running end-to-end tests" @echo "=============================" - $(POETRY) run pytest tests/integrations --cov=src --cov-append + $(POETRY) pytest tests/integrations --cov=src --cov-append test-audit: unit-test e2e-test @echo "=> Running test coverage report" @echo "=============================" - $(POETRY) run coverage report --show-missing --fail-under=$(MIN_TEST_COVERAGE) + $(POETRY) coverage report --show-missing --fail-under=$(MIN_TEST_COVERAGE) + +################# +# Data Commands # +################# sprint-data-export: @echo "=> Exporting project data from the sprint board" @echo "=====================================================" - $(POETRY) run analytics export gh_project_data \ + $(POETRY) analytics export gh_project_data \ --owner $(ORG) \ --project $(SPRINT_PROJECT) \ --output-file $(SPRINT_FILE) @@ -59,7 +109,7 @@ sprint-data-export: issue-data-export: @echo "=> Exporting issue data from the repository" @echo "=====================================================" - $(POETRY) run analytics export gh_issue_data \ + $(POETRY) analytics export gh_issue_data \ --owner $(ORG) \ --repo $(REPO) \ --output-file $(ISSUE_FILE) diff --git a/analytics/pyproject.toml b/analytics/pyproject.toml index d8121cd8e..1715b6859 100644 --- a/analytics/pyproject.toml +++ b/analytics/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] authors = ["widal001 "] description = "Python package for analyzing data related to the Simpler Grants Project" -name = "analytics" -packages = [{include = "analytics", from = "src"}] +name = "simpler-grants-gov-analytics" +packages = [{ include = "analytics", from = "src" }] readme = "README.md" version = "0.1.0" @@ -12,14 +12,14 @@ analytics = "analytics.cli:app" [tool.poetry.dependencies] dynaconf = "^3.2.4" kaleido = "0.2.1" -notebook = "^7.0.0" # Goal is to replace this with another method of presenting charts +notebook = "^7.0.0" # Goal is to replace this with another method of presenting charts pandas = "^2.0.3" pandas-stubs = "^2.0.2.230605" plotly = "^5.15.0" pydantic = "^2.0.3" python = "^3.11" slack-sdk = "^3.23.0" -typer = {extras = ["all"], version = "^0.9.0"} +typer = { extras = ["all"], version = "^0.9.0" } [tool.poetry.group.dev.dependencies] black = "^23.7.0" @@ -39,10 +39,7 @@ python_version = "3.11" [[tool.mypy.overrides]] ignore_missing_imports = true -module = [ - "plotly.*", - "dynaconf.*", -] +module = ["plotly.*", "dynaconf.*"] [tool.pylint."MESSAGE CONTROL"] disable = [ @@ -55,17 +52,17 @@ disable = [ [tool.ruff] ignore = [ - "ANN101", # missing type annotation for self - "ANN102", # missing type annotation for cls - "D203", # no blank line before class - "D212", # multi-line summary first line - "FIX002", # line contains TODO - "PD901", # pandas df variable name + "ANN101", # missing type annotation for self + "ANN102", # missing type annotation for cls + "D203", # no blank line before class + "D212", # multi-line summary first line + "FIX002", # line contains TODO + "PD901", # pandas df variable name "PLR0913", # Too many arguments to function call - "PTH123", # `open()` should be replaced by `Path.open()` - "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` - "TD003", # missing an issue link on TODO - "T201", # use of `print` detected + "PTH123", # `open()` should be replaced by `Path.open()` + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` + "TD003", # missing an issue link on TODO + "T201", # use of `print` detected ] line-length = 100 select = ["ALL"] diff --git a/analytics/src/analytics/cli.py b/analytics/src/analytics/cli.py index 2050c01bc..9c814105b 100644 --- a/analytics/src/analytics/cli.py +++ b/analytics/src/analytics/cli.py @@ -90,18 +90,19 @@ def calculate_sprint_burndown( post_results=post_results, ) + @metrics_app.command(name="sprint_burnup") def calculate_sprint_burnup( sprint_file: Annotated[str, SPRINT_FILE_ARG], issue_file: Annotated[str, ISSUE_FILE_ARG], sprint: Annotated[str, SPRINT_ARG], - unit: Annotated[Unit, UNIT_ARG] = Unit.points.value, # type: ignore[assignment] + unit: Annotated[Unit, UNIT_ARG] = Unit.points.value, # type: ignore[assignment] *, # makes the following args keyword only show_results: Annotated[bool, SHOW_RESULTS_ARG] = False, post_results: Annotated[bool, POST_RESULTS_ARG] = False, ) -> None: - """Calculate the burnup of a particular sprint""" - # load the input data + """Calculate the burnup of a particular sprint.""" + # load the input data sprint_data = SprintBoard.load_from_json_files( sprint_file=sprint_file, issue_file=issue_file, diff --git a/analytics/src/analytics/metrics/burndown.py b/analytics/src/analytics/metrics/burndown.py index 51391607c..07571ad6d 100644 --- a/analytics/src/analytics/metrics/burndown.py +++ b/analytics/src/analytics/metrics/burndown.py @@ -6,16 +6,18 @@ """ from __future__ import annotations -from typing import Literal +from typing import TYPE_CHECKING, Literal import pandas as pd import plotly.express as px from numpy import nan -from plotly.graph_objects import Figure from analytics.datasets.sprint_board import SprintBoard from analytics.metrics.base import BaseMetric, Statistic, Unit +if TYPE_CHECKING: + from plotly.graph_objects import Figure + class SprintBurndown(BaseMetric[SprintBoard]): """Calculates the running total of open issues per day in the sprint.""" diff --git a/analytics/src/analytics/metrics/burnup.py b/analytics/src/analytics/metrics/burnup.py index 0c808694b..a9fd90fdd 100644 --- a/analytics/src/analytics/metrics/burnup.py +++ b/analytics/src/analytics/metrics/burnup.py @@ -6,12 +6,10 @@ """ from __future__ import annotations -from typing import Literal +from typing import TYPE_CHECKING import pandas as pd import plotly.express as px -from numpy import nan -from plotly.graph_objects import Figure from analytics.datasets.sprint_board import SprintBoard from analytics.metrics.base import BaseMetric, Statistic, Unit @@ -21,6 +19,9 @@ get_tix_date_range, ) +if TYPE_CHECKING: + from plotly.graph_objects import Figure + class SprintBurnup(BaseMetric[SprintBoard]): """Calculates the running total of open issues per day in the sprint.""" @@ -183,15 +184,9 @@ def _isolate_data_for_this_sprint(self) -> pd.DataFrame: # """ # # create local copies of the key column names - # agg_col = self.opened_col if status == "opened" else self.closed_col - # unit_col = self.unit.value - # key_cols = [agg_col, unit_col] # # create a dummy column to sum per row if the unit is tasks # if self.unit == Unit.issues: - # df[unit_col] = 1 # # isolate the key columns, group by open or closed date, then sum the units - # df_agg = df[key_cols].groupby(agg_col, as_index=False).agg({unit_col: "sum"}) - # return df_agg.rename(columns={agg_col: self.date_col, unit_col: status}) # def _get_tix_date_range(self, df: pd.DataFrame) -> pd.DataFrame: # """ @@ -209,12 +204,5 @@ def _isolate_data_for_this_sprint(self) -> pd.DataFrame: # """ # # get earliest date an issue was opened and latest date one was closed - # sprint_end = self.dataset.sprint_end(self.sprint) - # opened_min = df[self.opened_col].min() - # closed_max = df[self.closed_col].max() - # closed_max = sprint_end if closed_max is nan else max(sprint_end, closed_max) # # creates a dataframe with one row for each day between min and max date # return pd.DataFrame( - # pd.date_range(opened_min, closed_max), - # columns=[self.date_col], - # ) diff --git a/analytics/src/analytics/metrics/utils.py b/analytics/src/analytics/metrics/utils.py index d7de7ad64..091762de0 100644 --- a/analytics/src/analytics/metrics/utils.py +++ b/analytics/src/analytics/metrics/utils.py @@ -9,9 +9,9 @@ from analytics.metrics.base import Unit -def get_daily_tix_counts_by_status(df: pd.DataFrame, - status: Literal["opened", "closed"], - unit: Unit) -> pd.DataFrame: +def get_daily_tix_counts_by_status( + df: pd.DataFrame, status: Literal["opened", "closed"], unit: Unit, +) -> pd.DataFrame: """ Count the number of issues or points opened or closed by date. @@ -30,9 +30,13 @@ def get_daily_tix_counts_by_status(df: pd.DataFrame, df_agg = df[key_cols].groupby(agg_col, as_index=False).agg({unit_col: "sum"}) return df_agg.rename(columns={agg_col: "date", unit_col: status}) -def get_tix_date_range(df: pd.DataFrame, open_col: str | None, - closed_col: str | None, - sprint_end: pd.Timestamp) -> pd.DataFrame: + +def get_tix_date_range( + df: pd.DataFrame, + open_col: str | None, + closed_col: str | None, + sprint_end: pd.Timestamp, +) -> pd.DataFrame: """ Get the data range over which issues were created and closed. @@ -52,9 +56,10 @@ def get_tix_date_range(df: pd.DataFrame, open_col: str | None, closed_max = sprint_end if pd.isna(closed_max) else max(sprint_end, closed_max) return pd.DataFrame( pd.date_range(opened_min, closed_max), - columns = ["date"], + columns=["date"], ) + def get_cum_sum_of_tix( dates: pd.DataFrame, opened: pd.DataFrame, @@ -83,5 +88,3 @@ def get_cum_sum_of_tix( df["total_open"] = df["delta"].cumsum() df["total_closed"] = df["closed"].cumsum() return df - - From b53a2496e0ddfc3022878fb25eb86a4c9cfe77b3 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 14:47:39 -0700 Subject: [PATCH 02/22] git restore --- analytics/src/analytics/cli.py | 7 +++---- analytics/src/analytics/metrics/burndown.py | 6 ++---- analytics/src/analytics/metrics/burnup.py | 20 ++++++++++++++++---- analytics/src/analytics/metrics/utils.py | 21 +++++++++------------ 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/analytics/src/analytics/cli.py b/analytics/src/analytics/cli.py index 9c814105b..2050c01bc 100644 --- a/analytics/src/analytics/cli.py +++ b/analytics/src/analytics/cli.py @@ -90,19 +90,18 @@ def calculate_sprint_burndown( post_results=post_results, ) - @metrics_app.command(name="sprint_burnup") def calculate_sprint_burnup( sprint_file: Annotated[str, SPRINT_FILE_ARG], issue_file: Annotated[str, ISSUE_FILE_ARG], sprint: Annotated[str, SPRINT_ARG], - unit: Annotated[Unit, UNIT_ARG] = Unit.points.value, # type: ignore[assignment] + unit: Annotated[Unit, UNIT_ARG] = Unit.points.value, # type: ignore[assignment] *, # makes the following args keyword only show_results: Annotated[bool, SHOW_RESULTS_ARG] = False, post_results: Annotated[bool, POST_RESULTS_ARG] = False, ) -> None: - """Calculate the burnup of a particular sprint.""" - # load the input data + """Calculate the burnup of a particular sprint""" + # load the input data sprint_data = SprintBoard.load_from_json_files( sprint_file=sprint_file, issue_file=issue_file, diff --git a/analytics/src/analytics/metrics/burndown.py b/analytics/src/analytics/metrics/burndown.py index 07571ad6d..51391607c 100644 --- a/analytics/src/analytics/metrics/burndown.py +++ b/analytics/src/analytics/metrics/burndown.py @@ -6,18 +6,16 @@ """ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +from typing import Literal import pandas as pd import plotly.express as px from numpy import nan +from plotly.graph_objects import Figure from analytics.datasets.sprint_board import SprintBoard from analytics.metrics.base import BaseMetric, Statistic, Unit -if TYPE_CHECKING: - from plotly.graph_objects import Figure - class SprintBurndown(BaseMetric[SprintBoard]): """Calculates the running total of open issues per day in the sprint.""" diff --git a/analytics/src/analytics/metrics/burnup.py b/analytics/src/analytics/metrics/burnup.py index a9fd90fdd..0c808694b 100644 --- a/analytics/src/analytics/metrics/burnup.py +++ b/analytics/src/analytics/metrics/burnup.py @@ -6,10 +6,12 @@ """ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import Literal import pandas as pd import plotly.express as px +from numpy import nan +from plotly.graph_objects import Figure from analytics.datasets.sprint_board import SprintBoard from analytics.metrics.base import BaseMetric, Statistic, Unit @@ -19,9 +21,6 @@ get_tix_date_range, ) -if TYPE_CHECKING: - from plotly.graph_objects import Figure - class SprintBurnup(BaseMetric[SprintBoard]): """Calculates the running total of open issues per day in the sprint.""" @@ -184,9 +183,15 @@ def _isolate_data_for_this_sprint(self) -> pd.DataFrame: # """ # # create local copies of the key column names + # agg_col = self.opened_col if status == "opened" else self.closed_col + # unit_col = self.unit.value + # key_cols = [agg_col, unit_col] # # create a dummy column to sum per row if the unit is tasks # if self.unit == Unit.issues: + # df[unit_col] = 1 # # isolate the key columns, group by open or closed date, then sum the units + # df_agg = df[key_cols].groupby(agg_col, as_index=False).agg({unit_col: "sum"}) + # return df_agg.rename(columns={agg_col: self.date_col, unit_col: status}) # def _get_tix_date_range(self, df: pd.DataFrame) -> pd.DataFrame: # """ @@ -204,5 +209,12 @@ def _isolate_data_for_this_sprint(self) -> pd.DataFrame: # """ # # get earliest date an issue was opened and latest date one was closed + # sprint_end = self.dataset.sprint_end(self.sprint) + # opened_min = df[self.opened_col].min() + # closed_max = df[self.closed_col].max() + # closed_max = sprint_end if closed_max is nan else max(sprint_end, closed_max) # # creates a dataframe with one row for each day between min and max date # return pd.DataFrame( + # pd.date_range(opened_min, closed_max), + # columns=[self.date_col], + # ) diff --git a/analytics/src/analytics/metrics/utils.py b/analytics/src/analytics/metrics/utils.py index 091762de0..d7de7ad64 100644 --- a/analytics/src/analytics/metrics/utils.py +++ b/analytics/src/analytics/metrics/utils.py @@ -9,9 +9,9 @@ from analytics.metrics.base import Unit -def get_daily_tix_counts_by_status( - df: pd.DataFrame, status: Literal["opened", "closed"], unit: Unit, -) -> pd.DataFrame: +def get_daily_tix_counts_by_status(df: pd.DataFrame, + status: Literal["opened", "closed"], + unit: Unit) -> pd.DataFrame: """ Count the number of issues or points opened or closed by date. @@ -30,13 +30,9 @@ def get_daily_tix_counts_by_status( df_agg = df[key_cols].groupby(agg_col, as_index=False).agg({unit_col: "sum"}) return df_agg.rename(columns={agg_col: "date", unit_col: status}) - -def get_tix_date_range( - df: pd.DataFrame, - open_col: str | None, - closed_col: str | None, - sprint_end: pd.Timestamp, -) -> pd.DataFrame: +def get_tix_date_range(df: pd.DataFrame, open_col: str | None, + closed_col: str | None, + sprint_end: pd.Timestamp) -> pd.DataFrame: """ Get the data range over which issues were created and closed. @@ -56,10 +52,9 @@ def get_tix_date_range( closed_max = sprint_end if pd.isna(closed_max) else max(sprint_end, closed_max) return pd.DataFrame( pd.date_range(opened_min, closed_max), - columns=["date"], + columns = ["date"], ) - def get_cum_sum_of_tix( dates: pd.DataFrame, opened: pd.DataFrame, @@ -88,3 +83,5 @@ def get_cum_sum_of_tix( df["total_open"] = df["delta"].cumsum() df["total_closed"] = df["closed"].cumsum() return df + + From a07bafd891d832d23a4c6c12548fc64929c48082 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 14:47:49 -0700 Subject: [PATCH 03/22] add docker files --- analytics/Dockerfile | 116 +++++++++++++++++++++++++++++++++++ analytics/docker-compose.yml | 15 +++++ 2 files changed, 131 insertions(+) create mode 100644 analytics/Dockerfile create mode 100644 analytics/docker-compose.yml diff --git a/analytics/Dockerfile b/analytics/Dockerfile new file mode 100644 index 000000000..081e9bc11 --- /dev/null +++ b/analytics/Dockerfile @@ -0,0 +1,116 @@ +# Use the official python3 image based on Debian 11 "Bullseye". +# https://hub.docker.com/_/python + +# The build stage that will be used to deploy to the various environments +# needs to be called `release` in order to integrate with the repo's +# top-level Makefile +FROM python:3-slim AS base + +# Install poetry, the package manager. +# https://python-poetry.org +RUN pip install --no-cache-dir poetry --upgrade + +RUN apt-get update \ + # Install security updates + # https://pythonspeed.com/articles/security-updates-in-docker/ + && apt-get upgrade --yes \ + && apt-get install --no-install-recommends --yes \ + build-essential \ + libpq-dev \ + postgresql \ + wget \ + # Reduce the image size by clear apt cached lists + # Complies with https://github.com/codacy/codacy-hadolint/blob/master/codacy-hadolint/docs/description/DL3009.md + && rm -fr /var/lib/apt/lists/* \ + && rm /etc/ssl/private/ssl-cert-snakeoil.key + +ARG RUN_UID +ARG RUN_USER + +# The following logic creates the RUN_USER home directory and the directory where +# we will be storing the application in the image. This runs when the user is not root +RUN : "${RUN_USER:?RUN_USER and RUN_UID need to be set and non-empty.}" && \ + [ "${RUN_USER}" = "root" ] || \ + (useradd --create-home --create --user-group --home "/home/${RUN_USER}" --uid ${RUN_UID} "${RUN_USER}" \ + && mkdir /analytics \ + && chown -R ${RUN_UID} "/home/${RUN_USER}" /analytics) + +#----------- +# Dev image +#----------- + +FROM base AS dev +ARG RUN_USER + +# In between ARG RUN_USER and USER ${RUN_USER}, the user is still root +# If there is anything that needs to be ran as root, this is the spot + +# Install graphviz which is used to generate ERD diagrams +RUN apt-get update && apt-get install --no-install-recommends --yes graphviz + +USER ${RUN_USER} +WORKDIR /analytics + +COPY pyproject.toml poetry.lock ./ +# Explicitly create a new virtualenv to avoid getting overridden by mounted .venv folders +RUN poetry config virtualenvs.in-project false && poetry env use python +# Install all dependencies including dev dependencies +RUN poetry install --no-root --with dev + +COPY . /analytics + +# Set the host to 0.0.0.0 to make the server available external +# to the Docker container that it's running in. +ENV HOST=0.0.0.0 + +# Run the application. +CMD ["poetry", "run", "python", "-m", "src"] + +#--------- +# Release +#--------- + +FROM base AS release +ARG RUN_USER + +# Gunicorn requires this workaround to create writable temporary directory in +# our readonly root file system. https://github.com/aws/containers-roadmap/issues/736 +RUN mkdir -p /tmp +VOLUME ["/tmp"] + +# TODO(https://github.com/navapbc/template-application-flask/issues/23) Productionize the Docker image + +WORKDIR /analytics + +COPY . /analytics + +# Remove any existing virtual environments that might exist. This +# might happen if testing out building the release image from a local machine +# that has a virtual environment within the project analytics folder. +RUN rm -fr /analytics/.venv + +# Set virtualenv location to be in project to be easy to find +# This will create a virtualenv in /analytics/.venv/ +# See https://python-poetry.org/docs/configuration/#virtualenvsin-project +# See https://python-poetry.org/docs/configuration/#using-environment-variables +ENV POETRY_VIRTUALENVS_IN_PROJECT=true + +# Install production runtime dependencies only +RUN poetry install --no-root --only main + +# Build the application binary (python wheel) defined in pyproject.toml +# Note that this will only copy over python files, and files stated in the +# include section in pyproject.toml. Also note that if you change the name or +# version section in pyproject.toml, you will need to change the dist/... to match +# or the application will not build +RUN poetry build --format wheel && poetry run pip install 'dist/simpler_grants_gov_analytics-0.1.0-py3-none-any.whl' + +# Add project's virtual env to the PATH so we can directly run poetry scripts +# defiend in pyproject.toml +ENV PATH="/analytics/.venv/bin:$PATH" + +# Set the host to 0.0.0.0 to make the server available external +# to the Docker container that it's running in. +ENV HOST=0.0.0.0 + +USER ${RUN_USER} diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml new file mode 100644 index 000000000..506682baf --- /dev/null +++ b/analytics/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + + grants-analytics: + build: + context: . + target: dev + args: + - RUN_UID=${RUN_UID:-4000} + - RUN_USER=${RUN_USER:-analytics} + container_name: grants-analytics + volumes: + - .:/analytics + - ~/.ssh:/home/${RUN_USER:-analytics}/.ssh From 5501a50db6c2e3c5db5d6d2621c8199ed89b92dd Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 15:29:46 -0700 Subject: [PATCH 04/22] checkpoint-kai-1711405786 --- analytics/Dockerfile | 3 --- analytics/Makefile | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 081e9bc11..8a57ab867 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -45,9 +45,6 @@ ARG RUN_USER # In between ARG RUN_USER and USER ${RUN_USER}, the user is still root # If there is anything that needs to be ran as root, this is the spot -# Install graphviz which is used to generate ERD diagrams -RUN apt-get update && apt-get install --no-install-recommends --yes graphviz - USER ${RUN_USER} WORKDIR /analytics diff --git a/analytics/Makefile b/analytics/Makefile index 42943685e..2eacd5b76 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -94,6 +94,15 @@ test-audit: unit-test e2e-test @echo "=============================" $(POETRY) coverage report --show-missing --fail-under=$(MIN_TEST_COVERAGE) +release-build: + docker buildx build \ + --target release \ + --platform=linux/amd64 \ + --build-arg RUN_USER=$(RUN_USER) \ + --build-arg RUN_UID=$(RUN_UID) \ + $(OPTS) \ + . + ################# # Data Commands # ################# From 86579dbb5357775d3252bc6d3da6f73b20d7c597 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 15:43:42 -0700 Subject: [PATCH 05/22] PYTHONPATH --- analytics/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 8a57ab867..e82d6f057 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -35,6 +35,9 @@ RUN : "${RUN_USER:?RUN_USER and RUN_UID need to be set and non-empty.}" && \ && mkdir /analytics \ && chown -R ${RUN_UID} "/home/${RUN_USER}" /analytics) +# Set PYTHONPATH so that the tests can find the source code. +ENV PYTHONPATH="/analytics/src/:$PYTHONPATH" + #----------- # Dev image #----------- From 3fafefed79d4ef9b608cff966d08f90d3e6d8407 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 15:53:39 -0700 Subject: [PATCH 06/22] ANALYTICS_SLACK_BOT_TOKEN --- analytics/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/analytics/Makefile b/analytics/Makefile index 2eacd5b76..10953a704 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -16,6 +16,11 @@ APP_NAME ?= grants-analytics # Required for CI to work properly SHELL = /bin/bash -o pipefail +ifdef CI + DOCKER_EXEC_ARGS := -T -e CI -e ANALYTICS_SLACK_BOT_TOKEN +else +endif + # By default, all python/poetry commands will run inside of the docker container # if you wish to run this natively, add PY_RUN_APPROACH=local to your environment vars # You can set this by either running `export PY_RUN_APPROACH=local` in your shell or add From c2e4ff35a8591c08fe8b309d181eea0708a43651 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 15:59:31 -0700 Subject: [PATCH 07/22] ANALYTICS_REPORTING_CHANNEL_ID --- analytics/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/Makefile b/analytics/Makefile index 10953a704..13b071acd 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -17,7 +17,7 @@ APP_NAME ?= grants-analytics SHELL = /bin/bash -o pipefail ifdef CI - DOCKER_EXEC_ARGS := -T -e CI -e ANALYTICS_SLACK_BOT_TOKEN + DOCKER_EXEC_ARGS := -T -e CI -e ANALYTICS_SLACK_BOT_TOKEN -e ANALYTICS_REPORTING_CHANNEL_ID else endif From be474a1109e964518d3c1a5d53d38c7a2e5281e7 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 16:01:45 -0700 Subject: [PATCH 08/22] local.env --- analytics/Makefile | 2 +- analytics/docker-compose.yml | 1 + analytics/local.env | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 analytics/local.env diff --git a/analytics/Makefile b/analytics/Makefile index 13b071acd..a267ae5a4 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -17,7 +17,7 @@ APP_NAME ?= grants-analytics SHELL = /bin/bash -o pipefail ifdef CI - DOCKER_EXEC_ARGS := -T -e CI -e ANALYTICS_SLACK_BOT_TOKEN -e ANALYTICS_REPORTING_CHANNEL_ID + DOCKER_EXEC_ARGS := -T -e CI else endif diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index 506682baf..dc53114d9 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -13,3 +13,4 @@ services: volumes: - .:/analytics - ~/.ssh:/home/${RUN_USER:-analytics}/.ssh + env_file: ./local.env diff --git a/analytics/local.env b/analytics/local.env new file mode 100644 index 000000000..f93437820 --- /dev/null +++ b/analytics/local.env @@ -0,0 +1,2 @@ +ANALYTICS_SLACK_BOT_TOKEN=FAKE +ANALYTICS_REPORTING_CHANNEL_ID=FAKE From 1e68afd82e12fc891f28ae6dcb777f4639764f0c Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 16:07:45 -0700 Subject: [PATCH 09/22] install gh CLI --- analytics/Dockerfile | 9 +++++++++ analytics/Makefile | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index e82d6f057..12db08021 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -24,6 +24,15 @@ RUN apt-get update \ && rm -fr /var/lib/apt/lists/* \ && rm /etc/ssl/private/ssl-cert-snakeoil.key +# Install gh CLI +# docs: https://github.com/cli/cli/blob/trunk/docs/install_linux.md +RUN mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh -y \ + && gh --version + ARG RUN_UID ARG RUN_USER diff --git a/analytics/Makefile b/analytics/Makefile index a267ae5a4..1ca444ad0 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -87,12 +87,12 @@ lint: unit-test: @echo "=> Running unit tests" @echo "=============================" - $(POETRY) pytest --cov=src + $(POETRY) pytest -s --cov=src e2e-test: @echo "=> Running end-to-end tests" @echo "=============================" - $(POETRY) pytest tests/integrations --cov=src --cov-append + $(POETRY) pytest -s tests/integrations --cov=src --cov-append test-audit: unit-test e2e-test @echo "=> Running test coverage report" @@ -133,7 +133,7 @@ gh-data-export: sprint-data-export issue-data-export sprint-burndown: @echo "=> Running sprint burndown report" @echo "=====================================================" - poetry run analytics calculate sprint_burndown \ + $(POETRY) analytics calculate sprint_burndown \ --sprint-file $(SPRINT_FILE) \ --issue-file $(ISSUE_FILE) \ --sprint "$(SPRINT)" \ @@ -143,7 +143,7 @@ sprint-burndown: percent-complete: @echo "=> Running percent complete deliverable" @echo "=====================================================" - poetry run analytics calculate deliverable_percent_complete \ + $(POETRY) analytics calculate deliverable_percent_complete \ --sprint-file $(SPRINT_FILE) \ --issue-file $(ISSUE_FILE) \ --unit $(UNIT) \ From 132a3d2ed2a29c5c8d43b19dac18df2a69abdd41 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Mon, 25 Mar 2024 16:25:09 -0700 Subject: [PATCH 10/22] gh CLI login process working --- analytics/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/analytics/Makefile b/analytics/Makefile index 1ca444ad0..f61ae1510 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -17,8 +17,9 @@ APP_NAME ?= grants-analytics SHELL = /bin/bash -o pipefail ifdef CI - DOCKER_EXEC_ARGS := -T -e CI + DOCKER_EXEC_ARGS := -T -e CI -e GH_TOKEN else + DOCKER_EXEC_ARGS := -e GH_TOKEN endif # By default, all python/poetry commands will run inside of the docker container @@ -69,7 +70,7 @@ install: check-prereqs poetry install login: - gh auth login + $(GITHUB) auth login build: docker-compose build @@ -87,12 +88,12 @@ lint: unit-test: @echo "=> Running unit tests" @echo "=============================" - $(POETRY) pytest -s --cov=src + $(POETRY) pytest --cov=src e2e-test: @echo "=> Running end-to-end tests" @echo "=============================" - $(POETRY) pytest -s tests/integrations --cov=src --cov-append + $(POETRY) pytest tests/integrations --cov=src --cov-append test-audit: unit-test e2e-test @echo "=> Running test coverage report" From 1ea26108b9fef8d1f06cf036a913a5a0cbcfc7a3 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Tue, 26 Mar 2024 11:14:42 -0700 Subject: [PATCH 11/22] workon env vars --- analytics/Makefile | 2 +- analytics/docker-compose.yml | 1 - analytics/local.env | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 analytics/local.env diff --git a/analytics/Makefile b/analytics/Makefile index f61ae1510..7b8e40431 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -17,7 +17,7 @@ APP_NAME ?= grants-analytics SHELL = /bin/bash -o pipefail ifdef CI - DOCKER_EXEC_ARGS := -T -e CI -e GH_TOKEN + DOCKER_EXEC_ARGS := -T -e CI -e GH_TOKEN -e ANALYTICS_SLACK_BOT_TOKEN -e ANALYTICS_REPORTING_CHANNEL_ID else DOCKER_EXEC_ARGS := -e GH_TOKEN endif diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index dc53114d9..506682baf 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -13,4 +13,3 @@ services: volumes: - .:/analytics - ~/.ssh:/home/${RUN_USER:-analytics}/.ssh - env_file: ./local.env diff --git a/analytics/local.env b/analytics/local.env deleted file mode 100644 index f93437820..000000000 --- a/analytics/local.env +++ /dev/null @@ -1,2 +0,0 @@ -ANALYTICS_SLACK_BOT_TOKEN=FAKE -ANALYTICS_REPORTING_CHANNEL_ID=FAKE From acaa5943d367307ccbc3de2617db4c9ea2fe1f0e Mon Sep 17 00:00:00 2001 From: "kai [they]" Date: Tue, 26 Mar 2024 13:39:54 -0700 Subject: [PATCH 12/22] Update Dockerfile --- analytics/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 12db08021..b8644734e 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -118,8 +118,5 @@ RUN poetry build --format wheel && poetry run pip install 'dist/simpler_grants_g # defiend in pyproject.toml ENV PATH="/analytics/.venv/bin:$PATH" -# Set the host to 0.0.0.0 to make the server available external -# to the Docker container that it's running in. -ENV HOST=0.0.0.0 USER ${RUN_USER} From 2746beee27d4c592d62453760682407438c760f9 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Tue, 26 Mar 2024 15:07:07 -0700 Subject: [PATCH 13/22] docs --- documentation/analytics/development.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/documentation/analytics/development.md b/documentation/analytics/development.md index 21ac2cdd4..8612ab7d5 100644 --- a/documentation/analytics/development.md +++ b/documentation/analytics/development.md @@ -40,11 +40,23 @@ Once you follow the steps above, check that you meet the prerequisites with: `ma 1. Set up the project: `make setup` -- This will install the required packages and prompt you to authenticate with GitHub 2. Create a `.secrets.toml` with the following details, see the next section to discover where these values can be found: + ```toml reporting_channel_id = "" slack_bot_token = "" ``` +3. Set a Github Token in your terminal, via `export GH_TOKEN=...`. Acquiring the token is a multi-step process: + + - Go to https://github.com/settings/tokens + - Create a token + - Give it the following scopes: + - repo + - read:org + - admin:public_key + - project + - Add `export GH_TOKEN=...` to your `zshrc` or similar + ### Configuring secrets #### Prerequisites From 585c40e9a47624e382cff82071dbddc4c5553b94 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 08:56:36 -0700 Subject: [PATCH 14/22] remove copied code --- analytics/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index b8644734e..5d327dcb9 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -68,13 +68,6 @@ RUN poetry install --no-root --with dev COPY . /analytics -# Set the host to 0.0.0.0 to make the server available external -# to the Docker container that it's running in. -ENV HOST=0.0.0.0 - -# Run the application. -CMD ["poetry", "run", "python", "-m", "src"] - #--------- # Release #--------- From 77139c14f6faabb73df519044883dc8e3922117d Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 11:41:39 -0700 Subject: [PATCH 15/22] add to development.md --- documentation/analytics/development.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation/analytics/development.md b/documentation/analytics/development.md index 8612ab7d5..7a4ecb8b9 100644 --- a/documentation/analytics/development.md +++ b/documentation/analytics/development.md @@ -57,6 +57,10 @@ Once you follow the steps above, check that you meet the prerequisites with: `ma - project - Add `export GH_TOKEN=...` to your `zshrc` or similar +### Docker vs Native + +This project run itself inside of docker by default. If you wish to run this natively, add PY_RUN_APPROACH=local to your environment variables. You can set this by either running `export PY_RUN_APPROACH=local` in your shell or add it to your ~/.zshrc file (and run `source ~/.zshrc`). + ### Configuring secrets #### Prerequisites From 673af6505a72bc36e7dc15346e400e20a5d7888b Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 14:24:26 -0700 Subject: [PATCH 16/22] Add ECR repo --- .../build-repository/.terraform.lock.hcl | 44 ++++++++++++++ infra/analytics/build-repository/main.tf | 59 +++++++++++++++++++ .../build-repository/shared.s3.tfbackend | 4 ++ 3 files changed, 107 insertions(+) create mode 100644 infra/analytics/build-repository/.terraform.lock.hcl create mode 100644 infra/analytics/build-repository/main.tf create mode 100644 infra/analytics/build-repository/shared.s3.tfbackend diff --git a/infra/analytics/build-repository/.terraform.lock.hcl b/infra/analytics/build-repository/.terraform.lock.hcl new file mode 100644 index 000000000..077ae85fb --- /dev/null +++ b/infra/analytics/build-repository/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.34.0" + constraints = "~> 5.34.0" + hashes = [ + "h1:1Y1JgV1z99QqAK06+atyfNqreZxyGZKbm4mZO4VhhT8=", + "zh:01bb20ae12b8c66f0cacec4f417a5d6741f018009f3a66077008e67cce127aa4", + "zh:3b0c9bdbbf846beef2c9573fc27898ceb71b69cf9d2f4b1dd2d0c2b539eab114", + "zh:5226ecb9c21c2f6fbf1d662ac82459ffcd4ad058a9ea9c6200750a21a80ca009", + "zh:6021b905d9b3cd3d7892eb04d405c6fa20112718de1d6ef7b9f1db0b0c97721a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9e61b8e0ccf923979cd2dc1f1140dbcb02f92248578e10c1996f560b6306317c", + "zh:ad6bf62cdcf531f2f92f6416822918b7ba2af298e4a0065c6baf44991fda982d", + "zh:b698b041ef38837753bbe5265dddbc70b76e8b8b34c5c10876e6aab0eb5eaf63", + "zh:bb799843c534f6a3f072a99d93a3b53ff97c58a96742be15518adf8127706784", + "zh:cebee0d942c37cd3b21e9050457cceb26d0a6ea886b855dab64bb67d78f863d1", + "zh:e061fdd1cb99e7c81fb4485b41ae000c6792d38f73f9f50aed0d3d5c2ce6dcfb", + "zh:eeb4943f82734946362696928336357cd1d36164907ae5905da0316a67e275e1", + "zh:ef09b6ad475efa9300327a30cbbe4373d817261c8e41e5b7391750b16ef4547d", + "zh:f01aab3881cd90b3f56da7c2a75f83da37fd03cc615fc5600a44056a7e0f9af7", + "zh:fcd0f724ebc4b56a499eb6c0fc602de609af18a0d578befa2f7a8df155c55550", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.3" + hashes = [ + "h1:gShzO1rJtADK9tDZMvMgjciVAzsBh39LNjtThCwX1Hg=", + "zh:03d81462f9578ec91ce8e26f887e34151eda0e100f57e9772dbea86363588239", + "zh:37ec2a20f6a3ec3a0fd95d3f3de26da6cb9534b30488bc45723e118a0911c0d8", + "zh:4eb5b119179539f2749ce9de0e1b9629d025990f062f4f4dddc161562bb89d37", + "zh:5a31bb58414f41bee5e09b939012df5b88654120b0238a89dfd6691ba197619a", + "zh:6221a05e52a6a2d4f520ffe7cbc741f4f6080e0855061b0ed54e8be4a84eb9b7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8bb068496b4679bef625e4710d9f3432e301c3a56602271f04e60eadf7f8a94c", + "zh:94742aa5378bab626ce34f79bcef6a373e4f86ea7a8b762e9f71270a899e0d00", + "zh:a485831b5a525cd8f40e8982fa37da40ff70b1ae092c8b755fcde123f0b1238d", + "zh:a647ff16d071eabcabd87ea8183eb90a775a0294ddd735d742075d62fff09193", + "zh:b74710c5954aaa3faf262c18d36a8c2407862d9f842c63e7fa92fa4de3d29df6", + "zh:fa73d83edc92af2e551857594c2232ba6a9e3603ad34b0a5940865202c08d8d7", + ] +} diff --git a/infra/analytics/build-repository/main.tf b/infra/analytics/build-repository/main.tf new file mode 100644 index 000000000..2e7aa7665 --- /dev/null +++ b/infra/analytics/build-repository/main.tf @@ -0,0 +1,59 @@ +data "aws_iam_role" "github_actions" { + name = module.project_config.github_actions_role_name +} + +locals { + # Set project tags that will be used to tag all resources. + tags = merge(module.project_config.default_tags, { + application = module.app_config.app_name + application_role = "build-repository" + description = "Backend resources required for storing built release candidate artifacts to be used for deploying to environments." + }) + + # Get list of AWS account ids for the application environments that + # will need access to the build repository + app_account_names = values(module.app_config.account_names_by_environment) + account_ids_by_name = data.external.account_ids_by_name.result + app_account_ids = [for account_name in local.app_account_names : local.account_ids_by_name[account_name] if contains(keys(local.account_ids_by_name), account_name)] +} + +terraform { + required_version = ">= 1.2.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.34.0" + } + } + + backend "s3" { + encrypt = "true" + } +} + +provider "aws" { + region = module.app_config.build_repository_config.region + default_tags { + tags = local.tags + } +} + +module "project_config" { + source = "../../project-config" +} + +module "app_config" { + source = "../app-config" +} + +data "external" "account_ids_by_name" { + program = ["../../../bin/account-ids-by-name.sh"] +} + +module "container_image_repository" { + source = "../../modules/container-image-repository" + name = module.app_config.image_repository_name + push_access_role_arn = data.aws_iam_role.github_actions.arn + app_account_ids = local.app_account_ids +} diff --git a/infra/analytics/build-repository/shared.s3.tfbackend b/infra/analytics/build-repository/shared.s3.tfbackend new file mode 100644 index 000000000..30c4c266f --- /dev/null +++ b/infra/analytics/build-repository/shared.s3.tfbackend @@ -0,0 +1,4 @@ +bucket = "simpler-grants-gov-315341936575-us-east-1-tf" +key = "infra/analytics/build-repository/shared.tfstate" +dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks" +region = "us-east-1" From de6f08f2d4a02040f11acbadc1667d73eff1a845 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 15:57:15 -0700 Subject: [PATCH 17/22] test deploy --- .github/workflows/cd-analytics.yml | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/cd-analytics.yml diff --git a/.github/workflows/cd-analytics.yml b/.github/workflows/cd-analytics.yml new file mode 100644 index 000000000..b74af2db5 --- /dev/null +++ b/.github/workflows/cd-analytics.yml @@ -0,0 +1,41 @@ +name: Deploy Analytics +# Need to set a default value for when the workflow is triggered from a git push +# which bypasses the default configuration for inputs +run-name: Deploy ${{ github.ref_name }} to Analytics ${{ inputs.environment || (github.event_name == 'release' && 'prod') || 'dev' }} + +on: + push: + branches: + - "main" + paths: + - "analytics/**" + # TEMPORARY: for testing purposes! + pull_request: + paths: + - .github/workflows/cd-analytics.yml + release: + types: [published] + workflow_dispatch: + inputs: + environment: + description: "target environment" + required: true + default: "dev" + type: choice + options: + - dev + - staging + - prod + +jobs: + analytics-checks: + name: Run Analyics Checks + uses: ./.github/workflows/ci-analytics.yml + + deploy: + name: Deploy + needs: analytics-checks + uses: ./.github/workflows/deploy.yml + with: + app_name: "analytics" + environment: ${{ inputs.environment || (github.event_name == 'release' && 'prod') || 'dev' }} From ffe1730c9972db36155e2c945183c751fbf4b7cb Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 16:23:02 -0700 Subject: [PATCH 18/22] testing --- analytics/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 5d327dcb9..2dd075fc1 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -111,5 +111,4 @@ RUN poetry build --format wheel && poetry run pip install 'dist/simpler_grants_g # defiend in pyproject.toml ENV PATH="/analytics/.venv/bin:$PATH" - USER ${RUN_USER} From 8442c218f73a02e6cb78c3c9ed9351f99c3979f2 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 16:31:05 -0700 Subject: [PATCH 19/22] test secrets --- .github/workflows/cd-analytics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cd-analytics.yml b/.github/workflows/cd-analytics.yml index b74af2db5..a48f5d29e 100644 --- a/.github/workflows/cd-analytics.yml +++ b/.github/workflows/cd-analytics.yml @@ -31,6 +31,7 @@ jobs: analytics-checks: name: Run Analyics Checks uses: ./.github/workflows/ci-analytics.yml + secrets: inherit deploy: name: Deploy From ee3e74b99f8bb3241eda724a26f2fbe81ea88d0a Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 16:43:39 -0700 Subject: [PATCH 20/22] has_database --- infra/analytics/app-config/outputs.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infra/analytics/app-config/outputs.tf b/infra/analytics/app-config/outputs.tf index 4a05f0332..398ba3cfb 100644 --- a/infra/analytics/app-config/outputs.tf +++ b/infra/analytics/app-config/outputs.tf @@ -17,3 +17,11 @@ output "build_repository_config" { output "environment_configs" { value = local.environment_configs } + +# This variable is slightly misnamed. It should really be called "has_migrations". +# It controls whether or not the `run-database-migrations.sh` script tries to run database +# migrations. The entire analytics application is going to have its schema controlled +# via ETL jobs, so we don't need to run migrations in the same way as the API. +output "has_database" { + value = false +} From d3c2798755ea7ae96879b241ffc64dff9da1b696 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 16:50:06 -0700 Subject: [PATCH 21/22] remove testing change --- .github/workflows/cd-analytics.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cd-analytics.yml b/.github/workflows/cd-analytics.yml index a48f5d29e..c06a391c1 100644 --- a/.github/workflows/cd-analytics.yml +++ b/.github/workflows/cd-analytics.yml @@ -9,10 +9,6 @@ on: - "main" paths: - "analytics/**" - # TEMPORARY: for testing purposes! - pull_request: - paths: - - .github/workflows/cd-analytics.yml release: types: [published] workflow_dispatch: From 5c33190dd1722646d469f73a085ede102ec44c45 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Mar 2024 16:50:24 -0700 Subject: [PATCH 22/22] remove testing change --- analytics/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 2dd075fc1..5d327dcb9 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -111,4 +111,5 @@ RUN poetry build --format wheel && poetry run pip install 'dist/simpler_grants_g # defiend in pyproject.toml ENV PATH="/analytics/.venv/bin:$PATH" + USER ${RUN_USER}