From 59c72aede543e1236ac016e4d0ec876576ab68bf Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Wed, 27 Nov 2024 03:52:55 -0600 Subject: [PATCH] [DRAFT] Automated Preprod Deploy workflow (#1845) * deploy-preprod backend workflow * python mini project for static assets deploy * don't write acl headers; we ignore them anyway * include static assets deploy in github workflow * update docker syntax; add vars to deploy-preprod workflow * remove depcheck workflow --- .github/workflows/depcheck.yml | 82 ------------------------ .github/workflows/deploy-preprod.yml | 62 ++++++++++++++++++ client-admin/Dockerfile | 6 +- client-admin/webpack.config.js | 2 - client-participation/Dockerfile | 8 +-- client-participation/webpack.config.js | 3 - client-report/Dockerfile | 6 +- client-report/writeHeadersJsonTask.js | 3 - deploy/.gitignore | 18 ++++++ deploy/README.md | 87 ++++++++++++++++++++++++++ deploy/deploy-static-assets.py | 87 ++++++++++++++++++++++++++ deploy/dev-requirements.txt | 4 ++ deploy/pyproject.toml | 14 +++++ deploy/requirements.txt | 1 + docker-compose.yml | 12 ++-- file-server/Dockerfile | 22 +++---- 16 files changed, 300 insertions(+), 117 deletions(-) delete mode 100644 .github/workflows/depcheck.yml create mode 100644 .github/workflows/deploy-preprod.yml create mode 100644 deploy/.gitignore create mode 100644 deploy/README.md create mode 100644 deploy/deploy-static-assets.py create mode 100644 deploy/dev-requirements.txt create mode 100644 deploy/pyproject.toml create mode 100644 deploy/requirements.txt diff --git a/.github/workflows/depcheck.yml b/.github/workflows/depcheck.yml deleted file mode 100644 index 057e95475..000000000 --- a/.github/workflows/depcheck.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: DepCheck -on: - pull_request: - types: ["opened", "reopened", "synchronize"] - paths: - - .github/workflows/depcheck.yml - - client-admin/** - -jobs: - run: - runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CI_COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }} - CI_REPO_NAME: ${{ github.repository }} - steps: - # See: https://github.community/t/if-expression-with-context-variable/16558/6 - - name: Check if secrets available - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - unset HAS_SECRET - if [ -n "$GITHUB_TOKEN" ]; then HAS_SECRET=true; fi - echo "name=HAS_SECRET" >> $GITHUB_ENV - - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: 18 - - # See: https://github.com/taskworld/commit-status - - name: Install commit-status CLI tool - run: npm install -g commit-status - - - name: Initiate commit status placeholders - if: github.event.pull_request.head.repo.full_name == github.repository - run: | - # commit-status - commit-status pending "DepCheck / dependencies" "Detecting unused packages..." - commit-status pending "DepCheck / devDependencies" "Detecting unused packages..." - - - uses: actions/checkout@v4 - - - name: Install depcheck CLI tool - run: npm install -g depcheck - - - name: Install project dependencies - working-directory: client-admin - run: npm install - - - name: Run depcheck and save output - id: depcheck - working-directory: client-admin - env: - # See: https://github.com/depcheck/depcheck#special - DEPCHECK_SPECIALS: "webpack,babel,eslint,prettier,bin" - # Why ignoring? - # - prettier: because it's needed by eslint-plugin-prettier, but peerDependencies aren't yet supported. - # See: https://github.com/depcheck/depcheck/issues/522 - # - webpack-cli: Needed for `webpack` command, but provide `webpack-cli` bin, which confuses depcheck. - DEPCHECK_IGNORES: "prettier,prettier-config-standard,webpack-cli" - run: | - # Note: Commit status descriptions can have 140 characters max. (We add an ellipsis in the final step as final char) - # Suppress failing exit codes with `true`. - depcheck --specials "$DEPCHECK_SPECIALS" --ignores "$DEPCHECK_IGNORES" --json > .results.json || true - echo ::set-output name=dependencies::$(cat .results.json | jq '.dependencies | join(", ") | .[:139]' --raw-output) - echo ::set-output name=devdependencies::$(cat .results.json | jq '.devDependencies | join(", ") | .[:139]' --raw-output) - - - name: Set commit status messages and success states - if: github.event.pull_request.head.repo.full_name == github.repository - run: | - if [ "${{ steps.depcheck.outputs.dependencies }}" = "" ]; then - commit-status success "DepCheck / dependencies" "No unused packages detected." - else - commit-status failure "DepCheck / dependencies" "${{ steps.depcheck.outputs.dependencies }}…" - fi - - if [ "${{ steps.depcheck.outputs.devdependencies }}" = "" ]; then - commit-status success "DepCheck / devDependencies" "No unused packages detected." - else - commit-status failure "DepCheck / devDependencies" "${{ steps.depcheck.outputs.devdependencies }}…" - fi diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml new file mode 100644 index 000000000..3d3642831 --- /dev/null +++ b/.github/workflows/deploy-preprod.yml @@ -0,0 +1,62 @@ +name: Deploy to Heroku Preprod + +on: + push: + branches: + - edge + +jobs: + deploy-backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Login to Heroku + uses: akhileshns/heroku-deploy@v3.13.15 + with: + heroku_api_key: ${{secrets.HEROKU_API_KEY}} + heroku_app_name: "polis-preprod" + heroku_email: ${{secrets.HEROKU_EMAIL}} + branch: "edge" + + - name: Deploy to Heroku + run: | + git push https://heroku:${{secrets.HEROKU_API_KEY}}@git.heroku.com/polis-preprod.git edge:main + + deploy-static: + runs-on: ubuntu-latest + needs: deploy-backend + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-west-2 + + - name: Build static assets + env: + EMBED_SERVICE_HOSTNAME: preprod.pol.is + ENABLE_TWITTER_WIDGETS: true + GA_TRACKING_ID: G-WVP78N35QR + SERVICE_URL: https://preprod.pol.is + run: | + docker compose create --build --force-recreate file-server + docker cp file-server:/app/build/build ./build + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install -r deploy/requirements.txt + + - name: Deploy to S3 + run: | + python deploy/deploy-static-assets.py --bucket edge.static-assets.pol.is diff --git a/client-admin/Dockerfile b/client-admin/Dockerfile index 3e51120f9..c3da0d223 100644 --- a/client-admin/Dockerfile +++ b/client-admin/Dockerfile @@ -7,12 +7,12 @@ FROM docker.io/node:18-alpine ARG ENABLE_TWITTER_WIDGETS ARG FB_APP_ID -ENV ENABLE_TWITTER_WIDGETS ${ENABLE_TWITTER_WIDGETS} -ENV FB_APP_ID ${FB_APP_ID} +ENV ENABLE_TWITTER_WIDGETS=${ENABLE_TWITTER_WIDGETS} +ENV FB_APP_ID=${FB_APP_ID} # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app diff --git a/client-admin/webpack.config.js b/client-admin/webpack.config.js index c5c64e1fe..66a59e500 100644 --- a/client-admin/webpack.config.js +++ b/client-admin/webpack.config.js @@ -98,7 +98,6 @@ module.exports = (env, options) => { function writeHeadersJsonHtml() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Type': 'text/html; charset=UTF-8', 'Cache-Control': 'no-cache' } @@ -107,7 +106,6 @@ module.exports = (env, options) => { function writeHeadersJsonJs() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Encoding': 'gzip', 'Content-Type': 'application/javascript', 'Cache-Control': diff --git a/client-participation/Dockerfile b/client-participation/Dockerfile index 968f3588f..a34663453 100644 --- a/client-participation/Dockerfile +++ b/client-participation/Dockerfile @@ -8,13 +8,13 @@ FROM docker.io/node:18-alpine ARG EMBED_SERVICE_HOSTNAME ARG FB_APP_ID ARG GA_TRACKING_ID -ENV EMBED_SERVICE_HOSTNAME ${EMBED_SERVICE_HOSTNAME} -ENV FB_APP_ID ${FB_APP_ID} -ENV GA_TRACKING_ID ${GA_TRACKING_ID} +ENV EMBED_SERVICE_HOSTNAME=${EMBED_SERVICE_HOSTNAME} +ENV FB_APP_ID=${FB_APP_ID} +ENV GA_TRACKING_ID=${GA_TRACKING_ID} # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app diff --git a/client-participation/webpack.config.js b/client-participation/webpack.config.js index ed549218a..e6578fa56 100644 --- a/client-participation/webpack.config.js +++ b/client-participation/webpack.config.js @@ -33,7 +33,6 @@ function writeHeadersJsonForOutputFiles(isDev) { function writeHeadersJsonHtml() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Type': 'text/html; charset=UTF-8', 'Cache-Control': 'no-cache' } @@ -42,7 +41,6 @@ function writeHeadersJsonForOutputFiles(isDev) { function writeHeadersJsonJs() { const headersData = { - 'x-amz-acl': 'public-read', ...(!isDev && { 'Content-Encoding': 'gzip' }), 'Content-Type': 'application/javascript', 'Cache-Control': @@ -54,7 +52,6 @@ function writeHeadersJsonForOutputFiles(isDev) { function writeHeadersJsonCss() { const headersData = { - 'x-amz-acl': 'public-read', ...(!isDev && { 'Content-Encoding': 'gzip' }), 'Content-Type': 'text/css', 'Cache-Control': diff --git a/client-report/Dockerfile b/client-report/Dockerfile index 5e1b71db3..ce0bdc443 100644 --- a/client-report/Dockerfile +++ b/client-report/Dockerfile @@ -7,7 +7,7 @@ FROM docker.io/node:18-alpine # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app @@ -23,7 +23,7 @@ COPY . . # Or may be passed in at build time with --build-arg GIT_HASH=$(git rev-parse --short HEAD) ARG GIT_HASH ARG SERVICE_URL -ENV GIT_HASH ${GIT_HASH:-placeholder} -ENV SERVICE_URL ${SERVICE_URL} +ENV GIT_HASH=${GIT_HASH:-placeholder} +ENV SERVICE_URL=${SERVICE_URL} CMD npm run build:prod diff --git a/client-report/writeHeadersJsonTask.js b/client-report/writeHeadersJsonTask.js index 2d3f54b0e..3cb98f0ee 100644 --- a/client-report/writeHeadersJsonTask.js +++ b/client-report/writeHeadersJsonTask.js @@ -13,7 +13,6 @@ module.exports = function writeHeadersJson() { function writeHeadersJsonCss() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Type': 'text/css', 'Cache-Control': 'no-transform,public,max-age=31536000,s-maxage=31536000' @@ -23,7 +22,6 @@ module.exports = function writeHeadersJson() { function writeHeadersJsonHtml() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Type': 'text/html; charset=UTF-8', 'Cache-Control': 'no-cache' } @@ -32,7 +30,6 @@ module.exports = function writeHeadersJson() { function writeHeadersJsonJs() { const headersData = { - 'x-amz-acl': 'public-read', 'Content-Type': 'application/javascript', 'Cache-Control': 'no-transform,public,max-age=31536000,s-maxage=31536000' diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 000000000..84f0a7a0c --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1,18 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +env/ +.env +venv/ +.venv +.pytest_cache/ +.mypy_cache/ + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 000000000..bc1e2c7e4 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,87 @@ +# Polis Deploy + +Tools for deploying Polis static assets to S3. + +## Prerequisites + +- python 3.8+ +- aws cli +- heroku cli + +### Push the backend code to Heroku + +```bash +heroku login +git push edge:main +``` + +_Replace `` with the appropriate heroku remote, e.g. heroku-preprod_ + +### Build the Polis static assets + +from the root of the project: + +```bash +make ENV_FILE= PROD build-web-assets +``` + +### AWS CLI + +Log into AWS SSO and configure your AWS CLI with the appropriate profile. + +#### First time setup (or to refresh credentials) + +```bash +aws configure sso +``` + +follow the prompts to configure your profile. +e.g. + +> SSO session name: polis-deploy +> +> SSO start URL: [aws-start-url] +> +> SSO region: us-east-1 +> +> SSO registration scopes: [enter for default] +> +> CLI default client Region: us-east-1 +> +> CLI default output format: json +> +> CLI profile name: polis-deploy + +#### Login with the above profile + +```bash +export AWS_PROFILE=polis-deploy + +aws sso login + +# Verify that you are logged in +aws sts get-caller-identity +``` + +## Python Setup + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +pip install -r dev-requirements.txt +``` + +## Usage + +```bash +python deploy-static-assets.py --bucket +``` + +_Replace `` with the appropriate bucket name, e.g. edge.static-assets.pol.is_ + +Or from the root of the project: + +```bash +python deploy/deploy-static-assets.py --bucket +``` diff --git a/deploy/deploy-static-assets.py b/deploy/deploy-static-assets.py new file mode 100644 index 000000000..2627eb3e2 --- /dev/null +++ b/deploy/deploy-static-assets.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import os +import boto3 +import mimetypes +import json +import argparse +import sys +from pathlib import Path + +# Map of file extensions to content types (only for special cases) +CONTENT_TYPES = { + ".woff": "application/x-font-woff", + ".woff2": "application/font-woff2", + ".ttf": "application/x-font-ttf", + ".otf": "application/x-font-opentype", + ".eot": "application/vnd.ms-fontobject", + ".svg": "image/svg+xml", +} + +# Cache settings +CACHE_BUSTER = "no-transform,public,max-age=31536000,s-maxage=31536000" + + +def get_content_type(file_path): + """Get content type based on file extension""" + ext = os.path.splitext(file_path)[1] + return CONTENT_TYPES.get(ext) or mimetypes.guess_type(file_path)[0] + + +def upload_file(s3_client, file_path, bucket, base_path): + """Upload single file to S3 with appropriate headers""" + relative_path = str(file_path).replace(base_path + "/", "") + + # Default upload arguments - no ACL + extra_args = {"CacheControl": CACHE_BUSTER} + + # Check for associated headersJson file + headers_path = str(file_path) + ".headersJson" + if os.path.exists(headers_path): + with open(headers_path) as f: + headers = json.load(f) + # Map only essential headers + header_mapping = { + "Cache-Control": "CacheControl", + "Content-Type": "ContentType", + "Content-Encoding": "ContentEncoding", + } + for old_key, new_key in header_mapping.items(): + if old_key in headers: + extra_args[new_key] = headers[old_key] + else: + # Set content type for files without headers + content_type = get_content_type(file_path) + if content_type: + extra_args["ContentType"] = content_type + + print(f"Uploading {relative_path} to {bucket}") + try: + s3_client.upload_file( + str(file_path), bucket, relative_path, ExtraArgs=extra_args + ) + except Exception as e: + print(f"Error uploading {relative_path}: {str(e)}") + raise + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--bucket", required=True, help="S3 bucket name") + args = parser.parse_args() + + s3_client = boto3.client("s3") + script_dir = os.path.dirname(os.path.abspath(__file__)) + build_path = os.path.normpath(os.path.join(script_dir, "..", "build")) + + if not os.path.exists(build_path): + print(f"Error: Build directory not found at {build_path}") + sys.exit(1) + + for file_path in Path(build_path).rglob("*"): + if file_path.is_file() and not str(file_path).endswith(".headersJson"): + upload_file(s3_client, file_path, args.bucket, build_path) + + +if __name__ == "__main__": + main() diff --git a/deploy/dev-requirements.txt b/deploy/dev-requirements.txt new file mode 100644 index 000000000..6403dcbbf --- /dev/null +++ b/deploy/dev-requirements.txt @@ -0,0 +1,4 @@ +black==24.2.0 +flake8==7.0.0 +mypy==1.8.0 +types-boto3 \ No newline at end of file diff --git a/deploy/pyproject.toml b/deploy/pyproject.toml new file mode 100644 index 000000000..e6854216e --- /dev/null +++ b/deploy/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "polis-deploy-tools" +version = "0.1.0" +requires-python = ">=3.8" + +[tool.black] +line-length = 88 +target-version = ['py38'] + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +check_untyped_defs = true diff --git a/deploy/requirements.txt b/deploy/requirements.txt new file mode 100644 index 000000000..6179f113f --- /dev/null +++ b/deploy/requirements.txt @@ -0,0 +1 @@ +boto3>=1.26.0 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e1417d280..ab5673ae6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,10 +54,10 @@ services: labels: polis_tag: ${TAG:-dev} environment: - - DATABASE_URL=${DATABASE_URL:?error} + - DATABASE_URL=${DATABASE_URL} - MATH_ENV=${MATH_ENV:-prod} - - WEBSERVER_USERNAME=${WEBSERVER_USERNAME:?error} - - WEBSERVER_PASS=${WEBSERVER_PASS:?error} + - WEBSERVER_USERNAME=${WEBSERVER_USERNAME} + - WEBSERVER_PASS=${WEBSERVER_PASS} networks: - "polis-net" @@ -70,9 +70,9 @@ services: labels: polis_tag: ${TAG:-dev} environment: - - POSTGRES_DB=${POSTGRES_DB:?error} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error} - - POSTGRES_USER=${POSTGRES_USER:?error} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_USER=${POSTGRES_USER} networks: - "polis-net" volumes: diff --git a/file-server/Dockerfile b/file-server/Dockerfile index dbc2cbfad..6f6e6e177 100644 --- a/file-server/Dockerfile +++ b/file-server/Dockerfile @@ -9,12 +9,12 @@ FROM docker.io/node:18-alpine AS client-admin ARG ENABLE_TWITTER_WIDGETS ARG FB_APP_ID -ENV ENABLE_TWITTER_WIDGETS ${ENABLE_TWITTER_WIDGETS} -ENV FB_APP_ID ${FB_APP_ID} +ENV ENABLE_TWITTER_WIDGETS=${ENABLE_TWITTER_WIDGETS} +ENV FB_APP_ID=${FB_APP_ID} # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app @@ -34,13 +34,13 @@ FROM docker.io/node:18-alpine AS client-participation ARG EMBED_SERVICE_HOSTNAME ARG FB_APP_ID ARG GA_TRACKING_ID -ENV EMBED_SERVICE_HOSTNAME ${EMBED_SERVICE_HOSTNAME} -ENV FB_APP_ID ${FB_APP_ID} -ENV GA_TRACKING_ID ${GA_TRACKING_ID} +ENV EMBED_SERVICE_HOSTNAME=${EMBED_SERVICE_HOSTNAME} +ENV FB_APP_ID=${FB_APP_ID} +ENV GA_TRACKING_ID=${GA_TRACKING_ID} # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app @@ -60,7 +60,7 @@ FROM docker.io/node:18-alpine AS client-report # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app @@ -75,8 +75,8 @@ COPY client-report/. . # GIT_HASH will be set properly when running `make` (see Makefile). ARG GIT_HASH ARG SERVICE_URL -ENV GIT_HASH ${GIT_HASH:-placeholder} -ENV SERVICE_URL ${SERVICE_URL} +ENV GIT_HASH=${GIT_HASH:-placeholder} +ENV SERVICE_URL=${SERVICE_URL} RUN npm run build:prod @@ -87,7 +87,7 @@ FROM docker.io/node:18-alpine # Set default NODE_ENV to production unless overridden at build time with --build-arg NODE_ENV=development ARG NODE_ENV -ENV NODE_ENV ${NODE_ENV:-production} +ENV NODE_ENV=${NODE_ENV:-production} WORKDIR /app