Dockerfy Openverse dev env

.github/CODEOWNERS

@@ -46,3 +46,4 @@ README.md                   @WordPress/openverse-maintainers
 docker-compose.yml          @WordPress/openverse-maintainers
 env.template                @WordPress/openverse-maintainers
 justfile                    @WordPress/openverse-maintainers
+ov                          @WordPress/openverse-maintainers

.gitignore

@@ -13,6 +13,7 @@ logs/
 
 # Environment
 .env
+.ovprofile
 
 # Python environments
 env/

docker/dev_env/Dockerfile

@@ -0,0 +1,46 @@
+# Use arch. Alpine is nice and small, but permissions and user management
+# are obscure, easy to break, and way too tedious to maintain for a development
+# environment that will inevitably have continuously changing requirements
+FROM docker.io/library/archlinux:latest
+
+# Dependency explanations
+# - git: required by some python package; contributors should use git on their host
+# - make: required by some pre-commit hooks for node-gyp
+# - perl: used in linting
+# - bash: used for scripts
+# - python: language runtime
+# - python-pipx: python CLI app installer
+# - docker*: used to interact with host docker socket
+# - nodejs, npm: language runtime and dependency manager (system npm required by node-based pre-commit hooks)
+# - just: command runner
+# - n: nodejs distribution manager
+# - pdm, pipenv: Python package managers
+# - pre-commit: pre-commit hook manager
+RUN pacman -Syyu --needed --noconfirm \
+  git bash perl curl base-devel \
+  python python-pipx \
+  docker docker-compose docker-buildx \
+  nodejs npm \
+  just \
+  && corepack enable \
+  && useradd -m opener -G docker \
+  && npm install -g n
+
+USER opener
+
+# pipx installed tools are user-dependent, so install these after switching to opener
+RUN pipx install pdm pre-commit
+
+ENV PATH="/home/opener/.local/bin:${PATH}"
+
+# Avoid overwriting `.venv`'s from the host
+ENV PDM_VENV_IN_PROJECT="False"
+
+ENV N_PREFIX="/home/opener/.local/share/n"
+ENV PATH="${N_PREFIX}/bin:${PATH}"
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["/bin/sh"]

docker/dev_env/bash_profile.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+# Place common shared utility functions here
+# Functions, not aliases, must be used. Aliases are not expanded in scripts
+# and as such will not be available in `entrypoint.sh` when the command
+# passed to the docker run is evaluated. That means that `j` below, for example,
+# wouldn't work if using `ov j`. However, as a function, it does work.
+
+# Personal aliases and utilities should go in an .ovprofile file at the monorepo root
+
+j() {
+  just "$@"
+}

docker/dev_env/entrypoint.sh

@@ -0,0 +1,76 @@
+#! /usr/bin/env bash
+
+set -e
+
+if [ "$UID" = "0" ]; then
+  printf "Do not run this container with the root user! Use the 'run.sh' script or pass --user to docker run.\n"
+  exit 1
+fi
+
+if [ ! -d "$OPENVERSE_PROJECT"/.git ]; then
+  printf "Repository not mounted to container!\n"
+  exit 1
+fi
+
+cd "$OPENVERSE_PROJECT" || exit 1
+
+suppress_output() {
+  "$@" 2>/dev/null 1>/dev/null
+}
+
+suppress_output corepack install
+
+if [ -z "$(n ls 2>/dev/null)" ]; then
+  printf "Installing the specific Node JS version required by Openverse frontend; this is only necessary the first time the toolkit runs\n"
+  n install auto
+fi
+
+if [ -n "$PNPM_HOME" ]; then
+  export PATH="$PNPM_HOME:$PATH"
+fi
+
+_python3s=("$HOME"/.local/share/pdm/python/[email protected].*/bin/python3)
+
+if [ ! -x "${_python3s[0]}" ]; then
+  printf "Installing the specific Python version required for pipenv environments; this is only necessary the first time the toolkit runs\n"
+  pdm python install 3.11
+  _python3s=("$HOME"/.local/share/pdm/python/[email protected].*/bin/python3)
+fi
+
+PYTHON_311=${_python3s[0]}
+export PYTHON_311
+
+if [ ! -x "$HOME"/.local/bin/python3.11 ]; then
+  ln -s "$PYTHON_311" "$HOME"/.local/bin/python3.11
+fi
+
+if [ -z "$(command -v pipenv)" ]; then
+  # Install pipenv with the specific python version required by environments
+  # still using it, otherwise it gets confused about which dependencies to use
+  pipx install pipenv --python "$PYTHON_311"
+fi
+
+if [ -n "$PDM_CACHE_DIR" ]; then
+  pdm config install.cache on
+fi
+
+_profiles=(
+  "$OPENVERSE_PROJECT"/.ovprofile
+  "$OPENVERSE_PROJECT"/docker/dev_env/bash_profile.sh
+)
+
+for profile in "${_profiles[@]}"; do
+  if [ -f "$profile" ]; then
+    if grep -q '^alias ' "$profile"; then
+      printf "Aliases in %s will not work. Use functions instead." "$profile" >>/dev/stderr
+      exit 1
+    fi
+
+    # SC1090 cannot be disabled directly
+    # Workaround from https://github.com/koalaman/shellcheck/issues/2038#issuecomment-680513830
+    # shellcheck source=/dev/null
+    source "$profile"
+  fi
+done
+
+"$@"

docker/dev_env/hooks/pre-commit

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+export TERM=xterm
+
+ov hook pre-commit

docker/dev_env/run.sh

@@ -0,0 +1,65 @@
+#! /usr/bin/env bash
+
+set -e
+
+host_docker_gid="$(getent group docker | cut -d: -f3)"
+
+suppress_output() {
+  "$@" 2>/dev/null 1>/dev/null
+}
+
+opener_home="/home/opener"
+
+run_args=(
+  -i
+  --rm
+  --name openverse-dev-env
+  --env "OPENVERSE_PROJECT=$OPENVERSE_PROJECT"
+  --env "TERM=xterm-256color"
+  --user "$UID:$host_docker_gid"
+  --network host
+  # Bind the repo to the same exact location inside the container so that pre-commit
+  # and others don't get confused about where files are supposed to be
+  -v "$OPENVERSE_PROJECT:$OPENVERSE_PROJECT:rw,z"
+  --workdir "$OPENVERSE_PROJECT"
+  # Save the home directory of the container so we can reuse it each time
+  --mount "type=volume,src=openverse-dev-env,target=$opener_home"
+  # Expose the host's docker socket to the container so the container can run docker/compose etc
+  -v /var/run/docker.sock:/var/run/docker.sock
+)
+
+# When running `ov` directly, `-t 0` will show that stdin is available, so
+# we should provision a TTY in the docker container (making it possible to
+# interact with the container directly)
+# However, when running in pre-commit (for example), there is no TTY, and
+# docker run will complain if `-t` requests a TTY when the execution
+# environment doesn't have one to attach.
+# In other words, only tell Docker to attach a TTY to the container when
+# there's one to attach in the first place.
+if [ -t 0 ]; then
+  run_args+=(-t)
+fi
+
+host_pnpm_store="$(pnpm store path 2>/dev/null || echo)"
+
+# Share the pnpm cache with the container, if it's available locally
+if [ "$host_pnpm_store" != "" ]; then
+  pnpm_home="$(dirname "$host_pnpm_store")"
+  run_args+=(
+    --env PNPM_HOME="$pnpm_home"
+    -v "$pnpm_home:$pnpm_home:rw,z"
+  )
+fi
+
+# Share the PDM cache with the container, if it's available locally
+# --quiet so PDM doesn't repeatedly fill the console with update messages
+# if they're enabled
+if [ "$(pdm config --quiet install.cache)" == "True" ]; then
+  host_pdm_cache="$(pdm config --quiet cache_dir)"
+  run_args+=(
+    --env "PDM_CACHE_DIR=$host_pdm_cache"
+    -v "$host_pdm_cache:$host_pdm_cache:rw,z"
+  )
+fi
+
+docker run "${run_args[@]}" openverse-dev-env:latest "$@"

justfile

@@ -62,6 +62,33 @@ install:
     just node-install
     just py-install
 
+@install-hooks:
+    bash -c "cp ./docker/dev_env/hooks/* ./.git/hooks"
+
+init-ovprofile:
+    #! /usr/bin/env bash
+    [[ -f ./.ovprofile ]] && echo '.ovprofile already exists! No changes made.' && exit 0 || cat <<-'EOPROFILE' > ./.ovprofile
+    #! /usr/bin/env bash
+
+    # Personal modifications to the ov dev environment go here
+    # Use bash functions to define aliases of your common workflow
+    # Shared aliases are available in docker/dev_env/bash_profile
+
+    # This is just an example bash function for demonstration. Feel free to delete it.
+    # ASCII art courtesy of http://patorjk.com/software/taag/#p=display&f=Big&t=Openverse
+    welcome_to_openverse() {
+        cat <<OPENVERSE
+      ___   ____   ___  ____   __ __    ___  ____    _____   ___
+     /   \ |    \ /  _]|    \ |  |  |  /  _]|    \  / ___/  /  _]
+    |     ||  o  )  [_ |  _  ||  |  | /  [_ |  D  )(   \_  /  [_
+    |  O  ||   _/    _]|  |  ||  |  ||    _]|    /  \__  ||    _]
+    |     ||  | |   [_ |  |  ||  :  ||   [_ |    \  /  \ ||   [_
+    |     ||  | |     ||  |  | \   / |     ||  .  \ \    ||     |
+     \___/ |__| |_____||__|__|  \_/  |_____||__|\_|  \___||_____|
+    OPENVERSE
+    }
+    EOPROFILE
+
 # Setup pre-commit as a Git hook
 precommit:
     #!/usr/bin/env bash

ov

@@ -0,0 +1,95 @@
+#! /usr/bin/env bash
+
+set -e
+
+# https://stackoverflow.com/a/1482133
+OPENVERSE_PROJECT="$(dirname "$(readlink -f -- "$0")")"
+export OPENVERSE_PROJECT
+
+_self="$OPENVERSE_PROJECT/ov"
+
+_cmd="$1"
+
+dev_env="$OPENVERSE_PROJECT"/docker/dev_env
+
+if [[ $_cmd == "help" || -z $_cmd ]]; then
+  cat <<-'EOF'
+Openverse development toolkit
+
+USAGE
+  ov <subcommand> [args]
+
+COMMANDS
+  ov init
+    Initialise the Openverse development toolkit for the first time
+    Alias for:
+
+        ov build && ov just install-hooks
+
+  ov build
+    Build the Openverse development toolkit Docker image
+
+  ov clean
+    Remove the Openverse development toolkit Docker image and volume
+    Use in conjunction with `ov init` to recreate the environment from
+    scratch:
+
+        ov clean && ov init
+
+  ov link [path]
+    Create a symlink from the path arg to the ov script. Path defaults
+    to the XDG-like ~/.local/bin/ov
+
+  ov hook HOOK
+    Run git hooks through pre-commit inside the development toolkit container
+
+  ov COMMAND
+    Run COMMAND inside the development toolkit container
+    Hints: The toolkit comes loaded with many tools for working with Openverse! Try
+    some of the following:
+
+      - ov just
+      - ov pdm
+      - ov pnpm
+      - ov python
+      - ov bash
+EOF
+
+  exit 0
+fi
+
+case "$_cmd" in
+init)
+  "$_self" build
+  "$_self" just install-hooks
+  "$_self" just init-ovprofile
+  ;;
+
+build)
+  docker build -t openverse-dev-env:latest "$dev_env"
+  ;;
+
+clean)
+  docker volume rm openverse-dev-env
+  ;;
+
+link)
+  # Assumes XDG-like bin directory... there's probably a better way? :thinking:
+  ln -s "$_self" "${2:$HOME/.local/bin/ov}"
+  ;;
+
+hook)
+  # Arguments match the implementation of hooks installed by pre-commit
+  "$_self" pre-commit hook-impl \
+    --config=.pre-commit-config.yaml \
+    --hook-type="$2" \
+    --hook-dir "$OPENVERSE_PROJECT"/.git/hooks \
+    --color=always \
+    -- "${@:3}"
+  ;;
+
+*)
+  "$dev_env"/run.sh "$@"
+  ;;
+
+esac