move files around for easier setup in docker

Dockerfile

@@ -1,25 +1,37 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.10.0
 
 RUN apt-get update && apt-get --assume-yes upgrade \
     && apt-get -qq install -y --no-install-recommends postgresql-common \
     && apt-get -qq install -y --no-install-recommends yes \
     && apt-get -qq install -y --no-install-recommends gnupg \
     && yes '' | sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh \
-    && apt-get -qq install -y --no-install-recommends postgresql-client-16 \
+    && apt-get -qq install -y --no-install-recommends postgresql-client-17 \
     && apt-get -qq install -y --no-install-recommends make \
     && apt-get -qq install -y --no-install-recommends g++ \
     && apt-get -qq install -y --no-install-recommends git \
     && apt-get -qq install -y --no-install-recommends zip \
     && apt-get -qq install -y --no-install-recommends unzip \
-    && apt-get -qq install -y --no-install-recommends python3-dev \
+    && apt-get -qq install -y --no-install-recommends parallel \
     && apt-get -qq install -y --no-install-recommends python3-pip \
+    && apt-get -qq install -y --no-install-recommends python3-dev \
+    && apt-get -qq install -y --no-install-recommends python3-venv \
     && apt-get -qq install -y --no-install-recommends python3-psycopg2 \
-    && pip3 install --upgrade numpy \
-    && pip3 install bcdata \
-    && pip3 install rasterstats \
     && rm -rf /var/lib/apt/lists/*
 
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install
+
+RUN python3 -m venv /opt/venv && \
+    /opt/venv/bin/python -m pip install -U pip && \
+    /opt/venv/bin/python -m pip install --no-cache-dir --upgrade numpy && \
+    /opt/venv/bin/python -m pip install --no-cache-dir rasterstats && \
+    /opt/venv/bin/python -m pip install --no-cache-dir bcdata
+
 WORKDIR /home/fwapg
+
 COPY ["db", "db/"]
 COPY ["extras", "extras/"]
-COPY [".env.docker", "Makefile", "./"]
+COPY [".env.docker", "Makefile", "./"]
+
+ENV PATH="/opt/venv/bin:$PATH"

Makefile

@@ -44,33 +44,11 @@ clean_db:
 	rm -Rf .make/db
 	$(PSQL) -f db/clean.sql
 
-# create database objects
-# ** the database must already exist **
+# create extensions/schemas/tables/views/functions
 .make/db:
 	mkdir -p .make
 	mkdir -p data
-	$(PSQL) -f db/schemas.sql
-	$(PSQL) -f db/extensions.sql
-	echo "ALTER DATABASE :db SET search_path TO public,whse_basemapping,usgs,hydrosheds" | \
-	  $(PSQL) -v db=$(shell echo $(DATABASE_URL) | cut -d "/" -f 4)
-	$(PSQL) -f db/tables.sql
-	$(PSQL) -f db/views.sql
-	$(PSQL) -f db/functions/CDB_MakeHexagon.sql
-	$(PSQL) -f db/functions/ST_Safe_Repair.sql
-	$(PSQL) -f db/functions/FWA_Downstream.sql
-	$(PSQL) -f db/functions/FWA_Upstream.sql
-	$(PSQL) -f db/functions/huc12.sql
-	$(PSQL) -f db/functions/hydroshed.sql
-	$(PSQL) -f db/functions/FWA_SliceWatershedAtPoint.sql
-	$(PSQL) -f db/functions/FWA_WatershedAtMeasure.sql
-	$(PSQL) -f db/functions/FWA_WatershedHex.sql
-	$(PSQL) -f db/functions/FWA_WatershedStream.sql
-	$(PSQL) -f db/functions/FWA_UpstreamBorderCrossings.sql
-	$(PSQL) -f db/functions/FWA_IndexPoint.sql
-	$(PSQL) -f db/functions/FWA_LocateAlong.sql
-	$(PSQL) -f db/functions/FWA_LocateAlongInterval.sql
-	$(PSQL) -f db/functions/FWA_UpstreamTrace.sql
-	$(PSQL) -f db/functions/postgisftw.sql  # pg_fs/pg_ts functions
+	cd db && ./create.sh && cd ..
 	touch $@
 
 # download

db/create.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -euxo pipefail
+
+PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
+
+$PSQL -f sql/schemas.sql
+$PSQL -f sql/extensions.sql
+
+echo "ALTER DATABASE :db SET search_path TO public,whse_basemapping,usgs,hydrosheds" |
+  $PSQL -v db=$(echo $DATABASE_URL | cut -d "/" -f 4)
+
+$PSQL -f sql/tables.sql
+$PSQL -f sql/views.sql
+$PSQL -f sql/functions/CDB_MakeHexagon.sql
+$PSQL -f sql/functions/ST_Safe_Repair.sql
+$PSQL -f sql/functions/FWA_Downstream.sql
+$PSQL -f sql/functions/FWA_Upstream.sql
+$PSQL -f sql/functions/huc12.sql
+$PSQL -f sql/functions/hydroshed.sql
+$PSQL -f sql/functions/FWA_SliceWatershedAtPoint.sql
+$PSQL -f sql/functions/FWA_WatershedAtMeasure.sql
+$PSQL -f sql/functions/FWA_WatershedHex.sql
+$PSQL -f sql/functions/FWA_WatershedStream.sql
+$PSQL -f sql/functions/FWA_UpstreamBorderCrossings.sql
+$PSQL -f sql/functions/FWA_IndexPoint.sql
+$PSQL -f sql/functions/FWA_LocateAlong.sql
+$PSQL -f sql/functions/FWA_LocateAlongInterval.sql
+$PSQL -f sql/functions/FWA_UpstreamTrace.sql
+$PSQL -f sql/functions/postgisftw.sql  # pg_fs/pg_ts functions

docker-compose.yml

@@ -1,7 +1,7 @@
-version: "3.9"  
 services:
   db:
-    image: postgis/postgis:16-3.4
+    build:
+      context: docker/postgis
     shm_size: 16gb
     container_name: fwapg-db
     command: postgres -c default_statistics_target=100 -c max_connections=100 -c max_locks_per_transaction=64 -c checkpoint_timeout=30min -c maintenance_work_mem=1GB -c effective_cache_size=16GB -c work_mem=500MB -c max_wal_size=10GB -c wal_buffers=16MB -c shared_buffers=8GB
@@ -17,7 +17,7 @@ services:
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
-      - PG_DATABASE=fwapg
+      - POSTGRES_DB=fwapg
   loader:
     build:
       context: .

docker.md

@@ -1,16 +1,16 @@
 # Docker
 
-Download the repo, create containers, create database, load fwa data:
+Download the repo, create containers, create database, load data:
 
     git clone https://github.com/smnorris/fwapg.git
     cd fwapg
-    docker-compose build
-    docker-compose up -d
-    docker-compose run --rm loader psql -c "CREATE DATABASE fwapg" postgres
-    docker-compose run --rm loader make --debug=basic
+    docker compose build
+    docker compose up -d
+    docker compose run --rm loader make --debug=basic
+
+(note that docker images specified in `docker-compose.yml` may not be available on ARM based systems)
 
-Note that docker images specified in `docker-compose.yml` may not be available on ARM based systems.
-As long as you do not remove the container `fwapg-db`, it will retain all the data you put in it.
+The database (`$PGDATA`) is written to `postgres-data` - even if containers are deleted, the database will be retained here.
 If you have shut down Docker or the container, start it up again with this command:
 
     docker-compose up -d
@@ -19,11 +19,16 @@ Connect to the db from your host OS via the port specified in `docker-compose.ym
 
     psql -p 8000 -U postgres fwapg
 
-Or see the FWA data in the browser as vector tiles/geojson features:
+Once data is loaded, see it in the browser as vector tiles/geojson features:
 
     http://localhost:7800/
     http://localhost:9000/
 
-Delete the containers (and associated fwa data):
+Stop the containers (without deleting):
+
+    docker compose stop
+
+Delete the containers:
+
+    docker compose down
 
-    docker-compose down

docker/postgis/Dockerfile

@@ -0,0 +1,28 @@
+#
+# NOTE: THIS DOCKERFILE IS GENERATED VIA "make update"! PLEASE DO NOT EDIT IT DIRECTLY.
+#
+
+FROM postgres:17-bullseye
+
+LABEL maintainer="PostGIS Project - https://postgis.net" \
+      org.opencontainers.image.description="PostGIS 3.5.0+dfsg-1.pgdg110+1 spatial database extension with PostgreSQL 17 bullseye" \
+      org.opencontainers.image.source="https://github.com/postgis/docker-postgis"
+
+ENV POSTGIS_MAJOR 3
+ENV POSTGIS_VERSION 3.5.0+dfsg-1.pgdg110+1
+
+RUN apt-get update \
+      && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
+      && apt-get install -y --no-install-recommends \
+           # ca-certificates: for accessing remote raster files;
+           #   fix: https://github.com/postgis/docker-postgis/issues/307
+           ca-certificates \
+           \
+           postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \
+           postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \
+      && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /docker-entrypoint-initdb.d
+COPY ./initdb-postgis.sh /docker-entrypoint-initdb.d/10_postgis.sh
+COPY ./update-postgis.sh /usr/local/bin
+

docker/postgis/initdb-postgis.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+# Perform all actions as $POSTGRES_USER
+export PGUSER="$POSTGRES_USER"
+
+# Create the 'template_postgis' template db
+"${psql[@]}" <<- 'EOSQL'
+CREATE DATABASE template_postgis IS_TEMPLATE true;
+EOSQL
+
+# Load PostGIS into both template_database and $POSTGRES_DB
+for DB in template_postgis "$POSTGRES_DB"; do
+	echo "Loading PostGIS extensions into $DB"
+	"${psql[@]}" --dbname="$DB" <<-'EOSQL'
+		CREATE EXTENSION IF NOT EXISTS postgis;
+		CREATE EXTENSION IF NOT EXISTS postgis_topology;
+		-- Reconnect to update pg_setting.resetval
+		-- See https://github.com/postgis/docker-postgis/issues/288
+		\c
+		CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+		CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
+EOSQL
+done

docker/postgis/update-postgis.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+# Perform all actions as $POSTGRES_USER
+export PGUSER="$POSTGRES_USER"
+
+POSTGIS_VERSION="${POSTGIS_VERSION%%+*}"
+
+# Load PostGIS into both template_database and $POSTGRES_DB
+for DB in template_postgis "$POSTGRES_DB" "${@}"; do
+    echo "Updating PostGIS extensions '$DB' to $POSTGIS_VERSION"
+    psql --dbname="$DB" -c "
+        -- Upgrade PostGIS (includes raster)
+        CREATE EXTENSION IF NOT EXISTS postgis VERSION '$POSTGIS_VERSION';
+        ALTER EXTENSION postgis  UPDATE TO '$POSTGIS_VERSION';
+
+        -- Upgrade Topology
+        CREATE EXTENSION IF NOT EXISTS postgis_topology VERSION '$POSTGIS_VERSION';
+        ALTER EXTENSION postgis_topology UPDATE TO '$POSTGIS_VERSION';
+
+        -- Install Tiger dependencies in case not already installed
+        CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+        -- Upgrade US Tiger Geocoder
+        CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder VERSION '$POSTGIS_VERSION';
+        ALTER EXTENSION postgis_tiger_geocoder UPDATE TO '$POSTGIS_VERSION';
+    "
+done