diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd12b1df881..202ba48cdc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,7 +209,7 @@ jobs: - 6379:6379 options: --entrypoint redis-server postgres: - image: postgres:14.15 + image: postgres:15.10 env: POSTGRES_USER: postgres POSTGRES_DB: postgres @@ -261,7 +261,7 @@ jobs: - 6379:6379 options: --entrypoint redis-server postgres: - image: postgres:14.15 + image: postgres:15.10 env: POSTGRES_USER: postgres POSTGRES_DB: postgres @@ -324,7 +324,7 @@ jobs: - 6379:6379 options: --entrypoint redis-server postgres: - image: postgres:14.15 + image: postgres:15.10 env: POSTGRES_USER: postgres POSTGRES_DB: postgres @@ -366,7 +366,7 @@ jobs: - 6379:6379 options: --entrypoint redis-server postgres: - image: postgres:14.15 + image: postgres:15.10 env: POSTGRES_USER: postgres POSTGRES_DB: postgres diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9f0d99562e7..ea8de8d7e34 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -34,7 +34,7 @@ env: jobs: e2e: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: @@ -48,7 +48,7 @@ jobs: - 6379:6379 options: --entrypoint redis-server postgres: - image: postgres:14.15 + image: postgres:15.10 env: POSTGRES_USER: postgres POSTGRES_DB: postgres @@ -62,14 +62,15 @@ jobs: - name: Update apt run: sudo apt-get update || exit 0 - - name: Install Cypress dependencies - run: sudo apt-get install --no-install-recommends -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb fonts-arphic-bkai00mp fonts-arphic-bsmi00lp fonts-arphic-gbsn00lp fonts-arphic-gkai00mp fonts-arphic-ukai fonts-arphic-uming ttf-wqy-zenhei ttf-wqy-microhei xfonts-wqy - - - name: Install postgresql-client - run: sudo apt-get install -y postgresql-client - - - name: Install graphicsmagick - run: sudo apt-get install -y graphicsmagick + - name: Install dependencies + run: | + sudo apt-get install \ + `# Cypress dependencies - see https://docs.cypress.io/app/get-started/install-cypress#UbuntuDebian` \ + libgtk2.0-0t64 libgtk-3-0t64 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb \ + `# Postgres client` \ + postgresql-client-16 \ + `# GraphicsMagick (not sure if needed)` \ + graphicsmagick - name: Install stripe-cli run: | @@ -249,6 +250,8 @@ jobs: - name: Setup DB working-directory: opencollective-frontend run: ./scripts/setup_db.sh + env: + LOCALDBUSER: ${{ env.PGUSER }} - name: Run E2E with Cypress working-directory: opencollective-frontend diff --git a/docs/postgres.md b/docs/postgres.md index 6261daaada9..99095cf8909 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -1,6 +1,6 @@ # PostgreSQL Database -You need to have PostgreSQL > 14.x. +You need to have PostgreSQL > 15.x. In production, we're currently running 16.4. @@ -57,7 +57,7 @@ If you don't want to run a local instance of PostgreSQL in your computer, you ca Create and run the container: ``` -docker run -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -d --name opencollective-postgres --shm-size=1g --memory=4g --cpus=2 postgres:14 +docker run -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -d --name opencollective-postgres --shm-size=1g --memory=4g --cpus=2 postgres:15 ``` Set the necessary environment variables: diff --git a/scripts/db_restore.sh b/scripts/db_restore.sh index 62a3ebe6996..a5515a90d9f 100755 --- a/scripts/db_restore.sh +++ b/scripts/db_restore.sh @@ -1,48 +1,47 @@ #!/bin/bash usage() { - echo "Usage: db_restore.sh -d DBNAME -U DBUSER --use-postgis -f DBDUMP_FILE"; - echo "e.g."; + echo "Usage: db_restore.sh -d DBNAME -U DBUSER --use-postgis -f DBDUMP_FILE" + echo "e.g." echo "> db_restore.sh -d opencollective_dvl -U opencollective -f test/dbdumps/opencollective_dvl.pgsql" - exit 0; + exit 0 } -while [[ $# -gt 0 ]] -do -key="$1" +while [[ $# -gt 0 ]]; do + key="$1" -case $key in - -d|--dbname) + case $key in + -d | --dbname) LOCALDBNAME="$2" shift # past argument ;; - -U|--username) + -U | --username) LOCALDBUSER="$2" shift # past argument ;; - --use-postgis) + --use-postgis) USE_POSTGIS=1 ;; - -f|--file) + -f | --file) DBDUMP_FILE="$2" shift # past argument ;; - *) - # unknown option + *) + # unknown option ;; -esac -shift # past argument or value + esac + shift # past argument or value done LOCALDBUSER=${LOCALDBUSER:-"opencollective"} LOCALDBNAME=${LOCALDBNAME:-"opencollective_dvl"} -DBDUMP_FILE=${DBDUMP_FILE:-"test/dbdumps/opencollective_dvl.pgsql"} +DBDUMP_FILE=${DBDUMP_FILE:-"./test/dbdumps/opencollective_dvl.pgsql"} echo "LOCALDBUSER=$LOCALDBUSER" echo "LOCALDBNAME=$LOCALDBNAME" echo "DBDUMP_FILE=$DBDUMP_FILE" -if [ -z "$LOCALDBNAME" ]; then usage; fi; +if [ -z "$LOCALDBNAME" ]; then usage; fi # kill all connections to the postgres server # echo "Killing all connections to database '$LOCALDBNAME'" @@ -53,8 +52,15 @@ if [ -z "$LOCALDBNAME" ]; then usage; fi; # where pg_stat_activity.datname = '$LOCALDBNAME' # EOF -dropdb -U postgres -h localhost --if-exists $LOCALDBNAME; -createdb -U postgres -h localhost $LOCALDBNAME 2> /dev/null +echo "Dropping '$LOCALDBNAME'" +dropdb -U postgres -h localhost --if-exists $LOCALDBNAME + +echo "Creating '$LOCALDBNAME'" +createdb -U postgres -h localhost $LOCALDBNAME 2>/dev/null + +# Update table permissions +echo "Updating table permissions" +psql -U postgres -h localhost $LOCALDBNAME -c "GRANT ALL ON SCHEMA public TO ${LOCALDBUSER};" # When restoring old backups, you may need to enable Postgis if [ "$USE_POSTGIS" = "1" ]; then @@ -72,13 +78,50 @@ fi set -e } | tee >/dev/null -# The first time we run it, we will trigger FK constraints errors -set +e -pg_restore -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${DBDUMP_FILE}" 2>/dev/null -set -e +PG_RESTORE_OPTIONS=(-U postgres -h localhost --no-acl --no-owner --role="${LOCALDBUSER}" -n public -O -d "${LOCALDBNAME}") +PG_RESTORE_COMMAND=pg_restore +if [ -n "$USE_DOCKER" ]; then + PG_RESTORE_COMMAND="sudo docker run --rm --network host -v $(dirname "$DBDUMP_FILE"):/backup postgres:14 pg_restore -h localhost -p 5432 -U opencollective" + DBDUMP_FILE="/backup/$(basename "$DBDUMP_FILE")" +fi + +# The first time we run it with the -s option (schema only) +echo "Restoring schema" +$PG_RESTORE_COMMAND "${PG_RESTORE_OPTIONS[@]}" -s "${DBDUMP_FILE}" + +# Disable triggers +echo "Disabling triggers" +psql -U postgres -h localhost "${LOCALDBNAME}" -c " +DO \$\$ +BEGIN + EXECUTE ( + SELECT string_agg( + format('ALTER TABLE %I.%I DISABLE TRIGGER ALL', schemaname, tablename), '; ' + ) + FROM pg_tables + WHERE schemaname = 'public' + ); +END \$\$; +" + +# Restore data (-a flag) +echo "Restoring data" +$PG_RESTORE_COMMAND "${PG_RESTORE_OPTIONS[@]}" -a "${DBDUMP_FILE}" -# So we run it twice :-) -pg_restore -U postgres -h localhost --no-acl --no-owner --role=${LOCALDBUSER} -n public -O -c -d "${LOCALDBNAME}" "${DBDUMP_FILE}" +# Re-enable triggers +echo "Re-enabling triggers" +psql -U postgres -h localhost "${LOCALDBNAME}" -c " +DO \$\$ +BEGIN + EXECUTE ( + SELECT string_agg( + format('ALTER TABLE %I.%I ENABLE TRIGGER ALL', schemaname, tablename), '; ' + ) + FROM pg_tables + WHERE schemaname = 'public' + ); +END \$\$; +" echo "DB restored to postgres://localhost/${LOCALDBNAME}" diff --git a/test/dbdumps/opencollective_dvl.pgsql b/test/dbdumps/opencollective_dvl.pgsql index cd2f59adde5..45e292a2301 100644 Binary files a/test/dbdumps/opencollective_dvl.pgsql and b/test/dbdumps/opencollective_dvl.pgsql differ