From ff87f0b91e2c1e5a960da8ebd5059f5dcb15899c Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 3 Apr 2024 10:27:36 +1300 Subject: [PATCH] fix: Configure cron db from environment (#628) The admin/cron connection is currently hard-coded to the `cron` database, but this needs to be configurable so that we can use the default DB (`postgres`) locally. Additionally, this PR combines the `pgbouncer` / `pg_cron` init scripts and uses the combined output in both docker compose and integration tests. --- docker-compose.yml | 5 +++-- postgres.Dockerfile => postgres/Dockerfile | 2 +- {init-scripts/postgres => postgres}/init.sql | 4 ++++ runner/src/provisioner/provisioner.ts | 15 ++++++++------- runner/tests/integration.test.ts | 4 ++-- runner/tests/testcontainers/postgres.ts | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) rename postgres.Dockerfile => postgres/Dockerfile (71%) rename {init-scripts/postgres => postgres}/init.sql (90%) diff --git a/docker-compose.yml b/docker-compose.yml index 16e2a6cfc..a609209c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: PGUSER: postgres PGPASSWORD: postgrespassword PGDATABASE: postgres + CRON_DATABASE: postgres PORT: 9180 AWS_REGION: eu-central-1 AWS_ACCESS_KEY_ID: @@ -78,10 +79,10 @@ services: - "6379:6379" postgres: - image: postgres:14 + build: + context: ./postgres restart: always volumes: - - ./init-scripts/postgres:/docker-entrypoint-initdb.d - postgres:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: postgrespassword diff --git a/postgres.Dockerfile b/postgres/Dockerfile similarity index 71% rename from postgres.Dockerfile rename to postgres/Dockerfile index 81851c84d..07796e6c3 100644 --- a/postgres.Dockerfile +++ b/postgres/Dockerfile @@ -4,7 +4,7 @@ RUN apt-get update && apt-get install -y postgresql-14-cron RUN echo "shared_preload_libraries = 'pg_cron'" >> /usr/share/postgresql/postgresql.conf.sample -RUN echo "CREATE EXTENSION pg_cron;" > /docker-entrypoint-initdb.d/init-pg-cron.sql +COPY ./init.sql /docker-entrypoint-initdb.d/ EXPOSE 5432 diff --git a/init-scripts/postgres/init.sql b/postgres/init.sql similarity index 90% rename from init-scripts/postgres/init.sql rename to postgres/init.sql index ffc461d02..6d1101992 100644 --- a/init-scripts/postgres/init.sql +++ b/postgres/init.sql @@ -1,3 +1,4 @@ +-- pgbouncer CREATE ROLE pgbouncer LOGIN; ALTER ROLE pgbouncer WITH PASSWORD 'pgbouncer'; CREATE OR REPLACE FUNCTION public.user_lookup(in i_username text, out uname text, out phash text) @@ -10,3 +11,6 @@ END; $$ LANGUAGE plpgsql SECURITY DEFINER; REVOKE ALL ON FUNCTION public.user_lookup(text) FROM public; GRANT EXECUTE ON FUNCTION public.user_lookup(text) TO pgbouncer; + +-- pg_cron +CREATE EXTENSION pg_cron; diff --git a/runner/src/provisioner/provisioner.ts b/runner/src/provisioner/provisioner.ts index 97d927b43..e74f38a31 100644 --- a/runner/src/provisioner/provisioner.ts +++ b/runner/src/provisioner/provisioner.ts @@ -7,7 +7,6 @@ import HasuraClient from '../hasura-client'; import PgClientClass from '../pg-client'; const DEFAULT_PASSWORD_LENGTH = 16; -const CRON_DATABASE = 'cron'; const adminDefaultPgClientGlobal = new PgClientClass({ user: process.env.PGUSER, @@ -20,7 +19,7 @@ const adminDefaultPgClientGlobal = new PgClientClass({ const adminCronPgClientGlobal = new PgClientClass({ user: process.env.PGUSER, password: process.env.PGPASSWORD, - database: CRON_DATABASE, + database: process.env.CRON_DATABASE, host: process.env.PGHOST, port: Number(process.env.PGPORT), }); @@ -35,13 +34,15 @@ export interface DatabaseConnectionParameters { interface Config { cronDatabase: string - // Allow overriding the default values for testing - postgresHost?: string - postgresPort?: number + // Override the host/port values returned by Hasura during testing/local development + hasuraHostOverride?: string + hasuraPortOverride?: number } const defaultConfig: Config = { cronDatabase: process.env.CRON_DATABASE, + hasuraHostOverride: process.env.HASURA_HOST_OVERRIDE, + hasuraPortOverride: process.env.HASURA_PORT_OVERRIDE ? Number(process.env.HASURA_PORT_OVERRIDE) : undefined }; export default class Provisioner { @@ -109,8 +110,8 @@ export default class Provisioner { user: userDbConnectionParameters.username, password: userDbConnectionParameters.password, database: this.config.cronDatabase, - host: this.config.postgresHost ?? userDbConnectionParameters.host, - port: this.config.postgresPort ?? userDbConnectionParameters.port, + host: this.config.hasuraHostOverride ?? userDbConnectionParameters.host, + port: this.config.hasuraPortOverride ?? userDbConnectionParameters.port, }); await userCronPgClient.query( diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 0342827e2..f009f2832 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -60,8 +60,8 @@ describe('Indexer integration', () => { pgClient, { cronDatabase: postgresContainer.getDatabase(), - postgresHost: postgresContainer.getIpAddress(), - postgresPort: Number(postgresContainer.getPort()), + hasuraHostOverride: postgresContainer.getIpAddress(), + hasuraPortOverride: Number(postgresContainer.getPort()), } ); diff --git a/runner/tests/testcontainers/postgres.ts b/runner/tests/testcontainers/postgres.ts index 658989dea..c36a88d79 100644 --- a/runner/tests/testcontainers/postgres.ts +++ b/runner/tests/testcontainers/postgres.ts @@ -17,7 +17,7 @@ export class PostgreSqlContainer { } public static async build (): Promise { - const container = await GenericContainer.fromDockerfile('../', 'postgres.Dockerfile').build(); + const container = await GenericContainer.fromDockerfile('../postgres').build(); return new PostgreSqlContainer(container); }