chore: improve test setup for pg errors / isolation (#1375)

* chore: improve pg error handling for tests * chore: changeset * chore: use error prop --------- Co-authored-by: typedarray <[email protected]>
ponder-sh · Dec 26, 2024 · 1687033 · 1687033
1 parent 93e5d4d
commit 1687033
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 25 deletions.
diff --git a/.changeset/healthy-buckets-swim.md b/.changeset/healthy-buckets-swim.md
@@ -0,0 +1,5 @@
+---
+"ponder": patch
+---
+
+Improved logs for Postgres pool errors.
diff --git a/packages/core/src/_test/setup.ts b/packages/core/src/_test/setup.ts
@@ -72,6 +72,14 @@ export async function setupIsolatedDatabase(context: TestContext) {
 
     const client = new pg.Client({ connectionString });
     await client.connect();
+    await client.query(
+      `
+      SELECT pg_terminate_backend(pg_stat_activity.pid)
+      FROM pg_stat_activity
+      WHERE pg_stat_activity.datname = $1 AND pid <> pg_backend_pid()
+      `,
+      [databaseName],
+    );
     await client.query(`DROP DATABASE IF EXISTS "${databaseName}"`);
     await client.query(`CREATE DATABASE "${databaseName}"`);
     await client.end();

diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts
@@ -216,27 +216,39 @@ export const createDatabase = ({
         : [equalMax, equalMax, equalMax];
 
     driver = {
-      internal: createPool({
-        ...preBuild.databaseConfig.poolConfig,
-        application_name: `${preBuild.namespace}_internal`,
-        max: internalMax,
-        statement_timeout: 10 * 60 * 1000, // 10 minutes to accommodate slow sync store migrations.
-      }),
-      user: createPool({
-        ...preBuild.databaseConfig.poolConfig,
-        application_name: `${preBuild.namespace}_user`,
-        max: userMax,
-      }),
-      readonly: createPool({
-        ...preBuild.databaseConfig.poolConfig,
-        application_name: `${preBuild.namespace}_readonly`,
-        max: readonlyMax,
-      }),
-      sync: createPool({
-        ...preBuild.databaseConfig.poolConfig,
-        application_name: "ponder_sync",
-        max: syncMax,
-      }),
+      internal: createPool(
+        {
+          ...preBuild.databaseConfig.poolConfig,
+          application_name: `${preBuild.namespace}_internal`,
+          max: internalMax,
+          statement_timeout: 10 * 60 * 1000, // 10 minutes to accommodate slow sync store migrations.
+        },
+        common.logger,
+      ),
+      user: createPool(
+        {
+          ...preBuild.databaseConfig.poolConfig,
+          application_name: `${preBuild.namespace}_user`,
+          max: userMax,
+        },
+        common.logger,
+      ),
+      readonly: createPool(
+        {
+          ...preBuild.databaseConfig.poolConfig,
+          application_name: `${preBuild.namespace}_readonly`,
+          max: readonlyMax,
+        },
+        common.logger,
+      ),
+      sync: createPool(
+        {
+          ...preBuild.databaseConfig.poolConfig,
+          application_name: "ponder_sync",
+          max: syncMax,
+        },
+        common.logger,
+      ),
     };
 
     qb = {

diff --git a/packages/core/src/utils/pg.ts b/packages/core/src/utils/pg.ts
@@ -1,3 +1,4 @@
+import type { Logger } from "@/common/logger.js";
 import pg, { type PoolConfig } from "pg";
 import { prettyPrint } from "./print.js";
 
@@ -60,20 +61,49 @@ class ReadonlyClient extends pg.Client {
   }
 }
 
-export function createPool(config: PoolConfig) {
-  return new pg.Pool({
+function createErrorHandler(logger: Logger) {
+  return (error: Error) => {
+    const client = (error as any).client as any | undefined;
+    const pid = (client?.processID as number | undefined) ?? "unknown";
+    const applicationName =
+      (client?.connectionParameters?.application_name as string | undefined) ??
+      "unknown";
+
+    logger.error({
+      service: "postgres",
+      msg: `Pool error (application_name: ${applicationName}, pid: ${pid})`,
+      error,
+    });
+
+    // NOTE: Errors thrown here cause an uncaughtException. It's better to just log and ignore -
+    // if the underlying problem persists, the process will crash due to downstream effects.
+  };
+}
+
+export function createPool(config: PoolConfig, logger: Logger) {
+  const pool = new pg.Pool({
     // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout
     statement_timeout: 2 * 60 * 1000, // 2 minutes
     ...config,
   });
+
+  const onError = createErrorHandler(logger);
+  pool.on("error", onError);
+
+  return pool;
 }
 
-export function createReadonlyPool(config: PoolConfig) {
-  return new pg.Pool({
+export function createReadonlyPool(config: PoolConfig, logger: Logger) {
+  const pool = new pg.Pool({
     // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout
     statement_timeout: 2 * 60 * 1000, // 2 minutes
     // @ts-expect-error: The custom Client is an undocumented option.
     Client: ReadonlyClient,
     ...config,
   });
+
+  const onError = createErrorHandler(logger);
+  pool.on("error", onError);
+
+  return pool;
 }