From fb3e679f335ad05c0a1ac751f1c037e0d962de9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juho=20H=C3=A4nninen?= Date: Mon, 9 Oct 2023 09:11:40 +0300 Subject: [PATCH 1/3] Limit concurrency to prevent pg pool problems --- package.json | 2 +- src/import.js | 2 +- src/knex.js | 2 +- yarn.lock | 35 +++++++++++++++-------------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index d81d4ae..4462af5 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "lodash-es": "^4.17.21", "p-map": "^2.1.0", "p-queue": "^7.4.1", - "pg": "^8.7.3", + "pg": "^8.11.3", "pg-connection-string": "^2.6.2", "react": "^16.8.6", "react-dom": "^16.8.6", diff --git a/src/import.js b/src/import.js index 78547d8..ed6a085 100644 --- a/src/import.js +++ b/src/import.js @@ -38,7 +38,7 @@ export async function importFile(filePath) { try { await startImport(fileName); - const queue = new PQueue({ concurrency: 50 }); // Promise queue for postgres insertions + const queue = new PQueue({ concurrency: 20 }); // Promise queue for postgres insertions console.log("Unpacking and processing the archive..."); const directory = await Open.file(filePath); diff --git a/src/knex.js b/src/knex.js index 2c9b531..a86a171 100644 --- a/src/knex.js +++ b/src/knex.js @@ -16,7 +16,7 @@ export function getKnex() { connection: PG_CONNECTION_STRING, pool: { min: 0, - max: 50, + max: 20, }, }); diff --git a/yarn.lock b/yarn.lock index e8f7292..15ac5f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4235,21 +4235,16 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== -pg-cloudflare@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.0.tgz#833d70870d610d14bf9df7afb40e1cba310c17a0" - integrity sha512-tGM8/s6frwuAIyRcJ6nWcIvd3+3NmUKIs6OjviIm1HPPFEt5MzQDOTBQyhPWg/m0kCl95M6gA1JaIXtS8KovOA== +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== pg-connection-string@2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.1.tgz#78c23c21a35dd116f48e12e23c0965e8d9e2cbfb" integrity sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg== -pg-connection-string@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.0.tgz#12a36cc4627df19c25cc1b9b736cc39ee1f73ae8" - integrity sha512-x14ibktcwlHKoHxx9X3uTVW9zIGR41ZB6QNhHb21OPNdCCO3NaRnpJuwKIQSR4u+Yqjx4HCvy7Hh7VSy1U4dGg== - pg-connection-string@^2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475" @@ -4260,10 +4255,10 @@ pg-int8@1.0.1: resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== -pg-pool@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" - integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== +pg-pool@^3.6.1: + version "3.6.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.1.tgz#5a902eda79a8d7e3c928b77abf776b3cb7d351f7" + integrity sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og== pg-protocol@^1.6.0: version "1.6.0" @@ -4281,20 +4276,20 @@ pg-types@^2.1.0: postgres-date "~1.0.4" postgres-interval "^1.1.0" -pg@^8.7.3: - version "8.11.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.0.tgz#a37e534e94b57a7ed811e926f23a7c56385f55d9" - integrity sha512-meLUVPn2TWgJyLmy7el3fQQVwft4gU5NGyvV0XbD41iU9Jbg8lCH4zexhIkihDzVHJStlt6r088G6/fWeNjhXA== +pg@^8.11.3: + version "8.11.3" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.3.tgz#d7db6e3fe268fcedd65b8e4599cda0b8b4bf76cb" + integrity sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g== dependencies: buffer-writer "2.0.0" packet-reader "1.0.0" - pg-connection-string "^2.6.0" - pg-pool "^3.6.0" + pg-connection-string "^2.6.2" + pg-pool "^3.6.1" pg-protocol "^1.6.0" pg-types "^2.1.0" pgpass "1.x" optionalDependencies: - pg-cloudflare "^1.1.0" + pg-cloudflare "^1.1.1" pgpass@1.x: version "1.0.5" From cbf3e3ba0c9a6fd7b0062bb0d6ff7f8065834905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Jyrki=C3=A4inen?= Date: Mon, 12 Feb 2024 13:55:56 +0200 Subject: [PATCH 2/3] fix local dump creation --- src/constants.js | 1 + src/sources/fetchExportFromFTP.js | 2 +- src/utils/createDbDump.js | 8 ++++---- src/utils/importDbDump.js | 6 ++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/constants.js b/src/constants.js index 5e4364f..e64fff6 100644 --- a/src/constants.js +++ b/src/constants.js @@ -43,6 +43,7 @@ export const SERVER_PORT = secretsEnv.SERVER_PORT || 3000; export const ADMIN_PASSWORD = secretsEnv.ADMIN_PASSWORD || "password"; export const PATH_PREFIX = secretsEnv.PATH_PREFIX || "/"; export const SCHEMA = "jore"; +export const DEFAULT_DATABASE = "postgres"; export const INTERMEDIATE_SCHEMA = "jore_new"; export const AZURE_UPLOAD_CONTAINER = secretsEnv.AZURE_UPLOAD_CONTAINER || "joredumps"; export const AZURE_STORAGE_ACCOUNT = secretsEnv.AZURE_STORAGE_ACCOUNT || ""; diff --git a/src/sources/fetchExportFromFTP.js b/src/sources/fetchExportFromFTP.js index 5dfa67d..c3042d5 100644 --- a/src/sources/fetchExportFromFTP.js +++ b/src/sources/fetchExportFromFTP.js @@ -38,7 +38,7 @@ export async function fetchExportFromFTP() { await client.cd(FTP_PATH); const files = await client.list(); - + const zips = files.filter(({ name }) => name.endsWith(".zip")); const newestFile = orderBy(zips, "name", "desc")[0]; const newestExportName = get(newestFile, "name", ""); diff --git a/src/utils/createDbDump.js b/src/utils/createDbDump.js index 15fadac..b18f15d 100644 --- a/src/utils/createDbDump.js +++ b/src/utils/createDbDump.js @@ -4,7 +4,7 @@ import fs from "fs-extra"; import { format } from "date-fns"; import pgConnectionString from "pg-connection-string"; -import { PG_CONNECTION_STRING } from "../constants.js"; +import { PG_CONNECTION_STRING, DEFAULT_DATABASE } from "../constants.js"; const { parse } = pgConnectionString; @@ -58,8 +58,8 @@ export const createDbDump = async () => { resolve(filePath); } else { const pgConnection = parse(PG_CONNECTION_STRING); - console.log(`Dumping the ${pgConnection.database} database into ${filePath}`); - + const database = pgConnection.database ? pgConnection.database : DEFAULT_DATABASE; + console.log(`Dumping the ${database} database into ${filePath}`); const dumpProcess = childProcess.spawn( "pg_dump", [`-f ${filePath}`, "-Fc", "-N '*old'", "-N '*new'"], @@ -71,7 +71,7 @@ export const createDbDump = async () => { PGPASSWORD: pgConnection.password, PGHOST: pgConnection.host, PGPORT: pgConnection.port, - PGDATABASE: pgConnection.database, + PGDATABASE: database, }, }, ); diff --git a/src/utils/importDbDump.js b/src/utils/importDbDump.js index 001461f..4532554 100644 --- a/src/utils/importDbDump.js +++ b/src/utils/importDbDump.js @@ -8,6 +8,7 @@ import { AZURE_STORAGE_KEY, AZURE_STORAGE_ACCOUNT, AZURE_UPLOAD_CONTAINER, + DEFAULT_DATABASE } from "../constants.js"; import { clearDb } from "../setup/clearDb.js"; import { deleteFiles } from "./createDbDump.js"; @@ -75,6 +76,7 @@ export const importDbDump = async () => { if (fileExists) { const pgConnection = parse(PG_CONNECTION_STRING); + const database = pgConnection.database ? pgConnection.database : DEFAULT_DATABASE; console.log(`Restoring db with ${filePath}`); const dumpProcess = childProcess.spawn( @@ -84,7 +86,7 @@ export const importDbDump = async () => { "--if-exists", "--no-owner", `-U ${pgConnection.user}`, - `-d ${pgConnection.database}`, + `-d ${database}`, "--single-transaction", `${filePath}`, ], @@ -96,7 +98,7 @@ export const importDbDump = async () => { PGPASSWORD: pgConnection.password, PGHOST: pgConnection.host, PGPORT: pgConnection.port, - PGDATABASE: pgConnection.database, + PGDATABASE: database, }, }, ); From f952d3a7331fad3ee0f0f088c0b0528c951dd088 Mon Sep 17 00:00:00 2001 From: Erkki Halinen Date: Mon, 12 Feb 2024 14:34:50 +0200 Subject: [PATCH 3/3] AB#31023: Fix createDbDump error when no dumps folder present --- src/utils/createDbDump.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/createDbDump.js b/src/utils/createDbDump.js index b18f15d..0a99176 100644 --- a/src/utils/createDbDump.js +++ b/src/utils/createDbDump.js @@ -15,7 +15,7 @@ const dumpsDir = path.join(cwd, "dumps"); export const deleteFiles = ({ filesDir, minFileCount }) => { fs.readdir(filesDir, (err, files) => { - if (files.length < minFileCount) return; + if (!files || files.length < minFileCount) return; files.forEach((file) => { const removableFile = path.join(filesDir, file); fs.stat(removableFile, async (err, stat) => {