From 732496364ae2ec0b814732802688e5b3b67ec27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juho=20H=C3=A4nninen?= Date: Mon, 9 Oct 2023 09:11:40 +0300 Subject: [PATCH 1/4] Limit concurrency to prevent pg pool problems (cherry picked from commit fb3e679f335ad05c0a1ac751f1c037e0d962de9b) --- package.json | 2 +- src/import.js | 2 +- src/knex.js | 2 +- yarn.lock | 35 +++++++++++++++-------------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index d81d4ae..4462af5 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "lodash-es": "^4.17.21", "p-map": "^2.1.0", "p-queue": "^7.4.1", - "pg": "^8.7.3", + "pg": "^8.11.3", "pg-connection-string": "^2.6.2", "react": "^16.8.6", "react-dom": "^16.8.6", diff --git a/src/import.js b/src/import.js index 78547d8..ed6a085 100644 --- a/src/import.js +++ b/src/import.js @@ -38,7 +38,7 @@ export async function importFile(filePath) { try { await startImport(fileName); - const queue = new PQueue({ concurrency: 50 }); // Promise queue for postgres insertions + const queue = new PQueue({ concurrency: 20 }); // Promise queue for postgres insertions console.log("Unpacking and processing the archive..."); const directory = await Open.file(filePath); diff --git a/src/knex.js b/src/knex.js index 2c9b531..a86a171 100644 --- a/src/knex.js +++ b/src/knex.js @@ -16,7 +16,7 @@ export function getKnex() { connection: PG_CONNECTION_STRING, pool: { min: 0, - max: 50, + max: 20, }, }); diff --git a/yarn.lock b/yarn.lock index e8f7292..15ac5f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4235,21 +4235,16 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== -pg-cloudflare@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.0.tgz#833d70870d610d14bf9df7afb40e1cba310c17a0" - integrity sha512-tGM8/s6frwuAIyRcJ6nWcIvd3+3NmUKIs6OjviIm1HPPFEt5MzQDOTBQyhPWg/m0kCl95M6gA1JaIXtS8KovOA== +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== pg-connection-string@2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.1.tgz#78c23c21a35dd116f48e12e23c0965e8d9e2cbfb" integrity sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg== -pg-connection-string@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.0.tgz#12a36cc4627df19c25cc1b9b736cc39ee1f73ae8" - integrity sha512-x14ibktcwlHKoHxx9X3uTVW9zIGR41ZB6QNhHb21OPNdCCO3NaRnpJuwKIQSR4u+Yqjx4HCvy7Hh7VSy1U4dGg== - pg-connection-string@^2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475" @@ -4260,10 +4255,10 @@ pg-int8@1.0.1: resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== -pg-pool@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" - integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== +pg-pool@^3.6.1: + version "3.6.1" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.1.tgz#5a902eda79a8d7e3c928b77abf776b3cb7d351f7" + integrity sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og== pg-protocol@^1.6.0: version "1.6.0" @@ -4281,20 +4276,20 @@ pg-types@^2.1.0: postgres-date "~1.0.4" postgres-interval "^1.1.0" -pg@^8.7.3: - version "8.11.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.0.tgz#a37e534e94b57a7ed811e926f23a7c56385f55d9" - integrity sha512-meLUVPn2TWgJyLmy7el3fQQVwft4gU5NGyvV0XbD41iU9Jbg8lCH4zexhIkihDzVHJStlt6r088G6/fWeNjhXA== +pg@^8.11.3: + version "8.11.3" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.3.tgz#d7db6e3fe268fcedd65b8e4599cda0b8b4bf76cb" + integrity sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g== dependencies: buffer-writer "2.0.0" packet-reader "1.0.0" - pg-connection-string "^2.6.0" - pg-pool "^3.6.0" + pg-connection-string "^2.6.2" + pg-pool "^3.6.1" pg-protocol "^1.6.0" pg-types "^2.1.0" pgpass "1.x" optionalDependencies: - pg-cloudflare "^1.1.0" + pg-cloudflare "^1.1.1" pgpass@1.x: version "1.0.5" From 8db3dee18b79aa9f58cb353329aae2e1c6e0acd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juho=20H=C3=A4nninen?= Date: Thu, 16 Nov 2023 13:52:06 +0200 Subject: [PATCH 2/4] Enable construction tags for routes that start in the future --- src/geometryMatcher.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/geometryMatcher.js b/src/geometryMatcher.js index 35c3c4a..f7adfc1 100644 --- a/src/geometryMatcher.js +++ b/src/geometryMatcher.js @@ -42,7 +42,9 @@ export const runGeometryMatcher = async (schema = SCHEMA) => { routeType = "TRAMBUS"; } - const profile = ROUTE_TYPE_PROFILES[routeType]; + // Use construction profile if route is not yet in use (dateBegin is in the future) + const useConstruction = new Date(shape.properties.date_begin) > new Date(); + const profile = ROUTE_TYPE_PROFILES[routeType] + (useConstruction ? CONSTRUCTION_PROFILE_SUFFIX : ""); let geometry; let confidence; From ffd5817d8001cf00c2ed369397e6f359495af7b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juho=20H=C3=A4nninen?= Date: Thu, 16 Nov 2023 13:57:51 +0200 Subject: [PATCH 3/4] Add prefix constant --- src/geometryMatcher.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/geometryMatcher.js b/src/geometryMatcher.js index f7adfc1..65631be 100644 --- a/src/geometryMatcher.js +++ b/src/geometryMatcher.js @@ -10,6 +10,8 @@ const ROUTE_TYPE_PROFILES = { TRAMBUS: "trambus", }; +const CONSTRUCTION_PROFILE_SUFFIX = "-with-construction"; + export const runGeometryMatcher = async (schema = SCHEMA) => { const startTime = process.hrtime(); From 62ea443868913b9fcd9520e9a0a744c3889ba74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juho=20H=C3=A4nninen?= Date: Mon, 20 Nov 2023 09:51:45 +0200 Subject: [PATCH 4/4] Check construction status after validating profile to prevent httpz 400 errors --- src/geometryMatcher.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/geometryMatcher.js b/src/geometryMatcher.js index 65631be..1207a5f 100644 --- a/src/geometryMatcher.js +++ b/src/geometryMatcher.js @@ -44,15 +44,18 @@ export const runGeometryMatcher = async (schema = SCHEMA) => { routeType = "TRAMBUS"; } - // Use construction profile if route is not yet in use (dateBegin is in the future) - const useConstruction = new Date(shape.properties.date_begin) > new Date(); - const profile = ROUTE_TYPE_PROFILES[routeType] + (useConstruction ? CONSTRUCTION_PROFILE_SUFFIX : ""); + let profile = ROUTE_TYPE_PROFILES[routeType] let geometry; let confidence; let fittedDataResult; if (profile) { + + // Use construction profile if route is not yet in use (dateBegin is in the future) + if (new Date(shape.properties.date_begin) > new Date()) { + profile += CONSTRUCTION_PROFILE_SUFFIX; + } // Request fitted geometry from map matcher for the profile fittedDataResult = await fetch(`${MAP_MATCHER_URL}match/${profile}`, { method: "POST",