From b95498f21f68ad588911c8b66fc7726da59d30bd Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 15:09:18 -0500 Subject: [PATCH 1/3] Set Timeout to 10s --- ingest-image/package-lock.json | 31 +++++++++++++++++++++++++++++++ ingest-image/package.json | 1 + ingest-image/task.js | 11 +++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ingest-image/package-lock.json b/ingest-image/package-lock.json index 487df67..0f90cbb 100644 --- a/ingest-image/package-lock.json +++ b/ingest-image/package-lock.json @@ -18,6 +18,7 @@ "rimraf": "^5.0.0", "sharp": "^0.32.0", "strtime": "^1.1.2", + "undici": "^5.24.0", "uuid": "^9.0.0" }, "devDependencies": { @@ -1824,6 +1825,17 @@ "ieee754": "^1.1.13" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3343,6 +3355,14 @@ "is-arrayish": "^0.3.1" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/streamx": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", @@ -3555,6 +3575,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/undici": { + "version": "5.24.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.24.0.tgz", + "integrity": "sha512-OKlckxBjFl0oXxcj9FU6oB8fDAaiRUq+D8jrFWGmOfI/gIyjk/IeS75LMzgYKUaeHzLUcYvf9bbJGSrUwTfwwQ==", + "dependencies": { + "busboy": "^1.6.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/ingest-image/package.json b/ingest-image/package.json index 2d62f36..b6a5c8d 100644 --- a/ingest-image/package.json +++ b/ingest-image/package.json @@ -22,6 +22,7 @@ "rimraf": "^5.0.0", "sharp": "^0.32.0", "strtime": "^1.1.2", + "undici": "^5.24.0", "uuid": "^9.0.0" }, "devDependencies": { diff --git a/ingest-image/task.js b/ingest-image/task.js index c909a67..bd83ffd 100644 --- a/ingest-image/task.js +++ b/ingest-image/task.js @@ -17,6 +17,13 @@ import { createHash } from 'node:crypto'; import { pipeline } from 'node:stream/promises'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; +import { fetch, setGlobalDispatcher, Agent } from 'undici'; + +setGlobalDispatcher(new Agent({ + connect: { + timeout: 10_000 + } +})); const region = process.env.AWS_DEFAULT_REGION || 'us-west-2'; @@ -136,6 +143,8 @@ export default class Task { async save_image(md) { try { + await this.copy_to_prod(md); + const imageAttempt = (await fetcher(this.ANIML_API_URL, { query: ` mutation CreateImageRecord($input: CreateImageInput!){ @@ -166,8 +175,6 @@ export default class Task { if (msg.includes('E11000')) msg = 'DUPLICATE_IMAGE'; const err = new Error(msg); await this.copy_to_dlb(err, md); - } else { - await this.copy_to_prod(md); } } catch (err) { From dc95221d44f54083b22b5a5591fa3108c7bbc8cf Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 15:17:09 -0500 Subject: [PATCH 2/3] Retain copy_to_prod --- ingest-image/task.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ingest-image/task.js b/ingest-image/task.js index bd83ffd..ff28257 100644 --- a/ingest-image/task.js +++ b/ingest-image/task.js @@ -143,8 +143,6 @@ export default class Task { async save_image(md) { try { - await this.copy_to_prod(md); - const imageAttempt = (await fetcher(this.ANIML_API_URL, { query: ` mutation CreateImageRecord($input: CreateImageInput!){ @@ -175,8 +173,9 @@ export default class Task { if (msg.includes('E11000')) msg = 'DUPLICATE_IMAGE'; const err = new Error(msg); await this.copy_to_dlb(err, md); + } else { + await this.copy_to_prod(md); } - } catch (err) { // backstop for unforeseen errors returned by the API // and errors resizing/copying the image to prod buckets. From 95753ffa82002d7841a7b8f688c6858dc34c0372 Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 16:07:30 -0500 Subject: [PATCH 3/3] Alernative Abort Approach --- ingest-image/package.json | 4 +--- ingest-image/task.js | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ingest-image/package.json b/ingest-image/package.json index b6a5c8d..72432e1 100644 --- a/ingest-image/package.json +++ b/ingest-image/package.json @@ -21,9 +21,7 @@ "mime-types": "^2.1.35", "rimraf": "^5.0.0", "sharp": "^0.32.0", - "strtime": "^1.1.2", - "undici": "^5.24.0", - "uuid": "^9.0.0" + "strtime": "^1.1.2" }, "devDependencies": { "eslint": "^8.33.0", diff --git a/ingest-image/task.js b/ingest-image/task.js index ff28257..ef17e13 100644 --- a/ingest-image/task.js +++ b/ingest-image/task.js @@ -17,13 +17,6 @@ import { createHash } from 'node:crypto'; import { pipeline } from 'node:stream/promises'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; -import { fetch, setGlobalDispatcher, Agent } from 'undici'; - -setGlobalDispatcher(new Agent({ - connect: { - timeout: 10_000 - } -})); const region = process.env.AWS_DEFAULT_REGION || 'us-west-2'; @@ -374,8 +367,13 @@ export default class Task { async function fetcher(url, body) { console.log('Posting metadata to API', JSON.stringify(body)); + + const controller = new AbortController(); + const id = setTimeout(() => controller.abort(), 10_000); + const res = await fetch(url, { method: 'POST', + signal: controller.signal, headers: { 'Content-Type': 'application/json', 'x-api-key': APIKEY @@ -383,6 +381,8 @@ async function fetcher(url, body) { body: JSON.stringify(body) }); + clearTimeout(id); + if (!res.ok) { const texterr = await res.text(); let jsonerr;