From c102db37091b2261c7d9555dcf07eb35c1600ce0 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sat, 21 Dec 2024 15:48:12 -0700 Subject: [PATCH 1/3] Add IP database scripts Added script to populate ip address database from maxmind --- apps/api/src/app/routes/route.middleware.ts | 6 +- apps/cron-tasks/project.json | 4 + apps/cron-tasks/src/config/db.config.ts | 6 +- apps/cron-tasks/src/config/env-config.ts | 9 +- apps/cron-tasks/src/geo-ip-updater.ts | 227 ++++++++++++++++++ apps/cron-tasks/src/save-analytics-summary.ts | 2 +- apps/cron-tasks/src/utils/slack.ts | 148 ------------ apps/cron-tasks/src/utils/utils.ts | 13 + package.json | 3 + yarn.lock | 42 +++- 10 files changed, 299 insertions(+), 161 deletions(-) create mode 100644 apps/cron-tasks/src/geo-ip-updater.ts delete mode 100644 apps/cron-tasks/src/utils/slack.ts create mode 100644 apps/cron-tasks/src/utils/utils.ts diff --git a/apps/api/src/app/routes/route.middleware.ts b/apps/api/src/app/routes/route.middleware.ts index 69be81c5..98e9954c 100644 --- a/apps/api/src/app/routes/route.middleware.ts +++ b/apps/api/src/app/routes/route.middleware.ts @@ -62,7 +62,7 @@ export function notFoundMiddleware(req: express.Request, res: express.Response, export function blockBotByUserAgentMiddleware(req: express.Request, res: express.Response, next: express.NextFunction) { const userAgent = req.get('User-Agent'); if (userAgent?.toLocaleLowerCase().includes('python')) { - logger.debug( + (res.log || logger).debug( { blocked: true, method: req.method, @@ -129,7 +129,7 @@ export async function checkAuth(req: express.Request, res: express.Response, nex req.log.error(`[AUTH][UNAUTHORIZED] User-Agent mismatch: ${req.session.userAgent} !== ${userAgent}`); req.session.destroy((err) => { if (err) { - logger.error({ ...getExceptionLog(err) }, '[AUTH][UNAUTHORIZED][ERROR] Error destroying session'); + (res.log || logger).error({ ...getExceptionLog(err) }, '[AUTH][UNAUTHORIZED][ERROR] Error destroying session'); } // TODO: Send email to user about potential suspicious activity next(new AuthenticationError('Unauthorized')); @@ -322,7 +322,7 @@ export function verifyCaptcha(req: express.Request, res: express.Response, next: if (res.success) { return next(); } - logger.warn({ token, res }, '[CAPTCHA][FAILED]'); + (res.log || logger).warn({ token, res }, '[CAPTCHA][FAILED]'); throw new InvalidCaptcha(); }) .catch(() => { diff --git a/apps/cron-tasks/project.json b/apps/cron-tasks/project.json index c48f803a..7233404d 100644 --- a/apps/cron-tasks/project.json +++ b/apps/cron-tasks/project.json @@ -14,6 +14,10 @@ { "entryName": "save-analytics-summary", "entryPath": "apps/cron-tasks/src/save-analytics-summary.ts" + }, + { + "entryName": "geo-ip-updater", + "entryPath": "apps/cron-tasks/src/geo-ip-updater.ts" } ], "tsConfig": "apps/cron-tasks/tsconfig.app.json", diff --git a/apps/cron-tasks/src/config/db.config.ts b/apps/cron-tasks/src/config/db.config.ts index 7ff82a9d..5358de6e 100644 --- a/apps/cron-tasks/src/config/db.config.ts +++ b/apps/cron-tasks/src/config/db.config.ts @@ -1,6 +1,6 @@ -import { getExceptionLog } from '@jetstream/api-config'; import { Prisma, PrismaClient } from '@prisma/client'; import { Pool } from 'pg'; +import { getExceptionLog } from '../utils/utils'; import { ENV } from './env-config'; import { logger } from './logger.config'; @@ -23,11 +23,11 @@ export const pgPool = new Pool({ pgPool.on('connect', (client) => { // logger.info('[DB][POOL] Connected'); client.on('error', (err) => { - logger.error(getExceptionLog(err), '[DB][CLIENT][ERROR] Unexpected error on client'); + logger.error(getExceptionLog, '[DB][CLIENT][ERROR] Unexpected error on client'); }); }); pgPool.on('error', (err, client) => { - logger.error(getExceptionLog(err), '[DB][POOL][ERROR] Unexpected error on idle client'); + logger.error(getExceptionLog, '[DB][POOL][ERROR] Unexpected error on idle client'); process.exit(-1); }); diff --git a/apps/cron-tasks/src/config/env-config.ts b/apps/cron-tasks/src/config/env-config.ts index 673e7e98..b9d79d54 100644 --- a/apps/cron-tasks/src/config/env-config.ts +++ b/apps/cron-tasks/src/config/env-config.ts @@ -1,5 +1,3 @@ -import { getExceptionLog } from '@jetstream/api-config'; -import { ensureBoolean } from '@jetstream/shared/utils'; import * as dotenv from 'dotenv'; import { readFileSync } from 'fs-extra'; import { join } from 'path'; @@ -10,7 +8,7 @@ try { VERSION = readFileSync(join(__dirname, '../../VERSION'), 'utf-8').trim(); console.info(`APP VERSION ${VERSION}`); } catch (ex) { - console.warn('COULD NOT READ VERSION FILE', getExceptionLog(ex)); + console.warn('COULD NOT READ VERSION FILE', ex); } export const ENV = { @@ -21,7 +19,7 @@ export const ENV = { ROLLBAR_SERVER_TOKEN: process.env.ROLLBAR_SERVER_TOKEN, // FIXME: there was a typo in env variables, using both temporarily as a safe fallback JETSTREAM_POSTGRES_DBURI: process.env.JETSTREAM_POSTGRES_DBURI || process.env.JESTREAM_POSTGRES_DBURI, - PRISMA_DEBUG: ensureBoolean(process.env.PRISMA_DEBUG), + PRISMA_DEBUG: process.env.PRISMA_DEBUG && process.env.PRISMA_DEBUG.toLocaleLowerCase().startsWith('t'), // MAILGUN MAILGUN_API_KEY: process.env.MAILGUN_API_KEY, @@ -30,4 +28,7 @@ export const ENV = { AMPLITUDE_API_KEY: process.env.AMPLITUDE_API_KEY, AMPLITUDE_SECRET_KEY: process.env.AMPLITUDE_SECRET_KEY, + + MAX_MIND_ACCOUNT_ID: process.env.MAX_MIND_ACCOUNT_ID, + MAX_MIND_LICENSE_KEY: process.env.MAX_MIND_LICENSE_KEY, }; diff --git a/apps/cron-tasks/src/geo-ip-updater.ts b/apps/cron-tasks/src/geo-ip-updater.ts new file mode 100644 index 00000000..e6da2ab8 --- /dev/null +++ b/apps/cron-tasks/src/geo-ip-updater.ts @@ -0,0 +1,227 @@ +import { exec } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { Open } from 'unzipper'; +import { promisify } from 'util'; +import { ENV } from './config/env-config'; +import { logger } from './config/logger.config'; +import { getExceptionLog } from './utils/utils'; + +/** +CREATE TABLE IF NOT EXISTS geo_ip.network ( + network cidr NOT NULL, + geoname_id int, + registered_country_geoname_id int, + represented_country_geoname_id int, + is_anonymous_proxy bool, + is_satellite_provider bool, + postal_code text, + latitude numeric, + longitude numeric, + accuracy_radius int, + is_anycast bool +); + +CREATE TABLE IF NOT EXISTS geo_ip.location ( + geoname_id int NOT NULL, + locale_code text NOT NULL, + continent_code text, + continent_name text, + country_iso_code text, + country_name text, + subdivision_1_iso_code text, + subdivision_1_name text, + subdivision_2_iso_code text, + subdivision_2_name text, + city_name text, + metro_code int, + time_zone text, + is_in_european_union bool NOT NULL, + PRIMARY KEY (geoname_id, locale_code) +); + +CREATE TABLE IF NOT EXISTS geo_ip.organization ( + network cidr NOT NULL, + autonomous_system_number int, + autonomous_system_organization text +); + +CREATE INDEX idx_geoip2_network_network ON geo_ip.network USING gist (network inet_ops); +CREATE INDEX idx_geoip2_network_geoname_id ON geo_ip.network(geoname_id); +CREATE INDEX idx_geoip2_location_locale_code ON geo_ip.location (locale_code); +CREATE INDEX idx_geoip2_organization_network ON geo_ip.organization USING gist (network inet_ops); + +*/ + +const execAsync = promisify(exec); + +if (!ENV.MAX_MIND_ACCOUNT_ID || !ENV.MAX_MIND_LICENSE_KEY) { + logger.error('Missing MaxMind credentials'); + process.exit(1); +} + +const ASN_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-ASN-CSV/download?suffix=zip'; +const ASN_FILENAME = 'GeoLite2-ASN.zip'; +const ASN_FILENAMES = ['GeoLite2-ASN-Blocks-IPv4.csv', 'GeoLite2-ASN-Blocks-IPv6.csv']; + +const CITY_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-City-CSV/download?suffix=zip'; +const CITY_FILENAME = 'GeoLite2-City.zip'; +const CITY_FILENAMES = ['GeoLite2-City-Locations-en.csv', 'GeoLite2-City-Blocks-IPv4.csv', 'GeoLite2-City-Blocks-IPv6.csv']; + +async function importCSVToTable(csvPath: string, tableName: string, schema: string): Promise { + const tempTableName = `${tableName}_temp`; + const fullTempTableName = `${schema}.${tempTableName}`; + const fullTableName = `${schema}.${tableName}`; + + try { + // Create temporary table with same structure + await execAsync(`psql "${ENV.JETSTREAM_POSTGRES_DBURI}" -c "CREATE TABLE ${fullTempTableName} (LIKE ${fullTableName} INCLUDING ALL)"`); + + // Import CSV data + await execAsync(`psql "${ENV.JETSTREAM_POSTGRES_DBURI}" -c "\\COPY ${fullTempTableName} FROM '${csvPath}' WITH (FORMAT CSV, HEADER)"`); + + // Atomic swap + await execAsync(` + psql "${ENV.JETSTREAM_POSTGRES_DBURI}" -c " + BEGIN; + DROP TABLE IF EXISTS ${fullTableName}; + ALTER TABLE ${fullTempTableName} RENAME TO ${tableName}; + COMMIT; + " + `); + + logger.info(`Successfully imported ${csvPath} to ${fullTableName}`); + } catch (error) { + logger.error(`Error importing ${csvPath}:`, error); + // Cleanup temp table if it exists + await execAsync(`psql "${ENV.JETSTREAM_POSTGRES_DBURI}" -c "DROP TABLE IF EXISTS ${fullTempTableName}"`).catch(() => { + // Ignore errors + logger.warn(`Failed to drop table ${fullTempTableName}`); + }); + throw error; + } +} + +async function processFile( + url: string, + zipFileName: string, + filenames: string[], + processor: (filename: string, filePath: string) => Promise +) { + const downloadDir = path.join(__dirname, '../../downloads'); + const zipFilePath = path.join(downloadDir, zipFileName); + + // Create downloads directory if it doesn't exist + if (!fs.existsSync(downloadDir)) { + fs.mkdirSync(downloadDir, { recursive: true }); + } + + let buffer: Buffer; + + // Check if file exists and is less than 24 hours old + if (fs.existsSync(zipFilePath)) { + const stats = fs.statSync(zipFilePath); + const fileAge = Date.now() - stats.mtime.getTime(); + const oneDayInMs = 24 * 60 * 60 * 1000; + + if (fileAge < oneDayInMs) { + logger.info(`Using existing file ${zipFilePath}`); + buffer = fs.readFileSync(zipFilePath); + } else { + buffer = await downloadFile(url, zipFilePath); + } + } else { + buffer = await downloadFile(url, zipFilePath); + } + + const directory = await Open.buffer(buffer); + + for (const entry of directory.files) { + const currentFilename = entry.path.split('/').reverse()[0]; + if (filenames.includes(currentFilename)) { + logger.info(`Extracting ${entry.path}...`); + const csvPath = path.join(downloadDir, currentFilename); + const writeStream = fs.createWriteStream(csvPath); + await new Promise((resolve, reject) => { + entry.stream().pipe(writeStream).on('finish', resolve).on('error', reject); + }); + await processor(currentFilename, csvPath); + // Cleanup CSV file + fs.unlinkSync(csvPath); + } + } +} + +async function downloadFile(url: string, savePath: string): Promise { + logger.info(`Downloading from ${url}...`); + + const response = await fetch(url, { + headers: { + Authorization: `Basic ${Buffer.from(`${ENV.MAX_MIND_ACCOUNT_ID}:${ENV.MAX_MIND_LICENSE_KEY}`).toString('base64')}`, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to download: ${response.statusText}`); + } + + const buffer = await streamToBuffer(response.body!); + fs.writeFileSync(savePath, buffer); + return buffer; +} + +async function streamToBuffer(stream: ReadableStream): Promise { + const chunks: Buffer[] = []; + const reader = stream.getReader(); + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(Buffer.from(value)); + } + + return Buffer.concat(chunks); +} + +async function processNetwork(csvPath: string) { + await importCSVToTable(csvPath, 'network', 'geo_ip'); +} + +async function processLocation(csvPath: string) { + await importCSVToTable(csvPath, 'location', 'geo_ip'); +} + +async function processASN(csvPath: string) { + await importCSVToTable(csvPath, 'organization', 'geo_ip'); +} + +async function main() { + try { + logger.info('Starting GeoIP database update...'); + + // Process ASN data + await processFile(ASN_URL, ASN_FILENAME, ASN_FILENAMES, async (filename, csvPath) => { + await processASN(csvPath); + }); + + // Process City data + await processFile(CITY_URL, CITY_FILENAME, CITY_FILENAMES, async (filename, csvPath) => { + if (filename.includes('Blocks')) { + await processNetwork(csvPath); + } else if (filename.includes('Locations')) { + await processLocation(csvPath); + } + }); + + logger.info('GeoIP database update completed successfully'); + } catch (error) { + logger.error(getExceptionLog(error), 'Error updating GeoIP database: %s', error.message); + throw error; + } +} + +main().catch((error) => { + logger.error(getExceptionLog(error), 'Fatal error: %s', error.message); + process.exit(1); +}); diff --git a/apps/cron-tasks/src/save-analytics-summary.ts b/apps/cron-tasks/src/save-analytics-summary.ts index 36867eaa..f6730b8f 100644 --- a/apps/cron-tasks/src/save-analytics-summary.ts +++ b/apps/cron-tasks/src/save-analytics-summary.ts @@ -1,7 +1,7 @@ -import { getExceptionLog } from '@jetstream/api-config'; import { prisma } from './config/db.config'; import { logger } from './config/logger.config'; import { getAmplitudeChart } from './utils/amplitude-dashboard-api'; +import { getExceptionLog } from './utils/utils'; const CHART_IDS = { LOAD: { diff --git a/apps/cron-tasks/src/utils/slack.ts b/apps/cron-tasks/src/utils/slack.ts deleted file mode 100644 index f3c0dfca..00000000 --- a/apps/cron-tasks/src/utils/slack.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { getExceptionLog } from '@jetstream/api-config'; -import SlackNotify from 'slack-notify'; -import { logger } from '../config/logger.config'; -import { DeleteResult } from './types'; -const SLACK_WEBHOOK_URL = 'https://hooks.slack.com/services/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'; -const channel = '#auth0-notifications'; - -export interface SlackResponse { - ok: true; - channel: string; - ts: string; // transaction id, aka message id -} - -export async function sendInactiveUserWarning( - numUsersNotified: number, - { - failedCount, - successAuth0UpdateCount, - successEmailCount, - }: { successEmailCount: number; successAuth0UpdateCount: number; failedCount: number } -): Promise { - const slack = SlackNotify(SLACK_WEBHOOK_URL); - return await slack.alert({ - channel, - text: `${numUsersNotified} users were notified of future account deletion.`, - blocks: [ - { - type: 'header', - text: { - type: 'plain_text', - text: `Users notified of account deletion :sob:`, - emoji: true, - }, - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*${numUsersNotified} users were notified of future account deletion.*`, - }, - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: ` - - *failedCount*: ${failedCount} - - *successAuth0UpdateCount*: ${successAuth0UpdateCount} - - *successEmailCount*: ${successEmailCount} - `, - }, - }, - ], - }); -} - -export async function logExceptionToSlack(message: string, errors: any): Promise { - try { - const slack = SlackNotify(SLACK_WEBHOOK_URL); - await slack.alert({ - channel, - text: `Cron Error`, - blocks: [ - { - type: 'header', - text: { - type: 'plain_text', - text: `:warning: There was an error :warning:`, - emoji: true, - }, - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: message, - }, - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: `\`\`\` - ${JSON.stringify(errors, null, 2)} - \`\`\``, - }, - }, - ], - }); - } catch (ex) { - logger.error(getExceptionLog(ex), '[ERROR] Sending to slack'); - } -} - -export async function accountDeletionInitialNotification(results: DeleteResult[]): Promise { - const slack = SlackNotify(SLACK_WEBHOOK_URL); - - const blocks: any[] = [ - { - type: 'header', - text: { - type: 'plain_text', - text: `:skull_and_crossbones: ${results.length} Inactive users deleted :skull_and_crossbones:`, - emoji: true, - }, - }, - { - type: 'divider', - }, - ]; - - results.forEach((result) => { - blocks.push({ - type: 'section', - fields: [ - { - type: 'mrkdwn', - text: `*Auth0 Id:*\n${result.auth0Id}`, - }, - { - type: 'mrkdwn', - text: `*Local Database Id:*\n${result.localDatabaseId}`, - }, - { - type: 'mrkdwn', - text: `*auth0Success:*\n${result.auth0Success}`, - }, - { - type: 'mrkdwn', - text: `*Local Delete Success:*\n${result.localDeleteSuccess}`, - }, - { - type: 'mrkdwn', - text: `*Org Count:*\n${result.orgCount}`, - }, - ], - }); - blocks.push({ - type: 'divider', - }); - }); - - return await slack.alert({ - channel, - text: `${results.length} inactive users have been deleted.`, - blocks, - }); -} diff --git a/apps/cron-tasks/src/utils/utils.ts b/apps/cron-tasks/src/utils/utils.ts new file mode 100644 index 00000000..beb5a6ac --- /dev/null +++ b/apps/cron-tasks/src/utils/utils.ts @@ -0,0 +1,13 @@ +export function getExceptionLog(error: unknown, includeStack = false) { + const status = (error as any) /** UserFacingError */?.apiRequestError?.status || (error as any) /** ApiRequestError */?.status; + if (error instanceof Error) { + return { + error: error.message, + status, + stack: includeStack ? error.stack : undefined, + }; + } + return { + error, + }; +} diff --git a/package.json b/package.json index d9afa4c9..76059bd6 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "start:e2e": "node dist/apps/api/main.js", "start:web-extension": "nx run jetstream-web-extension:serve", "start:cron:save-analytics-summary": "node dist/apps/cron-tasks/save-analytics-summary.js", + "start:geo-ip-db-update": "node dist/apps/cron-tasks/geo-ip-updater.js", "build": "cross-env NODE_ENV=production npm-run-all db:generate build:core build:landing generate:version", "build:pre-deploy": "cross-env NODE_ENV=production npm-run-all --parallel rollbar:upload-sourcemaps db:migrate", "build:core": "NODE_OPTIONS=--max_old_space_size=8192 nx run-many --output-style=dynamic --target=build --parallel=3 --projects=jetstream,api,download-zip-sw --configuration=production", @@ -264,6 +265,7 @@ "@swc/helpers": "0.5.11", "@tanstack/react-virtual": "^3.4.0", "@tippyjs/react": "^4.2.6", + "@types/unzipper": "^0.10.10", "amplitude-js": "^8.21.9", "axios": "^1.7.7", "bcrypt": "^5.1.1", @@ -347,6 +349,7 @@ "split.js": "^1.6.5", "tiny-request-router": "^1.2.2", "tslib": "^2.3.0", + "unzipper": "^0.12.3", "uuid": "^9.0.1", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "xmlbuilder2": "^3.1.1", diff --git a/yarn.lock b/yarn.lock index 0f1bd9dc..22161b8f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9606,6 +9606,13 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-3.0.3.tgz#acaab0f919ce69cce629c2d4ed2eb4adc1b6c20c" integrity sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q== +"@types/unzipper@^0.10.10": + version "0.10.10" + resolved "https://registry.yarnpkg.com/@types/unzipper/-/unzipper-0.10.10.tgz#4407f7f633db0c5cf20f05257352cb8197fb9e5a" + integrity sha512-jKJdNxhmCHTZsaKW5x0qjn6rB+gHk0w5VFbEKsw84i+RJqXZyfTmGnpjDcKqzMpjz7VVLsUBMtO5T3mVidpt0g== + dependencies: + "@types/node" "*" + "@types/uuid@^9.0.1", "@types/uuid@^9.0.8": version "9.0.8" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" @@ -11296,7 +11303,7 @@ bl@^5.0.0: inherits "^2.0.4" readable-stream "^3.4.0" -bluebird@3.7.2: +bluebird@3.7.2, bluebird@~3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -13360,6 +13367,13 @@ dotenv@^16.4.4, dotenv@^16.4.5, dotenv@~16.4.5: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== +duplexer2@~0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + integrity sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA== + dependencies: + readable-stream "^2.0.2" + duplexer@~0.1.1: version "0.1.2" resolved "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz" @@ -15934,7 +15948,7 @@ graceful-fs@4.2.10, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -graceful-fs@^4.1.11, graceful-fs@^4.2.11: +graceful-fs@^4.1.11, graceful-fs@^4.2.11, graceful-fs@^4.2.2: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -22131,6 +22145,19 @@ readable-stream@^2.0.1, readable-stream@^2.2.2, readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^2.0.2: + version "2.3.8" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz" @@ -24936,6 +24963,17 @@ unplugin@^1.3.1: webpack-sources "^3.2.3" webpack-virtual-modules "^0.6.0" +unzipper@^0.12.3: + version "0.12.3" + resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.12.3.tgz#31958f5eed7368ed8f57deae547e5a673e984f87" + integrity sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA== + dependencies: + bluebird "~3.7.2" + duplexer2 "~0.1.4" + fs-extra "^11.2.0" + graceful-fs "^4.2.2" + node-int64 "^0.4.0" + upath@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/upath/-/upath-2.0.1.tgz#50c73dea68d6f6b990f51d279ce6081665d61a8b" From 7b8d0743a5cc31e659c51044851821aad560189c Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sat, 21 Dec 2024 15:49:24 -0700 Subject: [PATCH 2/3] fix logger error unwrapping --- apps/cron-tasks/src/config/db.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cron-tasks/src/config/db.config.ts b/apps/cron-tasks/src/config/db.config.ts index 5358de6e..9870288f 100644 --- a/apps/cron-tasks/src/config/db.config.ts +++ b/apps/cron-tasks/src/config/db.config.ts @@ -23,11 +23,11 @@ export const pgPool = new Pool({ pgPool.on('connect', (client) => { // logger.info('[DB][POOL] Connected'); client.on('error', (err) => { - logger.error(getExceptionLog, '[DB][CLIENT][ERROR] Unexpected error on client'); + logger.error(getExceptionLog(err), '[DB][CLIENT][ERROR] Unexpected error on client'); }); }); pgPool.on('error', (err, client) => { - logger.error(getExceptionLog, '[DB][POOL][ERROR] Unexpected error on idle client'); + logger.error(getExceptionLog(err), '[DB][POOL][ERROR] Unexpected error on idle client'); process.exit(-1); }); From 2682489816f0f70511662f10332a70017269bc34 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sat, 21 Dec 2024 15:56:32 -0700 Subject: [PATCH 3/3] fix error logging --- apps/cron-tasks/src/geo-ip-updater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cron-tasks/src/geo-ip-updater.ts b/apps/cron-tasks/src/geo-ip-updater.ts index e6da2ab8..22a957ae 100644 --- a/apps/cron-tasks/src/geo-ip-updater.ts +++ b/apps/cron-tasks/src/geo-ip-updater.ts @@ -92,7 +92,7 @@ async function importCSVToTable(csvPath: string, tableName: string, schema: stri logger.info(`Successfully imported ${csvPath} to ${fullTableName}`); } catch (error) { - logger.error(`Error importing ${csvPath}:`, error); + logger.error(getExceptionLog(error), `Error importing ${csvPath}: %s`, error.message); // Cleanup temp table if it exists await execAsync(`psql "${ENV.JETSTREAM_POSTGRES_DBURI}" -c "DROP TABLE IF EXISTS ${fullTempTableName}"`).catch(() => { // Ignore errors