From 8beec5c57f8f41d51b4f2140e168481677d247a8 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 1 Aug 2024 12:48:57 -0400 Subject: [PATCH 1/4] support configurable chunk size --- src/push/kibana_api.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/push/kibana_api.ts b/src/push/kibana_api.ts index bea7329e..e827ccea 100644 --- a/src/push/kibana_api.ts +++ b/src/push/kibana_api.ts @@ -37,7 +37,12 @@ import { import { generateURL } from './utils'; // Default chunk size for bulk put / delete -export const CHUNK_SIZE = 100; +export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE) || 100; +if (CHUNK_SIZE > 250) { + throw new Error( + 'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250' + ); +} export type PutResponse = { createdMonitors: string[]; From 642946cff212edb191b74278a669529ee1ba1822 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 1 Aug 2024 13:05:37 -0400 Subject: [PATCH 2/4] update error --- src/push/kibana_api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/push/kibana_api.ts b/src/push/kibana_api.ts index e827ccea..a11e2ec4 100644 --- a/src/push/kibana_api.ts +++ b/src/push/kibana_api.ts @@ -24,7 +24,7 @@ */ import { PushOptions } from '../common_types'; -import { safeNDJSONParse } from '../helpers'; +import { safeNDJSONParse, error } from '../helpers'; import { MonitorHashID, MonitorSchema } from './monitor'; import { formatFailedMonitors, @@ -39,7 +39,7 @@ import { generateURL } from './utils'; // Default chunk size for bulk put / delete export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE) || 100; if (CHUNK_SIZE > 250) { - throw new Error( + throw error( 'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250' ); } From c736dcc0de42ea0fdd92247c8393148e1ac475ab Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 1 Aug 2024 13:50:33 -0400 Subject: [PATCH 3/4] move error further up --- src/push/index.ts | 10 ++++++++-- src/push/kibana_api.ts | 7 +------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/push/index.ts b/src/push/index.ts index 15a4309c..9b85ab7c 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -61,6 +61,11 @@ import { import { log } from '../core/logger'; export async function push(monitors: Monitor[], options: PushOptions) { + if (parseInt(process.env.CHUNK_SIZE) > 250) { + throw error( + 'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250' + ); + } const duplicates = trackDuplicates(monitors); if (duplicates.size > 0) { throw error(formatDuplicateError(duplicates)); @@ -218,8 +223,9 @@ export function validateSettings(opts: PushOptions) { - CLI '--schedule ' - Config file 'monitors.schedule' field`; } else if (opts.schedule && !ALLOWED_SCHEDULES.includes(opts.schedule)) { - reason = `Set default schedule(${opts.schedule - }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`; + reason = `Set default schedule(${ + opts.schedule + }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`; } if (!reason) return; diff --git a/src/push/kibana_api.ts b/src/push/kibana_api.ts index a11e2ec4..4815a90b 100644 --- a/src/push/kibana_api.ts +++ b/src/push/kibana_api.ts @@ -24,7 +24,7 @@ */ import { PushOptions } from '../common_types'; -import { safeNDJSONParse, error } from '../helpers'; +import { safeNDJSONParse } from '../helpers'; import { MonitorHashID, MonitorSchema } from './monitor'; import { formatFailedMonitors, @@ -38,11 +38,6 @@ import { generateURL } from './utils'; // Default chunk size for bulk put / delete export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE) || 100; -if (CHUNK_SIZE > 250) { - throw error( - 'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250' - ); -} export type PutResponse = { createdMonitors: string[]; From 1169efbf8579562bd838d2f425161242329beb14 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 1 Aug 2024 15:02:50 -0400 Subject: [PATCH 4/4] add test adjust test confirm if pushed --- __tests__/push/index.test.ts | 34 +++++++++++++++++++++++++++++++++- src/push/index.ts | 5 ++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/__tests__/push/index.test.ts b/__tests__/push/index.test.ts index 95e33a53..151762ac 100644 --- a/__tests__/push/index.test.ts +++ b/__tests__/push/index.test.ts @@ -200,8 +200,8 @@ heartbeat.monitors: ` ); const output = await runPush(); - expect(output).toContain(`Aborted: Duplicate monitors found`); await rm(heartbeatYml, { force: true }); + expect(output).toContain(`Aborted: Duplicate monitors found`); }); it('format duplicate monitors', () => { @@ -218,6 +218,38 @@ heartbeat.monitors: expect(formatDuplicateError(duplicates as Set)).toMatchSnapshot(); }); + it('error on invalid CHUNK SIZE', async () => { + await fakeProjectSetup( + { id: 'test-project', space: 'dummy', url: server.PREFIX }, + { locations: ['test-loc'], schedule: 3 } + ); + const output = await runPush(undefined, { CHUNK_SIZE: '251' }); + expect(output).toContain( + 'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250' + ); + }); + + it('respects valid CHUNK SIZE', async () => { + await fakeProjectSetup( + { id: 'test-project', space: 'dummy', url: server.PREFIX }, + { locations: ['test-loc'], schedule: 3 } + ); + const testJourney = join(PROJECT_DIR, 'chunk.journey.ts'); + await writeFile( + testJourney, + `import {journey, monitor} from '../../../'; + journey('a', () => monitor.use({ tags: ['chunk'] })); + journey('b', () => monitor.use({ tags: ['chunk'] }));` + ); + const output = await runPush([...DEFAULT_ARGS, '--tags', 'chunk'], { + CHUNK_SIZE: '1', + }); + await rm(testJourney, { force: true }); + expect(output).toContain('Added(2)'); + expect(output).toContain('creating or updating 1 monitors'); + expect(output).toContain('✓ Pushed:'); + }); + ['8.5.0', '8.6.0'].forEach(version => { describe('API: ' + version, () => { let server: Server; diff --git a/src/push/index.ts b/src/push/index.ts index 9b85ab7c..fe4f9dbb 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -223,9 +223,8 @@ export function validateSettings(opts: PushOptions) { - CLI '--schedule ' - Config file 'monitors.schedule' field`; } else if (opts.schedule && !ALLOWED_SCHEDULES.includes(opts.schedule)) { - reason = `Set default schedule(${ - opts.schedule - }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`; + reason = `Set default schedule(${opts.schedule + }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`; } if (!reason) return;