From e1d3cb9c146fa5e1d9d00aa9fa0adfc7394dd67f Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 4 Oct 2023 19:30:45 -0700 Subject: [PATCH] [Reporting] Improve language for error when CSV row total was indeterminable (#167843) ## Summary Closes https://github.com/elastic/kibana/issues/153250 --- .../__snapshots__/generate_csv.test.ts.snap | 28 +- .../kbn-generate-csv/src/generate_csv.test.ts | 1557 +++++++++-------- packages/kbn-generate-csv/src/generate_csv.ts | 10 +- packages/kbn-generate-csv/src/i18n_texts.ts | 6 + 4 files changed, 810 insertions(+), 791 deletions(-) diff --git a/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap b/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap index c10911d7687d3..da0f6a4560640 100644 --- a/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap +++ b/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap @@ -1,71 +1,71 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`fields from job.columns (7.13+ generated) cells can be multi-value 1`] = ` +exports[`CsvGenerator fields from job.columns (7.13+ generated) cells can be multi-value 1`] = ` "product,category coconut,\\"cool, rad\\" " `; -exports[`fields from job.columns (7.13+ generated) columns can be top-level fields such as _id and _index 1`] = ` +exports[`CsvGenerator fields from job.columns (7.13+ generated) columns can be top-level fields such as _id and _index 1`] = ` "\\"_id\\",\\"_index\\",product,category \\"my-cool-id\\",\\"my-cool-index\\",coconut,\\"cool, rad\\" " `; -exports[`fields from job.columns (7.13+ generated) default column names come from tabify 1`] = ` +exports[`CsvGenerator fields from job.columns (7.13+ generated) default column names come from tabify 1`] = ` "\\"_id\\",\\"_index\\",\\"_score\\",category,product \\"my-cool-id\\",\\"my-cool-index\\",\\"'-\\",\\"cool, rad\\",coconut " `; -exports[`fields from job.searchSource.getFields() (7.12 generated) cells can be multi-value 1`] = ` +exports[`CsvGenerator fields from job.searchSource.getFields() (7.12 generated) cells can be multi-value 1`] = ` "\\"_id\\",sku \\"my-cool-id\\",\\"This is a cool SKU., This is also a cool SKU.\\" " `; -exports[`fields from job.searchSource.getFields() (7.12 generated) provides top-level underscored fields as columns 1`] = ` +exports[`CsvGenerator fields from job.searchSource.getFields() (7.12 generated) provides top-level underscored fields as columns 1`] = ` "\\"_id\\",\\"_index\\",date,message \\"my-cool-id\\",\\"my-cool-index\\",\\"2020-12-31T00:14:28.000Z\\",\\"it's nice to see you\\" " `; -exports[`fields from job.searchSource.getFields() (7.12 generated) sorts the fields when they are to be used as table column names 1`] = ` +exports[`CsvGenerator fields from job.searchSource.getFields() (7.12 generated) sorts the fields when they are to be used as table column names 1`] = ` "\\"_id\\",\\"_index\\",\\"_score\\",date,\\"message_t\\",\\"message_u\\",\\"message_v\\",\\"message_w\\",\\"message_x\\",\\"message_y\\",\\"message_z\\" \\"my-cool-id\\",\\"my-cool-index\\",\\"'-\\",\\"2020-12-31T00:14:28.000Z\\",\\"test field T\\",\\"test field U\\",\\"test field V\\",\\"test field W\\",\\"test field X\\",\\"test field Y\\",\\"test field Z\\" " `; -exports[`formats a search result to CSV content 1`] = ` +exports[`CsvGenerator formats a search result to CSV content 1`] = ` "date,ip,message \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is a great message!\\" " `; -exports[`formats an empty search result to CSV content 1`] = ` +exports[`CsvGenerator formats an empty search result to CSV content 1`] = ` "date,ip,message " `; -exports[`formulas can check for formulas, without escaping them 1`] = ` +exports[`CsvGenerator formulas can check for formulas, without escaping them 1`] = ` "date,ip,message \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"=SUM(A1:A2)\\" " `; -exports[`formulas escapes formula values in a cell, doesn't warn the csv contains formulas 1`] = ` +exports[`CsvGenerator formulas escapes formula values in a cell, doesn't warn the csv contains formulas 1`] = ` "date,ip,message \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"'=SUM(A1:A2)\\" " `; -exports[`formulas escapes formula values in a header, doesn't warn the csv contains formulas 1`] = ` +exports[`CsvGenerator formulas escapes formula values in a header, doesn't warn the csv contains formulas 1`] = ` "date,ip,\\"'=SUM(A1:A2)\\" \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is great data\\" " `; -exports[`keeps order of the columns during the scroll 1`] = ` +exports[`CsvGenerator keeps order of the columns during the scroll 1`] = ` "\\"_id\\",\\"_index\\",\\"_score\\",a,b \\"'-\\",\\"'-\\",\\"'-\\",a1,b1 \\"'-\\",\\"'-\\",\\"'-\\",\\"'-\\",b2 @@ -73,7 +73,7 @@ exports[`keeps order of the columns during the scroll 1`] = ` " `; -exports[`uses the pit ID to page all the data 1`] = ` +exports[`CsvGenerator uses the pit ID to page all the data 1`] = ` "date,ip,message \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\" \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\" @@ -178,7 +178,7 @@ exports[`uses the pit ID to page all the data 1`] = ` " `; -exports[`warns if max size was reached 1`] = ` +exports[`CsvGenerator warns if max size was reached 1`] = ` "date,ip,message \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\" \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\" diff --git a/packages/kbn-generate-csv/src/generate_csv.test.ts b/packages/kbn-generate-csv/src/generate_csv.test.ts index 874c58aeb8469..52f15d6a82c49 100644 --- a/packages/kbn-generate-csv/src/generate_csv.test.ts +++ b/packages/kbn-generate-csv/src/generate_csv.test.ts @@ -32,394 +32,110 @@ const createMockJob = (baseObj: any = {}): JobParams => ({ ...baseObj, }); -let mockEsClient: IScopedClusterClient; -let mockDataClient: IScopedSearchClient; -let mockConfig: CsvConfig; -let mockLogger: jest.Mocked; -let uiSettingsClient: IUiSettingsClient; -let stream: jest.Mocked; -let content: string; - -const searchSourceMock = { - ...searchSourceInstanceMock, - getSearchRequestBody: jest.fn(() => ({})), -}; - -const mockSearchSourceService: jest.Mocked = { - create: jest.fn().mockReturnValue(searchSourceMock), - createEmpty: jest.fn().mockReturnValue(searchSourceMock), - telemetry: jest.fn(), - inject: jest.fn(), - extract: jest.fn(), - getAllMigrations: jest.fn(), -}; - -const mockPitId = 'oju9fs3698s3902f02-8qg3-u9w36oiewiuyew6'; - -const getMockRawResponse = (hits: Array> = [], total = hits.length) => ({ - took: 1, - timed_out: false, - pit_id: mockPitId, - _shards: { total: 1, successful: 1, failed: 0, skipped: 0 }, - hits: { hits, total, max_score: 0 }, -}); - -const mockDataClientSearchDefault = jest.fn().mockImplementation( - (): Rx.Observable<{ rawResponse: SearchResponse }> => - Rx.of({ - rawResponse: getMockRawResponse(), - }) -); - -const mockFieldFormatsRegistry = { - deserialize: jest - .fn() - .mockImplementation(() => ({ id: 'string', convert: jest.fn().mockImplementation(identity) })), -} as unknown as FieldFormatsRegistry; - -beforeEach(async () => { - content = ''; - stream = { write: jest.fn((chunk) => (content += chunk)) } as unknown as typeof stream; - mockEsClient = elasticsearchServiceMock.createScopedClusterClient(); - mockDataClient = dataPluginMock.createStartContract().search.asScoped({} as any); - mockDataClient.search = mockDataClientSearchDefault; - - mockEsClient.asCurrentUser.openPointInTime = jest.fn().mockResolvedValueOnce({ id: mockPitId }); - - uiSettingsClient = uiSettingsServiceMock - .createStartContract() - .asScopedToClient(savedObjectsClientMock.create()); - uiSettingsClient.get = jest.fn().mockImplementation((key): any => { - switch (key) { - case UI_SETTINGS_CSV_QUOTE_VALUES: - return true; - case UI_SETTINGS_CSV_SEPARATOR: - return ','; - case UI_SETTINGS_DATEFORMAT_TZ: - return 'Browser'; - } - }); - - mockConfig = { - checkForFormulas: true, - escapeFormulaValues: true, - maxSizeBytes: 180000, - useByteOrderMarkEncoding: false, - scroll: { size: 500, duration: '30s' }, +describe('CsvGenerator', () => { + let mockEsClient: IScopedClusterClient; + let mockDataClient: IScopedSearchClient; + let mockConfig: CsvConfig; + let mockLogger: jest.Mocked; + let uiSettingsClient: IUiSettingsClient; + let stream: jest.Mocked; + let content: string; + + const searchSourceMock = { + ...searchSourceInstanceMock, + getSearchRequestBody: jest.fn(() => ({})), }; - searchSourceMock.getField = jest.fn((key: string) => { - switch (key) { - case 'pit': - return { id: mockPitId }; - case 'index': - return { - fields: { - getByName: jest.fn(() => []), - getByType: jest.fn(() => []), - }, - metaFields: ['_id', '_index', '_type', '_score'], - getFormatterForField: jest.fn(), - getIndexPattern: () => 'logstash-*', - }; - } - }); - - mockLogger = loggingSystemMock.createLogger(); -}); - -it('formats an empty search result to CSV content', async () => { - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - const csvResult = await generateCsv.generateData(); - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(false); -}); - -it('formats a search result to CSV content', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - fields: { - date: `["2020-12-31T00:14:28.000Z"]`, - ip: `["110.135.176.89"]`, - message: `["This is a great message!"]`, - }, - } as unknown as estypes.SearchHit, - ]), - }) - ); - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - const csvResult = await generateCsv.generateData(); - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(false); -}); - -const HITS_TOTAL = 100; - -it('calculates the bytes of the content', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse( - range(0, HITS_TOTAL).map( - () => - ({ - fields: { - message: ['this is a great message'], - }, - } as unknown as estypes.SearchHit) - ) - ), - }) - ); - - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - const csvResult = await generateCsv.generateData(); - expect(csvResult.max_size_reached).toBe(false); - expect(csvResult.warnings).toEqual([]); -}); - -it('warns if max size was reached', async () => { - const TEST_MAX_SIZE = 500; - mockConfig = { - checkForFormulas: true, - escapeFormulaValues: true, - maxSizeBytes: TEST_MAX_SIZE, - useByteOrderMarkEncoding: false, - scroll: { size: 500, duration: '30s' }, + const mockSearchSourceService: jest.Mocked = { + create: jest.fn().mockReturnValue(searchSourceMock), + createEmpty: jest.fn().mockReturnValue(searchSourceMock), + telemetry: jest.fn(), + inject: jest.fn(), + extract: jest.fn(), + getAllMigrations: jest.fn(), }; - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse( - range(0, HITS_TOTAL).map( - () => - ({ - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: ['super cali fragile istic XPLA docious'], - }, - } as unknown as estypes.SearchHit) - ) - ), - }) - ); - - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - const csvResult = await generateCsv.generateData(); - expect(csvResult.max_size_reached).toBe(true); - expect(csvResult.warnings).toEqual([]); - expect(content).toMatchSnapshot(); -}); - -it('uses the pit ID to page all the data', async () => { - mockDataClient.search = jest - .fn() - .mockImplementationOnce(() => - Rx.of({ - rawResponse: getMockRawResponse( - range(0, HITS_TOTAL / 10).map( - () => - ({ - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: ['hit from the initial search'], - }, - } as unknown as estypes.SearchHit) - ), - HITS_TOTAL - ), - }) - ) - .mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse( - range(0, HITS_TOTAL / 10).map( - () => - ({ - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: ['hit from a subsequent scroll'], - }, - } as unknown as estypes.SearchHit) - ) - ), - }) - ); - - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - const csvResult = await generateCsv.generateData(); - expect(csvResult.warnings).toEqual([]); - expect(content).toMatchSnapshot(); - - expect(mockDataClient.search).toHaveBeenCalledTimes(10); - expect(mockDataClient.search).toBeCalledWith( - { params: { body: {}, ignore_throttled: undefined } }, - { strategy: 'es', transport: { maxRetries: 0, requestTimeout: '30s' } } - ); - - expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledTimes(1); - expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( - { - ignore_unavailable: true, - index: 'logstash-*', - keep_alive: '30s', - }, - { maxRetries: 0, requestTimeout: '30s' } - ); - - expect(mockEsClient.asCurrentUser.closePointInTime).toHaveBeenCalledTimes(1); - expect(mockEsClient.asCurrentUser.closePointInTime).toHaveBeenCalledWith({ - body: { id: mockPitId }, + const mockPitId = 'oju9fs3698s3902f02-8qg3-u9w36oiewiuyew6'; + + const getMockRawResponse = ( + hits: Array> = [], + total = hits.length + ) => ({ + took: 1, + timed_out: false, + pit_id: mockPitId, + _shards: { total: 1, successful: 1, failed: 0, skipped: 0 }, + hits: { hits, total, max_score: 0 }, }); -}); -it('keeps order of the columns during the scroll', async () => { - mockDataClient.search = jest - .fn() - .mockImplementationOnce(() => - Rx.of({ - rawResponse: getMockRawResponse( - [{ fields: { a: ['a1'], b: ['b1'] } } as unknown as estypes.SearchHit], - 3 - ), - }) - ) - .mockImplementationOnce(() => - Rx.of({ - rawResponse: getMockRawResponse( - [{ fields: { b: ['b2'] } } as unknown as estypes.SearchHit], - 3 - ), - }) - ) - .mockImplementationOnce(() => + const mockDataClientSearchDefault = jest.fn().mockImplementation( + (): Rx.Observable<{ rawResponse: SearchResponse }> => Rx.of({ - rawResponse: getMockRawResponse( - [{ fields: { a: ['a3'], c: ['c3'] } } as unknown as estypes.SearchHit], - 3 - ), + rawResponse: getMockRawResponse(), }) - ); - - const generateCsv = new CsvGenerator( - createMockJob({ searchSource: {}, columns: [] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream ); - await generateCsv.generateData(); - expect(content).toMatchSnapshot(); -}); + const mockFieldFormatsRegistry = { + deserialize: jest.fn().mockImplementation(() => ({ + id: 'string', + convert: jest.fn().mockImplementation(identity), + })), + } as unknown as FieldFormatsRegistry; + + beforeEach(async () => { + content = ''; + stream = { write: jest.fn((chunk) => (content += chunk)) } as unknown as typeof stream; + mockEsClient = elasticsearchServiceMock.createScopedClusterClient(); + mockDataClient = dataPluginMock.createStartContract().search.asScoped({} as any); + mockDataClient.search = mockDataClientSearchDefault; + + mockEsClient.asCurrentUser.openPointInTime = jest.fn().mockResolvedValueOnce({ id: mockPitId }); + + uiSettingsClient = uiSettingsServiceMock + .createStartContract() + .asScopedToClient(savedObjectsClientMock.create()); + uiSettingsClient.get = jest.fn().mockImplementation((key): any => { + switch (key) { + case UI_SETTINGS_CSV_QUOTE_VALUES: + return true; + case UI_SETTINGS_CSV_SEPARATOR: + return ','; + case UI_SETTINGS_DATEFORMAT_TZ: + return 'Browser'; + } + }); -describe('fields from job.searchSource.getFields() (7.12 generated)', () => { - it('cells can be multi-value', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, + mockConfig = { + checkForFormulas: true, + escapeFormulaValues: true, + maxSizeBytes: 180000, + useByteOrderMarkEncoding: false, + scroll: { size: 500, duration: '30s' }, + }; + + searchSourceMock.getField = jest.fn((key: string) => { + switch (key) { + case 'pit': + return { id: mockPitId }; + case 'index': + return { fields: { - sku: [`This is a cool SKU.`, `This is also a cool SKU.`], + getByName: jest.fn(() => []), + getByType: jest.fn(() => []), }, - }, - ]), - }) - ); + metaFields: ['_id', '_index', '_type', '_score'], + getFormatterForField: jest.fn(), + getIndexPattern: () => 'logstash-*', + }; + } + }); + mockLogger = loggingSystemMock.createLogger(); + }); + + it('formats an empty search result to CSV content', async () => { const generateCsv = new CsvGenerator( - createMockJob({ searchSource: {}, columns: ['_id', 'sku'] }), + createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, { es: mockEsClient, @@ -434,39 +150,27 @@ describe('fields from job.searchSource.getFields() (7.12 generated)', () => { mockLogger, stream ); - await generateCsv.generateData(); - + const csvResult = await generateCsv.generateData(); expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(false); }); - it('provides top-level underscored fields as columns', async () => { + it('formats a search result to CSV content', async () => { mockDataClient.search = jest.fn().mockImplementation(() => Rx.of({ rawResponse: getMockRawResponse([ { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, fields: { - date: ['2020-12-31T00:14:28.000Z'], - message: [`it's nice to see you`], + date: `["2020-12-31T00:14:28.000Z"]`, + ip: `["110.135.176.89"]`, + message: `["This is a great message!"]`, }, - }, + } as unknown as estypes.SearchHit, ]), }) ); - const generateCsv = new CsvGenerator( - createMockJob({ - searchSource: { - query: { query: '', language: 'kuery' }, - sort: [{ '@date': 'desc' }], - index: '93f4bc50-6662-11eb-98bc-f550e2308366', - fields: ['_id', '_index', '@date', 'message'], - filter: [], - }, - columns: ['_id', '_index', 'date', 'message'], - }), + createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, { es: mockEsClient, @@ -481,46 +185,31 @@ describe('fields from job.searchSource.getFields() (7.12 generated)', () => { mockLogger, stream ); - const csvResult = await generateCsv.generateData(); - expect(content).toMatchSnapshot(); expect(csvResult.csv_contains_formulas).toBe(false); }); - it('sorts the fields when they are to be used as table column names', async () => { + const HITS_TOTAL = 100; + + it('calculates the bytes of the content', async () => { mockDataClient.search = jest.fn().mockImplementation(() => Rx.of({ - rawResponse: getMockRawResponse([ - { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, - fields: { - date: ['2020-12-31T00:14:28.000Z'], - message_z: [`test field Z`], - message_y: [`test field Y`], - message_x: [`test field X`], - message_w: [`test field W`], - message_v: [`test field V`], - message_u: [`test field U`], - message_t: [`test field T`], - }, - }, - ]), + rawResponse: getMockRawResponse( + range(0, HITS_TOTAL).map( + () => + ({ + fields: { + message: ['this is a great message'], + }, + } as unknown as estypes.SearchHit) + ) + ), }) ); const generateCsv = new CsvGenerator( - createMockJob({ - searchSource: { - query: { query: '', language: 'kuery' }, - sort: [{ '@date': 'desc' }], - index: '93f4bc50-6662-11eb-98bc-f550e2308366', - fields: ['*'], - filter: [], - }, - }), + createMockJob({ columns: ['message'] }), mockConfig, { es: mockEsClient, @@ -535,34 +224,40 @@ describe('fields from job.searchSource.getFields() (7.12 generated)', () => { mockLogger, stream ); - const csvResult = await generateCsv.generateData(); - - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(false); + expect(csvResult.max_size_reached).toBe(false); + expect(csvResult.warnings).toEqual([]); }); -}); -describe('fields from job.columns (7.13+ generated)', () => { - it('cells can be multi-value', async () => { + it('warns if max size was reached', async () => { + const TEST_MAX_SIZE = 500; + mockConfig = { + checkForFormulas: true, + escapeFormulaValues: true, + maxSizeBytes: TEST_MAX_SIZE, + useByteOrderMarkEncoding: false, + scroll: { size: 500, duration: '30s' }, + }; + mockDataClient.search = jest.fn().mockImplementation(() => Rx.of({ - rawResponse: getMockRawResponse([ - { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, - fields: { - product: 'coconut', - category: [`cool`, `rad`], - }, - }, - ]), + rawResponse: getMockRawResponse( + range(0, HITS_TOTAL).map( + () => + ({ + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: ['super cali fragile istic XPLA docious'], + }, + } as unknown as estypes.SearchHit) + ) + ), }) ); const generateCsv = new CsvGenerator( - createMockJob({ searchSource: {}, columns: ['product', 'category'] }), + createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, { es: mockEsClient, @@ -577,30 +272,51 @@ describe('fields from job.columns (7.13+ generated)', () => { mockLogger, stream ); - await generateCsv.generateData(); - + const csvResult = await generateCsv.generateData(); + expect(csvResult.max_size_reached).toBe(true); + expect(csvResult.warnings).toEqual([]); expect(content).toMatchSnapshot(); }); - it('columns can be top-level fields such as _id and _index', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, - fields: { - product: 'coconut', - category: [`cool`, `rad`], - }, - }, - ]), - }) - ); + it('uses the pit ID to page all the data', async () => { + mockDataClient.search = jest + .fn() + .mockImplementationOnce(() => + Rx.of({ + rawResponse: getMockRawResponse( + range(0, HITS_TOTAL / 10).map( + () => + ({ + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: ['hit from the initial search'], + }, + } as unknown as estypes.SearchHit) + ), + HITS_TOTAL + ), + }) + ) + .mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse( + range(0, HITS_TOTAL / 10).map( + () => + ({ + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: ['hit from a subsequent scroll'], + }, + } as unknown as estypes.SearchHit) + ) + ), + }) + ); const generateCsv = new CsvGenerator( - createMockJob({ searchSource: {}, columns: ['_id', '_index', 'product', 'category'] }), + createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, { es: mockEsClient, @@ -615,28 +331,60 @@ describe('fields from job.columns (7.13+ generated)', () => { mockLogger, stream ); - await generateCsv.generateData(); - + const csvResult = await generateCsv.generateData(); + expect(csvResult.warnings).toEqual([]); expect(content).toMatchSnapshot(); - }); - it('default column names come from tabify', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - _id: 'my-cool-id', - _index: 'my-cool-index', - _version: 4, - fields: { - product: 'coconut', - category: [`cool`, `rad`], - }, - }, - ]), - }) + expect(mockDataClient.search).toHaveBeenCalledTimes(10); + expect(mockDataClient.search).toBeCalledWith( + { params: { body: {}, ignore_throttled: undefined } }, + { strategy: 'es', transport: { maxRetries: 0, requestTimeout: '30s' } } + ); + + expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledTimes(1); + expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( + { + ignore_unavailable: true, + index: 'logstash-*', + keep_alive: '30s', + }, + { maxRetries: 0, requestTimeout: '30s' } ); + expect(mockEsClient.asCurrentUser.closePointInTime).toHaveBeenCalledTimes(1); + expect(mockEsClient.asCurrentUser.closePointInTime).toHaveBeenCalledWith({ + body: { id: mockPitId }, + }); + }); + + it('keeps order of the columns during the scroll', async () => { + mockDataClient.search = jest + .fn() + .mockImplementationOnce(() => + Rx.of({ + rawResponse: getMockRawResponse( + [{ fields: { a: ['a1'], b: ['b1'] } } as unknown as estypes.SearchHit], + 3 + ), + }) + ) + .mockImplementationOnce(() => + Rx.of({ + rawResponse: getMockRawResponse( + [{ fields: { b: ['b2'] } } as unknown as estypes.SearchHit], + 3 + ), + }) + ) + .mockImplementationOnce(() => + Rx.of({ + rawResponse: getMockRawResponse( + [{ fields: { a: ['a3'], c: ['c3'] } } as unknown as estypes.SearchHit], + 3 + ), + }) + ); + const generateCsv = new CsvGenerator( createMockJob({ searchSource: {}, columns: [] }), mockConfig, @@ -657,34 +405,403 @@ describe('fields from job.columns (7.13+ generated)', () => { expect(content).toMatchSnapshot(); }); -}); -describe('formulas', () => { - const TEST_FORMULA = '=SUM(A1:A2)'; + describe('fields from job.searchSource.getFields() (7.12 generated)', () => { + it('cells can be multi-value', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + sku: [`This is a cool SKU.`, `This is also a cool SKU.`], + }, + }, + ]), + }) + ); - it(`escapes formula values in a cell, doesn't warn the csv contains formulas`, async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: [TEST_FORMULA], + const generateCsv = new CsvGenerator( + createMockJob({ searchSource: {}, columns: ['_id', 'sku'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + }); + + it('provides top-level underscored fields as columns', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + date: ['2020-12-31T00:14:28.000Z'], + message: [`it's nice to see you`], + }, }, - } as unknown as estypes.SearchHit, - ]), - }) - ); + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ + searchSource: { + query: { query: '', language: 'kuery' }, + sort: [{ '@date': 'desc' }], + index: '93f4bc50-6662-11eb-98bc-f550e2308366', + fields: ['_id', '_index', '@date', 'message'], + filter: [], + }, + columns: ['_id', '_index', 'date', 'message'], + }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const csvResult = await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(false); + }); + + it('sorts the fields when they are to be used as table column names', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + date: ['2020-12-31T00:14:28.000Z'], + message_z: [`test field Z`], + message_y: [`test field Y`], + message_x: [`test field X`], + message_w: [`test field W`], + message_v: [`test field V`], + message_u: [`test field U`], + message_t: [`test field T`], + }, + }, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ + searchSource: { + query: { query: '', language: 'kuery' }, + sort: [{ '@date': 'desc' }], + index: '93f4bc50-6662-11eb-98bc-f550e2308366', + fields: ['*'], + filter: [], + }, + }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const csvResult = await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(false); + }); + }); + + describe('fields from job.columns (7.13+ generated)', () => { + it('cells can be multi-value', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + product: 'coconut', + category: [`cool`, `rad`], + }, + }, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ searchSource: {}, columns: ['product', 'category'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + }); + + it('columns can be top-level fields such as _id and _index', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + product: 'coconut', + category: [`cool`, `rad`], + }, + }, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ searchSource: {}, columns: ['_id', '_index', 'product', 'category'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + }); + + it('default column names come from tabify', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + _id: 'my-cool-id', + _index: 'my-cool-index', + _version: 4, + fields: { + product: 'coconut', + category: [`cool`, `rad`], + }, + }, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ searchSource: {}, columns: [] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + }); + }); + + describe('formulas', () => { + const TEST_FORMULA = '=SUM(A1:A2)'; + + it(`escapes formula values in a cell, doesn't warn the csv contains formulas`, async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: [TEST_FORMULA], + }, + } as unknown as estypes.SearchHit, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', 'message'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const csvResult = await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(false); + }); + + it(`escapes formula values in a header, doesn't warn the csv contains formulas`, async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + [TEST_FORMULA]: 'This is great data', + }, + } as unknown as estypes.SearchHit, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', TEST_FORMULA] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const csvResult = await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(false); + }); + + it('can check for formulas, without escaping them', async () => { + mockConfig = { + checkForFormulas: true, + escapeFormulaValues: false, + maxSizeBytes: 180000, + useByteOrderMarkEncoding: false, + scroll: { size: 500, duration: '30s' }, + }; + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: getMockRawResponse([ + { + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: [TEST_FORMULA], + }, + } as unknown as estypes.SearchHit, + ]), + }) + ); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', 'message'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const csvResult = await generateCsv.generateData(); + + expect(content).toMatchSnapshot(); + expect(csvResult.csv_contains_formulas).toBe(true); + }); + }); + + it('can override ignoring frozen indices', async () => { + const originalGet = uiSettingsClient.get; + uiSettingsClient.get = jest.fn().mockImplementation((key): any => { + if (key === 'search:includeFrozen') { + return true; + } + return originalGet(key); + }); const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), + createMockJob({}), mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, + { es: mockEsClient, data: mockDataClient, uiSettings: uiSettingsClient }, { searchSourceStart: mockSearchSourceService, fieldFormatsRegistry: mockFieldFormatsRegistry, @@ -694,29 +811,49 @@ describe('formulas', () => { stream ); - const csvResult = await generateCsv.generateData(); + await generateCsv.generateData(); - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(false); + expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( + { + ignore_unavailable: true, + ignore_throttled: false, + index: 'logstash-*', + keep_alive: '30s', + }, + { maxRetries: 0, requestTimeout: '30s' } + ); + + expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( + { + ignore_unavailable: true, + ignore_throttled: false, + index: 'logstash-*', + keep_alive: '30s', + }, + { maxRetries: 0, requestTimeout: '30s' } + ); + + expect(mockDataClient.search).toBeCalledWith( + { + params: { + body: {}, + }, + }, + { strategy: 'es', transport: { maxRetries: 0, requestTimeout: '30s' } } + ); }); - it(`escapes formula values in a header, doesn't warn the csv contains formulas`, async () => { - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - [TEST_FORMULA]: 'This is great data', - }, - } as unknown as estypes.SearchHit, - ]), + it('adds a warning if export was unable to close the PIT', async () => { + mockEsClient.asCurrentUser.closePointInTime = jest.fn().mockRejectedValueOnce( + new esErrors.ResponseError({ + statusCode: 419, + warnings: [], + meta: { context: 'test' } as any, }) ); const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', TEST_FORMULA] }), + createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, { es: mockEsClient, @@ -732,34 +869,33 @@ describe('formulas', () => { stream ); - const csvResult = await generateCsv.generateData(); - - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(false); + await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` + Object { + "content_type": "text/csv", + "csv_contains_formulas": false, + "error_code": undefined, + "max_size_reached": false, + "metrics": Object { + "csv": Object { + "rows": 0, + }, + }, + "warnings": Array [ + "Unable to close the Point-In-Time used for search. Check the Kibana server logs.", + ], + } + `); }); - it('can check for formulas, without escaping them', async () => { - mockConfig = { - checkForFormulas: true, - escapeFormulaValues: false, - maxSizeBytes: 180000, - useByteOrderMarkEncoding: false, - scroll: { size: 500, duration: '30s' }, - }; - mockDataClient.search = jest.fn().mockImplementation(() => - Rx.of({ - rawResponse: getMockRawResponse([ - { - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: [TEST_FORMULA], - }, - } as unknown as estypes.SearchHit, - ]), - }) - ); - + it('will return partial data if the scroll or search fails', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => { + throw new esErrors.ResponseError({ + statusCode: 500, + meta: {} as any, + body: 'my error', + warnings: [], + }); + }); const generateCsv = new CsvGenerator( createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, @@ -776,226 +912,39 @@ describe('formulas', () => { mockLogger, stream ); - - const csvResult = await generateCsv.generateData(); - - expect(content).toMatchSnapshot(); - expect(csvResult.csv_contains_formulas).toBe(true); - }); -}); - -it('can override ignoring frozen indices', async () => { - const originalGet = uiSettingsClient.get; - uiSettingsClient.get = jest.fn().mockImplementation((key): any => { - if (key === 'search:includeFrozen') { - return true; - } - return originalGet(key); - }); - - const generateCsv = new CsvGenerator( - createMockJob({}), - mockConfig, - { es: mockEsClient, data: mockDataClient, uiSettings: uiSettingsClient }, - { searchSourceStart: mockSearchSourceService, fieldFormatsRegistry: mockFieldFormatsRegistry }, - new CancellationToken(), - mockLogger, - stream - ); - - await generateCsv.generateData(); - - expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( - { - ignore_unavailable: true, - ignore_throttled: false, - index: 'logstash-*', - keep_alive: '30s', - }, - { maxRetries: 0, requestTimeout: '30s' } - ); - - expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith( - { - ignore_unavailable: true, - ignore_throttled: false, - index: 'logstash-*', - keep_alive: '30s', - }, - { maxRetries: 0, requestTimeout: '30s' } - ); - - expect(mockDataClient.search).toBeCalledWith( - { - params: { - body: {}, - }, - }, - { strategy: 'es', transport: { maxRetries: 0, requestTimeout: '30s' } } - ); -}); - -it('adds a warning if export was unable to close the PIT', async () => { - mockEsClient.asCurrentUser.closePointInTime = jest.fn().mockRejectedValueOnce( - new esErrors.ResponseError({ - statusCode: 419, - warnings: [], - meta: { context: 'test' } as any, - }) - ); - - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - - await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` - Object { - "content_type": "text/csv", - "csv_contains_formulas": false, - "error_code": undefined, - "max_size_reached": false, - "metrics": Object { - "csv": Object { - "rows": 0, - }, - }, - "warnings": Array [ - "Unable to close the Point-In-Time used for search. Check the Kibana server logs.", - ], - } - `); -}); - -it('will return partial data if the scroll or search fails', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => { - throw new esErrors.ResponseError({ - statusCode: 500, - meta: {} as any, - body: 'my error', - warnings: [], - }); - }); - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` - Object { - "content_type": "text/csv", - "csv_contains_formulas": false, - "error_code": undefined, - "max_size_reached": false, - "metrics": Object { - "csv": Object { - "rows": 0, + await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` + Object { + "content_type": "text/csv", + "csv_contains_formulas": false, + "error_code": undefined, + "max_size_reached": false, + "metrics": Object { + "csv": Object { + "rows": 0, + }, }, - }, - "warnings": Array [ - "Received a 500 response from Elasticsearch: my error", - "Encountered an error with the number of CSV rows generated from the search: expected NaN, received 0.", - ], - } - `); - expect(mockLogger.error.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - "CSV export search error: ResponseError: my error", - ], + "warnings": Array [ + "Received a 500 response from Elasticsearch: my error", + "Encountered an error with the number of CSV rows generated from the search: expected rows were indeterminable, received 0.", + ], + } + `); + expect(mockLogger.error.mock.calls).toMatchInlineSnapshot(` Array [ - [ResponseError: my error], - ], - ] - `); -}); - -it('handles unknown errors', async () => { - mockDataClient.search = jest.fn().mockImplementation(() => { - throw new Error('An unknown error'); + Array [ + "CSV export search error: ResponseError: my error", + ], + Array [ + [ResponseError: my error], + ], + ] + `); }); - const generateCsv = new CsvGenerator( - createMockJob({ columns: ['date', 'ip', 'message'] }), - mockConfig, - { - es: mockEsClient, - data: mockDataClient, - uiSettings: uiSettingsClient, - }, - { - searchSourceStart: mockSearchSourceService, - fieldFormatsRegistry: mockFieldFormatsRegistry, - }, - new CancellationToken(), - mockLogger, - stream - ); - await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` - Object { - "content_type": "text/csv", - "csv_contains_formulas": false, - "error_code": undefined, - "max_size_reached": false, - "metrics": Object { - "csv": Object { - "rows": 0, - }, - }, - "warnings": Array [ - "Encountered an unknown error: An unknown error", - "Encountered an error with the number of CSV rows generated from the search: expected NaN, received 0.", - ], - } - `); -}); - -describe('error codes', () => { - it('returns the expected error code when authentication expires', async () => { - mockDataClient.search = jest - .fn() - .mockImplementationOnce(() => - Rx.of({ - rawResponse: getMockRawResponse( - range(0, 5).map(() => ({ - _index: 'lasdf', - _id: 'lasdf123', - fields: { - date: ['2020-12-31T00:14:28.000Z'], - ip: ['110.135.176.89'], - message: ['super cali fragile istic XPLA docious'], - }, - })), - 10 - ), - }) - ) - .mockImplementationOnce(() => { - throw new esErrors.ResponseError({ statusCode: 403, meta: {} as any, warnings: [] }); - }); + it('handles unknown errors', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => { + throw new Error('An unknown error'); + }); const generateCsv = new CsvGenerator( createMockJob({ columns: ['date', 'ip', 'message'] }), mockConfig, @@ -1012,25 +961,85 @@ describe('error codes', () => { mockLogger, stream ); - - const { error_code: errorCode, warnings } = await generateCsv.generateData(); - expect(errorCode).toBe('authentication_expired_error'); - expect(warnings).toMatchInlineSnapshot(` - Array [ - "This report contains partial CSV results because the authentication token expired. Export a smaller amount of data or increase the timeout of the authentication token.", - "Encountered an error with the number of CSV rows generated from the search: expected 10, received 5.", - ] + await expect(generateCsv.generateData()).resolves.toMatchInlineSnapshot(` + Object { + "content_type": "text/csv", + "csv_contains_formulas": false, + "error_code": undefined, + "max_size_reached": false, + "metrics": Object { + "csv": Object { + "rows": 0, + }, + }, + "warnings": Array [ + "Encountered an unknown error: An unknown error", + "Encountered an error with the number of CSV rows generated from the search: expected rows were indeterminable, received 0.", + ], + } `); + }); - expect(mockLogger.error.mock.calls).toMatchInlineSnapshot(` - Array [ + describe('error codes', () => { + it('returns the expected error code when authentication expires', async () => { + mockDataClient.search = jest + .fn() + .mockImplementationOnce(() => + Rx.of({ + rawResponse: getMockRawResponse( + range(0, 5).map(() => ({ + _index: 'lasdf', + _id: 'lasdf123', + fields: { + date: ['2020-12-31T00:14:28.000Z'], + ip: ['110.135.176.89'], + message: ['super cali fragile istic XPLA docious'], + }, + })), + 10 + ), + }) + ) + .mockImplementationOnce(() => { + throw new esErrors.ResponseError({ statusCode: 403, meta: {} as any, warnings: [] }); + }); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', 'message'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + const { error_code: errorCode, warnings } = await generateCsv.generateData(); + expect(errorCode).toBe('authentication_expired_error'); + expect(warnings).toMatchInlineSnapshot(` Array [ - "CSV export search error: ResponseError: Response Error", - ], + "This report contains partial CSV results because the authentication token expired. Export a smaller amount of data or increase the timeout of the authentication token.", + "Encountered an error with the number of CSV rows generated from the search: expected 10, received 5.", + ] + `); + + expect(mockLogger.error.mock.calls).toMatchInlineSnapshot(` Array [ - [ResponseError: Response Error], - ], - ] - `); + Array [ + "CSV export search error: ResponseError: Response Error", + ], + Array [ + [ResponseError: Response Error], + ], + ] + `); + }); }); }); diff --git a/packages/kbn-generate-csv/src/generate_csv.ts b/packages/kbn-generate-csv/src/generate_csv.ts index 3e4d324dcfced..9d3b85b214c94 100644 --- a/packages/kbn-generate-csv/src/generate_csv.ts +++ b/packages/kbn-generate-csv/src/generate_csv.ts @@ -475,9 +475,13 @@ export class CsvGenerator { `ES scroll returned fewer total hits than expected! ` + `Search result total hits: ${totalRecords}. Row count: ${this.csvRowCount}` ); - warnings.push( - i18nTexts.csvRowCountError({ expected: totalRecords ?? NaN, received: this.csvRowCount }) - ); + if (totalRecords || totalRecords === 0) { + warnings.push( + i18nTexts.csvRowCountError({ expected: totalRecords, received: this.csvRowCount }) + ); + } else { + warnings.push(i18nTexts.csvRowCountIndeterminable({ received: this.csvRowCount })); + } } return { diff --git a/packages/kbn-generate-csv/src/i18n_texts.ts b/packages/kbn-generate-csv/src/i18n_texts.ts index 8b492aae7fae1..b9a863a18d2df 100644 --- a/packages/kbn-generate-csv/src/i18n_texts.ts +++ b/packages/kbn-generate-csv/src/i18n_texts.ts @@ -37,6 +37,12 @@ export const i18nTexts = { 'Encountered an error with the number of CSV rows generated from the search: expected {expected}, received {received}.', values: { expected, received }, }), + csvRowCountIndeterminable: ({ received }: { expected?: number; received: number }) => + i18n.translate('generateCsv.indeterminableRowCount', { + defaultMessage: + 'Encountered an error with the number of CSV rows generated from the search: expected rows were indeterminable, received {received}.', + values: { received }, + }), csvUnableToClosePit: () => i18n.translate('generateCsv.csvUnableToClosePit', { defaultMessage: