From 031846ae67184175cc57da4192735c913f764b4c Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 1 Feb 2019 17:01:45 -0700 Subject: [PATCH 1/4] Fix date formatting on server for CSV export --- .../common/field_formats/types/date_server.js | 84 +++++++++++++++++++ .../kibana/server/field_formats/register.js | 4 +- .../export_types/csv/server/execute_job.js | 36 ++++---- .../csv/server/lib/format_csv_values.js | 31 ++++--- 4 files changed, 121 insertions(+), 34 deletions(-) create mode 100644 src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js diff --git a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js new file mode 100644 index 0000000000000..87c21cae57447 --- /dev/null +++ b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { memoize } from 'lodash'; +import moment from 'moment-timezone'; + +export function createDateOnServerFormat(FieldFormat) { + return class DateFormat extends FieldFormat { + constructor(params, getConfig) { + super(params); + + this.getConfig = getConfig; + this._memoizedConverter = memoize(val => { + if (val == null) { + return '-'; + } + + /* On the server, importing moment returns a new instance. Unlike on + * the client side, it doesn't have the dateFormat:tz configuration + * baked in. + * We need to set the timezone manually here. The date is taken in as + * UTC and converted into the desired timezone. */ + let date; + if (this._timeZone === 'Browser') { + /* FIXME this assumes to use the server timezone since we don't know + * what the timezone of the browser. The CSV generation URL does not + * yet have a param for browserTimezone. */ + date = moment.utc(val); + } else { + date = moment.utc(val).tz(this._timeZone); + } + + if (date.isValid()) { + console.log({ date: date.format(this._memoizedPattern) }); + return date.format(this._memoizedPattern); + } else { + return val; + } + }); + } + + getParamDefaults() { + return { + pattern: this.getConfig('dateFormat'), + timezone: this.getConfig('dateFormat:tz'), + }; + } + + _convert(val) { + // don't give away our ref to converter so we can hot-swap when config changes + const pattern = this.param('pattern'); + const timezone = this.param('timezone'); + + const timezoneChanged = this._timeZone !== timezone; + const datePatternChanged = this._memoizedPattern !== pattern; + if (timezoneChanged || datePatternChanged) { + this._timeZone = timezone; + this._memoizedPattern = pattern; + } + + return this._memoizedConverter(val); + } + + static id = 'date'; + static title = 'Date'; + static fieldType = 'date'; + }; +} diff --git a/src/legacy/core_plugins/kibana/server/field_formats/register.js b/src/legacy/core_plugins/kibana/server/field_formats/register.js index f1cf735028f52..acefdbd9bf6d5 100644 --- a/src/legacy/core_plugins/kibana/server/field_formats/register.js +++ b/src/legacy/core_plugins/kibana/server/field_formats/register.js @@ -19,7 +19,7 @@ import { createUrlFormat } from '../../common/field_formats/types/url'; import { createBytesFormat } from '../../common/field_formats/types/bytes'; -import { createDateFormat } from '../../common/field_formats/types/date'; +import { createDateOnServerFormat } from '../../common/field_formats/types/date_server'; import { createDurationFormat } from '../../common/field_formats/types/duration'; import { createIpFormat } from '../../common/field_formats/types/ip'; import { createNumberFormat } from '../../common/field_formats/types/number'; @@ -34,7 +34,7 @@ import { createStaticLookupFormat } from '../../common/field_formats/types/stati export function registerFieldFormats(server) { server.registerFieldFormat(createUrlFormat); server.registerFieldFormat(createBytesFormat); - server.registerFieldFormat(createDateFormat); + server.registerFieldFormat(createDateOnServerFormat); server.registerFieldFormat(createDurationFormat); server.registerFieldFormat(createIpFormat); server.registerFieldFormat(createNumberFormat); diff --git a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js index 4ef1658d88f8e..abe3b380210ca 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js @@ -27,17 +27,23 @@ function executeJobFn(server) { metaFields, conflictedTypesFields, headers: serializedEncryptedHeaders, - basePath + basePath, } = job; let decryptedHeaders; try { decryptedHeaders = await crypto.decrypt(serializedEncryptedHeaders); } catch (e) { - throw new Error(i18n.translate('xpack.reporting.exportTypes.csv.executeJob.failedToDecryptReportJobDataErrorMessage', { - defaultMessage: 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report.', - values: { encryptionKey: 'xpack.reporting.encryptionKey' } - })); + throw new Error( + i18n.translate( + 'xpack.reporting.exportTypes.csv.executeJob.failedToDecryptReportJobDataErrorMessage', + { + defaultMessage: + 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report.', + values: { encryptionKey: 'xpack.reporting.encryptionKey' }, + } + ) + ); } const fakeRequest = { @@ -54,16 +60,19 @@ function executeJobFn(server) { const savedObjects = server.savedObjects; const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(fakeRequest); const uiSettings = server.uiSettingsServiceFactory({ - savedObjectsClient + savedObjectsClient, }); const fieldFormats = await server.fieldFormatServiceFactory(uiSettings); const formatsMap = fieldFormatMapFactory(indexPatternSavedObject, fieldFormats); - const separator = await uiSettings.get('csv:separator'); - const quoteValues = await uiSettings.get('csv:quoteValues'); - const maxSizeBytes = config.get('xpack.reporting.csv.maxSizeBytes'); - const scroll = config.get('xpack.reporting.csv.scroll'); + const settings = { + separator: await uiSettings.get('csv:separator'), + quoteValues: await uiSettings.get('csv:quoteValues'), + maxSizeBytes: config.get('xpack.reporting.csv.maxSizeBytes'), + scroll: config.get('xpack.reporting.csv.scroll'), + timezone: await uiSettings.get('dateFormat:tz'), + }; const { content, maxSizeReached, size } = await generateCsv({ searchRequest, @@ -73,12 +82,7 @@ function executeJobFn(server) { conflictedTypesFields, callEndpoint, cancellationToken, - settings: { - separator, - quoteValues, - maxSizeBytes, - scroll - } + settings, }); return { diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js b/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js index 4092480d25500..b54659da617d0 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js @@ -8,24 +8,23 @@ import { isObject, isNull, isUndefined } from 'lodash'; export function createFormatCsvValues(escapeValue, separator, fields, formatsMap) { return function formatCsvValues(values) { - return fields.map((field) => { - let value = values[field]; + return fields + .map(field => { + const value = values[field]; + if (isNull(value) || isUndefined(value)) { + return ''; + } - if (isNull(value) || isUndefined(value)) { - return ''; - } + let formattedValue = value; + if (formatsMap.has(field)) { + const formatter = formatsMap.get(field); + formattedValue = formatter.convert(value); + } - if (formatsMap.has(field)) { - const formatter = formatsMap.get(field); - value = formatter.convert(value); - } - - if (isObject(value)) { - return JSON.stringify(value); - } - - return value.toString(); - }) + return formattedValue; + }) + .map(value => (isObject(value) ? JSON.stringify(value) : value)) + .map(value => value.toString()) .map(escapeValue) .join(separator); }; From f9023f6c03a534bbe9a53cc82a58e0ae889a4f40 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 5 Feb 2019 14:09:20 -0700 Subject: [PATCH 2/4] remove stray console.log --- .../kibana/common/field_formats/types/date_server.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js index 87c21cae57447..dbfc01460abf3 100644 --- a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js +++ b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js @@ -47,7 +47,6 @@ export function createDateOnServerFormat(FieldFormat) { } if (date.isValid()) { - console.log({ date: date.format(this._memoizedPattern) }); return date.format(this._memoizedPattern); } else { return val; From 36850d088772ec0a8e8fd59f49283bddbea12709 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 5 Feb 2019 14:27:46 -0700 Subject: [PATCH 3/4] allow async to act in parallel --- .../export_types/csv/server/execute_job.js | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js index abe3b380210ca..39b633ee20b59 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js @@ -59,30 +59,43 @@ function executeJobFn(server) { }; const savedObjects = server.savedObjects; const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(fakeRequest); - const uiSettings = server.uiSettingsServiceFactory({ + const uiConfig = server.uiSettingsServiceFactory({ savedObjectsClient, }); - const fieldFormats = await server.fieldFormatServiceFactory(uiSettings); - const formatsMap = fieldFormatMapFactory(indexPatternSavedObject, fieldFormats); + const [formatsMap, uiSettings] = await Promise.all([ + (async () => { + const fieldFormats = await server.fieldFormatServiceFactory(uiConfig); + return fieldFormatMapFactory(indexPatternSavedObject, fieldFormats); + })(), + (async () => { + const [separator, quoteValues, timezone] = await Promise.all([ + uiConfig.get('csv:separator'), + uiConfig.get('csv:quoteValues'), + uiConfig.get('dateFormat:tz'), + ]); - const settings = { - separator: await uiSettings.get('csv:separator'), - quoteValues: await uiSettings.get('csv:quoteValues'), - maxSizeBytes: config.get('xpack.reporting.csv.maxSizeBytes'), - scroll: config.get('xpack.reporting.csv.scroll'), - timezone: await uiSettings.get('dateFormat:tz'), - }; + return { + separator, + quoteValues, + timezone, + }; + })(), + ]); const { content, maxSizeReached, size } = await generateCsv({ searchRequest, fields, - formatsMap, metaFields, conflictedTypesFields, callEndpoint, cancellationToken, - settings, + formatsMap, + settings: { + ...uiSettings, + maxSizeBytes: config.get('xpack.reporting.csv.maxSizeBytes'), + scroll: config.get('xpack.reporting.csv.scroll'), + }, }); return { From 45a3f6138b284d4fbbbf8206e985d932c9f9e544 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 5 Feb 2019 18:06:05 -0700 Subject: [PATCH 4/4] Log a warning when "Browser" is the timezone --- .../kibana/common/field_formats/types/date_server.js | 5 ++--- .../reporting/export_types/csv/server/execute_job.js | 9 ++++++++- .../export_types/csv/server/lib/generate_csv.js | 4 ++-- .../export_types/csv/server/lib/hit_iterator.js | 6 +++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js index dbfc01460abf3..c39d13a07111c 100644 --- a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js +++ b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js @@ -38,9 +38,8 @@ export function createDateOnServerFormat(FieldFormat) { * UTC and converted into the desired timezone. */ let date; if (this._timeZone === 'Browser') { - /* FIXME this assumes to use the server timezone since we don't know - * what the timezone of the browser. The CSV generation URL does not - * yet have a param for browserTimezone. */ + // Assume a warning has been logged this can be unpredictable. It + // would be too verbose to log anything here. date = moment.utc(val); } else { date = moment.utc(val).tz(this._timeZone); diff --git a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js index 39b633ee20b59..d0351311f3eb8 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js @@ -15,7 +15,10 @@ function executeJobFn(server) { const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); const crypto = cryptoFactory(server); const config = server.config(); - const logger = createTaggedLogger(server, ['reporting', 'csv', 'debug']); + const logger = { + debug: createTaggedLogger(server, ['reporting', 'csv', 'debug']), + warn: createTaggedLogger(server, ['reporting', 'csv', 'warning']), + }; const generateCsv = createGenerateCsv(logger); const serverBasePath = config.get('server.basePath'); @@ -75,6 +78,10 @@ function executeJobFn(server) { uiConfig.get('dateFormat:tz'), ]); + if (timezone === 'Browser') { + logger.warn(`Kibana Advanced Setting "dateFormat:tz" is set to "Browser". Dates will be formatted as UTC to avoid ambiguity.`); + } + return { separator, quoteValues, diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js index 2d0133238bf82..56a5ce3d80bbb 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js @@ -49,7 +49,7 @@ export function createGenerateCsv(logger) { } if (!builder.tryAppend(formatCsvValues(flattenHit(hit)) + '\n')) { - logger('max Size Reached'); + logger.warn('max Size Reached'); maxSizeReached = true; cancellationToken.cancel(); break; @@ -59,7 +59,7 @@ export function createGenerateCsv(logger) { await iterator.return(); } const size = builder.getSizeInBytes(); - logger(`finished generating, total size in bytes: ${size}`); + logger.debug(`finished generating, total size in bytes: ${size}`); return { content: builder.getString(), diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js b/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js index 8c71b7244489e..5ad6182568721 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js @@ -30,7 +30,7 @@ async function parseResponse(request) { export function createHitIterator(logger) { return async function* hitIterator(scrollSettings, callEndpoint, searchRequest, cancellationToken) { - logger('executing search request'); + logger.debug('executing search request'); function search(index, body) { return parseResponse(callEndpoint('search', { index, @@ -41,7 +41,7 @@ export function createHitIterator(logger) { } function scroll(scrollId) { - logger('executing scroll request'); + logger.debug('executing scroll request'); return parseResponse(callEndpoint('scroll', { scrollId, scroll: scrollSettings.duration @@ -49,7 +49,7 @@ export function createHitIterator(logger) { } function clearScroll(scrollId) { - logger('executing clearScroll request'); + logger.debug('executing clearScroll request'); return callEndpoint('clearScroll', { scrollId: [ scrollId ] });