From 3dff74a7dedeb5005e446efdbfe87f6d8ca31cfa Mon Sep 17 00:00:00 2001 From: Miki Date: Wed, 6 Sep 2023 16:38:45 -0700 Subject: [PATCH] [Console] Add support for JSON with long numerals (#4562) Also: * Add support for parsing and stringifying JSON with long numerals into `@osd/std` * Upgrade to `@opensearch/opensearch@2.3.1` which supports long numerals * Add support for long numerals to `http/fetch` Signed-off-by: Miki Signed-off-by: Kawika Avilla --- CHANGELOG.md | 1 + package.json | 2 +- packages/osd-opensearch-archiver/package.json | 3 +- packages/osd-opensearch/package.json | 2 +- packages/osd-std/README.md | 72 +++- .../src/__snapshots__/json.test.ts.snap | 19 ++ packages/osd-std/src/index.ts | 1 + packages/osd-std/src/json.test.ts | 176 ++++++++++ packages/osd-std/src/json.ts | 321 ++++++++++++++++++ src/core/public/http/fetch.test.ts | 35 ++ src/core/public/http/fetch.ts | 6 +- src/core/public/http/types.ts | 6 + .../opensearch/client/cluster_client.test.ts | 182 +++++++--- .../opensearch/client/cluster_client.ts | 19 +- src/core/server/opensearch/client/mocks.ts | 10 + .../client/scoped_cluster_client.test.ts | 14 +- .../client/scoped_cluster_client.ts | 15 +- .../send_request_to_opensearch.test.ts | 21 ++ .../send_request_to_opensearch.ts | 5 +- .../public/lib/opensearch/opensearch.ts | 1 + .../__snapshots__/utils.test.ts.snap | 118 +++++++ .../public/lib/utils/__tests__/utils.test.ts | 68 ++++ src/plugins/console/public/lib/utils/index.ts | 5 +- .../console/public/services/storage.ts | 5 +- .../api/console/proxy/create_handler.ts | 22 +- .../api/console/proxy/tests/body.test.ts | 4 +- .../api/console/proxy/tests/headers.test.ts | 4 +- .../api/console/proxy/tests/params.test.ts | 8 +- .../console/proxy/tests/query_string.test.ts | 16 +- .../json_xjson_translation_tools/index.ts | 5 +- yarn.lock | 8 +- 31 files changed, 1098 insertions(+), 76 deletions(-) create mode 100644 packages/osd-std/src/__snapshots__/json.test.ts.snap create mode 100644 packages/osd-std/src/json.test.ts create mode 100644 packages/osd-std/src/json.ts create mode 100644 src/plugins/console/public/lib/utils/__tests__/__snapshots__/utils.test.ts.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index d9f6017becb4..edfdd5f2aa8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Enable plugins to augment visualizations with additional data and context ([#4361](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4361)) - Dashboard De-Angularization ([#4502](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4502)) - New management overview page and rename stack management to dashboard management ([#4287](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4287)) +- [Console] Add support for JSON with long numerals ([#4562](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4562)) - [Vis Augmenter] Update base vis height in view events flyout ([#4535](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4535)) ### ๐Ÿ› Bug Fixes diff --git a/package.json b/package.json index 2944fd9d768f..cdacb6227531 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "@hapi/vision": "^6.1.0", "@hapi/wreck": "^17.1.0", "@opensearch-project/opensearch": "^1.1.0", - "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.2.0", + "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.3.1", "@osd/ace": "1.0.0", "@osd/analytics": "1.0.0", "@osd/apm-config-loader": "1.0.0", diff --git a/packages/osd-opensearch-archiver/package.json b/packages/osd-opensearch-archiver/package.json index 740c94bfd786..9f93b0a0e431 100644 --- a/packages/osd-opensearch-archiver/package.json +++ b/packages/osd-opensearch-archiver/package.json @@ -12,7 +12,8 @@ }, "dependencies": { "@osd/dev-utils": "1.0.0", - "@opensearch-project/opensearch": "^1.1.0" + "@osd/std": "1.0.0", + "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.3.1" }, "devDependencies": {} } diff --git a/packages/osd-opensearch/package.json b/packages/osd-opensearch/package.json index 89e24f46dc43..c7263e1979a8 100644 --- a/packages/osd-opensearch/package.json +++ b/packages/osd-opensearch/package.json @@ -12,7 +12,7 @@ "osd:watch": "../../scripts/use_node scripts/build --watch" }, "dependencies": { - "@opensearch-project/opensearch": "^1.1.0", + "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.3.1", "@osd/dev-utils": "1.0.0", "abort-controller": "^3.0.0", "chalk": "^4.1.0", diff --git a/packages/osd-std/README.md b/packages/osd-std/README.md index 3730735cb5a4..24f888979d25 100644 --- a/packages/osd-std/README.md +++ b/packages/osd-std/README.md @@ -1,3 +1,73 @@ # `@osd/std` โ€” OpenSearch Dashboards standard library -This package is a set of utilities that can be used both on server-side and client-side. \ No newline at end of file +This package is a set of utilities that can be used both on server-side and client-side. + +## API + +#### `assertNever` + +Can be used in switch statements to ensure we perform exhaustive checks. + +#### `deepFreeze` + +Apply `Object.freeze` to a value recursively and convert the return type to `Readonly` variant recursively. + +#### `get` + +Retrieve the value for the specified path of an object. + +#### `getFlattenedObject` + +Flatten a deeply nested object to a map of dot-separated paths, pointing to all of its primitive values and arrays. + +#### `stringify` and `parse` + +Drop-in replacement for `JSON.stringify` and `JSON.parse`, capable of handling long numerals and `BigInt` values. + +#### `mapToObject` + +Convert a map to an object. + +#### `mapValuesOfMap` + +Create a new `Map` populated with the results of calling a provided function on every element in the input `Map`. + +#### `groupIntoMap` + +Group elements of an `Array` into a `Map` based on a provided function. + +#### `merge` + +Deeply merge two objects, omitting undefined values, and not deeply merging arrays. + +#### `pick` + +Create a new `Object` of specified keys and their values from an input `Object`. + +#### `withTimeout` + +Apply a `timeout` duration to a `Promise` before throwing an `Error` with the provided message. + +#### `firstValueFrom` and `lastValueFrom` + +Get a `Promise` that resolves as soon as the first or last value arrives from an observable. + +#### `unset` + +Unset a (potentially nested) key from given object. + +#### `modifyUrl` + +Get an `Object` resulting from applying a provided function to the meaningful parts of a URL. + +#### `isRelativeUrl` + +Determine if a url is relative. + +#### `getUrlOrigin` + +Get the origin URL of a provided URL. + +#### `validateObject` + +Deeply validate that an `Object` does not contain any `__proto__` or `constructor.prototype` keys, or circular references. \ No newline at end of file diff --git a/packages/osd-std/src/__snapshots__/json.test.ts.snap b/packages/osd-std/src/__snapshots__/json.test.ts.snap new file mode 100644 index 000000000000..42854db1dd78 --- /dev/null +++ b/packages/osd-std/src/__snapshots__/json.test.ts.snap @@ -0,0 +1,19 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`json can apply a replacer and spaces values while stringifying BigInts 1`] = ` +"{ + \\"\\\\\\": 18014398509481982\\": \\"\\", + \\"positive\\": 54043195528445946, + \\"negative\\": -54043195528445946, + \\"array\\": [ + -54043195528445946, + 54043195528445946, + [ + \\"]]>\\" + ] + ], + \\"number\\": \\"5d9d89cc6b13\\" +}" +`; + +exports[`json can handle BigInt values while stringifying 1`] = `"{\\"\\\\\\": 18014398509481982\\":\\"[ -18014398509481982, 18014398509481982 ]\\",\\"positive\\":18014398509481982,\\"negative\\":-18014398509481982,\\"array\\":[-18014398509481982,18014398509481982],\\"number\\":102931203123987}"`; diff --git a/packages/osd-std/src/index.ts b/packages/osd-std/src/index.ts index 0b3c65d8cc04..170c819a2b0a 100644 --- a/packages/osd-std/src/index.ts +++ b/packages/osd-std/src/index.ts @@ -42,3 +42,4 @@ export { unset } from './unset'; export { getFlattenedObject } from './get_flattened_object'; export { validateObject } from './validate_object'; export * from './rxjs_7'; +export { parse, stringify } from './json'; diff --git a/packages/osd-std/src/json.test.ts b/packages/osd-std/src/json.test.ts new file mode 100644 index 000000000000..33abd71d91d2 --- /dev/null +++ b/packages/osd-std/src/json.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { stringify, parse } from './json'; + +describe('json', () => { + it('can parse', () => { + const input = { + a: [ + { A: 1 }, + { B: '2' }, + { C: [1, 2, 3, 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'] }, + ], + b: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + c: { + i: {}, + ii: [], + iii: '', + iv: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + }, + }; + const result = parse(JSON.stringify(input)); + expect(result).toEqual(input); + }); + + it('can stringify', () => { + const input = { + a: [ + { A: 1 }, + { B: '2' }, + { C: [1, 2, 3, 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'] }, + ], + b: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + c: { + i: {}, + ii: [], + iii: '', + iv: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + }, + }; + const result = stringify(input); + expect(result).toEqual(JSON.stringify(input)); + }); + + it('can apply a reviver while parsing', () => { + const input = { + A: 255, + B: { + i: [[]], + ii: 'Lorem ipsum', + iii: {}, + rand: Math.random(), + }, + }; + const text = JSON.stringify(input); + function reviver(this: any, key: string, val: any) { + if (Array.isArray(val) && toString.call(this) === '[object Object]') this._hasArrays = true; + else if (typeof val === 'string') val = ``; + else if (typeof val === 'number') val = val.toString(16); + else if (toString.call(this) === '[object Object]' && key === 'rand' && val === input.B.rand) + this._found = true; + return val; + } + + expect(parse(text, reviver)).toEqual(JSON.parse(text, reviver)); + }); + + it('can apply a replacer and spaces while stringifying', () => { + const input = { + A: 255, + B: { + i: [[]], + ii: 'Lorem ipsum', + iii: {}, + rand: Math.random(), + }, + }; + + function replacer(this: any, key: string, val: any) { + if (Array.isArray(val) && val.length === 0) val.push(''); + else if (typeof val === 'string') val = ``; + else if (typeof val === 'number') val = val.toString(16); + else if (toString.call(this) === '[object Object]' && key === 'rand' && val === input.B.rand) + val = 1; + return val; + } + + expect(stringify(input, replacer, 2)).toEqual(JSON.stringify(input, replacer, 2)); + }); + + it('can handle long numerals while parsing', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const text = + `{` + + // The space before and after the values, and the lack of spaces before comma are intentional + `"\\":${longPositive}": "[ ${longNegative.toString()}, ${longPositive.toString()} ]", ` + + `"positive": ${longPositive.toString()}, ` + + `"array": [ ${longNegative.toString()}, ${longPositive.toString()} ], ` + + `"negative": ${longNegative.toString()},` + + `"number": 102931203123987` + + `}`; + + const result = parse(text); + expect(result.positive).toBe(longPositive); + expect(result.negative).toBe(longNegative); + expect(result.array).toEqual([longNegative, longPositive]); + expect(result['":' + longPositive]).toBe( + `[ ${longNegative.toString()}, ${longPositive.toString()} ]` + ); + expect(result.number).toBe(102931203123987); + }); + + it('can handle BigInt values while stringifying', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const input = { + [`": ${longPositive}`]: `[ ${longNegative.toString()}, ${longPositive.toString()} ]`, + positive: longPositive, + negative: longNegative, + array: [longNegative, longPositive], + number: 102931203123987, + }; + + expect(stringify(input)).toMatchSnapshot(); + }); + + it('can apply a reviver on long numerals while parsing', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const text = + `{` + + // The space before and after the values, and the lack of spaces before comma are intentional + `"\\":${longPositive}": "[ ${longNegative.toString()}, ${longPositive.toString()} ]", ` + + `"positive": ${longPositive.toString()}, ` + + `"array": [ ${longNegative.toString()}, ${longPositive.toString()} ], ` + + `"negative": ${longNegative.toString()},` + + `"number": 102931203123987` + + `}`; + + const reviver = (key: string, val: any) => (typeof val === 'bigint' ? val * 3n : val); + + const result = parse(text, reviver); + expect(result.positive).toBe(longPositive * 3n); + expect(result.negative).toBe(longNegative * 3n); + expect(result.array).toEqual([longNegative * 3n, longPositive * 3n]); + expect(result['":' + longPositive]).toBe( + `[ ${longNegative.toString()}, ${longPositive.toString()} ]` + ); + expect(result.number).toBe(102931203123987); + }); + + it('can apply a replacer and spaces values while stringifying BigInts', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const input = { + [`": ${longPositive}`]: `[ ${longNegative.toString()}, ${longPositive.toString()} ]`, + positive: longPositive, + negative: longNegative, + array: [longNegative, longPositive, []], + number: 102931203123987, + }; + + function replacer(this: any, key: string, val: any) { + if (typeof val === 'bigint') val = val * 3n; + else if (Array.isArray(val) && val.length === 0) val.push(''); + else if (typeof val === 'string') val = ``; + else if (typeof val === 'number') val = val.toString(16); + return val; + } + + expect(stringify(input, replacer, 4)).toMatchSnapshot(); + }); +}); diff --git a/packages/osd-std/src/json.ts b/packages/osd-std/src/json.ts new file mode 100644 index 000000000000..7c619dcd1656 --- /dev/null +++ b/packages/osd-std/src/json.ts @@ -0,0 +1,321 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* In JavaScript, a `Number` is a 64-bit floating-point value which can store 16 digits. However, the + * serializer and deserializer will need to cater to numeric values generated by other languages which + * can have up to 19 digits. Native JSON parser and stringifier, incapable of handling the extra + * digits, corrupt the values, making them unusable. + * + * To work around this limitation, the deserializer converts long sequences of digits into strings and + * marks them before applying the parser. During the parsing, string values that begin with the mark + * are converted to `BigInt` values. + * Similarly, during stringification, the serializer converts `BigInt` values to marked strings and + * when done, it replaces them with plain numerals. + * + * `Number.MAX_SAFE_INTEGER`, 9,007,199,254,740,991, is the largest number that the native methods can + * parse and stringify, and any numeral greater than that would need to be translated using the + * workaround; all 17-digits or longer and only tail-end of the 16-digits need translation. It would + * be unfair to all the 16-digit numbers if the translation applied to `\d{16,}` only to cover the + * less than 10%. Hence, a RegExp is created to only match numerals too long to be a number. + * + * To make the explanation simpler, let's assume that MAX_SAFE_INTEGER is 8921 which has 4 digits. + * Starting from the right, we take each digit onwards, `[-9]`: + * 1) 7922 - 7929: 792[2-9]\d{0} + * 2) 7930 - 7999: 79[3-9]\d{1} + * 9) 9 + 1 = 10 which results in a rollover; no need to do anything. + * 8) 9000 - 9999: [9-9]\d{3} + * Finally we add anything 5 digits or longer: `\d{5,} + * + * Note: A better solution would use AST but considering its performance penalty, RegExp is the next + * best thing. + */ +const maxIntAsString = String(Number.MAX_SAFE_INTEGER); +const maxIntLength = maxIntAsString.length; +// Sub-patterns for each digit +const longNumeralMatcherTokens = [`\\d{${maxIntAsString.length + 1},}`]; +for (let i = 0; i < maxIntLength; i++) { + if (maxIntAsString[i] !== '9') { + longNumeralMatcherTokens.push( + maxIntAsString.substring(0, i) + + `[${parseInt(maxIntAsString[i], 10) + 1}-9]` + + `\\d{${maxIntLength - i - 1}}` + ); + } +} + +/* The matcher that looks for `": , ...}` and `[..., , ...]` + * + * The pattern starts by looking for `":` not immediately preceded by a `\`. That should be + * followed by any of the numeric sub-patterns. A comma, end of an array, end of an object, or + * the end of the input are the only acceptable elements after it. + * + * Note: This RegExp can result in false-positive hits on the likes of `{"key": "[ ]"}` and + * those are cleaned out during parsing. + */ +const longNumeralMatcher = new RegExp( + `((?:\\[|,|(? { + // coverage:ignore-line + if (!length || length < 0) return []; + const choices = []; + const arr = markerChars; + const arrLength = arr.length; + const temp = Array(length); + + (function fill(pos, start) { + if (pos === length) return choices.push(temp.join('')); + + for (let i = start; i < arrLength; i++) { + temp[pos] = arr[i]; + fill(pos + 1, i); + } + })(0, 0); + + return choices; +}; + +/* Experiments with different combinations of various lengths, until one is found to not be in + * the input string. + */ +const getMarker = (text: string): { marker: string; length: number } => { + let marker; + let length = 0; + do { + length++; + getMarkerChoices(length).some((markerChoice) => { + if (text.indexOf(markerChoice) === -1) { + marker = markerChoice; + return true; + } + }); + } while (!marker); + + return { + marker, + length, + }; +}; + +const parseStringWithLongNumerals = ( + text: string, + reviver?: ((this: any, key: string, value: any) => any) | null +): any => { + const { marker, length } = getMarker(text); + + let hadException; + let obj; + let markedJSON = text.replace(longNumeralMatcher, `$1"${marker}$2"$3`); + const markedValueMatcher = new RegExp(`^${marker}-?\\d+$`); + + /* Convert marked values to BigInt values. + * The `startsWith` is purely for performance, to avoid running `test` if not needed. + */ + const convertMarkedValues = (val: any) => + typeof val === 'string' && val.startsWith(marker) && markedValueMatcher.test(val) + ? BigInt(val.substring(length)) + : val; + + /* For better performance, instead of testing for existence of `reviver` on each value, two almost + * identical functions are used. + */ + const parseMarkedText = reviver + ? (markedText: string) => + JSON.parse(markedText, function (key, val) { + return reviver.call(this, key, convertMarkedValues(val)); + }) + : (markedText: string) => JSON.parse(markedText, (key, val) => convertMarkedValues(val)); + + /* RegExp cannot replace AST and the process of marking adds quotes. So, any false-positive hit + * will make the JSON string unparseable. + * + * To find those instances, we try to parse and watch for the location of any errors. If an error + * is caused by the marking, we remove that single marking and try again. + */ + do { + try { + hadException = false; + obj = parseMarkedText(markedJSON); + } catch (e) { + hadException = true; + /* There are two types of exception objects that can be raised: + * 1) a proper object with lineNumber and columnNumber which we can use + * 2) a textual message with the position that we need to parse + */ + let { lineNumber, columnNumber } = e; + if (!lineNumber || !columnNumber) { + const match = e?.message?.match?.(/^Unexpected token.*at position (\d+)$/); + if (match) { + lineNumber = 1; + // The position is zero-indexed; adding 1 to normalize it for the -2 that comes later + columnNumber = parseInt(match[1], 10) + 1; + } + } + + if (lineNumber < 1 || columnNumber < 2) { + /* The problem is not with this replacement. + * Note: This will never happen because the outer parse would have already thrown. + */ + // coverage:ignore-line + throw e; + } + + /* We need to skip e.lineNumber - 1 number of `\n` occurrences. + * Then, we need to go to e.columnNumber - 2 to look for `"\d+"`; we need to `-1` to + * account for the quote but an additional `-1` is needed because columnNumber starts from 1. + */ + const re = new RegExp( + `^((?:.*\\n){${lineNumber - 1}}[^\\n]{${columnNumber - 2}})"${marker}(-?\\d+)"` + ); + if (!re.test(markedJSON)) { + /* The exception is not caused by adding the marker. + * Note: This will never happen because the outer parse would have already thrown. + */ + // coverage:ignore-line + throw e; + } + + // We have found a bad replacement; let's remove it. + markedJSON = markedJSON.replace(re, '$1$2'); + } + } while (hadException); + + return obj; +}; + +const stringifyObjectWithBigInts = ( + obj: any, + candidate: string, + replacer?: ((this: any, key: string, value: any) => any) | null, + space?: string | number +): string => { + const { marker } = getMarker(candidate); + + /* The matcher that looks for "" + * Because we have made sure that `marker` was never present in the original object, we can + * carelessly assume every "" is due to our marking. + */ + const markedBigIntMatcher = new RegExp(`"${marker}(-?\\d+)"`, 'g'); + + /* Convert BigInt values to a string and mark them. + * Can't be bothered with Number values outside the safe range because they are already corrupted. + * + * For better performance, instead of testing for existence of `replacer` on each value, two almost + * identical functions are used. + */ + const addMarkerToBigInts = replacer + ? function (this: any, key: string, val: any) { + // replacer is called before marking because marking changes the type + const newVal = replacer.call(this, key, val); + return typeof newVal === 'bigint' ? `${marker}${newVal.toString()}` : newVal; + } + : (key: string, val: any) => (typeof val === 'bigint' ? `${marker}${val.toString()}` : val); + + return ( + JSON.stringify(obj, addMarkerToBigInts, space) + // Replace marked substrings with just the numerals + .replace(markedBigIntMatcher, '$1') + ); +}; + +export const stringify = ( + obj: any, + replacer?: ((this: any, key: string, value: any) => any) | null, + space?: string | number +): string => { + let text; + let numeralsAreNumbers = true; + /* For better performance, instead of testing for existence of `replacer` on each value, two almost + * identical functions are used. + * + * Note: Converting BigInt values to numbers, `Number()` is much faster that `parseInt()`. Since we + * check the `type`, it is safe to just use `Number()`. + */ + const checkForBigInts = replacer + ? function (this: any, key: string, val: any) { + if (typeof val === 'bigint') { + numeralsAreNumbers = false; + return replacer.call(this, key, Number(val)); + } + return replacer.call(this, key, val); + } + : (key: string, val: any) => { + if (typeof val === 'bigint') { + numeralsAreNumbers = false; + return Number(val); + } + return val; + }; + + /* While this is a check for possibly having BigInt values, if none were found, the results is + * sufficient to fulfill the purpose of the function. However, if BigInt values were found, we will + * use `stringifyObjectWithBigInts` to do this again. + * + * The goal was not to punish every object that doesn't have a BigInt with the more expensive + * `stringifyObjectWithBigInts`. Those with BigInt values are also not unduly burdened because we + * still need it in its string form to find a suitable marker. + */ + text = JSON.stringify(obj, checkForBigInts, space); + + if (!numeralsAreNumbers) { + text = stringifyObjectWithBigInts(obj, text, replacer, space); + } + + return text; +}; + +export const parse = ( + text: string, + reviver?: ((this: any, key: string, value: any) => any) | null +): any => { + let obj; + let numeralsAreNumbers = true; + const inspectValueForLargeNumerals = (val: any) => { + if ( + numeralsAreNumbers && + typeof val === 'number' && + (val < Number.MAX_SAFE_INTEGER || val > Number.MAX_SAFE_INTEGER) + ) { + numeralsAreNumbers = false; + } + + // This function didn't have to have a return value but having it makes the rest cleaner + return val; + }; + + /* For better performance, instead of testing for existence of `reviver` on each value, two almost + * identical functions are used. + */ + const checkForLargeNumerals = reviver + ? function (this: any, key: string, val: any) { + return inspectValueForLargeNumerals(reviver.call(this, key, val)); + } + : (key: string, val: any) => inspectValueForLargeNumerals(val); + + /* While this is a check for possibly having BigInt values, if none were found, the results is + * sufficient to fulfill the purpose of the function. However, if BigInt values were found, we will + * use `stringifyObjectWithBigInts` to do this again. + * + * The goal was not to punish every object that doesn't have a BigInt with the more expensive + * `stringifyObjectWithBigInts`. Those with BigInt values are also not unduly burdened because we + * still need it in its string form to find a suitable marker. + */ + obj = JSON.parse(text, checkForLargeNumerals); + + if (!numeralsAreNumbers) { + obj = parseStringWithLongNumerals(text, reviver); + } + + return obj; +}; diff --git a/src/core/public/http/fetch.test.ts b/src/core/public/http/fetch.test.ts index efc8d4aa31bb..20f070dbba80 100644 --- a/src/core/public/http/fetch.test.ts +++ b/src/core/public/http/fetch.test.ts @@ -825,4 +825,39 @@ describe('Fetch', () => { expect(usedSpy).toHaveBeenCalledTimes(2); }); }); + + describe('long numerals', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + + it('should use alternate parser on JSON responses when asked to', async () => { + fetchMock.get('*', { + body: `{"long-max": ${longPositive}, "long-min": ${longNegative}}`, + status: 200, + headers: { + 'Content-Type': 'application/json', + }, + }); + + await expect(fetchInstance.fetch('/my/path', { withLongNumerals: true })).resolves.toEqual({ + 'long-max': longPositive, + 'long-min': longNegative, + }); + }); + + it('should use alternate parser on non-JSON responses when asked to', async () => { + fetchMock.get('*', { + body: `{"long-max": ${longPositive}, "long-min": ${longNegative}}`, + status: 200, + headers: { + 'Content-Type': 'text', + }, + }); + + await expect(fetchInstance.fetch('/my/path', { withLongNumerals: true })).resolves.toEqual({ + 'long-max': longPositive, + 'long-min': longNegative, + }); + }); + }); }); diff --git a/src/core/public/http/fetch.ts b/src/core/public/http/fetch.ts index 694372c46d99..767d58643003 100644 --- a/src/core/public/http/fetch.ts +++ b/src/core/public/http/fetch.ts @@ -31,7 +31,7 @@ import { omitBy } from 'lodash'; import { format } from 'url'; import { BehaviorSubject } from 'rxjs'; -import { isRelativeUrl } from '@osd/std'; +import { isRelativeUrl, parse } from '@osd/std'; import { IBasePath, @@ -190,12 +190,12 @@ export class Fetch { if (NDJSON_CONTENT.test(contentType)) { body = await response.blob(); } else if (JSON_CONTENT.test(contentType)) { - body = await response.json(); + body = fetchOptions.withLongNumerals ? parse(await response.text()) : await response.json(); } else { const text = await response.text(); try { - body = JSON.parse(text); + body = fetchOptions.withLongNumerals ? parse(text) : JSON.parse(text); } catch (err) { body = text; } diff --git a/src/core/public/http/types.ts b/src/core/public/http/types.ts index ab046e6d2d5a..3b7dff71c811 100644 --- a/src/core/public/http/types.ts +++ b/src/core/public/http/types.ts @@ -257,6 +257,12 @@ export interface HttpFetchOptions extends HttpRequestInit { * response information. When `false`, the return type will just be the parsed response body. Defaults to `false`. */ asResponse?: boolean; + + /** + * When `true`, if the response has a JSON mime type, the {@link HttpResponse} will use an alternate JSON parser + * that converts long numerals to BigInts. Defaults to `false`. + */ + withLongNumerals?: boolean; } /** diff --git a/src/core/server/opensearch/client/cluster_client.test.ts b/src/core/server/opensearch/client/cluster_client.test.ts index 1510d2b148fe..81f55b987805 100644 --- a/src/core/server/opensearch/client/cluster_client.test.ts +++ b/src/core/server/opensearch/client/cluster_client.test.ts @@ -99,8 +99,16 @@ describe('ClusterClient', () => { const scopedClusterClient = clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ headers: expect.any(Object) }); + const expected = { headers: expect.any(Object) }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); expect(scopedClusterClient.asInternalUser).toBe(clusterClient.asInternalUser); expect(scopedClusterClient.asCurrentUser).toBe(scopedClient.child.mock.results[0].value); @@ -113,7 +121,7 @@ describe('ClusterClient', () => { const scopedClusterClient1 = clusterClient.asScoped(request); const scopedClusterClient2 = clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenCalledTimes(2 * 2); expect(scopedClusterClient1).not.toBe(scopedClusterClient2); expect(scopedClusterClient1.asInternalUser).toBe(scopedClusterClient2.asInternalUser); @@ -135,10 +143,18 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, foo: 'bar', 'x-opaque-id': expect.any(String) }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('creates a scoped facade with filtered auth headers', () => { @@ -155,10 +171,18 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, authorization: 'auth', 'x-opaque-id': expect.any(String) }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respects auth headers precedence', () => { @@ -179,10 +203,18 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, authorization: 'auth', 'x-opaque-id': expect.any(String) }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('includes the `customHeaders` from the config without filtering them', () => { @@ -200,15 +232,23 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, foo: 'bar', hello: 'dolly', 'x-opaque-id': expect.any(String), }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('adds the x-opaque-id header based on the request id', () => { @@ -225,13 +265,21 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, 'x-opaque-id': 'my-fake-id', }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respect the precedence of auth headers over config headers', () => { @@ -251,15 +299,23 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, foo: 'auth', hello: 'dolly', 'x-opaque-id': expect.any(String), }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respect the precedence of request headers over config headers', () => { @@ -279,15 +335,23 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, foo: 'request', hello: 'dolly', 'x-opaque-id': expect.any(String), }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respect the precedence of config headers over default headers', () => { @@ -304,13 +368,21 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { [headerKey]: 'foo', 'x-opaque-id': expect.any(String), }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respect the precedence of request headers over default headers', () => { @@ -327,13 +399,21 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { [headerKey]: 'foo', 'x-opaque-id': expect.any(String), }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('respect the precedence of x-opaque-id header over config headers', () => { @@ -355,13 +435,21 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, 'x-opaque-id': 'from request', }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('filter headers when called with a `FakeRequest`', () => { @@ -380,10 +468,18 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, authorization: 'auth' }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); it('does not add auth headers when called with a `FakeRequest`', () => { @@ -404,10 +500,18 @@ describe('ClusterClient', () => { clusterClient.asScoped(request); - expect(scopedClient.child).toHaveBeenCalledTimes(1); - expect(scopedClient.child).toHaveBeenCalledWith({ + const expected = { headers: { ...DEFAULT_HEADERS, foo: 'bar' }, - }); + }; + expect(scopedClient.child).toHaveBeenCalledTimes(2); + expect(scopedClient.child).toHaveBeenNthCalledWith(1, expect.objectContaining(expected)); + expect(scopedClient.child).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + ...expected, + enableLongNumeralSupport: true, + }) + ); }); }); diff --git a/src/core/server/opensearch/client/cluster_client.ts b/src/core/server/opensearch/client/cluster_client.ts index ac2348921658..8ea87bb910f7 100644 --- a/src/core/server/opensearch/client/cluster_client.ts +++ b/src/core/server/opensearch/client/cluster_client.ts @@ -90,10 +90,27 @@ export class ClusterClient implements ICustomClusterClient { asScoped(request: ScopeableRequest) { const scopedHeaders = this.getScopedHeaders(request); + const scopedClient = this.rootScopedClient.child({ headers: scopedHeaders, }); - return new ScopedClusterClient(this.asInternalUser, scopedClient); + + const asInternalUserWithLongNumeralsSupport = this.asInternalUser.child({ + // @ts-expect-error - Remove ignoring after https://github.com/opensearch-project/opensearch-js/pull/598 is included in a release + enableLongNumeralSupport: true, + }); + + const scopedClientWithLongNumeralsSupport = this.rootScopedClient.child({ + headers: scopedHeaders, + // @ts-expect-error - Remove ignoring after https://github.com/opensearch-project/opensearch-js/pull/598 is included in a release + enableLongNumeralSupport: true, + }); + return new ScopedClusterClient( + this.asInternalUser, + scopedClient, + asInternalUserWithLongNumeralsSupport, + scopedClientWithLongNumeralsSupport + ); } public async close() { diff --git a/src/core/server/opensearch/client/mocks.ts b/src/core/server/opensearch/client/mocks.ts index 7b055d5b03cf..40b731f0f3bf 100644 --- a/src/core/server/opensearch/client/mocks.ts +++ b/src/core/server/opensearch/client/mocks.ts @@ -112,12 +112,16 @@ const createClientMock = (): OpenSearchClientMock => export interface ScopedClusterClientMock { asInternalUser: OpenSearchClientMock; asCurrentUser: OpenSearchClientMock; + asInternalUserWithLongNumeralsSupport: OpenSearchClientMock; + asCurrentUserWithLongNumeralsSupport: OpenSearchClientMock; } const createScopedClusterClientMock = () => { const mock: ScopedClusterClientMock = { asInternalUser: createClientMock(), asCurrentUser: createClientMock(), + asInternalUserWithLongNumeralsSupport: createClientMock(), + asCurrentUserWithLongNumeralsSupport: createClientMock(), }; return mock; @@ -126,12 +130,16 @@ const createScopedClusterClientMock = () => { export interface ClusterClientMock { asInternalUser: OpenSearchClientMock; asScoped: jest.MockedFunction<() => ScopedClusterClientMock>; + asInternalUserWithLongNumeralsSupport: OpenSearchClientMock; + asCurrentUserWithLongNumeralsSupport: jest.MockedFunction<() => ScopedClusterClientMock>; } const createClusterClientMock = () => { const mock: ClusterClientMock = { asInternalUser: createClientMock(), asScoped: jest.fn(), + asInternalUserWithLongNumeralsSupport: createClientMock(), + asCurrentUserWithLongNumeralsSupport: jest.fn(), }; mock.asScoped.mockReturnValue(createScopedClusterClientMock()); @@ -145,6 +153,8 @@ const createCustomClusterClientMock = () => { const mock: CustomClusterClientMock = { asInternalUser: createClientMock(), asScoped: jest.fn(), + asInternalUserWithLongNumeralsSupport: createClientMock(), + asCurrentUserWithLongNumeralsSupport: jest.fn(), close: jest.fn(), }; diff --git a/src/core/server/opensearch/client/scoped_cluster_client.test.ts b/src/core/server/opensearch/client/scoped_cluster_client.test.ts index 396075d35316..5e3d222c3be2 100644 --- a/src/core/server/opensearch/client/scoped_cluster_client.test.ts +++ b/src/core/server/opensearch/client/scoped_cluster_client.test.ts @@ -36,7 +36,12 @@ describe('ScopedClusterClient', () => { const internalClient = opensearchClientMock.createOpenSearchClient(); const scopedClient = opensearchClientMock.createOpenSearchClient(); - const scopedClusterClient = new ScopedClusterClient(internalClient, scopedClient); + const scopedClusterClient = new ScopedClusterClient( + internalClient, + scopedClient, + internalClient, + scopedClient + ); expect(scopedClusterClient.asInternalUser).toBe(internalClient); }); @@ -45,7 +50,12 @@ describe('ScopedClusterClient', () => { const internalClient = opensearchClientMock.createOpenSearchClient(); const scopedClient = opensearchClientMock.createOpenSearchClient(); - const scopedClusterClient = new ScopedClusterClient(internalClient, scopedClient); + const scopedClusterClient = new ScopedClusterClient( + internalClient, + scopedClient, + internalClient, + scopedClient + ); expect(scopedClusterClient.asCurrentUser).toBe(scopedClient); }); diff --git a/src/core/server/opensearch/client/scoped_cluster_client.ts b/src/core/server/opensearch/client/scoped_cluster_client.ts index d4db3ce3606c..4453243bd3ba 100644 --- a/src/core/server/opensearch/client/scoped_cluster_client.ts +++ b/src/core/server/opensearch/client/scoped_cluster_client.ts @@ -49,12 +49,25 @@ export interface IScopedClusterClient { * on behalf of the user that initiated the request to the OpenSearch Dashboards server. */ readonly asCurrentUser: OpenSearchClient; + /** + * A {@link OpenSearchClient | client}, with support for long numerals, to be used to + * query the opensearch cluster on behalf of the internal OpenSearch Dashboards user. + */ + readonly asInternalUserWithLongNumeralsSupport: OpenSearchClient; + /** + * A {@link OpenSearchClient | client}, with support for long numerals, to be used to + * query the opensearch cluster on behalf of the user that initiated the request to + * the OpenSearch Dashboards server. + */ + readonly asCurrentUserWithLongNumeralsSupport: OpenSearchClient; } /** @internal **/ export class ScopedClusterClient implements IScopedClusterClient { constructor( public readonly asInternalUser: OpenSearchClient, - public readonly asCurrentUser: OpenSearchClient + public readonly asCurrentUser: OpenSearchClient, + public readonly asInternalUserWithLongNumeralsSupport: OpenSearchClient, + public readonly asCurrentUserWithLongNumeralsSupport: OpenSearchClient ) {} } diff --git a/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.test.ts b/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.test.ts index eaa171132785..fbeadf4cc7de 100644 --- a/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.test.ts +++ b/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.test.ts @@ -86,6 +86,27 @@ describe('test sendRequestToOpenSearch', () => { }); }); + it('test request success, json with long numerals', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const mockHttpResponse = createMockHttpResponse( + 200, + 'ok', + [['Content-Type', 'application/json, utf-8']], + { + 'long-max': longPositive, + 'long-min': longNegative, + } + ); + + jest.spyOn(opensearch, 'send').mockResolvedValue(mockHttpResponse); + sendRequestToOpenSearch(dummyArgs).then((result) => { + const value = (result as any)[0].response.value; + expect(value).toMatch(new RegExp(`"long-max": ${longPositive}[,\n]`)); + expect(value).toMatch(new RegExp(`"long-min": ${longNegative}[,\n]`)); + }); + }); + it('test request success, text', () => { const mockHttpResponse = createMockHttpResponse( 200, diff --git a/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.ts b/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.ts index 4e1ae7267542..1cb992a7a99c 100644 --- a/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.ts +++ b/src/plugins/console/public/application/hooks/use_send_current_request_to_opensearch/send_request_to_opensearch.ts @@ -28,6 +28,7 @@ * under the License. */ +import { stringify } from '@osd/std'; import { HttpFetchError, HttpSetup } from 'opensearch-dashboards/public'; import { extractDeprecationMessages } from '../../../lib/utils'; import { XJson } from '../../../../../opensearch_ui_shared/public'; @@ -116,7 +117,7 @@ export function sendRequestToOpenSearch( const contentType = httpResponse.response.headers.get('Content-Type') as BaseResponseType; let value = ''; if (contentType.includes('application/json')) { - value = JSON.stringify(httpResponse.body, null, 2); + value = stringify(httpResponse.body, null, 2); } else { value = httpResponse.body; } @@ -155,7 +156,7 @@ export function sendRequestToOpenSearch( if (httpError.body) { contentType = httpResponse.headers.get('Content-Type') as string; if (contentType?.includes('application/json')) { - value = JSON.stringify(httpError.body, null, 2); + value = stringify(httpError.body, null, 2); } else { value = httpError.body; } diff --git a/src/plugins/console/public/lib/opensearch/opensearch.ts b/src/plugins/console/public/lib/opensearch/opensearch.ts index b0158945eb25..d1ba8797e474 100644 --- a/src/plugins/console/public/lib/opensearch/opensearch.ts +++ b/src/plugins/console/public/lib/opensearch/opensearch.ts @@ -57,6 +57,7 @@ export async function send( body: data, prependBasePath: true, asResponse: true, + withLongNumerals: true, }); } diff --git a/src/plugins/console/public/lib/utils/__tests__/__snapshots__/utils.test.ts.snap b/src/plugins/console/public/lib/utils/__tests__/__snapshots__/utils.test.ts.snap new file mode 100644 index 000000000000..48739d834c9a --- /dev/null +++ b/src/plugins/console/public/lib/utils/__tests__/__snapshots__/utils.test.ts.snap @@ -0,0 +1,118 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Utils class formatRequestBodyDoc changed with indenting 1`] = ` +Object { + "changed": true, + "data": Array [ + "{ + \\"data\\": { + \\"long-max\\": 18014398509481982, + \\"long-min\\": -18014398509481982, + \\"num\\": 2 + } +}", + "{ + \\"'\\\\\\"key\\\\\\"'\\": \\"\\"\\"'oddly' 'quoted \\"value\\"'\\"\\"\\", + \\"obj\\": \\"\\"\\"{\\"nested\\": \\"inner-value\\", \\"inner-max\\": 18014398509481982\\"\\"\\" +}", + "{ + \\"version\\": true, + \\"size\\": 500, + \\"sort\\": [ + { + \\"time\\": { + \\"order\\": \\"desc\\", + \\"unmapped_type\\": \\"boolean\\" + } + } + ], + \\"aggs\\": { + \\"2\\": { + \\"date_histogram\\": { + \\"field\\": \\"time\\", + \\"fixed_interval\\": \\"30s\\", + \\"time_zone\\": \\"America/Los_Angeles\\", + \\"min_doc_count\\": 1 + } + } + }, + \\"_source\\": { + \\"excludes\\": [] + }, + \\"query\\": { + \\"bool\\": { + \\"filter\\": [ + { + \\"match_all\\": {} + }, + { + \\"range\\": { + \\"time\\": { + \\"gte\\": \\"2222-22-22T22:22:22.222Z\\", + \\"lte\\": \\"3333-33-33T33:33:33.333Z\\", + \\"format\\": \\"strict_date_optional_time\\" + } + } + } + ] + } + } +}", + "{ + \\"version\\": true, + \\"size\\": 500, + \\"sort\\": [ + { + \\"time\\": { + \\"order\\": \\"desc\\", + \\"unmapped_type\\": \\"boolean\\" + } + } + ], + \\"aggs\\": { + \\"2\\": { + \\"date_histogram\\": { + \\"field\\": \\"time\\", + \\"fixed_interval\\": \\"30s\\", + \\"time_zone\\": \\"America/Los_Angeles\\", + \\"min_doc_count\\": 1 + } + } + }, + \\"_source\\": { + \\"excludes\\": [] + }, + \\"query\\": { + \\"bool\\": { + \\"filter\\": [ + { + \\"match_all\\": {} + }, + { + \\"range\\": { + \\"time\\": { + \\"gte\\": \\"2222-22-22T22:22:22.222Z\\", + \\"lte\\": \\"3333-33-33T33:33:33.333Z\\", + \\"format\\": \\"strict_date_optional_time\\" + } + } + } + ] + } + } +}", + ], +} +`; + +exports[`Utils class formatRequestBodyDoc changed with no-indenting 1`] = ` +Object { + "changed": true, + "data": Array [ + "{\\"data\\":{\\"long-max\\":18014398509481982,\\"long-min\\":-18014398509481982,\\"num\\":2}}", + "{\\"'\\\\\\"key\\\\\\"'\\":\\"'oddly' 'quoted \\\\\\"value\\\\\\"'\\",\\"obj\\":\\"{\\\\\\"nested\\\\\\": \\\\\\"inner-value\\\\\\", \\\\\\"inner-max\\\\\\": 18014398509481982\\"}", + "{\\"version\\":true,\\"size\\":500,\\"sort\\":[{\\"time\\":{\\"order\\":\\"desc\\",\\"unmapped_type\\":\\"boolean\\"}}],\\"aggs\\":{\\"2\\":{\\"date_histogram\\":{\\"field\\":\\"time\\",\\"fixed_interval\\":\\"30s\\",\\"time_zone\\":\\"America/Los_Angeles\\",\\"min_doc_count\\":1}}},\\"_source\\":{\\"excludes\\":[]},\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_all\\":{}},{\\"range\\":{\\"time\\":{\\"gte\\":\\"2222-22-22T22:22:22.222Z\\",\\"lte\\":\\"3333-33-33T33:33:33.333Z\\",\\"format\\":\\"strict_date_optional_time\\"}}}]}}}", + "{\\"version\\":true,\\"size\\":500,\\"sort\\":[{\\"time\\":{\\"order\\":\\"desc\\",\\"unmapped_type\\":\\"boolean\\"}}],\\"aggs\\":{\\"2\\":{\\"date_histogram\\":{\\"field\\":\\"time\\",\\"fixed_interval\\":\\"30s\\",\\"time_zone\\":\\"America/Los_Angeles\\",\\"min_doc_count\\":1}}},\\"_source\\":{\\"excludes\\":[]},\\"query\\":{\\"bool\\":{\\"filter\\":[{\\"match_all\\":{}},{\\"range\\":{\\"time\\":{\\"gte\\":\\"2222-22-22T22:22:22.222Z\\",\\"lte\\":\\"3333-33-33T33:33:33.333Z\\",\\"format\\":\\"strict_date_optional_time\\"}}}]}}}", + ], +} +`; diff --git a/src/plugins/console/public/lib/utils/__tests__/utils.test.ts b/src/plugins/console/public/lib/utils/__tests__/utils.test.ts index 7f5f0c0de53f..6004d963e27c 100644 --- a/src/plugins/console/public/lib/utils/__tests__/utils.test.ts +++ b/src/plugins/console/public/lib/utils/__tests__/utils.test.ts @@ -29,6 +29,7 @@ */ import * as utils from '../'; +import { stringify } from '@osd/std'; describe('Utils class', () => { test('extract deprecation messages', function () { @@ -104,4 +105,71 @@ describe('Utils class', () => { 'e", f"', ]); }); + + describe('formatRequestBodyDoc', () => { + const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n; + const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n; + const sample = { + version: true, + size: 500, + sort: [{ time: { order: 'desc', unmapped_type: 'boolean' } }], + aggs: { + '2': { + date_histogram: { + field: 'time', + fixed_interval: '30s', + time_zone: 'America/Los_Angeles', + min_doc_count: 1, + }, + }, + }, + _source: { excludes: [] }, + query: { + bool: { + filter: [ + { + match_all: {}, + }, + { + range: { + time: { + gte: '2222-22-22T22:22:22.222Z', + lte: '3333-33-33T33:33:33.333Z', + format: 'strict_date_optional_time', + }, + }, + }, + ], + }, + }, + }; + const objStringSample = { + [`'"key"'`]: `'oddly' 'quoted "value"'`, + obj: `{"nested": "inner-value", "inner-max": ${longPositive}`, + }; + const data = [ + stringify({ data: { 'long-max': longPositive, 'long-min': longNegative, num: 2 } }), + JSON.stringify(objStringSample), + JSON.stringify(sample), + JSON.stringify(sample, null, 3), + ]; + + test('changed with indenting', () => { + expect(utils.formatRequestBodyDoc(data, true)).toMatchSnapshot(); + }); + + test('changed with no-indenting', () => { + expect(utils.formatRequestBodyDoc(data, false)).toMatchSnapshot(); + }); + + test('unchanged with indenting', () => { + const result = utils.formatRequestBodyDoc([JSON.stringify(sample, null, 2)], true); + expect(result.changed).toStrictEqual(false); + }); + + test('unchanged with no-indenting', () => { + const result = utils.formatRequestBodyDoc([JSON.stringify(sample)], false); + expect(result.changed).toStrictEqual(false); + }); + }); }); diff --git a/src/plugins/console/public/lib/utils/index.ts b/src/plugins/console/public/lib/utils/index.ts index 93a0688ae725..7ea4cd9b893f 100644 --- a/src/plugins/console/public/lib/utils/index.ts +++ b/src/plugins/console/public/lib/utils/index.ts @@ -28,6 +28,7 @@ * under the License. */ +import { parse, stringify } from '@osd/std'; import _ from 'lodash'; import { XJson } from '../../../../opensearch_ui_shared/public'; @@ -42,7 +43,7 @@ export function textFromRequest(request: any) { } export function jsonToString(data: any, indent: boolean) { - return JSON.stringify(data, null, indent ? 2 : 0); + return stringify(data, null, indent ? 2 : 0); } export function formatRequestBodyDoc(data: string[], indent: boolean) { @@ -51,7 +52,7 @@ export function formatRequestBodyDoc(data: string[], indent: boolean) { for (let i = 0; i < data.length; i++) { const curDoc = data[i]; try { - let newDoc = jsonToString(JSON.parse(collapseLiteralStrings(curDoc)), indent); + let newDoc = jsonToString(parse(collapseLiteralStrings(curDoc)), indent); if (indent) { newDoc = expandLiteralStrings(newDoc); } diff --git a/src/plugins/console/public/services/storage.ts b/src/plugins/console/public/services/storage.ts index f143b0cefcfe..870904a4b3d8 100644 --- a/src/plugins/console/public/services/storage.ts +++ b/src/plugins/console/public/services/storage.ts @@ -29,6 +29,7 @@ */ import { transform, keys, startsWith } from 'lodash'; +import { parse, stringify } from '@osd/std'; type IStorageEngine = typeof window.localStorage; @@ -40,12 +41,12 @@ export class Storage { constructor(private readonly engine: IStorageEngine, private readonly prefix: string) {} encode(val: any) { - return JSON.stringify(val); + return stringify(val); } decode(val: any) { if (typeof val === 'string') { - return JSON.parse(val); + return parse(val); } } diff --git a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts index 0a401ded813b..f2155011cd74 100644 --- a/src/plugins/console/server/routes/api/console/proxy/create_handler.ts +++ b/src/plugins/console/server/routes/api/console/proxy/create_handler.ts @@ -31,8 +31,8 @@ import { OpenSearchDashboardsRequest, RequestHandler } from 'opensearch-dashboards/server'; import { trimStart } from 'lodash'; import { Readable } from 'stream'; - -import { ApiResponse } from '@opensearch-project/opensearch/'; +import { stringify } from '@osd/std'; +import { ApiResponse } from '@opensearch-project/opensearch'; // eslint-disable-next-line @osd/eslint/no-restricted-paths import { ensureRawRequest } from '../../../../../../../core/server/http/router'; @@ -90,7 +90,7 @@ export const createHandler = ({ const { path, method, dataSourceId } = query; const client = dataSourceId ? await ctx.dataSource.opensearch.getClient(dataSourceId) - : ctx.core.opensearch.client.asCurrentUser; + : ctx.core.opensearch.client.asCurrentUserWithLongNumeralsSupport; let opensearchResponse: ApiResponse; if (!pathFilters.some((re) => re.test(path))) { @@ -116,14 +116,24 @@ export const createHandler = ({ { headers: requestHeaders } ); - const { statusCode, body: responseContent, warnings } = opensearchResponse; + const { + statusCode, + body: responseContent, + warnings, + headers: responseHeaders, + } = opensearchResponse; if (method.toUpperCase() !== 'HEAD') { + /* If a response is a parse JSON object, we need to use a custom `stringify` to handle BigInt + * values. + */ + const isJSONResponse = responseHeaders?.['content-type']?.includes?.('application/json'); return response.custom({ statusCode: statusCode!, - body: responseContent, + body: isJSONResponse ? stringify(responseContent) : responseContent, headers: { warning: warnings || '', + ...(isJSONResponse ? { 'Content-Type': 'application/json; charset=utf-8' } : {}), }, }); } @@ -139,7 +149,7 @@ export const createHandler = ({ } catch (e: any) { const isResponseErrorFlag = isResponseError(e); if (!isResponseError) log.error(e); - const errorMessage = isResponseErrorFlag ? JSON.stringify(e.meta.body) : e.message; + const errorMessage = isResponseErrorFlag ? stringify(e.meta.body) : e.message; // core http route handler has special logic that asks for stream readable input to pass error opaquely const errorResponseBody = new Readable({ read() { diff --git a/src/plugins/console/server/routes/api/console/proxy/tests/body.test.ts b/src/plugins/console/server/routes/api/console/proxy/tests/body.test.ts index 95b243dd4734..59008fc81684 100644 --- a/src/plugins/console/server/routes/api/console/proxy/tests/body.test.ts +++ b/src/plugins/console/server/routes/api/console/proxy/tests/body.test.ts @@ -50,7 +50,9 @@ describe('Console Proxy Route', () => { const requestHandlerContextMock = coreMock.createRequestHandlerContext(); opensearchClient = requestHandlerContextMock.opensearch.client; - opensearchClient.asCurrentUser.transport.request.mockResolvedValueOnce(mockResponse); + opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mockResolvedValueOnce( + mockResponse + ); const handler = createHandler(getProxyRouteHandlerDeps({})); return handler( diff --git a/src/plugins/console/server/routes/api/console/proxy/tests/headers.test.ts b/src/plugins/console/server/routes/api/console/proxy/tests/headers.test.ts index a1964d160e2c..7a6b3f70f490 100644 --- a/src/plugins/console/server/routes/api/console/proxy/tests/headers.test.ts +++ b/src/plugins/console/server/routes/api/console/proxy/tests/headers.test.ts @@ -85,7 +85,9 @@ describe('Console Proxy Route', () => { opensearchDashboardsResponseFactory ); - const [[, opts]] = opensearchClient.asCurrentUser.transport.request.mock.calls; + const [ + [, opts], + ] = opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mock.calls; const headers = opts?.headers; expect(headers).to.have.property('x-forwarded-for'); expect(headers!['x-forwarded-for']).to.be('0.0.0.0'); diff --git a/src/plugins/console/server/routes/api/console/proxy/tests/params.test.ts b/src/plugins/console/server/routes/api/console/proxy/tests/params.test.ts index 80523c8031df..2e191425752a 100644 --- a/src/plugins/console/server/routes/api/console/proxy/tests/params.test.ts +++ b/src/plugins/console/server/routes/api/console/proxy/tests/params.test.ts @@ -75,7 +75,9 @@ describe('Console Proxy Route', () => { ); const mockResponse = opensearchServiceMock.createSuccessTransportRequestPromise('foo'); - opensearchClient.asCurrentUser.transport.request.mockResolvedValueOnce(mockResponse); + opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mockResolvedValueOnce( + mockResponse + ); const { status } = await handler( { core: requestHandlerContextMock, dataSource: {} as any }, @@ -93,7 +95,9 @@ describe('Console Proxy Route', () => { ); const mockResponse = opensearchServiceMock.createSuccessTransportRequestPromise('foo'); - opensearchClient.asCurrentUser.transport.request.mockResolvedValueOnce(mockResponse); + opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mockResolvedValueOnce( + mockResponse + ); const { status } = await handler( { core: requestHandlerContextMock, dataSource: {} as any }, diff --git a/src/plugins/console/server/routes/api/console/proxy/tests/query_string.test.ts b/src/plugins/console/server/routes/api/console/proxy/tests/query_string.test.ts index 9aefea1182cc..edb3ec6e6418 100644 --- a/src/plugins/console/server/routes/api/console/proxy/tests/query_string.test.ts +++ b/src/plugins/console/server/routes/api/console/proxy/tests/query_string.test.ts @@ -63,7 +63,9 @@ describe('Console Proxy Route', () => { describe('contains full url', () => { it('treats the url as a path', async () => { await request('GET', 'http://evil.com/test'); - const [[args]] = opensearchClient.asCurrentUser.transport.request.mock.calls; + const [ + [args], + ] = opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mock.calls; expect(args.path).toBe('/http://evil.com/test?pretty=true'); }); @@ -71,14 +73,18 @@ describe('Console Proxy Route', () => { describe('starts with a slash', () => { it('keeps as it is', async () => { await request('GET', '/index/id'); - const [[args]] = opensearchClient.asCurrentUser.transport.request.mock.calls; + const [ + [args], + ] = opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mock.calls; expect(args.path).toBe('/index/id?pretty=true'); }); }); describe(`doesn't start with a slash`, () => { it('adds slash to path before sending request', async () => { await request('GET', 'index/id'); - const [[args]] = opensearchClient.asCurrentUser.transport.request.mock.calls; + const [ + [args], + ] = opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mock.calls; expect(args.path).toBe('/index/id?pretty=true'); }); }); @@ -86,7 +92,9 @@ describe('Console Proxy Route', () => { describe(`contains query parameter`, () => { it('adds slash to path before sending request', async () => { await request('GET', '_cat/tasks?v'); - const [[args]] = opensearchClient.asCurrentUser.transport.request.mock.calls; + const [ + [args], + ] = opensearchClient.asCurrentUserWithLongNumeralsSupport.transport.request.mock.calls; expect(args.path).toBe('/_cat/tasks?v=&pretty=true'); }); }); diff --git a/src/plugins/opensearch_ui_shared/__packages_do_not_import__/xjson/json_xjson_translation_tools/index.ts b/src/plugins/opensearch_ui_shared/__packages_do_not_import__/xjson/json_xjson_translation_tools/index.ts index 9f42e669b280..9860be7800e4 100644 --- a/src/plugins/opensearch_ui_shared/__packages_do_not_import__/xjson/json_xjson_translation_tools/index.ts +++ b/src/plugins/opensearch_ui_shared/__packages_do_not_import__/xjson/json_xjson_translation_tools/index.ts @@ -28,12 +28,13 @@ * under the License. */ +import { parse, stringify } from '@osd/std'; import { extractJSONStringValues } from './parser'; export function collapseLiteralStrings(data: string) { const splitData = data.split(`"""`); for (let idx = 1; idx < splitData.length - 1; idx += 2) { - splitData[idx] = JSON.stringify(splitData[idx]); + splitData[idx] = stringify(splitData[idx]); } return splitData.join(''); } @@ -79,7 +80,7 @@ export function expandLiteralStrings(data: string) { (candidate[candidate.length - 2] === '"' && candidate[candidate.length - 3] === '\\'); if (!skip && candidate.match(/\\./)) { - result += `"""${JSON.parse(candidate)}"""`; + result += `"""${parse(candidate)}"""`; } else { result += candidate; } diff --git a/yarn.lock b/yarn.lock index d71aa1d447e5..88f7dd435331 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2497,10 +2497,10 @@ mkdirp "^1.0.4" rimraf "^3.0.2" -"@opensearch-project/opensearch-next@npm:@opensearch-project/opensearch@^2.2.0": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-2.2.1.tgz#a400203afa6512ef73945663163a404763a10f5a" - integrity sha512-8zfQX1acL9eWG+ohIc9nJVT9LSqXCdbVEJs0rCPRtji3XF6ahzsiKmGNTeWLxCPDxWCjAIWq9t95xP3Y5Egi6Q== +"@opensearch-project/opensearch-next@npm:@opensearch-project/opensearch@^2.3.1": + version "2.3.1" + resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-2.3.1.tgz#3596e2f1f0615a7555102f6f941f0e0ec645c2cd" + integrity sha512-Kg8tddAx6sinStnNi6IeGilfvLWlonIxaRdVNiJcNPr1yMqd0c9TSegn18zKr0Pb0IM9xBIGBSkRPuh67ZN6Hw== dependencies: aws4 "^1.11.0" debug "^4.3.1"