feat: convert arrays to indexed object (#27)

* add `convertArrays` option * add cli options for convert arrays * add tests for converting arrays * refactor: use `replacer` of JSON.stringify instead * refactor: use object options for LogBuilder.build * fix: add convertArrays to loki options * docs: note about convertArrays * fix: doc --------- Co-authored-by: Joel Edwardson <[email protected]> Co-authored-by: joeledwardson <[email protected]>
Julien-R44 · May 12, 2024 · 6078a16 · 6078a16
1 parent 6d76363
commit 6078a16
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -108,6 +108,10 @@ The interval at which batched logs are sent in seconds. Defaults to `5`.
 
 Defaults to `false`. If true, the timestamp in the pino log will be replaced with `Date.now()`. Be careful when using this option with `batching` enabled, as the logs will be sent in batches, and the timestamp will be the time of the batch, not the time of the log.
 
+#### `convertArrays`
+
+Defaults to `false`. As documented in the [Loki documentation](https://grafana.com/docs/loki/latest/query/log_queries/#json), Loki JSON parser will skip arrays. Setting this options to `true` will convert arrays to object with index as key. For example, `["foo", "bar"]` will be converted to `{ "0": "foo", "1": "bar" }`.
+
 ## CLI usage
 ```shell
 npm install -g pino-loki
@@ -127,6 +131,7 @@ Options:
   -s, --silenceErrors            If false, errors will be displayed in the console
   -r, --replaceTimestamp         Replace pino logs timestamps with Date.now()
   -l, --labels <label>           Additional labels to be added to all Loki logs
+  -a, --convertArrays            If true, arrays will be converted to objects
   -pl, --propsLabels <labels>    Fields in log line to convert to Loki labels (comma separated values)
   --no-stdout                    Disable output to stdout
   -h, --help                     display help for command

diff --git a/src/cli.ts b/src/cli.ts
@@ -16,6 +16,7 @@ program
   .option('-s, --silenceErrors', 'If false, errors will be displayed in the console')
   .option('-r, --replaceTimestamp', 'Replace pino logs timestamps with Date.now()')
   .option('-l, --labels <label>', 'Additional labels to be added to all Loki logs')
+  .option('-a, --convertArrays', 'If true, arrays will be converted to objects')
   .option(
     '-pl, --propsLabels <labels>',
     'Fields in log line to convert to Loki labels (comma separated values)',
@@ -37,6 +38,7 @@ export const createPinoLokiConfigFromArgs = () => {
     replaceTimestamp: opts.replaceTimestamp,
     labels: opts.labels ? JSON.parse(opts.labels) : undefined,
     propsToLabels: opts.propsLabels ? opts.propsLabels.split(',') : [],
+    convertArrays: opts.convertArrays,
   }
 
   if (opts.user && opts.password) {

diff --git a/src/index.ts b/src/index.ts
@@ -17,6 +17,7 @@ function resolveOptions(options: LokiOptions) {
     interval: options.interval ?? 5,
     replaceTimestamp: options.replaceTimestamp ?? false,
     propsToLabels: options.propsToLabels ?? [],
+    convertArrays: options.convertArrays ?? false,
   }
 }
 

diff --git a/src/log_builder/index.ts b/src/log_builder/index.ts
@@ -49,6 +49,22 @@ export class LogBuilder {
     return (time * missingFactor).toString()
   }
 
+  /**
+   * Stringify the log object. If convertArrays is true then it will convert
+   * arrays to objects with indexes as keys.
+   */
+  #stringifyLog(log: PinoLog, convertArrays?: boolean): string {
+    return JSON.stringify(log, (_, value) => {
+      if (!convertArrays) return value
+
+      if (Array.isArray(value)) {
+        return Object.fromEntries(value.map((value, index) => [index, value]))
+      }
+
+      return value
+    })
+  }
+
   #buildLabelsFromProps(log: PinoLog) {
     const labels: Record<string, string> = {}
 
@@ -71,26 +87,27 @@ export class LogBuilder {
   /**
    * Build a loki log entry from a pino log
    */
-  build(
-    log: PinoLog,
-    replaceTimestamp?: boolean,
-    additionalLabels?: Record<string, string>,
-  ): LokiLog {
-    const status = this.statusFromLevel(log.level)
-    const time = this.#buildTimestamp(log, replaceTimestamp)
-    const propsLabels = this.#buildLabelsFromProps(log)
-
-    const hostname = log.hostname
-    log.hostname = undefined
+  build(options: {
+    log: PinoLog
+    replaceTimestamp?: boolean
+    additionalLabels?: Record<string, string>
+    convertArrays?: boolean
+  }): LokiLog {
+    const status = this.statusFromLevel(options.log.level)
+    const time = this.#buildTimestamp(options.log, options.replaceTimestamp)
+    const propsLabels = this.#buildLabelsFromProps(options.log)
+
+    const hostname = options.log.hostname
+    options.log.hostname = undefined
 
     return {
       stream: {
         level: status,
         hostname,
-        ...additionalLabels,
+        ...options.additionalLabels,
         ...propsLabels,
       },
-      values: [[time, JSON.stringify(log)]],
+      values: [[time, this.#stringifyLog(options.log, options.convertArrays)]],
     }
   }
 }
diff --git a/src/log_pusher/index.ts b/src/log_pusher/index.ts
@@ -60,7 +60,12 @@ export class LogPusher {
     }
 
     const lokiLogs = logs.map((log) =>
-      this.#logBuilder.build(log, this.#options.replaceTimestamp, this.#options.labels),
+      this.#logBuilder.build({
+        log,
+        replaceTimestamp: this.#options.replaceTimestamp,
+        additionalLabels: this.#options.labels,
+        convertArrays: this.#options.convertArrays,
+      }),
     )
 
     debug(`[LogPusher] pushing ${lokiLogs.length} logs to Loki`)

diff --git a/src/types/index.ts b/src/types/index.ts
@@ -114,4 +114,11 @@ export interface LokiOptions {
    * Select log message's props to set as Loki labels
    */
   propsToLabels?: string[]
+
+  /**
+   * Convert arrays in log messages to objects with index as key
+   *
+   * @default false
+   */
+  convertArrays?: boolean
 }
diff --git a/tests/unit/log_builder.spec.ts b/tests/unit/log_builder.spec.ts
@@ -50,8 +50,12 @@ test.group('Log Builder', () => {
       v: 1,
     }
 
-    const lokiLog = logBuilder.build(log, false, {
-      application: 'MY-APP',
+    const lokiLog = logBuilder.build({
+      log,
+      replaceTimestamp: false,
+      additionalLabels: {
+        application: 'MY-APP',
+      },
     })
 
     assert.deepEqual(lokiLog.stream.level, 'info')
@@ -74,7 +78,11 @@ test.group('Log Builder', () => {
 
     await sleep(1000)
 
-    const lokiLog = logBuilder.build(log, true, { application: 'MY-APP' })
+    const lokiLog = logBuilder.build({
+      log,
+      replaceTimestamp: true,
+      additionalLabels: { application: 'MY-APP' },
+    })
     const currentTime = new Date().getTime() * 1_000_000
 
     assert.closeTo(+lokiLog.values[0][0], +currentTime, 10_000_000)
@@ -92,7 +100,11 @@ test.group('Log Builder', () => {
       appId: 123,
       buildId: 'aaaa',
     }
-    const lokiLog = logBuilder.build(log, true, { application: 'MY-APP' })
+    const lokiLog = logBuilder.build({
+      log,
+      replaceTimestamp: true,
+      additionalLabels: { application: 'MY-APP' },
+    })
     assert.equal(lokiLog.stream.appId, 123)
     assert.equal(lokiLog.stream.buildId, 'aaaa')
   })
@@ -102,14 +114,11 @@ test.group('Log Builder', () => {
 
     const now = nanoseconds().toString()
 
-    const log: PinoLog = {
-      hostname: 'localhost',
-      level: 30,
-      msg: 'hello world',
-      time: now,
-    }
-
-    const lokiLog = logBuilder.build(log, false, { application: 'MY-APP' })
+    const lokiLog = logBuilder.build({
+      log: { hostname: 'localhost', level: 30, msg: 'hello world', time: now },
+      replaceTimestamp: false,
+      additionalLabels: { application: 'MY-APP' },
+    })
     assert.deepEqual(lokiLog.values[0][0], now)
   })
 
@@ -118,15 +127,40 @@ test.group('Log Builder', () => {
 
     const now = new Date().getTime().toString()
 
-    const log: PinoLog = {
-      hostname: 'localhost',
+    const lokiLog = logBuilder.build({
+      log: { hostname: 'localhost', level: 30, msg: 'hello world', time: now },
+      replaceTimestamp: false,
+      additionalLabels: { application: 'MY-APP' },
+    })
+
+    assert.deepEqual(lokiLog.values[0][0], now + '000000')
+  })
+
+  test('convert arrays to indexed keys', ({ assert }) => {
+    const logBuilder = new LogBuilder()
+
+    const log1 = logBuilder.build({
+      log: { level: 30, msg: 'hello world', additional: [['x', 'y', 'z'], { a: 1, b: 2 }] },
+      replaceTimestamp: true,
+      convertArrays: true,
+    })
+
+    assert.deepEqual(JSON.parse(log1.values[0][1]), {
       level: 30,
       msg: 'hello world',
-      time: now,
-    }
+      additional: { 0: { 0: 'x', 1: 'y', 2: 'z' }, 1: { a: 1, b: 2 } },
+    })
 
-    const lokiLog = logBuilder.build(log, false, { application: 'MY-APP' })
+    const log2 = logBuilder.build({
+      log: { level: 30, msg: 'hello world', additional: { foo: { bar: [{ a: 1, b: 2 }] } } },
+      replaceTimestamp: true,
+      convertArrays: true,
+    })
 
-    assert.deepEqual(lokiLog.values[0][0], now + '000000')
+    assert.deepEqual(JSON.parse(log2.values[0][1]), {
+      level: 30,
+      msg: 'hello world',
+      additional: { foo: { bar: { 0: { a: 1, b: 2 } } } },
+    })
   })
 })