Change node messageId, and update core in order to remove transient d…

…ependencies from nodejs (#670)

.changeset/short-balloons-worry.md

@@ -0,0 +1,4 @@
+---
+'@segment/analytics-core': patch
+---
+Allow consumers to inject custom messageId into EventFactory, allowing us to remove node transient dependency on md5 library. Change node messageId to format "node-next-[unix epoch time]-[uuid]".

packages/core/package.json

@@ -32,7 +32,6 @@
   },
   "packageManager": "[email protected]",
   "dependencies": {
-    "@lukeed/uuid": "^2.0.0",
     "dset": "^3.1.2",
     "tslib": "^2.4.0"
   }

packages/core/src/events/__tests__/index.test.ts

@@ -1,5 +1,3 @@
-import uuid from '@lukeed/uuid'
-import { range, uniq } from 'lodash'
 import { EventFactory } from '..'
 import { User } from '../../user'
 import { CoreSegmentEvent } from '../..'
@@ -15,7 +13,10 @@ describe('Event Factory', () => {
       anonymousId: () => undefined,
       id: () => 'foo',
     }
-    factory = new EventFactory(user)
+    factory = new EventFactory({
+      user,
+      createMessageId: () => 'foo',
+    })
   })
 
   describe('alias', () => {
@@ -78,8 +79,11 @@ describe('Event Factory', () => {
 
     it('uses userId / anonymousId from the user class (if specified)', function () {
       factory = new EventFactory({
-        id: () => 'abc',
-        anonymousId: () => '123',
+        createMessageId: () => 'foo',
+        user: {
+          id: () => 'abc',
+          anonymousId: () => '123',
+        },
       })
       const group = factory.group('my_group_id')
       expect(group.userId).toBe('abc')
@@ -149,44 +153,14 @@ describe('Event Factory', () => {
 
     test('adds a message id', () => {
       const track = factory.track('Order Completed', shoes)
-      expect(track.messageId).toContain('ajs-next')
+      expect(typeof track.messageId).toBe('string')
     })
 
     test('adds a timestamp', () => {
       const track = factory.track('Order Completed', shoes)
       expect(track.timestamp).toBeInstanceOf(Date)
     })
 
-    test('adds a random message id even when random is mocked', () => {
-      jest.useFakeTimers()
-      jest.spyOn(uuid, 'v4').mockImplementation(() => 'abc-123')
-      // fake timer and fake uuid => equal
-      expect(factory.track('Order Completed', shoes).messageId).toEqual(
-        factory.track('Order Completed', shoes).messageId
-      )
-
-      // restore uuid function => not equal
-      jest.restoreAllMocks()
-      expect(factory.track('Order Completed', shoes).messageId).not.toEqual(
-        factory.track('Order Completed', shoes).messageId
-      )
-
-      // restore timers function => not equal
-      jest.useRealTimers()
-
-      expect(factory.track('Order Completed', shoes).messageId).not.toEqual(
-        factory.track('Order Completed', shoes).messageId
-      )
-    })
-
-    test('message ids are random', () => {
-      const ids = range(0, 200).map(
-        () => factory.track('Order Completed', shoes).messageId
-      )
-
-      expect(uniq(ids)).toHaveLength(200)
-    })
-
     test('sets an user id', () => {
       user.id = () => '007'
 
@@ -390,7 +364,6 @@ describe('Event Factory', () => {
       }
       const normalized = factory['normalize'](msg)
 
-      expect(normalized.messageId?.length).toBeGreaterThanOrEqual(41) // 'ajs-next-md5(content + [UUID])'
       delete normalized.messageId
 
       expect(normalized.timestamp).toBeInstanceOf(Date)

packages/core/src/events/index.ts

@@ -1,5 +1,4 @@
 export * from './interfaces'
-import { v4 as uuid } from '@lukeed/uuid'
 import { dset } from 'dset'
 import { ID, User } from '../user'
 import {
@@ -9,14 +8,20 @@ import {
   CoreSegmentEvent,
   CoreOptions,
 } from './interfaces'
-import md5 from 'spark-md5'
 import { validateEvent } from '../validation/assertions'
 
+interface EventFactorySettings {
+  createMessageId: () => string
+  user?: User
+}
+
 export class EventFactory {
+  createMessageId: EventFactorySettings['createMessageId']
   user?: User
 
-  constructor(user?: User) {
-    this.user = user
+  constructor(settings: EventFactorySettings) {
+    this.user = settings.user
+    this.createMessageId = settings.createMessageId
   }
 
   track(
@@ -240,11 +245,9 @@ export class EventFactory {
       ...overrides,
     }
 
-    const messageId = 'ajs-next-' + md5.hash(JSON.stringify(body) + uuid())
-
     const evt: CoreSegmentEvent = {
       ...body,
-      messageId,
+      messageId: this.createMessageId(),
     }
 
     validateEvent(evt)

packages/node/package.json

@@ -33,6 +33,7 @@
     "publish-prerelease": "sh scripts/prerelease.sh"
   },
   "dependencies": {
+    "@lukeed/uuid": "^2.0.0",
     "@segment/analytics-core": "1.1.1",
     "node-fetch": "^2.6.7",
     "tslib": "^2.4.0"

packages/node/src/app/analytics-node.ts

@@ -19,6 +19,7 @@ import {
 import { AnalyticsSettings, validateSettings } from './settings'
 import { version } from '../../package.json'
 import { configureNodePlugin } from '../plugins/segmentio'
+import { createNodeEventFactory } from '../lib/create-node-event-factory'
 
 // create a derived class since we may want to add node specific things to Context later
 export class Context extends CoreContext {}
@@ -82,7 +83,8 @@ export class Analytics
   constructor(settings: AnalyticsSettings) {
     super()
     validateSettings(settings)
-    this._eventFactory = new EventFactory()
+
+    this._eventFactory = createNodeEventFactory()
     this.queue = new EventQueue(new NodePriorityQueue())
 
     const flushInterval = settings.flushInterval ?? 10000

packages/node/src/lib/__tests__/get-message-id.test.ts

@@ -0,0 +1,21 @@
+import { createMessageId } from '../get-message-id'
+
+// https://gist.github.com/johnelliott/cf77003f72f889abbc3f32785fa3df8d
+const uuidv4Regex =
+  /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+
+describe(createMessageId, () => {
+  it('returns a string in the format "node-next-[unix epoch time]-[uuid v4]"', () => {
+    const msg = createMessageId().split('-')
+    expect(msg.length).toBe(8)
+
+    expect(`${msg[0]}-${msg[1]}`).toBe('node-next')
+
+    const epochTimeSeg = msg[2]
+    expect(typeof parseInt(epochTimeSeg)).toBe('number')
+    expect(epochTimeSeg.length > 10).toBeTruthy()
+
+    const uuidSeg = msg.slice(3).join('-')
+    expect(uuidSeg).toEqual(expect.stringMatching(uuidv4Regex))
+  })
+})

packages/node/src/lib/create-node-event-factory.ts

@@ -0,0 +1,7 @@
+import { EventFactory } from '@segment/analytics-core'
+import { createMessageId } from './get-message-id'
+
+export const createNodeEventFactory = () =>
+  new EventFactory({
+    createMessageId,
+  })

packages/node/src/lib/get-message-id.ts

@@ -0,0 +1,10 @@
+import { v4 } from '@lukeed/uuid/secure'
+
+/**
+ * get a unique messageId with a very low chance of collisions
+ * using @lukeed/uuid/secure uses the node crypto module, which is the fastest
+ * @example "node-next-1668208232027-743be593-7789-4b74-8078-cbcc8894c586"
+ */
+export const createMessageId = (): string => {
+  return `node-next-${Date.now()}-${v4()}`
+}

packages/node/src/plugins/segmentio/__tests__/index.test.ts

@@ -1,15 +1,16 @@
 const fetcher = jest.fn()
 jest.mock('node-fetch', () => fetcher)
 
-import { CoreContext, EventFactory } from '@segment/analytics-core'
+import { CoreContext } from '@segment/analytics-core'
+import { createNodeEventFactory } from '../../../lib/create-node-event-factory'
 import {
   createError,
   createSuccess,
 } from '../../../__tests__/test-helpers/factories'
 import { configureNodePlugin } from '../index'
 
 const bodyPropertyMatchers = {
-  messageId: expect.stringMatching(/^ajs-next-[\w\d]+$/),
+  messageId: expect.stringMatching(/^node-next-\d*-\w*-\w*-\w*-\w*-\w*/),
   timestamp: expect.stringMatching(
     /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
   ),
@@ -49,7 +50,7 @@ function validateFetcherInputs(...contexts: CoreContext[]) {
 }
 
 describe('SegmentNodePlugin', () => {
-  const eventFactory = new EventFactory()
+  const eventFactory = createNodeEventFactory()
   const realSetTimeout = setTimeout
 
   beforeEach(() => {

yarn.lock

@@ -1827,7 +1827,6 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@segment/analytics-core@workspace:packages/core"
   dependencies:
-    "@lukeed/uuid": ^2.0.0
     dset: ^3.1.2
     tslib: ^2.4.0
   languageName: unknown
@@ -1895,6 +1894,7 @@ __metadata:
   resolution: "@segment/analytics-node@workspace:packages/node"
   dependencies:
     "@internal/config": 0.0.0
+    "@lukeed/uuid": ^2.0.0
     "@segment/analytics-core": 1.1.1
     "@types/node": ^14
     node-fetch: ^2.6.7