diff --git a/code/lib/core-server/src/build-static.ts b/code/lib/core-server/src/build-static.ts index b9567b6d4530..7c444c4ae6fa 100644 --- a/code/lib/core-server/src/build-static.ts +++ b/code/lib/core-server/src/build-static.ts @@ -5,7 +5,7 @@ import { dedent } from 'ts-dedent'; import global from 'global'; import { logger } from '@storybook/node-logger'; -import { telemetry } from '@storybook/telemetry'; +import { telemetry, getPrecedingUpgrade } from '@storybook/telemetry'; import type { BuilderOptions, CLIOptions, @@ -173,11 +173,14 @@ export async function buildStaticStandalone( effects.push( initializedStoryIndexGenerator.then(async (generator) => { const storyIndex = await generator?.getIndex(); - const payload = storyIndex - ? { - storyIndex: summarizeIndex(storyIndex), - } - : undefined; + const payload = { + precedingUpgrade: await getPrecedingUpgrade(), + }; + if (storyIndex) { + Object.assign(payload, { + storyIndex: summarizeIndex(storyIndex), + }); + } await telemetry('build', payload, { configDir: options.configDir }); }) ); diff --git a/code/lib/core-server/src/utils/doTelemetry.ts b/code/lib/core-server/src/utils/doTelemetry.ts index ab4085349451..4b262f20affc 100644 --- a/code/lib/core-server/src/utils/doTelemetry.ts +++ b/code/lib/core-server/src/utils/doTelemetry.ts @@ -1,5 +1,5 @@ import type { CoreConfig, Options } from '@storybook/types'; -import { telemetry } from '@storybook/telemetry'; +import { telemetry, getPrecedingUpgrade } from '@storybook/telemetry'; import { useStorybookMetadata } from './metadata'; import type { StoryIndexGenerator } from './StoryIndexGenerator'; import { summarizeIndex } from './summarizeIndex'; @@ -15,12 +15,15 @@ export async function doTelemetry( initializedStoryIndexGenerator.then(async (generator) => { const storyIndex = await generator?.getIndex(); const { versionCheck, versionUpdates } = options; - const payload = storyIndex - ? { - versionStatus: versionUpdates ? versionStatus(versionCheck) : 'disabled', - storyIndex: summarizeIndex(storyIndex), - } - : undefined; + const payload = { + precedingUpgrade: await getPrecedingUpgrade(), + }; + if (storyIndex) { + Object.assign(payload, { + versionStatus: versionUpdates ? versionStatus(versionCheck) : 'disabled', + storyIndex: summarizeIndex(storyIndex), + }); + } telemetry('dev', payload, { configDir: options.configDir }); }); } diff --git a/code/lib/core-server/src/withTelemetry.test.ts b/code/lib/core-server/src/withTelemetry.test.ts index d887909ecc72..121b60718afa 100644 --- a/code/lib/core-server/src/withTelemetry.test.ts +++ b/code/lib/core-server/src/withTelemetry.test.ts @@ -63,10 +63,10 @@ describe('when command fails', () => { withTelemetry('dev', { presetOptions: {} as any }, run) ).rejects.toThrow(error); - expect(telemetry).toHaveBeenCalledTimes(1); - expect(telemetry).not.toHaveBeenCalledWith( + expect(telemetry).toHaveBeenCalledTimes(2); + expect(telemetry).toHaveBeenCalledWith( 'error', - { eventType: 'dev', error }, + { eventType: 'dev' }, expect.objectContaining({}) ); }); @@ -88,7 +88,7 @@ describe('when command fails', () => { ); }); - it('does not send error message when telemetry is disabled', async () => { + it('does not send full error message when telemetry is disabled', async () => { jest.mocked(loadAllPresets).mockResolvedValueOnce({ apply: async () => ({ disableTelemetry: true } as any), }); @@ -132,10 +132,10 @@ describe('when command fails', () => { withTelemetry('dev', { presetOptions: {} as any }, run) ).rejects.toThrow(error); - expect(telemetry).toHaveBeenCalledTimes(1); - expect(telemetry).not.toHaveBeenCalledWith( + expect(telemetry).toHaveBeenCalledTimes(2); + expect(telemetry).toHaveBeenCalledWith( 'error', - { eventType: 'dev', error }, + { eventType: 'dev' }, expect.objectContaining({}) ); }); @@ -169,10 +169,10 @@ describe('when command fails', () => { withTelemetry('dev', { presetOptions: {} as any }, run) ).rejects.toThrow(error); - expect(telemetry).toHaveBeenCalledTimes(1); - expect(telemetry).not.toHaveBeenCalledWith( + expect(telemetry).toHaveBeenCalledTimes(2); + expect(telemetry).toHaveBeenCalledWith( 'error', - { eventType: 'dev', error }, + { eventType: 'dev' }, expect.objectContaining({}) ); }); diff --git a/code/lib/core-server/src/withTelemetry.ts b/code/lib/core-server/src/withTelemetry.ts index fbd77d0129fb..a59e6fd4a184 100644 --- a/code/lib/core-server/src/withTelemetry.ts +++ b/code/lib/core-server/src/withTelemetry.ts @@ -1,7 +1,7 @@ import prompts from 'prompts'; import type { CLIOptions, CoreConfig } from '@storybook/types'; import { loadAllPresets, cache } from '@storybook/core-common'; -import { telemetry } from '@storybook/telemetry'; +import { telemetry, getPrecedingUpgrade } from '@storybook/telemetry'; import type { EventType } from '@storybook/telemetry'; type TelemetryOptions = { @@ -26,11 +26,13 @@ const promptCrashReports = async () => { return enableCrashReports; }; -async function shouldSendError({ cliOptions, presetOptions }: TelemetryOptions) { - if (cliOptions?.disableTelemetry) return false; +type ErrorLevel = 'none' | 'error' | 'full'; + +async function getErrorLevel({ cliOptions, presetOptions }: TelemetryOptions): Promise { + if (cliOptions?.disableTelemetry) return 'none'; // If we are running init or similar, we just have to go with true here - if (!presetOptions) return true; + if (!presetOptions) return 'full'; // should we load the preset? const presets = await loadAllPresets({ @@ -42,18 +44,18 @@ async function shouldSendError({ cliOptions, presetOptions }: TelemetryOptions) // If the user has chosen to enable/disable crash reports in main.js // or disabled telemetry, we can return that const core = await presets.apply('core'); - if (core?.enableCrashReports !== undefined) return core.enableCrashReports; - if (core?.disableTelemetry) return false; + if (core?.enableCrashReports !== undefined) return core.enableCrashReports ? 'full' : 'error'; + if (core?.disableTelemetry) return 'none'; // Deal with typo, remove in future version (7.1?) const valueFromCache = (await cache.get('enableCrashReports')) ?? (await cache.get('enableCrashreports')); - if (valueFromCache !== undefined) return valueFromCache; + if (valueFromCache !== undefined) return valueFromCache ? 'full' : 'error'; const valueFromPrompt = await promptCrashReports(); - if (valueFromPrompt !== undefined) return valueFromPrompt; + if (valueFromPrompt !== undefined) return valueFromPrompt ? 'full' : 'error'; - return true; + return 'full'; } export async function withTelemetry( @@ -67,14 +69,17 @@ export async function withTelemetry( await run(); } catch (error) { try { - if (await shouldSendError(options)) { + const errorLevel = await getErrorLevel(options); + if (errorLevel !== 'none') { + const precedingUpgrade = await getPrecedingUpgrade(); + await telemetry( 'error', - { eventType, error }, + { eventType, precedingUpgrade, error: errorLevel === 'full' ? error : undefined }, { immediate: true, configDir: options.cliOptions?.configDir || options.presetOptions?.configDir, - enableCrashReports: true, + enableCrashReports: errorLevel === 'full', } ); } diff --git a/code/lib/telemetry/src/event-cache.test.ts b/code/lib/telemetry/src/event-cache.test.ts new file mode 100644 index 000000000000..7d1fd463bd94 --- /dev/null +++ b/code/lib/telemetry/src/event-cache.test.ts @@ -0,0 +1,175 @@ +import { getPrecedingUpgrade } from './event-cache'; + +expect.addSnapshotSerializer({ + print: (val: any) => JSON.stringify(val, null, 2), + test: (val) => typeof val !== 'string', +}); + +describe('event-cache', () => { + const init = { body: { eventType: 'init', eventId: 'init' }, timestamp: 1 }; + const upgrade = { body: { eventType: 'upgrade', eventId: 'upgrade' }, timestamp: 2 }; + const dev = { body: { eventType: 'dev', eventId: 'dev' }, timestamp: 3 }; + const build = { body: { eventType: 'build', eventId: 'build' }, timestamp: 3 }; + const error = { body: { eventType: 'build', eventId: 'error' }, timestamp: 4 }; + const versionUpdate = { + body: { eventType: 'version-update', eventId: 'version-update' }, + timestamp: 5, + }; + + describe('data handling', () => { + it('errors', async () => { + const preceding = await getPrecedingUpgrade({ + init: { timestamp: 1, body: { ...init.body, error: {} } }, + }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 1, + "eventType": "init", + "eventId": "init" + } + `); + }); + + it('session IDs', async () => { + const preceding = await getPrecedingUpgrade({ + init: { timestamp: 1, body: { ...init.body, sessionId: 100 } }, + }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 1, + "eventType": "init", + "eventId": "init", + "sessionId": 100 + } + `); + }); + + it('extra fields', async () => { + const preceding = await getPrecedingUpgrade({ + init: { timestamp: 1, body: { ...init.body, foobar: 'baz' } }, + }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 1, + "eventType": "init", + "eventId": "init" + } + `); + }); + }); + + describe('no intervening dev events', () => { + it('no upgrade events', async () => { + const preceding = await getPrecedingUpgrade({}); + expect(preceding).toBeUndefined(); + }); + + it('init', async () => { + const preceding = await getPrecedingUpgrade({ init }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 1, + "eventType": "init", + "eventId": "init" + } + `); + }); + + it('upgrade', async () => { + const preceding = await getPrecedingUpgrade({ upgrade }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 2, + "eventType": "upgrade", + "eventId": "upgrade" + } + `); + }); + + it('both init and upgrade', async () => { + const preceding = await getPrecedingUpgrade({ init, upgrade }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 2, + "eventType": "upgrade", + "eventId": "upgrade" + } + `); + }); + }); + + describe('intervening dev events', () => { + it('no upgrade events', async () => { + const preceding = await getPrecedingUpgrade({ dev }); + expect(preceding).toBeUndefined(); + }); + + it('init', async () => { + const preceding = await getPrecedingUpgrade({ init, dev }); + expect(preceding).toBeUndefined(); + }); + + it('upgrade', async () => { + const preceding = await getPrecedingUpgrade({ upgrade, dev }); + expect(preceding).toBeUndefined(); + }); + + it('init followed by upgrade', async () => { + const preceding = await getPrecedingUpgrade({ init, upgrade, dev }); + expect(preceding).toBeUndefined(); + }); + + it('both init and upgrade with intervening dev', async () => { + const secondUpgrade = { + body: { eventType: 'upgrade', eventId: 'secondUpgrade' }, + timestamp: 4, + }; + const preceding = await getPrecedingUpgrade({ init, dev, upgrade: secondUpgrade }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 4, + "eventType": "upgrade", + "eventId": "secondUpgrade" + } + `); + }); + + it('both init and upgrade with non-intervening dev', async () => { + const earlyDev = { + body: { eventType: 'dev', eventId: 'earlyDev' }, + timestamp: -1, + }; + const preceding = await getPrecedingUpgrade({ dev: earlyDev, init, upgrade }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 2, + "eventType": "upgrade", + "eventId": "upgrade" + } + `); + }); + }); + + describe('intervening other events', () => { + it('build', async () => { + const preceding = await getPrecedingUpgrade({ upgrade, build }); + expect(preceding).toBeUndefined(); + }); + + it('error', async () => { + const preceding = await getPrecedingUpgrade({ upgrade, error }); + expect(preceding).toBeUndefined(); + }); + + it('version-update', async () => { + const preceding = await getPrecedingUpgrade({ upgrade, 'version-update': versionUpdate }); + expect(preceding).toMatchInlineSnapshot(` + { + "timestamp": 2, + "eventType": "upgrade", + "eventId": "upgrade" + } + `); + }); + }); +}); diff --git a/code/lib/telemetry/src/event-cache.ts b/code/lib/telemetry/src/event-cache.ts new file mode 100644 index 000000000000..4e511d3b644b --- /dev/null +++ b/code/lib/telemetry/src/event-cache.ts @@ -0,0 +1,61 @@ +import { cache } from '@storybook/core-common'; +import type { EventType } from './types'; + +interface UpgradeSummary { + timestamp: number; + eventType?: EventType; + eventId?: string; + sessionId?: string; +} + +let operation: Promise = Promise.resolve(); + +const setHelper = async (eventType: EventType, body: any) => { + const lastEvents = (await cache.get('lastEvents')) || {}; + lastEvents[eventType] = { body, timestamp: Date.now() }; + await cache.set('lastEvents', lastEvents); +}; + +export const set = async (eventType: EventType, body: any) => { + await operation; + operation = setHelper(eventType, body); + return operation; +}; + +export const get = async (eventType: EventType) => { + const lastEvents = await cache.get('lastEvents'); + return lastEvents?.[eventType]; +}; + +const upgradeFields = (event: any): UpgradeSummary => { + const { body, timestamp } = event; + return { + timestamp, + eventType: body?.eventType, + eventId: body?.eventId, + sessionId: body?.sessionId, + }; +}; + +const UPGRADE_EVENTS: EventType[] = ['init', 'upgrade']; +const RUN_EVENTS: EventType[] = ['build', 'dev', 'error']; + +const lastEvent = (lastEvents: Record, eventTypes: EventType[]) => { + const descendingEvents = eventTypes + .map((eventType) => lastEvents?.[eventType]) + .filter(Boolean) + .sort((a, b) => b.timestamp - a.timestamp); + return descendingEvents.length > 0 ? descendingEvents[0] : undefined; +}; + +export const getPrecedingUpgrade = async (events: any = undefined) => { + const lastEvents = events || (await cache.get('lastEvents')) || {}; + const lastUpgradeEvent = lastEvent(lastEvents, UPGRADE_EVENTS); + const lastRunEvent = lastEvent(lastEvents, RUN_EVENTS); + + if (!lastUpgradeEvent) return undefined; + + return !lastRunEvent?.timestamp || lastUpgradeEvent.timestamp > lastRunEvent.timestamp + ? upgradeFields(lastUpgradeEvent) + : undefined; +}; diff --git a/code/lib/telemetry/src/index.ts b/code/lib/telemetry/src/index.ts index d4ccab43aa1c..a1da9bc69531 100644 --- a/code/lib/telemetry/src/index.ts +++ b/code/lib/telemetry/src/index.ts @@ -11,6 +11,8 @@ export * from './types'; export { getStorybookCoreVersion } from './package-json'; +export { getPrecedingUpgrade } from './event-cache'; + export const telemetry = async ( eventType: EventType, payload: Payload = {}, diff --git a/code/lib/telemetry/src/telemetry.ts b/code/lib/telemetry/src/telemetry.ts index d8d895395b7e..a2cbe0e43917 100644 --- a/code/lib/telemetry/src/telemetry.ts +++ b/code/lib/telemetry/src/telemetry.ts @@ -3,6 +3,7 @@ import retry from 'fetch-retry'; import { nanoid } from 'nanoid'; import type { Options, TelemetryData } from './types'; import { getAnonymousProjectId } from './anonymous-id'; +import { set as saveToCache } from './event-cache'; const URL = process.env.STORYBOOK_TELEMETRY_URL || 'https://storybook.js.org/event-log'; @@ -24,7 +25,7 @@ export async function sendTelemetry( // the server actually gets the request and stores it anyway. // flatten the data before we send it - const { payload, metadata, ...rest } = data; + const { eventType, payload, metadata, ...rest } = data; const context = options.stripMetadata ? {} : { @@ -32,7 +33,7 @@ export async function sendTelemetry( inCI: Boolean(process.env.CI), }; const eventId = nanoid(); - const body = { ...rest, eventId, sessionId, metadata, payload, context }; + const body = { ...rest, eventType, eventId, sessionId, metadata, payload, context }; let request: Promise; try { @@ -48,13 +49,15 @@ export async function sendTelemetry( ? options.retryDelay : 1000), }); - tasks.push(request); + tasks.push(request); if (options.immediate) { await Promise.all(tasks); } else { await request; } + + await saveToCache(eventType, body); } catch (err) { // } finally {