feat(tree-explorer): add treeview telemetry and refactor to use a mor…

…e standardized format VSCODE-651 (#891) Co-authored-by: Alena Khineika <[email protected]>
mongodb-js · Dec 9, 2024 · 7879cf9 · 7879cf9
1 parent fabbc8c
commit 7879cf9
Show file tree

Hide file tree

Showing 11 changed files with 177 additions and 63 deletions.
diff --git a/src/commands/index.ts b/src/commands/index.ts
@@ -85,4 +85,6 @@ enum EXTENSION_COMMANDS {
   SHOW_EXPORT_TO_LANGUAGE_RESULT = 'mdb.showExportToLanguageResult',
 }
 
+export type ExtensionCommand = EXTENSION_COMMANDS;
+
 export default EXTENSION_COMMANDS;
diff --git a/src/documentSource.ts b/src/documentSource.ts
@@ -2,5 +2,5 @@ export enum DocumentSource {
   DOCUMENT_SOURCE_TREEVIEW = 'treeview',
   DOCUMENT_SOURCE_PLAYGROUND = 'playground',
   DOCUMENT_SOURCE_COLLECTIONVIEW = 'collectionview',
-  DOCUMENT_SOURCE_QUERY_WITH_COPILOT_CODELENS = 'query with copilot codelens',
+  DOCUMENT_SOURCE_CODELENS = 'codelens',
 }
diff --git a/src/editors/queryWithCopilotCodeLensProvider.ts b/src/editors/queryWithCopilotCodeLensProvider.ts
@@ -31,9 +31,11 @@ export class QueryWithCopilotCodeLensProvider
       prompt: 'Describe the query you would like to generate',
       placeHolder:
         'e.g. Find the document in sample_mflix.users with the name of Kayden Washington',
-      messagePrefix: '/query',
+      command: 'query',
       isNewChat: true,
-      source: DocumentSource.DOCUMENT_SOURCE_QUERY_WITH_COPILOT_CODELENS,
+      telemetry: {
+        source: DocumentSource.DOCUMENT_SOURCE_CODELENS,
+      },
     };
 
     return [

diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts
@@ -25,7 +25,7 @@ import {
 } from './explorer';
 import ExportToLanguageCodeLensProvider from './editors/exportToLanguageCodeLensProvider';
 import { type ExportToLanguageResult } from './types/playgroundType';
-import EXTENSION_COMMANDS from './commands';
+import type { ExtensionCommand } from './commands';
 import type FieldTreeItem from './explorer/fieldTreeItem';
 import type IndexListTreeItem from './explorer/indexListTreeItem';
 import { LanguageServerController } from './language';
@@ -40,21 +40,20 @@ import WebviewController from './views/webviewController';
 import { createIdFactory, generateId } from './utils/objectIdHelper';
 import { ConnectionStorage } from './storage/connectionStorage';
 import type StreamProcessorTreeItem from './explorer/streamProcessorTreeItem';
-import type {
-  ParticipantCommand,
-  RunParticipantCodeCommandArgs,
-} from './participant/participant';
+import type { RunParticipantCodeCommandArgs } from './participant/participant';
 import ParticipantController from './participant/participant';
 import type { OpenSchemaCommandArgs } from './participant/prompts/schema';
 import { QueryWithCopilotCodeLensProvider } from './editors/queryWithCopilotCodeLensProvider';
 import type {
   SendMessageToParticipantOptions,
   SendMessageToParticipantFromInputOptions,
+  ParticipantCommand,
 } from './participant/participantTypes';
 import {
   COPILOT_CHAT_EXTENSION_ID,
   COPILOT_EXTENSION_ID,
 } from './participant/constants';
+import EXTENSION_COMMANDS from './commands';
 
 // This class is the top-level controller for our extension.
 // Commands which the extensions handles are defined in the function `activate`.
@@ -402,7 +401,7 @@ export default class MDBExtensionController implements vscode.Disposable {
   };
 
   registerParticipantCommand = (
-    command: string,
+    command: ExtensionCommand,
     commandHandler: (...args: any[]) => Promise<boolean>
   ): void => {
     const commandHandlerWithTelemetry = (args: any[]): Promise<boolean> => {
@@ -420,7 +419,7 @@ export default class MDBExtensionController implements vscode.Disposable {
   };
 
   registerCommand = (
-    command: string,
+    command: ExtensionCommand,
     commandHandler: (...args: any[]) => Promise<boolean>
   ): void => {
     const commandHandlerWithTelemetry = (args: any[]): Promise<boolean> => {

diff --git a/src/participant/participant.ts b/src/participant/participant.ts
@@ -52,10 +52,13 @@ import { PromptHistory } from './prompts/promptHistory';
 import type {
   SendMessageToParticipantOptions,
   SendMessageToParticipantFromInputOptions,
+  ParticipantCommand,
+  ParticipantCommandType,
 } from './participantTypes';
 import { DEFAULT_EXPORT_TO_LANGUAGE_DRIVER_SYNTAX } from '../editors/exportToLanguageCodeLensProvider';
 import { EXPORT_TO_LANGUAGE_ALIASES } from '../editors/playgroundSelectionCodeActionProvider';
 import { CollectionTreeItem, DatabaseTreeItem } from '../explorer';
+import { DocumentSource } from '../documentSource';
 
 const log = createLogger('participant');
 
@@ -72,8 +75,6 @@ export type RunParticipantCodeCommandArgs = {
   runnableContent: string;
 };
 
-export type ParticipantCommand = '/query' | '/schema' | '/docs';
-
 const MAX_MARKDOWN_LIST_LENGTH = 10;
 
 export default class ParticipantController {
@@ -121,6 +122,7 @@ export default class ParticipantController {
       participantId: this._participant?.id,
     });
     this._participant.onDidReceiveFeedback(this.handleUserFeedback.bind(this));
+
     return this._participant;
   }
 
@@ -142,6 +144,8 @@ export default class ParticipantController {
       message,
       isNewChat = false,
       isPartialQuery = false,
+      telemetry,
+      command,
       ...otherOptions
     } = options;
 
@@ -151,10 +155,28 @@ export default class ParticipantController {
         'workbench.action.chat.clearHistory'
       );
     }
+    const commandPrefix = command ? `/${command} ` : '';
+    const query = `@MongoDB ${commandPrefix}${message}`;
+
+    if (telemetry) {
+      if (isNewChat) {
+        this._telemetryService.trackParticipantChatOpenedFromAction({
+          ...telemetry,
+          command,
+        });
+      }
+      if (!isPartialQuery) {
+        this._telemetryService.trackParticipantPromptSubmittedFromAction({
+          ...telemetry,
+          command: command ?? 'generic',
+          input_length: query.length,
+        });
+      }
+    }
 
     return await vscode.commands.executeCommand('workbench.action.chat.open', {
       ...otherOptions,
-      query: `@MongoDB ${message}`,
+      query,
       isPartialQuery,
     });
   }
@@ -163,27 +185,34 @@ export default class ParticipantController {
     options: SendMessageToParticipantFromInputOptions
   ): Promise<unknown> {
     const {
-      messagePrefix = '',
-      isNewChat = false,
-      isPartialQuery = false,
-      source,
+      isNewChat,
+      isPartialQuery,
+      telemetry,
+      command,
       ...inputBoxOptions
     } = options;
 
-    this._telemetryService.trackCopilotParticipantSubmittedFromInputBox({
-      source,
-    });
-
     const message = await vscode.window.showInputBox({
       ...inputBoxOptions,
     });
 
+    if (telemetry) {
+      this._telemetryService.trackParticipantInputBoxSubmitted({
+        ...telemetry,
+        input_length: message?.length,
+        dismissed: message === undefined,
+        command,
+      });
+    }
+
     if (message === undefined || message.trim() === '') {
       return Promise.resolve();
     }
 
     return this.sendMessageToParticipant({
-      message: `${messagePrefix ? `${messagePrefix} ` : ''}${message}`,
+      message,
+      telemetry,
+      command,
       isNewChat,
       isPartialQuery,
     });
@@ -198,13 +227,21 @@ export default class ParticipantController {
       await this.sendMessageToParticipant({
         message: `I want to ask questions about the \`${databaseName}\` database.`,
         isNewChat: true,
+        telemetry: {
+          source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW,
+          source_details: 'database',
+        },
       });
     } else if (treeItem instanceof CollectionTreeItem) {
       const { databaseName, collectionName } = treeItem;
 
       await this.sendMessageToParticipant({
         message: `I want to ask questions about the \`${databaseName}\` database's \`${collectionName}\` collection.`,
         isNewChat: true,
+        telemetry: {
+          source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW,
+          source_details: 'collection',
+        },
       });
     } else {
       throw new Error('Unsupported tree item type');
@@ -233,7 +270,7 @@ export default class ParticipantController {
           })
       ),
     });
-    this._telemetryService.trackCopilotParticipantPrompt(modelInput.stats);
+    this._telemetryService.trackParticipantPrompt(modelInput.stats);
 
     const modelResponse = await model.sendRequest(
       modelInput.messages,
@@ -413,7 +450,7 @@ export default class ParticipantController {
         stream,
       });
 
-    this._telemetryService.trackCopilotParticipantResponse({
+    this._telemetryService.trackParticipantResponse({
       command: 'generic',
       has_cta: false,
       found_namespace: false,
@@ -1380,7 +1417,7 @@ export default class ParticipantController {
       ],
     });
 
-    this._telemetryService.trackCopilotParticipantResponse({
+    this._telemetryService.trackParticipantResponse({
       command: 'schema',
       has_cta: true,
       found_namespace: true,
@@ -1491,7 +1528,7 @@ export default class ParticipantController {
         token,
       });
 
-    this._telemetryService.trackCopilotParticipantResponse({
+    this._telemetryService.trackParticipantResponse({
       command: 'query',
       has_cta: false,
       found_namespace: true,
@@ -1597,7 +1634,7 @@ export default class ParticipantController {
 
     this._streamGenericDocsLink(stream);
 
-    this._telemetryService.trackCopilotParticipantResponse({
+    this._telemetryService.trackParticipantResponse({
       command: 'docs/copilot',
       has_cta: true,
       found_namespace: false,
@@ -1677,7 +1714,7 @@ export default class ParticipantController {
         }
       }
 
-      this._telemetryService.trackCopilotParticipantResponse({
+      this._telemetryService.trackParticipantResponse({
         command: 'docs/chatbot',
         has_cta: !!docsResult.responseReferences,
         found_namespace: false,
@@ -1795,10 +1832,7 @@ export default class ParticipantController {
       return true;
     } catch (error) {
       const message = formatError(error).message;
-      this._telemetryService.trackCopilotParticipantError(
-        error,
-        'exportToPlayground'
-      );
+      this._telemetryService.trackParticipantError(error, 'exportToPlayground');
       void vscode.window.showErrorMessage(
         `An error occurred exporting to a playground: ${message}`
       );
@@ -1905,9 +1939,9 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i
           return await this.handleGenericRequest(...args);
       }
     } catch (error) {
-      this._telemetryService.trackCopilotParticipantError(
+      this._telemetryService.trackParticipantError(
         error,
-        request.command || 'generic'
+        (request.command as ParticipantCommandType) || 'generic'
       );
       // Re-throw other errors so they show up in the UI.
       throw error;
@@ -1956,7 +1990,7 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i
       'unhelpfulReason' in feedback
         ? (feedback.unhelpfulReason as string)
         : undefined;
-    this._telemetryService.trackCopilotParticipantFeedback({
+    this._telemetryService.trackParticipantFeedback({
       feedback: chatResultFeedbackKindToTelemetryValue(feedback.kind),
       reason: unhelpfulReason,
       response_type: (feedback.result as ChatResult)?.metadata.intent,

diff --git a/src/participant/participantTypes.ts b/src/participant/participantTypes.ts
@@ -1,14 +1,40 @@
 import type * as vscode from 'vscode';
 import type { DocumentSource } from '../documentSource';
 
+export type ParticipantCommandType = 'query' | 'schema' | 'docs';
+export type ParticipantCommand = `/${ParticipantCommandType}`;
+
+export type ParticipantRequestType = ParticipantCommandType | 'generic';
+
+export type ParticipantResponseType =
+  | 'query'
+  | 'schema'
+  | 'docs'
+  | 'docs/chatbot'
+  | 'docs/copilot'
+  | 'exportToPlayground'
+  | 'generic'
+  | 'emptyRequest'
+  | 'cancelledRequest'
+  | 'askToConnect'
+  | 'askForNamespace';
+
+type TelemetryMetadata = {
+  source: DocumentSource;
+  source_details?: 'database' | 'collection';
+};
+
+/** Based on options from Copilot's chat open command IChatViewOpenOptions */
 export type SendMessageToParticipantOptions = {
   message: string;
+  command?: ParticipantCommandType;
   isNewChat?: boolean;
   isPartialQuery?: boolean;
+  telemetry?: TelemetryMetadata;
 };
 
-export type SendMessageToParticipantFromInputOptions = {
-  messagePrefix?: string;
-  source?: DocumentSource;
-} & Omit<SendMessageToParticipantOptions, 'message'> &
+export type SendMessageToParticipantFromInputOptions = Pick<
+  SendMessageToParticipantOptions,
+  'isNewChat' | 'isPartialQuery' | 'command' | 'telemetry'
+> &
   vscode.InputBoxOptions;
diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts
@@ -5,6 +5,7 @@ import type {
   ParticipantPromptProperties,
 } from '../../telemetry/telemetryService';
 import { PromptHistory } from './promptHistory';
+import type { ParticipantCommandType } from '../participantTypes';
 
 export interface PromptArgsBase {
   request: {
@@ -175,7 +176,7 @@ export abstract class PromptBase<TArgs extends PromptArgsBase> {
       ),
       user_input_length: request.prompt.length,
       has_sample_documents: hasSampleDocs,
-      command: request.command || 'generic',
+      command: (request.command as ParticipantCommandType) || 'generic',
       history_size: context?.history.length || 0,
       internal_purpose: this.internalPurposeForTelemetry,
     };

diff --git a/src/participant/prompts/promptHistory.ts b/src/participant/prompts/promptHistory.ts
@@ -1,6 +1,7 @@
 import * as vscode from 'vscode';
 import { ParticipantErrorTypes } from '../participantErrorTypes';
-import type { ChatResult, ParticipantResponseType } from '../constants';
+import type { ChatResult } from '../constants';
+import type { ParticipantResponseType } from '../participantTypes';
 
 export class PromptHistory {
   private static _handleChatResponseTurn({