feat: add frontend context logging (#1115)

* feat: add frontend context logging

* add batch call

* udpate comment

* created_at

* move to const file

package.json

@@ -1,6 +1,6 @@
 {
     "name": "querybook",
-    "version": "3.15.5",
+    "version": "3.15.6",
     "description": "A Big Data Webapp",
     "private": true,
     "scripts": {

querybook/server/const/event_log.py

@@ -1,12 +1,19 @@
 from enum import Enum
+from typing import TypedDict
 
 
 class EventType(Enum):
     # an api request
-    API = "api"
+    API = "API"
     # websocket event
-    WEBSOCKET = "websocket"
+    WEBSOCKET = "WEBSOCKET"
     # a UI element gets viewed
-    VIEW = "view"
+    VIEW = "VIEW"
     # a UI element gets clicked
-    CLICK = "click"
+    CLICK = "CLICK"
+
+
+class FrontendEvent(TypedDict):
+    timestamp: int
+    event_data: dict
+    event_type: str  # value of EventType

querybook/server/datasources/__init__.py

@@ -13,6 +13,7 @@
 from . import utils
 from . import table_upload
 from . import tag
+from . import event_log
 
 # Flake8 :(
 admin
@@ -30,3 +31,4 @@
 utils
 table_upload
 tag
+event_log

querybook/server/datasources/event_log.py

@@ -0,0 +1,18 @@
+from app.datasource import register
+from const.event_log import EventType, FrontendEvent
+from lib.event_logger import event_logger
+
+
+@register("/event_log/", methods=["POST"], api_logging=False)
+def log_frontend_event(events: list[FrontendEvent]):
+    """Log a list of frontend events.
+
+    Args:
+        events (list[FrontendEvent]): a list of frontend events
+    """
+    for event in events:
+        event_logger.log(
+            event_type=EventType(event["type"]),
+            event_data=event["data"],
+            timestamp=event["timestamp"],
+        )

querybook/server/lib/event_logger/__init__.py

@@ -13,14 +13,13 @@ def __init__(self):
         logger_name = QuerybookSettings.EVENT_LOGGER_NAME
         self.logger = get_event_logger_class(logger_name)
 
-    def log(
-        self,
-        event_type: EventType,
-        event_data: dict,
-    ):
+    def log(self, event_type: EventType, event_data: dict, timestamp: int = None):
         try:
             self.logger.log(
-                uid=current_user.id, event_type=event_type, event_data=event_data
+                uid=current_user.id,
+                event_type=event_type,
+                event_data=event_data,
+                timestamp=timestamp,
             )
         except Exception as e:
             # catch any potential exceptions to avoid event logging

querybook/server/lib/event_logger/base_event_logger.py

@@ -75,13 +75,16 @@ def _should_log_api_request(self, route: str, method: str) -> bool:
         return True
 
     @abstractmethod
-    def log(self, uid: int, event_type: EventType, event_data: dict) -> None:
+    def log(
+        self, uid: int, event_type: EventType, event_data: dict, timestamp: int = None
+    ) -> None:
         """Log an event to some data store
 
         Args:
             uid (int): id of the user who performed the action
             event_type (EventType): action event type, e.g. CLICK, VIEW
             event_data (dict): addtional info of the event in JSON format.
+            timestamp (int): timestamp in milliseconds
         """
         raise NotImplementedError()
 

querybook/server/lib/event_logger/loggers/console_event_logger.py

@@ -17,8 +17,16 @@ class ConsoleEventLogger(BaseEventLogger):
     def logger_name(self) -> str:
         return "console"
 
-    def log(self, uid: int, event_type: EventType, event_data: dict):
-        now = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+    def log(
+        self, uid: int, event_type: EventType, event_data: dict, timestamp: int = None
+    ):
+        now = (
+            datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+            if timestamp is None
+            else datetime.utcfromtimestamp(timestamp / 1000).strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+        )
         event = f"created_at: {now}, uid={uid}, event_type={event_type}, event_data={event_data}"
 
         LOG.info(f"{COLOR_YELLOW}{self.__class__.__name__} - {event}{COLOR_RESET}")

querybook/server/lib/event_logger/loggers/db_event_logger.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 from const.event_log import EventType
 from lib.event_logger.base_event_logger import BaseEventLogger
 from models.event_log import EventLog
@@ -62,7 +64,19 @@ def _api_deny_list(self) -> list:
             },
         ]
 
-    def log(self, uid: int, event_type: EventType, event_data: dict):
+    def log(
+        self, uid: int, event_type: EventType, event_data: dict, timestamp: int = None
+    ):
+        created_at = (
+            datetime.utcfromtimestamp(timestamp / 1000)
+            if timestamp is not None
+            else None
+        )
         EventLog.create(
-            {"uid": uid, "event_type": event_type, "event_data": event_data}
+            {
+                "uid": uid,
+                "event_type": event_type,
+                "event_data": event_data,
+                "created_at": created_at,
+            }
         )

querybook/server/lib/event_logger/loggers/null_event_logger.py

@@ -9,8 +9,7 @@ class NullEventLogger(BaseEventLogger):
     def logger_name(self) -> str:
         return "null"
 
-    def log(self, uid: int, event_type: EventType, event_data: dict) -> None:
-        pass
-
-    def log_api_request(self, uid: int, route: str, method: str, params: dict) -> None:
+    def log(
+        self, uid: int, event_type: EventType, event_data: dict, timestamp: int
+    ) -> None:
         pass

querybook/webapp/components/ChangeLog/ChangeLog.tsx

@@ -1,7 +1,9 @@
 import * as React from 'react';
 import { useParams } from 'react-router-dom';
 
+import { ComponentType } from 'const/analytics';
 import { IChangeLogItem } from 'const/changeLog';
+import { useTrackView } from 'hooks/useTrackView';
 import localStore from 'lib/local-store';
 import { CHANGE_LOG_KEY, ChangeLogValue } from 'lib/local-store/const';
 import { sanitizeAndExtraMarkdown } from 'lib/markdown';
@@ -29,6 +31,7 @@ const ChangeLogMarkdown: React.FC<{ markdown: string }> = ({ markdown }) => {
 };
 
 export const ChangeLog: React.FunctionComponent = () => {
+    useTrackView(ComponentType.CHANGE_LOG);
     const { date: changeLogDate } = useParams();
     const [changeLogContent, setChangeLogContent] = React.useState<string[]>(
         []

querybook/webapp/components/EnvironmentAppSidebar/EntitySidebar.tsx

@@ -7,6 +7,8 @@ import { QueryEngineStatusButton } from 'components/QueryEngineStatusButton/Quer
 import { QueryExecutionButton } from 'components/QueryExecutionButton/QueryExecutionButton';
 import { SearchContainer } from 'components/Search/SearchContainer';
 import { UserMenu } from 'components/UserMenu/UserMenu';
+import { ComponentType, ElementType } from 'const/analytics';
+import { trackClick } from 'lib/analytics';
 import { queryMetastoresSelector } from 'redux/dataSources/selector';
 import { currentEnvironmentSelector } from 'redux/environment/selector';
 import { IconButton } from 'ui/Button/IconButton';
@@ -41,6 +43,14 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                                             location.pathname ===
                                             `/${environment.name}/`
                                         }
+                                        onClick={() =>
+                                            trackClick({
+                                                component:
+                                                    ComponentType.LEFT_SIDEBAR,
+                                                element:
+                                                    ElementType.HOME_BUTTON,
+                                            })
+                                        }
                                     />
                                 </Link>
                                 <SearchContainer />
@@ -53,6 +63,14 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                                             `/${environment.name}/adhoc/`
                                         )}
                                         title="Adhoc"
+                                        onClick={() =>
+                                            trackClick({
+                                                component:
+                                                    ComponentType.LEFT_SIDEBAR,
+                                                element:
+                                                    ElementType.ADHOC_BUTTON,
+                                            })
+                                        }
                                     />
                                 </Link>
                                 <Link
@@ -66,6 +84,14 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                                             `/${environment.name}/doc_schedules/`
                                         )}
                                         title="Scheds"
+                                        onClick={() =>
+                                            trackClick({
+                                                component:
+                                                    ComponentType.LEFT_SIDEBAR,
+                                                element:
+                                                    ElementType.SCHEDS_BUTTON,
+                                            })
+                                        }
                                     />
                                 </Link>
                             </>
@@ -79,6 +105,10 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                         tooltipPos="right"
                         active={selectedEntity === 'datadoc'}
                         onClick={() => {
+                            trackClick({
+                                component: ComponentType.LEFT_SIDEBAR,
+                                element: ElementType.DOCS_BUTTON,
+                            });
                             onSelectEntity('datadoc');
                         }}
                         title="Docs"
@@ -89,7 +119,13 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                             tooltip="Tables"
                             tooltipPos="right"
                             active={selectedEntity === 'table'}
-                            onClick={() => onSelectEntity('table')}
+                            onClick={() => {
+                                trackClick({
+                                    component: ComponentType.LEFT_SIDEBAR,
+                                    element: ElementType.TABLES_BUTTON,
+                                });
+                                onSelectEntity('table');
+                            }}
                             title="Tables"
                         />
                     ) : null}
@@ -98,11 +134,23 @@ export const EntitySidebar: React.FunctionComponent<IEntitySidebarProps> =
                         tooltip="Snippets"
                         tooltipPos="right"
                         active={selectedEntity === 'snippet'}
-                        onClick={() => onSelectEntity('snippet')}
+                        onClick={() => {
+                            trackClick({
+                                component: ComponentType.LEFT_SIDEBAR,
+                                element: ElementType.SNIPS_BUTTON,
+                            });
+                            onSelectEntity('snippet');
+                        }}
                         title="Snips"
                     />
                     <QueryExecutionButton
-                        onClick={() => onSelectEntity('execution')}
+                        onClick={() => {
+                            trackClick({
+                                component: ComponentType.LEFT_SIDEBAR,
+                                element: ElementType.EXECS_BUTTON,
+                            });
+                            onSelectEntity('execution');
+                        }}
                         active={selectedEntity === 'execution'}
                     />
                 </div>

querybook/webapp/components/InfoMenuButton/InfoMenuButton.tsx

@@ -1,5 +1,7 @@
 import * as React from 'react';
 
+import { ComponentType, ElementType } from 'const/analytics';
+import { trackClick } from 'lib/analytics';
 import localStore from 'lib/local-store';
 import { CHANGE_LOG_KEY, ChangeLogValue } from 'lib/local-store/const';
 import { navigateWithinEnv } from 'lib/utils/query-string';
@@ -105,7 +107,13 @@ export const InfoMenuButton: React.FunctionComponent = () => {
         <div className="InfoMenuButton">
             <IconButton
                 className="InfoMenuButton-button"
-                onClick={() => setShowPanel(true)}
+                onClick={() => {
+                    trackClick({
+                        component: ComponentType.LEFT_SIDEBAR,
+                        element: ElementType.HELP_BUTTON,
+                    });
+                    setShowPanel(true);
+                }}
                 ref={buttonRef}
                 icon={'HelpCircle'}
                 tooltip={'Logs, Tips, Shortcuts, & FAQs'}

querybook/webapp/components/Landing/Landing.tsx

@@ -3,8 +3,10 @@ import React from 'react';
 import { useDispatch } from 'react-redux';
 
 import { QuerybookSidebarUIGuide } from 'components/UIGuide/QuerybookSidebarUIGuide';
+import { ComponentType } from 'const/analytics';
 import { useShallowSelector } from 'hooks/redux/useShallowSelector';
 import { useBrowserTitle } from 'hooks/useBrowserTitle';
+import { useTrackView } from 'hooks/useTrackView';
 import { titleize } from 'lib/utils';
 import { navigateWithinEnv } from 'lib/utils/query-string';
 import { fetchDataDocs } from 'redux/dataDoc/action';
@@ -115,6 +117,7 @@ const DefaultLanding: React.FC = ({ children }) => {
 };
 
 const Landing: React.FC = () => {
+    useTrackView(ComponentType.LANDING_PAGE);
     useBrowserTitle();
 
     const customLandingConfig = window.CUSTOM_LANDING_PAGE;

querybook/webapp/components/QueryEngineStatusButton/QueryEngineStatusButton.tsx

@@ -9,12 +9,14 @@ import React, {
 import { useDispatch, useSelector } from 'react-redux';
 
 import { QueryEngineStatusViewer } from 'components/QueryEngineStatusViewer/QueryEngineStatusViewer';
+import { ComponentType, ElementType } from 'const/analytics';
 import { IQueryEngine, QueryEngineStatus } from 'const/queryEngine';
 import {
     queryEngineStatusToIconStatus,
     queryEngineStatusToMessage,
 } from 'const/queryStatusIcon';
 import { TooltipDirection } from 'const/tooltip';
+import { trackClick } from 'lib/analytics';
 import { capitalize, titleize } from 'lib/utils';
 import { fetchAllSystemStatus } from 'redux/queryEngine/action';
 import {
@@ -221,7 +223,13 @@ export const QueryEngineStatusButton: React.FC<IProps> = ({
             >
                 <IconButton
                     className="query-engine-status-button"
-                    onClick={() => setShowPanel(true)}
+                    onClick={() => {
+                        trackClick({
+                            component: ComponentType.LEFT_SIDEBAR,
+                            element: ElementType.STATUS_BUTTON,
+                        });
+                        setShowPanel(true);
+                    }}
                     ref={buttonRef}
                     icon={'Activity'}
                     tooltip={`Summary: ${queryEngineStatusToMessage[overallWorstQueryEngineStatus]}. Click to see details.`}

querybook/webapp/components/Search/SearchContainer.tsx

@@ -1,12 +1,18 @@
 import React from 'react';
 
+import { ComponentType, ElementType } from 'const/analytics';
+import { trackClick } from 'lib/analytics';
 import { navigateWithinEnv } from 'lib/utils/query-string';
 import { IconButton } from 'ui/Button/IconButton';
 
 import './SearchContainer.scss';
 
 export const SearchContainer: React.FC = () => {
     const navigateToSearch = React.useCallback(() => {
+        trackClick({
+            component: ComponentType.LEFT_SIDEBAR,
+            element: ElementType.ADHOC_BUTTON,
+        });
         navigateWithinEnv('/search/', { isModal: true });
     }, []);