From fce31397957d25d682a84ab0d115498bd95ad57e Mon Sep 17 00:00:00 2001 From: Samik Roy Date: Mon, 7 Jun 2021 20:54:37 +0530 Subject: [PATCH 1/6] Update WorkspaceUsage.json Removed second to minute conversion as the unit is set to 24, which indicates seconds. --- Workbooks/WorkspaceUsage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Workbooks/WorkspaceUsage.json b/Workbooks/WorkspaceUsage.json index 49e4e11e261..7168d338b47 100644 --- a/Workbooks/WorkspaceUsage.json +++ b/Workbooks/WorkspaceUsage.json @@ -2852,7 +2852,8 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", + "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(), + ,2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", "size": 0, "timeContext": { "durationMs": 2592000000 From 1bf23342ad57cac3525f29886e7242085d21028a Mon Sep 17 00:00:00 2001 From: Samik Roy Date: Mon, 7 Jun 2021 21:08:51 +0530 Subject: [PATCH 2/6] Update WorkspaceUsage.json --- Workbooks/WorkspaceUsage.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Workbooks/WorkspaceUsage.json b/Workbooks/WorkspaceUsage.json index 7168d338b47..49e4e11e261 100644 --- a/Workbooks/WorkspaceUsage.json +++ b/Workbooks/WorkspaceUsage.json @@ -2852,8 +2852,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(), - ,2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", + "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", "size": 0, "timeContext": { "durationMs": 2592000000 From c3cac31cbb1675b0e2414158854d1392aa4c0245 Mon Sep 17 00:00:00 2001 From: Samik Roy Date: Mon, 7 Jun 2021 21:12:04 +0530 Subject: [PATCH 3/6] Update WorkspaceUsage.json Removing second to minute conversion (/60) as selected unit is second (unit : 24) --- Workbooks/WorkspaceUsage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workbooks/WorkspaceUsage.json b/Workbooks/WorkspaceUsage.json index 49e4e11e261..c5e347578f2 100644 --- a/Workbooks/WorkspaceUsage.json +++ b/Workbooks/WorkspaceUsage.json @@ -2852,7 +2852,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))/60 ),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", + "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", "size": 0, "timeContext": { "durationMs": 2592000000 From 3154db76cc8421b9cf6f0aef4ce9e85745d739ec Mon Sep 17 00:00:00 2001 From: Samik Roy Date: Tue, 8 Jun 2021 18:28:27 +0530 Subject: [PATCH 4/6] Updated for Workspace Usage workbook Updated for Workspace Usage workbook. --- Workbooks/WorkbooksMetadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workbooks/WorkbooksMetadata.json b/Workbooks/WorkbooksMetadata.json index 2a1a6996d37..f91276e1a34 100644 --- a/Workbooks/WorkbooksMetadata.json +++ b/Workbooks/WorkbooksMetadata.json @@ -1319,7 +1319,7 @@ "dataTypesDependencies": [], "dataConnectorsDependencies": [], "previewImagesFileNames": [ "WorkspaceUsageBlack.png", "WorkspaceUsageWhite.png"], - "version": "1.1", + "version": "1.2", "title": "Workspace Usage Report", "templateRelativePath": "WorkspaceUsage.json", "subtitle": "", From 8b135f6125922f2d782b94309170d7b89588df45 Mon Sep 17 00:00:00 2001 From: Samik Roy Date: Thu, 10 Jun 2021 22:05:15 +0530 Subject: [PATCH 5/6] Update WorkspaceUsage.json --- Workbooks/WorkspaceUsage.json | 44 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Workbooks/WorkspaceUsage.json b/Workbooks/WorkspaceUsage.json index c5e347578f2..c40c4057bf7 100644 --- a/Workbooks/WorkspaceUsage.json +++ b/Workbooks/WorkspaceUsage.json @@ -358,7 +358,7 @@ { "type": 1, "content": { - "json": "## Help File\r\n\r\nMore details in the Wiki: https://github.com/CliveW-MSFT/KQLpublic/wiki/Workbook-Usage\r\n\r\n## Usage\r\n\t- Please select your Subscription and Workspace\r\n\t- Time Range: is the time you wish to query back to. i.e 7days from now, into the past.\r\n\t- Help is available in various parts of this Workbook.\r\n\t- Select the Detail Level - is a toggle to reduce the page load time of certain queries.\r\n## Categories\r\n\t- Azure Monitor Logs (Workspace)\r\n\t- Azure Sentinel\r\n\t- Azure Security Center\r\n## Solutions\r\n|Solution|Description|\r\n|---|---|\r\n| Workspace Infomation | info about the workspace, usage and statistics|\r\n|Latency | Which Tables or machines have latency issues, average, minimun and maximum values|\r\n|Costs Analysis| Looking again at the tables in the [Workspace Info] tab, but also you can select a Price of your choosing (£,$). Sub menus have been added to aid drill-down to sepfic products or data|\r\n|Azure Sentinel | specifc data about Sentinel|\r\n| Regular Checks | Daily, Weekly and Monthly suggested checks you can look at.|\r\n\r\n## Data Sources\r\n\t- Azure Resource Graph (ARG) and Various tables within Azure Monitor Logs (Workspace).\r\n\r\n## Anomoly Detection\r\nSome reports now show anomoly detection, this explains the Score: https://docs.microsoft.com/en-us/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n\r\nAnomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly." + "json": "## Help File\r\n\r\nMore details in the Wiki: https://github.com/CliveW-MSFT/KQLpublic/wiki/Workbook-Usage\r\n\r\n## Usage\r\n\t- Please select your Subscription and Workspace\r\n\t- Time Range: is the time you wish to query back to. i.e 7days from now, into the past.\r\n\t- Help is available in various parts of this Workbook.\r\n\t- Select the Detail Level - is a toggle to reduce the page load time of certain queries.\r\n## Categories\r\n\t- Azure Monitor Logs (Workspace)\r\n\t- Azure Sentinel\r\n\t- Azure Security Center\r\n## Solutions\r\n|Solution|Description|\r\n|---|---|\r\n| Workspace Infomation | info about the workspace, usage and statistics|\r\n|Latency | Which Tables or machines have latency issues, average, minimun and maximum values|\r\n|Costs Analysis| Looking again at the tables in the [Workspace Info] tab, but also you can select a Price of your choosing (£,$). Sub menus have been added to aid drill-down to sepfic products or data|\r\n|Azure Sentinel | specifc data about Sentinel|\r\n| Regular Checks | Daily, Weekly and Monthly suggested checks you can look at.|\r\n\r\n## Data Sources\r\n\t- Azure Resource Graph (ARG) and Various tables within Azure Monitor Logs (Workspace).\r\n\r\n## Anomoly Detection\r\nSome reports now show anomoly detection, this explains the Score: https://docs.microsoft.com/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n\r\nAnomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly." }, "conditionalVisibilities": [ { @@ -393,7 +393,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "Operation | where OperationCategory == 'Data Collection Status'\r\n\r\n// https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage#troubleshooting-why-log-analytics-is-no-longer-collecting-data", + "query": "Operation | where OperationCategory == 'Data Collection Status'\r\n\r\n// https://docs.microsoft.com/azure/azure-monitor/platform/manage-cost-storage#troubleshooting-why-log-analytics-is-no-longer-collecting-data", "size": 4, "title": "Troubleshoot Workspace: Data Collection Issue", "timeContext": { @@ -424,7 +424,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "Usage | where IsBillable | summarize DataGB = sum(Quantity / 1000.) //| where DataGB > 50\r\n\r\n// https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage#create-an-alert-when-data-collection-is-high", + "query": "Usage | where IsBillable | summarize DataGB = sum(Quantity / 1000.) //| where DataGB > 50\r\n\r\n// https://docs.microsoft.com/azure/azure-monitor/platform/manage-cost-storage#create-an-alert-when-data-collection-is-high", "size": 4, "title": "Troubleshoot Workspace: Data sent today", "timeContext": { @@ -487,7 +487,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//https://docs.microsoft.com/en-us/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation \r\n| summarize count() by Level", + "query": "//https://docs.microsoft.com/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation \r\n| summarize count() by Level", "size": 4, "title": "Troubleshoot Workspace: Ingestion Operations by type", "timeContext": { @@ -520,7 +520,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//https://docs.microsoft.com/en-us/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation | where Category == \"Ingestion\" | where Operation == \"Ingestion rate\" | where Level == \"Warning\"", + "query": "//https://docs.microsoft.com/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation | where Category == \"Ingestion\" | where Operation == \"Ingestion rate\" | where Level == \"Warning\"", "size": 4, "title": "Troubleshoot Workspace: warning alert when the ingestion volume rate has reached 80% of the limit", "timeContext": { @@ -553,7 +553,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//https://docs.microsoft.com/en-us/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation | where Category == \"Ingestion\" | where Operation == \"Data Collection\" | where Level == \"Warning\"", + "query": "//https://docs.microsoft.com/azure/azure-monitor/platform/monitor-workspace\r\n_LogOperation | where Category == \"Ingestion\" | where Operation == \"Data Collection\" | where Level == \"Warning\"", "size": 4, "title": "Troubleshoot Workspace: warning alert when the data collection has reached the daily limit", "timeContext": { @@ -1600,7 +1600,7 @@ { "type": 1, "content": { - "json": "You need to enable this for Workspace Query auditing \r\nhttps://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit", + "json": "You need to enable this for Workspace Query auditing \r\nhttps://docs.microsoft.com/azure/azure-monitor/log-query/query-audit", "style": "warning" }, "name": "text - 12" @@ -1679,7 +1679,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// needs https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| extend code_ = case(ResponseCode == 200,\"200 OK\", ResponseCode == 400,\"400 Bad Request\", ResponseCode == 401,\"401 Unauthorized\",ResponseCode == 503,\"503 Service Unavailable\",ResponseCode == 504,\"504 Gateway Timeout\",\r\n//else\r\nstrcat(\"Unknown or undefined code: \", ResponseCode))\r\n| summarize count() by ResponseCode, code_\r\n| order by count_ desc\r\n", + "query": "// needs https://docs.microsoft.com/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| extend code_ = case(ResponseCode == 200,\"200 OK\", ResponseCode == 400,\"400 Bad Request\", ResponseCode == 401,\"401 Unauthorized\",ResponseCode == 503,\"503 Service Unavailable\",ResponseCode == 504,\"504 Gateway Timeout\",\r\n//else\r\nstrcat(\"Unknown or undefined code: \", ResponseCode))\r\n| summarize count() by ResponseCode, code_\r\n| order by count_ desc\r\n", "size": 1, "title": "LAQuery Diagnostics: count by Status", "timeContext": { @@ -1705,7 +1705,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// needs https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| summarize count() by RequestClientApp\r\n| order by count_ desc\r\n", + "query": "// needs https://docs.microsoft.com/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| summarize count() by RequestClientApp\r\n| order by count_ desc\r\n", "size": 1, "title": "LAQuery Diagnostics: count by product ", "timeContext": { @@ -1731,7 +1731,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// needs https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| summarize dcount(QueryText) by AADEmail\r\n| where isnotempty(AADEmail)\r\n| order by dcount_QueryText desc\r\n", + "query": "// needs https://docs.microsoft.com/azure/azure-monitor/log-query/query-audit\r\nLAQueryLogs\r\n| summarize dcount(QueryText) by AADEmail\r\n| where isnotempty(AADEmail)\r\n| order by dcount_QueryText desc\r\n", "size": 1, "title": "LAQuery Diagnostics: count by User", "timeContext": { @@ -2831,7 +2831,7 @@ { "type": 1, "content": { - "json": "In this report we measure the latency of a specific Table by comparing the result of the ingestion_time() function to the TimeGenerated property. \r\n
\r\n\r\nSource: https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-ingestion-time", + "json": "In this report we measure the latency of a specific Table by comparing the result of the ingestion_time() function to the TimeGenerated property. \r\n
\r\n\r\nSource: https://docs.microsoft.com/azure/azure-monitor/platform/data-ingestion-time", "style": "info" }, "conditionalVisibilities": [ @@ -2852,7 +2852,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/en-us/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/en-us/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/en-us/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", + "query": "//\r\n// Add enrichment information about selected Tables\r\n//\r\nlet enrich = datatable (TableName:string,Information:string,link:string)\r\n [\r\n \"Usage\",\"This is common Usage data in all Workspaces\",\"https://docs.microsoft.com/azure/azure-monitor/platform/manage-cost-storage\",\r\n \"Operation\",\"This is common Operations data in all Workspaces\",\"\",\r\n \"OfficeActivity\",\"This is a PUSH data source, typically <30mins (max ~1day)\",\"https://docs.microsoft.com/office/office-365-management-api/troubleshooting-the-office-365-management-activity-api#what-is-the-maximum-time-i-will-have-to-wait-before-a-notification-is-sent-about-a-given-office-365-event\",\r\n \"Perf\",\"Operational Data source, this can have limited value to Azure Sentinel\",\"https://docs.microsoft.com/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Event\",\"Windows Server Event Logs\",\"https://docs.microsoft.com/azure/azure-monitor/platform/agent-data-sources\",\r\n \"Syslog\", \"Syslog messages using Common Event Format (CEF) streamed from variety of security solutions.Learn more\",\"https://docs.microsoft.com/azure/sentinel/connect-data-sources\",\r\n \"SecurityAlert\",\"Security Alerts from various sources\",\"https://docs.microsoft.com/azure/sentinel/connect-data-sources\",\r\n \"Alert\",\"Alerts from Azure Monitor sources\",\"\",\r\n \"AWSCloudTrail\",\"Azure Sentinel: AWS Cloud trail connector\",\"\",\r\n \"Anomalies\",\"Azure Sentinel: This table contains anomalies generated by the active Anomaly analytics rules in Azure Sentinel.\",\"\",\r\n \"AADNonInteractiveUserSignInLogs\",\"Non-interactive Azure Active Directory sign-in logs from user\",\"\",\r\n \"SecurityEvent\",\"Data from Azure Security Center or Azure Sentinel\",\"https://docs.microsoft.com/azure/sentinel/connect-data-sources\",\r\n \"CommonSecurityLog\",\"CEF data for Azure Sentinel, from multiple vendors\",\"https://docs.microsoft.com/azure/sentinel/connect-data-sources\",\r\n \"SecurityIncident\",\"Incident events from Azure Sentinel, also see SecurityAlert\",\"\",\r\n \"Heartbeat\", \"MMA info from Log Analytics agents\",\"\",\r\n \"LAQueryLogs\",\"Auditing of the workspace has been set\",\"https://docs.microsoft.com/azure/azure-monitor/log-query/query-audit\",\r\n \"VMConnection\",\"Azure Monitor for VMs or Service Map\",\"https://docs.microsoft.com/azure/azure-monitor/reference/tables/vmconnection\",\r\n \"KubeServices\",\"AKS data sources\",\"\",\r\n \"Update\",\"Patching info\",\"\", \r\n \"ThreatIntelligenceIndicator\",\"Azure Sentinel TI data\",\"\", \r\n \"AuditLogs\",\"Azure AAD audit information\",\"\", \r\n \"SigninLogs\",\"Azure AAD Signin information\",\"\", \r\n \"AzureActivity\",\"Azure Activity Logs\",\"\",\r\n \"HuntingBookmark\",\"Azure Sentinel - Hunting book mark data\",\"\", \r\n \"UserPeerAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"UserAccessAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"IdentityInfo\",\"Azure Sentinel - UEBA\",\"\", \r\n \"BehaviorAnalytics\",\"Azure Sentinel - UEBA\",\"\", \r\n \"DeviceEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Multiple event types, including events triggered by security controls such as Windows Defender Antivirus and exploit protection.\",\"\",\r\n \"DeviceFileEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains File creation, modification, and other file system events.\",\"\", \r\n \"DeviceImageLoadEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains DLL loading events\",\"\", \r\n \"DeviceInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Machine information, including OS information.\",\"\", \r\n \"DeviceLogonEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Sign-ins and other authentication events\",\"\", \r\n \"DeviceNetworkEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network connection and related events.\",\"\", \r\n \"DeviceNetworkInfo\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Network properties of machines, including adapters, IP and MAC addresses, as well as connected networks and domains.\",\"\", \r\n \"DeviceProcessEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Process creation and related events.\",\"\", \r\n \"DeviceRegistryEvents\",\"Azure Sentinel: This table is part of Microsoft Defender for Endpoints with Azure Sentinel. This table contains Creation and modification of registry entries.\",\"\", \r\n \"DnsInventory\",\"DNS Analytics (Preview)\",\"\", \r\n \"W3CIISLog\",\"IIS log data from Azure Monitor Logs\",\"\"\r\n ]\r\n; \r\n// Gather and calculate Latency information\r\nunion withsource = TableName1 *\r\n| summarize \r\n ['average E2E IngestionLatency'] = round(avg(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n , ['minimun E2E IngestionLatency'] = round(min(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2) \r\n , ['maximum E2E IngestionLatency'] = round(max(todouble(datetime_diff(\"Second\",ingestion_time(),TimeGenerated))),2)\r\n by TableName = TableName1\r\n| sort by ['average E2E IngestionLatency'] desc\r\n// join to enrichment and remove unwanted columns \r\n| join kind= fullouter (enrich) on TableName\r\n| where isnotempty(TableName)\r\n| project-away TableName1\r\n", "size": 0, "timeContext": { "durationMs": 2592000000 @@ -3425,7 +3425,7 @@ { "type": 1, "content": { - "json": "### Workspace Pricing\r\n\r\nPrice Parameter: Please provide an estimated cost so that we can get a value in the \"Estimated Table Price\" column, in the [Workspace Info] tab. \r\ni.e If your Log Analytics PAYG is £2.00 per GB enter 2.00. You can enter an interger (without a currency symbol), in format 1.0, 2.0, 3.1 etc... \r\nIf you want to see the combined Log Analytics + Azure Sentinel estimated cost please use both PAYG prices i.e. 2.00 + 2.00 = 4.00. Please see the licence pages for both products.\r\n\r\nBase value on 7th July 2020, using an example of Azure Sentinel ($2.46 US Central PAYG) + Log Analytics ($2.76 US Central PAYG) \r\n\r\n\r\nNote: No capacity reservation is taken into account.\r\n\r\n\r\n### Anomoly Detection\r\nSome reports now show anomoly detection, this explains the Score:\r\nhttps://docs.microsoft.com/en-us/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n\r\nAnomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly.", + "json": "### Workspace Pricing\r\n\r\nPrice Parameter: Please provide an estimated cost so that we can get a value in the \"Estimated Table Price\" column, in the [Workspace Info] tab. \r\ni.e If your Log Analytics PAYG is £2.00 per GB enter 2.00. You can enter an interger (without a currency symbol), in format 1.0, 2.0, 3.1 etc... \r\nIf you want to see the combined Log Analytics + Azure Sentinel estimated cost please use both PAYG prices i.e. 2.00 + 2.00 = 4.00. Please see the licence pages for both products.\r\n\r\nBase value on 7th July 2020, using an example of Azure Sentinel ($2.46 US Central PAYG) + Log Analytics ($2.76 US Central PAYG) \r\n\r\n\r\nNote: No capacity reservation is taken into account.\r\n\r\n\r\n### Anomoly Detection\r\nSome reports now show anomoly detection, this explains the Score:\r\nhttps://docs.microsoft.com/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n\r\nAnomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly.", "style": "info" }, "customWidth": "48", @@ -3446,7 +3446,7 @@ { "type": 1, "content": { - "json": "### Azure Security Center \r\n\r\nASC allows for 500MB/day (0.5GBytes) of *free* data to be sent by each Computer. Please see the licencing link at the bottom of this text box.\r\nYou have two variables\r\n1. The total allowed: < number of computers> * 0.5GB (this is a pooled metric). \t\r\n\te.g 4 computers * 0.5GB = 2.0GB, the maximum value which is allowed to be sent for free. Any data over this number will have the standard (Azure Sentinel + Log Analytics) charge applied. \r\n2. The actual data sent by each computer\r\n\te.g 4 * computers * 0.1GB = 0.4GB, which is allowed for free.\r\n\r\nTo fully calculate the Azure Sentinel average GBytes per day, if ASC data is present, we need to take #1 from the total or #2. This is shown as the \"Revised GiB Number\". \r\n\r\n### Note: Licencing is subject to change, you must fully investigate and understand from the current documentation the implications.\r\nhttps://azure.microsoft.com/en-us/pricing/details/security-center/ correct as of 17th September 2020. \r\nRESOURCE TYPE\tFREE TIER\tSTANDARD TIER (a.k.a \"Azure Defender ON\")\r\nVirtual Machine\tFree\t$0.02/Server/Hour\r\nIncluded data - 500 MB/day", + "json": "### Azure Security Center \r\n\r\nASC allows for 500MB/day (0.5GBytes) of *free* data to be sent by each Computer. Please see the licencing link at the bottom of this text box.\r\nYou have two variables\r\n1. The total allowed: < number of computers> * 0.5GB (this is a pooled metric). \t\r\n\te.g 4 computers * 0.5GB = 2.0GB, the maximum value which is allowed to be sent for free. Any data over this number will have the standard (Azure Sentinel + Log Analytics) charge applied. \r\n2. The actual data sent by each computer\r\n\te.g 4 * computers * 0.1GB = 0.4GB, which is allowed for free.\r\n\r\nTo fully calculate the Azure Sentinel average GBytes per day, if ASC data is present, we need to take #1 from the total or #2. This is shown as the \"Revised GiB Number\". \r\n\r\n### Note: Licencing is subject to change, you must fully investigate and understand from the current documentation the implications.\r\nhttps://azure.microsoft.com/pricing/details/security-center/ correct as of 17th September 2020. \r\nRESOURCE TYPE\tFREE TIER\tSTANDARD TIER (a.k.a \"Azure Defender ON\")\r\nVirtual Machine\tFree\t$0.02/Server/Hour\r\nIncluded data - 500 MB/day", "style": "info" }, "customWidth": "50", @@ -3685,7 +3685,7 @@ { "type": 1, "content": { - "json": "### Capacity Reservation Insight\r\nThis report, shows the Average GB/day (based on the TimeRange parameter), which is rounded up to the nearest whole number, this is the number the Azure Pricing Calculator requires. https://azure.microsoft.com/en-us/pricing/calculator/ for Azure Sentinel.\r\n- The Pay as You Go (PAYG_estimate) is the GB/day multiplied by the [Azure Sentinel Price] parameter, default is 2.0 (the price in $ for EAST US). This is location and currency neutral, so adjust to your own preferred value. This is the Daily estimate of the price.\r\n- The Pay as You Go (PAYG_estimate_mthly) is the Monthly (31day) estimate of the price.\r\n- CR_Estimate_Monthly is the (PAYG_estimate_mthly) minus any discount for the Capacity Reservation Tier that is *currently* set." + "json": "### Capacity Reservation Insight\r\nThis report, shows the Average GB/day (based on the TimeRange parameter), which is rounded up to the nearest whole number, this is the number the Azure Pricing Calculator requires. https://azure.microsoft.com/pricing/calculator/ for Azure Sentinel.\r\n- The Pay as You Go (PAYG_estimate) is the GB/day multiplied by the [Azure Sentinel Price] parameter, default is 2.0 (the price in $ for EAST US). This is location and currency neutral, so adjust to your own preferred value. This is the Daily estimate of the price.\r\n- The Pay as You Go (PAYG_estimate_mthly) is the Monthly (31day) estimate of the price.\r\n- CR_Estimate_Monthly is the (PAYG_estimate_mthly) minus any discount for the Capacity Reservation Tier that is *currently* set." }, "name": "text - 5 - Copy" }, @@ -3762,7 +3762,7 @@ { "type": 1, "content": { - "json": "### Capacity Reservation for your Average GB/day value? \r\nThis report helps you decide:\r\n- A green circle in a Status column 🟢 indicates your are on the Optimal setting, based on your average data ingestion (GB/day) and Pay As You Go/Capacity reservation settings.\r\n- A recommend value of \"0\" / zero, means that you are on a setting (SKU) such as Free, PerGB etc.. rather than a Capacity Reservation setting.\r\n- If the 'recommend' Azure Sentinel or Workspace setting doesn't match the 'current' setting you will get a Blue Cirlce in the Status columns 🔵.\r\n- Please read https://azure.microsoft.com/en-us/pricing/details/azure-sentinel/ and https://azure.microsoft.com/en-us/pricing/details/monitor/ before deciding. " + "json": "### Capacity Reservation for your Average GB/day value? \r\nThis report helps you decide:\r\n- A green circle in a Status column 🟢 indicates your are on the Optimal setting, based on your average data ingestion (GB/day) and Pay As You Go/Capacity reservation settings.\r\n- A recommend value of \"0\" / zero, means that you are on a setting (SKU) such as Free, PerGB etc.. rather than a Capacity Reservation setting.\r\n- If the 'recommend' Azure Sentinel or Workspace setting doesn't match the 'current' setting you will get a Blue Cirlce in the Status columns 🔵.\r\n- Please read https://azure.microsoft.com/pricing/details/azure-sentinel/ and https://azure.microsoft.com/pricing/details/monitor/ before deciding. " }, "name": "text - 5" }, @@ -3879,7 +3879,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// https://docs.microsoft.com/en-us/azure/azure-monitor/logs/manage-cost-storage#log-analytics-and-security-center\r\nlet ascTables = dynamic(['WindowsEvent', 'SecurityAlert', 'SecurityBaseline', 'SecurityBaselineSummary', 'SecurityDetection', 'SecurityEvent','WindowsFirewall', 'MaliciousIPCommunication', 'LinuxAuditLog', 'SysmonEvent', 'ProtectionStatus']);\r\nunion withsource = tt *\r\n| where TimeGenerated > ago(30d)\r\n| where _IsBillable == True\r\n// Calculate the daily GiB size per billable Table and unique Server count \r\n| summarize allGBytes =sum(_BilledSize)/(1024*1024*1024), ascGBytes=sumif(_BilledSize, Type in (ascTables))/(1024*1024*1024), ascComputerCount = dcountif(Computer,tt in (ascTables)) by bin(TimeGenerated, 1d)\r\n// Calculate daily GiB average, and average ASC server count. Each Server is allowed max 0.5GB each\r\n| summarize avg(allGBytes), avg(ascGBytes),ascComputers=round(avg(ascComputerCount),0),ascMaxGB = avg(ascComputerCount) * 0.5\r\n// If ASC sends less than the allowed max, revise the GB averge per day, else use the max allowed (all servers * 0.5)\r\n| extend revisedNumber = iif(avg_ascGBytes <= ascMaxGB, allMinusASCused = (avg_allGBytes - avg_ascGBytes),allMinusASCused = (avg_allGBytes - ascMaxGB) )\r\n| extend withinPool = iif(avg_ascGBytes >= ascMaxGB,\"Yes\",\"No\")\r\n| extend overBy = iif(round(avg_ascGBytes,2) - round(ascMaxGB,2) < 0 , toreal(0), round(avg_ascGBytes,2) - round(ascMaxGB,2) )\r\n| project ['Raw Average GiB'] = avg_allGBytes,\r\n ['ASC has used, GiB'] = strcat(round(avg_ascGBytes,1),\" GiB of \" ,round(ascMaxGB,1), \" GiB across \", ascComputers , \" Computers\"), \r\n ['Revised GiB Number']= round(revisedNumber,2),\r\n ['Help'] =\"Please toggle [Show Help] to Yes, for more information\", \r\n ['Sent more than allocation?'] = withinPool,\r\n ['Overage in GiB'] = overBy\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n", + "query": "// https://docs.microsoft.com/azure/azure-monitor/logs/manage-cost-storage#log-analytics-and-security-center\r\nlet ascTables = dynamic(['WindowsEvent', 'SecurityAlert', 'SecurityBaseline', 'SecurityBaselineSummary', 'SecurityDetection', 'SecurityEvent','WindowsFirewall', 'MaliciousIPCommunication', 'LinuxAuditLog', 'SysmonEvent', 'ProtectionStatus']);\r\nunion withsource = tt *\r\n| where TimeGenerated > ago(30d)\r\n| where _IsBillable == True\r\n// Calculate the daily GiB size per billable Table and unique Server count \r\n| summarize allGBytes =sum(_BilledSize)/(1024*1024*1024), ascGBytes=sumif(_BilledSize, Type in (ascTables))/(1024*1024*1024), ascComputerCount = dcountif(Computer,tt in (ascTables)) by bin(TimeGenerated, 1d)\r\n// Calculate daily GiB average, and average ASC server count. Each Server is allowed max 0.5GB each\r\n| summarize avg(allGBytes), avg(ascGBytes),ascComputers=round(avg(ascComputerCount),0),ascMaxGB = avg(ascComputerCount) * 0.5\r\n// If ASC sends less than the allowed max, revise the GB averge per day, else use the max allowed (all servers * 0.5)\r\n| extend revisedNumber = iif(avg_ascGBytes <= ascMaxGB, allMinusASCused = (avg_allGBytes - avg_ascGBytes),allMinusASCused = (avg_allGBytes - ascMaxGB) )\r\n| extend withinPool = iif(avg_ascGBytes >= ascMaxGB,\"Yes\",\"No\")\r\n| extend overBy = iif(round(avg_ascGBytes,2) - round(ascMaxGB,2) < 0 , toreal(0), round(avg_ascGBytes,2) - round(ascMaxGB,2) )\r\n| project ['Raw Average GiB'] = avg_allGBytes,\r\n ['ASC has used, GiB'] = strcat(round(avg_ascGBytes,1),\" GiB of \" ,round(ascMaxGB,1), \" GiB across \", ascComputers , \" Computers\"), \r\n ['Revised GiB Number']= round(revisedNumber,2),\r\n ['Help'] =\"Please toggle [Show Help] to Yes, for more information\", \r\n ['Sent more than allocation?'] = withinPool,\r\n ['Overage in GiB'] = overBy\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n", "size": 4, "aggregation": 3, "title": "Average GiB per day: past 30days only", @@ -4297,7 +4297,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// source: https://docs.microsoft.com/en-us/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet minimal = dynamic([1102,4624,4625,4657,4663,4688,4700,4702,4719,4720,4722,4723,4724,4727,4728,4732,4735,4737,4739,4740,4754,4755,\r\n4756,4767,4799,4825,4946,4948,4956,5024,5033,8001,8002,8003,8004,8005,8006,8007,8222]);\r\nSecurityEvent\r\n| where EventID in (minimal)\r\n| summarize Count = count(), BilledSize= sum(_BilledSize) by EventID, Activity\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize\r\n//https://docs.microsoft.com/en-us/azure/data-explorer/anomaly-detection#time-series-anomaly-detection", + "query": "//\r\n// source: https://docs.microsoft.com/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet minimal = dynamic([1102,4624,4625,4657,4663,4688,4700,4702,4719,4720,4722,4723,4724,4727,4728,4732,4735,4737,4739,4740,4754,4755,\r\n4756,4767,4799,4825,4946,4948,4956,5024,5033,8001,8002,8003,8004,8005,8006,8007,8222]);\r\nSecurityEvent\r\n| where EventID in (minimal)\r\n| summarize Count = count(), BilledSize= sum(_BilledSize) by EventID, Activity\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize\r\n//https://docs.microsoft.com/azure/data-explorer/anomaly-detection#time-series-anomaly-detection", "size": 1, "title": "ASC : \"minimal\" data set - {TimeRange:label}. anomaly-detection:{AnomolyTuning:label}", "timeContext": { @@ -4458,7 +4458,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// source: https://docs.microsoft.com/en-us/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet common = dynamic([1,299,300,324,340,403,404,410,411,412,413,431,500,501,1100,1102,1107,1108,4608,4610,4611,4614,4622,\r\n4624,4625,4634,4647,4648,4649,4657,4661,4662,4663,4665,4666,4667,4688,4670,4672,4673,4674,4675,4689,4697,\r\n4700,4702,4704,4705,4716,4717,4718,4719,4720,4722,4723,4724,4725,4726,4727,4728,4729,4733,4732,4735,4737,\r\n4738,4739,4740,4742,4744,4745,4746,4750,4751,4752,4754,4755,4756,4757,4760,4761,4762,4764,4767,4768,4771,\r\n4774,4778,4779,4781,4793,4797,4798,4799,4800,4801,4802,4803,4825,4826,4870,4886,4887,4888,4893,4898,4902,\r\n4904,4905,4907,4931,4932,4933,4946,4948,4956,4985,5024,5033,5059,5136,5137,5140,5145,5632,6144,6145,6272,\r\n6273,6278,6416,6423,6424,8001,8002,8003,8004,8005,8006,8007,8222,26401,30004]);\r\nSecurityEvent\r\n| where EventID in (common)\r\n| summarize Count = count(), BilledSize= sum(_BilledSize) by EventID, Activity\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize", + "query": "//\r\n// source: https://docs.microsoft.com/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet common = dynamic([1,299,300,324,340,403,404,410,411,412,413,431,500,501,1100,1102,1107,1108,4608,4610,4611,4614,4622,\r\n4624,4625,4634,4647,4648,4649,4657,4661,4662,4663,4665,4666,4667,4688,4670,4672,4673,4674,4675,4689,4697,\r\n4700,4702,4704,4705,4716,4717,4718,4719,4720,4722,4723,4724,4725,4726,4727,4728,4729,4733,4732,4735,4737,\r\n4738,4739,4740,4742,4744,4745,4746,4750,4751,4752,4754,4755,4756,4757,4760,4761,4762,4764,4767,4768,4771,\r\n4774,4778,4779,4781,4793,4797,4798,4799,4800,4801,4802,4803,4825,4826,4870,4886,4887,4888,4893,4898,4902,\r\n4904,4905,4907,4931,4932,4933,4946,4948,4956,4985,5024,5033,5059,5136,5137,5140,5145,5632,6144,6145,6272,\r\n6273,6278,6416,6423,6424,8001,8002,8003,8004,8005,8006,8007,8222,26401,30004]);\r\nSecurityEvent\r\n| where EventID in (common)\r\n| summarize Count = count(), BilledSize= sum(_BilledSize) by EventID, Activity\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize", "size": 1, "title": "ASC : \"common\" data set - {TimeRange:label}. anomaly-detection:{AnomolyTuning:label}", "timeContext": { @@ -4631,7 +4631,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "//\r\n// source: https://docs.microsoft.com/en-us/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet common = dynamic([1,299,300,324,340,403,404,410,411,412,413,431,500,501,1100,1102,1107,1108,4608,4610,4611,4614,4622,\r\n4624,4625,4634,4647,4648,4649,4657,4661,4662,4663,4665,4666,4667,4688,4670,4672,4673,4674,4675,4689,4697,\r\n4700,4702,4704,4705,4716,4717,4718,4719,4720,4722,4723,4724,4725,4726,4727,4728,4729,4733,4732,4735,4737,\r\n4738,4739,4740,4742,4744,4745,4746,4750,4751,4752,4754,4755,4756,4757,4760,4761,4762,4764,4767,4768,4771,\r\n4774,4778,4779,4781,4793,4797,4798,4799,4800,4801,4802,4803,4825,4826,4870,4886,4887,4888,4893,4898,4902,\r\n4904,4905,4907,4931,4932,4933,4946,4948,4956,4985,5024,5033,5059,5136,5137,5140,5145,5632,6144,6145,6272,\r\n6273,6278,6416,6423,6424,8001,8002,8003,8004,8005,8006,8007,8222,26401,30004]);\r\nSecurityEvent\r\n| where EventID !in (common)\r\n| summarize Count = count(), BilledSize=sum(_BilledSize) by EventID, Activity, TimeGenerated\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n //\r\n // anomalies - seasonality baseline that captures the repetitive pattern. Outliers can be clearly spotted in the Score\r\n //\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize\r\n", + "query": "//\r\n// source: https://docs.microsoft.com/azure/security-center/security-center-enable-data-collection#data-collection-tier\r\n//\r\nlet common = dynamic([1,299,300,324,340,403,404,410,411,412,413,431,500,501,1100,1102,1107,1108,4608,4610,4611,4614,4622,\r\n4624,4625,4634,4647,4648,4649,4657,4661,4662,4663,4665,4666,4667,4688,4670,4672,4673,4674,4675,4689,4697,\r\n4700,4702,4704,4705,4716,4717,4718,4719,4720,4722,4723,4724,4725,4726,4727,4728,4729,4733,4732,4735,4737,\r\n4738,4739,4740,4742,4744,4745,4746,4750,4751,4752,4754,4755,4756,4757,4760,4761,4762,4764,4767,4768,4771,\r\n4774,4778,4779,4781,4793,4797,4798,4799,4800,4801,4802,4803,4825,4826,4870,4886,4887,4888,4893,4898,4902,\r\n4904,4905,4907,4931,4932,4933,4946,4948,4956,4985,5024,5033,5059,5136,5137,5140,5145,5632,6144,6145,6272,\r\n6273,6278,6416,6423,6424,8001,8002,8003,8004,8005,8006,8007,8222,26401,30004]);\r\nSecurityEvent\r\n| where EventID !in (common)\r\n| summarize Count = count(), BilledSize=sum(_BilledSize) by EventID, Activity, TimeGenerated\r\n| order by EventID asc \r\n| join \r\n( \r\n SecurityEvent\r\n //\r\n // anomalies - seasonality baseline that captures the repetitive pattern. Outliers can be clearly spotted in the Score\r\n //\r\n | make-series Trend = count() on TimeGenerated from startofday({TimeRange:start}) to startofday({TimeRange:end}) step {TimeRange:grain} by EventID\r\n | extend (anomalies, score, baseline) = series_decompose_anomalies(Trend, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n | extend Score = score[-1]\r\n | extend expectedEventCounts=baseline[-1], actualEventCount=Trend[-1], Score = score[-1], Trend\r\n | project-away score\r\n) on EventID\r\n| where Score > {AnomolyTuning} or Score < toreal(strcat(\"-\",{AnomolyTuning}))\r\n| project-away EventID1\r\n| project Activity, expectedEventCounts, actualEventCount, Score, Trend, baseline, anomalies, Count, BilledSize\r\n", "size": 0, "title": "ASC : possible \"all\" data set - EventsIDs found that are not in \"common\" or \"minmal\" - {TimeRange:label}. anomaly-detection:{AnomolyTuning:label}", "timeContext": { @@ -6827,7 +6827,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "union withsource = _TableName *\r\n| make-series bill_ = sum(_BilledSize) on TimeGenerated from ago(90d) to now() step 1d by _TableName\r\n| extend (anomalies, score, baseline) = series_decompose_anomalies(bill_, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n| where anomalies[-1] == 1 or anomalies[-1] == -1 \r\n| extend Score = score[-1]\r\n| where Score > 3 or Score < -3\r\n| project [\"Table Name\"] = _TableName, expectedCounts=baseline[-1], actualCount=bill_[-1] , Score = score[-1], Trend = bill_, Baseline = baseline\r\n\r\n// https://docs.microsoft.com/en-us/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n// Anomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly.", + "query": "union withsource = _TableName *\r\n| make-series bill_ = sum(_BilledSize) on TimeGenerated from ago(90d) to now() step 1d by _TableName\r\n| extend (anomalies, score, baseline) = series_decompose_anomalies(bill_, 1.5, 7, 'linefit', 1, 'ctukey', 0.01)\r\n| where anomalies[-1] == 1 or anomalies[-1] == -1 \r\n| extend Score = score[-1]\r\n| where Score > 3 or Score < -3\r\n| project [\"Table Name\"] = _TableName, expectedCounts=baseline[-1], actualCount=bill_[-1] , Score = score[-1], Trend = bill_, Baseline = baseline\r\n\r\n// https://docs.microsoft.com/azure/data-explorer/anomaly-detection#time-series-anomaly-detection\r\n// Anomaly scores above 1.5 or below -1.5 indicate a mild anomaly rise or decline respectively. Anomaly scores above 3.0 or below -3.0 indicate a strong anomaly.", "size": 1, "title": "BilledSize Anomalies: 90day, fixed look back period. anomaly-detection: Strong", "timeContext": { @@ -7950,7 +7950,7 @@ { "type": 1, "content": { - "json": "https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/plan/appendix-l--events-to-monitor", + "json": "https://docs.microsoft.com/windows-server/identity/ad-ds/plan/appendix-l--events-to-monitor", "style": "info" }, "name": "text - 24" From 3a4a85d055069e8388c5fb249dba134f0e4913cf Mon Sep 17 00:00:00 2001 From: v-rucdu Date: Tue, 15 Jun 2021 18:30:37 +0530 Subject: [PATCH 6/6] Removed locale reference to fix failing validation --- Workbooks/WorkspaceUsage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workbooks/WorkspaceUsage.json b/Workbooks/WorkspaceUsage.json index c40c4057bf7..73e1a7999ec 100644 --- a/Workbooks/WorkspaceUsage.json +++ b/Workbooks/WorkspaceUsage.json @@ -4247,7 +4247,7 @@ { "type": 1, "content": { - "json": "The following three charts help you understand SecuityEvent data if its present.\r\n- SecurityEvents are captured in Groups: All, Common, Minimal and None\r\n- Using https://docs.microsoft.com/en-gb/azure/data-explorer/kusto/query/series-decompose-anomaliesfunction we will look at this data\r\n- There are displays for Activity and EventIds that are captured in _Minimal_ and _Common_. Note _Minimal_ is a subset of _Common_\r\n- _\"All\"_ is a special case, as there isn't a list of those, I have listed any EventIds found, but only those not within _Common_\r\n- Using anomoliesfunction, we look at the expected count, actual count, give that a score (the higer varienences are marked in Red), a baseline and and Trend for the TimeRange ", + "json": "The following three charts help you understand SecuityEvent data if its present.\r\n- SecurityEvents are captured in Groups: All, Common, Minimal and None\r\n- Using https://docs.microsoft.com/azure/data-explorer/kusto/query/series-decompose-anomaliesfunction we will look at this data\r\n- There are displays for Activity and EventIds that are captured in _Minimal_ and _Common_. Note _Minimal_ is a subset of _Common_\r\n- _\"All\"_ is a special case, as there isn't a list of those, I have listed any EventIds found, but only those not within _Common_\r\n- Using anomoliesfunction, we look at the expected count, actual count, give that a score (the higer varienences are marked in Red), a baseline and and Trend for the TimeRange ", "style": "info" }, "conditionalVisibilities": [