From fc9a7f75daca16bc35298468ba1ca9e274cbfeb6 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 22:53:57 -0400 Subject: [PATCH 01/31] New audit logs --- .../Get-CippAuditLogSearchResults.ps1 | 23 +++++ .../AuditLogs/Get-CippAuditLogSearches.ps1 | 23 +++++ .../AuditLogs/New-CippAuditLogSearch.ps1 | 24 ++++- .../Webhooks/Push-AuditLogTenant.ps1 | 95 +++++++------------ .../Start-AuditLogOrchestrator.ps1 | 43 ++++++--- .../Webhooks/Test-CIPPAuditLogRules.ps1 | 13 ++- 6 files changed, 137 insertions(+), 84 deletions(-) create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 new file mode 100644 index 000000000000..5dedbd706609 --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 @@ -0,0 +1,23 @@ +function Get-CippAuditLogSearchResults { + <# + .SYNOPSIS + Get the results of an audit log search + .DESCRIPTION + Get the results of an audit log search from the Graph API + .PARAMETER TenantFilter + The tenant to filter on. + .PARAMETER QueryId + The ID of the query to get the results for. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter, + [Parameter(ValueFromPipelineByPropertyName = $true, Mandatory = $true)] + [Alias('id')] + [string]$QueryId + ) + + process { + New-GraphGetRequest -uri ('https://graph.microsoft.com/beta/security/auditLog/queries/{0}/records' -f $QueryId) -AsApp $true -tenantid $TenantFilter + } +} diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 new file mode 100644 index 000000000000..8c2aa9e9a7ff --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 @@ -0,0 +1,23 @@ +function Get-CippAuditLogSearches { + <# + .SYNOPSIS + Get the available audit log searches + .DESCRIPTION + Query the Graph API for available audit log searches. + .PARAMETER TenantFilter + The tenant to filter on. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter, + [Parameter()] + [switch]$ReadyToProcess + ) + $Queries = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -AsApp $true -tenantid $TenantFilter + if ($ReadyToProcess.IsPresent) { + $AuditLogSearchesTable = Get-CippTable -TableName 'AuditLogSearches' + $PendingQueries = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "Tenant eq '$TenantFilter' and CippStatus eq 'Pending'" + $Queries = $Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' } + } + return $Queries +} diff --git a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 index e28c6b742cac..39fa70846ce5 100644 --- a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 @@ -28,6 +28,8 @@ function New-CippAuditLogSearch { The object IDs to filter on. .PARAMETER AdministrativeUnitFilters The administrative units to filter on. + .PARAMETER ProcessLogs + Store the search in the CIPP AuditLogSearches table for alert processing. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -117,7 +119,9 @@ function New-CippAuditLogSearch { [Parameter()] [string[]]$ObjectIdFilters, [Parameter()] - [string[]]$AdministrativeUnitFilters + [string[]]$AdministrativeUnitFilters, + [Parameter()] + [switch]$ProcessLogs ) $SearchParams = @{ @@ -151,6 +155,22 @@ function New-CippAuditLogSearch { } if ($PSCmdlet.ShouldProcess('Create a new audit log search for tenant ' + $TenantFilter)) { - New-GraphPOSTRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -body ($SearchParams | ConvertTo-Json -Compress) -tenantid $TenantFilter -AsApp $true + $Query = New-GraphPOSTRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -body ($SearchParams | ConvertTo-Json -Compress) -tenantid $TenantFilter -AsApp $true + + if ($ProcessLogs.IsPresent) { + $Entity = [PSCustomObject]@{ + PartitionKey = [string]'Search' + RowKey = [string]$Query.id + Tenant = [string]$TenantFilter + DisplayName = [string]$DisplayName + StartTime = [datetime]$StartTime.ToUniversalTime() + EndTime = [datetime]$EndTime.ToUniversalTime() + Query = [string]($Query | ConvertTo-Json -Compress) + CippStatus = [string]'Pending' + } + $Table = Get-CIPPTable -TableName 'AuditLogSearches' + Add-CIPPAzDataTableEntity @Table -Entity $Entity -Force | Out-Null + } + return $Query } } diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 8525ff065e34..482a3e596f14 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -1,78 +1,49 @@ function Push-AuditLogTenant { Param($Item) - # Get Table contexts - $AuditBundleTable = Get-CippTable -tablename 'AuditLogBundles' $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' - $WebhookTable = Get-CippTable -tablename 'webhookTable' $ConfigTable = Get-CippTable -TableName 'WebhookRules' # Query CIPPURL for linking $CIPPURL = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'" | Select-Object -First 1 -ExpandProperty CIPPURL - # Get all webhooks for the tenant - $Webhooks = Get-CIPPAzDataTableEntity @WebhookTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and Version eq '3'" | Where-Object { $_.Resource -match '^Audit' } - # Get webhook rules $ConfigEntries = Get-CIPPAzDataTableEntity @ConfigTable - - # Date filter for existing bundles - $LastHour = (Get-Date).AddHours(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss') - - $NewBundles = [System.Collections.Generic.List[object]]::new() - foreach ($Webhook in $Webhooks) { - # only process webhooks that are configured in the webhookrules table - $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } - if ($Configuration.Type -notcontains $Webhook.Resource) { - continue - } - - $TenantFilter = $Webhook.PartitionKey - $LogType = $Webhook.Resource - Write-Information "Querying for $LogType on $TenantFilter" - $ContentBundleQuery = @{ - TenantFilter = $TenantFilter - ContentType = $LogType - StartTime = $Item.StartTime - EndTime = $Item.EndTime - } - try { - $LogBundles = Get-CIPPAuditLogContentBundles @ContentBundleQuery - $ExistingBundles = Get-CIPPAzDataTableEntity @AuditBundleTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and ContentType eq '$LogType' and Timestamp ge datetime'$($LastHour)'" - - foreach ($Bundle in $LogBundles) { - if ($ExistingBundles.RowKey -notcontains $Bundle.contentId) { - $NewBundles.Add([PSCustomObject]@{ - PartitionKey = $TenantFilter - RowKey = $Bundle.contentId - DefaultDomainName = $TenantFilter - ContentType = $Bundle.contentType - ContentUri = $Bundle.contentUri - ContentCreated = $Bundle.contentCreated - ContentExpiration = $Bundle.contentExpiration - CIPPURL = [string]$CIPPURL - ProcessingStatus = 'Pending' - MatchedRules = '' - MatchedLogs = 0 - }) + $LogSearchesTable = Get-CippTable -TableName 'AuditLogSearches' + + $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } + if ($Configuration) { + $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.TenantFilter -ReadyToProcess + foreach ($Search in $LogSearches) { + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and RowKey eq '$($Search.id)'" + $SearchEntity.Status = 'Processing' + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + try { + # Test the audit log rules against the search results + $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.TenantFilter -SearchId $Search.id + + $SearchEntity.CippStatus = 'Completed' + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs + $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs + } catch { + $SearchEntity.CippStatus = 'Failed' + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $_.InvocationInfo.PositionMessage + } + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + $DataToProcess = ($AuditLogTest).DataToProcess + Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items" + if ($DataToProcess) { + foreach ($AuditLog in $DataToProcess) { + Write-Information "Processing $($AuditLog.operation)" + $Webhook = @{ + Data = $AuditLog + CIPPURL = [string]$CIPPURL + TenantFilter = $Item.TenantFilter + } + Invoke-CippWebhookProcessing @Webhook } } - } catch { - Write-Information "Could not get audit log content bundles for $TenantFilter - $LogType, $($_.Exception.Message)" - } - } - - if (($NewBundles | Measure-Object).Count -gt 0) { - Add-CIPPAzDataTableEntity @AuditBundleTable -Entity $NewBundles -Force - Write-Information ($NewBundles | ConvertTo-Json -Depth 5 -Compress) - - $Batch = $NewBundles | Select-Object @{Name = 'ContentId'; Expression = { $_.RowKey } }, @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogBundleProcessing' } } - $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' - Batch = @($Batch) - SkipLog = $true } - $InstanceId = Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) - Write-Host "Started orchestration with ID = '$InstanceId'" } } diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index b17c0c00f3ad..e5e87bfb738b 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -6,29 +6,46 @@ function Start-AuditLogOrchestrator { [CmdletBinding(SupportsShouldProcess = $true)] param() try { - $webhookTable = Get-CIPPTable -tablename webhookTable - $Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } - if (($Webhooks | Measure-Object).Count -eq 0) { - Write-Information 'No webhook subscriptions found. Exiting.' - return - } + #$webhookTable = Get-CIPPTable -tablename webhookTable + #$Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } + #if (($Webhooks | Measure-Object).Count -eq 0) { + # Write-Information 'No webhook subscriptions found. Exiting.' + # return + #} - $StartTime = (Get-Date).AddMinutes(-30) + $StartTime = (Get-Date).AddMinutes(-15) $EndTime = Get-Date - $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($Webhooks | Sort-Object -Property PartitionKey -Unique | Measure-Object).Count + $TenantList = Get-Tenants -IncludeErrors + $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($TenantList | Measure-Object).Count - $Batch = $Webhooks | Sort-Object -Property PartitionKey -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } }, @{Name = 'StartTime'; Expression = { $StartTime } }, @{Name = 'EndTime'; Expression = { $EndTime } } + #$Batch = $Webhooks | Sort-Object -Property PartitionKey -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } }, @{Name = 'StartTime'; Expression = { $StartTime } }, @{Name = 'EndTime'; Expression = { $EndTime } } $InputObject = [PSCustomObject]@{ OrchestratorName = 'AuditLogs' - Batch = @($Batch) + QueueFunction = @{ + FunctionName = 'GetTenants' + TenantParams = @{ + IncludeErrors = $true + } + QueueId = $Queue.RowKey + DurableFunction = 'AuditLogTenant' + } SkipLog = $true } - if ($PSCmdlet.ShouldProcess('Start-AuditLogPolling', 'Starting Audit Log Polling')) { + if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) } + + foreach ($Tenant in $TenantList) { + try { + $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs + } catch { + Write-LogMessage -API 'Audit Logs' -message 'Error creating audit log search' -sev Error -LogData (Get-CippException -Exception $_) + Write-Information ( 'Audit logs error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + } + } } catch { - Write-LogMessage -API 'Webhooks' -message 'Error processing webhooks' -sev Error -LogData (Get-CippException -Exception $_) - Write-Information ( 'Webhook error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + Write-LogMessage -API 'Audit Logs' -message 'Error processing audit logs' -sev Error -LogData (Get-CippException -Exception $_) + Write-Information ( 'Audit logs error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) } } diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index 33ef01e6589c..eaa7c4e68b9a 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -4,10 +4,7 @@ function Test-CIPPAuditLogRules { [Parameter(Mandatory = $true)] $TenantFilter, [Parameter(Mandatory = $true)] - $ContentUri, - [Parameter(Mandatory = $true)] - [ValidateSet('Audit.AzureActiveDirectory', 'Audit.Exchange')] - $LogType + $SearchId ) $Results = [PSCustomObject]@{ @@ -41,13 +38,15 @@ function Test-CIPPAuditLogRules { TenantFilter = $TenantFilter ContentUri = $ContentUri } - Write-Information 'Getting data from Office 365 Management Activity API' - $Data = Get-CIPPAuditLogContent @AuditLogQuery + #Write-Information 'Getting data from Office 365 Management Activity API' + #$Data = Get-CIPPAuditLogContent @AuditLogQuery + Write-Information 'Getting audit records from Graph API' + $Data = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -SearchId $SearchId $LogCount = ($Data | Measure-Object).Count Write-Information "Logs to process: $LogCount" $Results.TotalLogs = $LogCount if ($LogCount -gt 0) { - $PreProcessedData = $Data | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties -ErrorAction SilentlyContinue + $PreProcessedData = $Data.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties -ErrorAction SilentlyContinue $LocationTable = Get-CIPPTable -TableName 'knownlocationdb' $ProcessedData = foreach ($Data in $PreProcessedData) { try { From 78d3c9a761b11f2b68bf59ae7ad11d4fb1090cc6 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 23:06:12 -0400 Subject: [PATCH 02/31] audit logs --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 9 +++++---- .../Start-AuditLogOrchestrator.ps1 | 2 -- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 482a3e596f14..fb54d1a362b6 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -3,6 +3,7 @@ function Push-AuditLogTenant { $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' $ConfigTable = Get-CippTable -TableName 'WebhookRules' + $TenantFilter = $Item.defaultDomainName # Query CIPPURL for linking $CIPPURL = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'" | Select-Object -First 1 -ExpandProperty CIPPURL @@ -13,14 +14,14 @@ function Push-AuditLogTenant { $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } if ($Configuration) { - $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.TenantFilter -ReadyToProcess + $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.defaultDomainName -ReadyToProcess foreach ($Search in $LogSearches) { - $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and RowKey eq '$($Search.id)'" + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.defaultDomainName)' and RowKey eq '$($Search.id)'" $SearchEntity.Status = 'Processing' Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force try { # Test the audit log rules against the search results - $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.TenantFilter -SearchId $Search.id + $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.defaultDomainName -SearchId $Search.id $SearchEntity.CippStatus = 'Completed' $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) @@ -39,7 +40,7 @@ function Push-AuditLogTenant { $Webhook = @{ Data = $AuditLog CIPPURL = [string]$CIPPURL - TenantFilter = $Item.TenantFilter + TenantFilter = $Item.defaultDomainName } Invoke-CippWebhookProcessing @Webhook } diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index e5e87bfb738b..2aa307705e5a 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -40,8 +40,6 @@ function Start-AuditLogOrchestrator { try { $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs } catch { - Write-LogMessage -API 'Audit Logs' -message 'Error creating audit log search' -sev Error -LogData (Get-CippException -Exception $_) - Write-Information ( 'Audit logs error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) } } } catch { From c4b1f35e21cfe9c65d1ab31bf9ec8d73ed83791f Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 23:09:36 -0400 Subject: [PATCH 03/31] try / catch --- .../Webhooks/Push-AuditLogTenant.ps1 | 60 ++++++++++--------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index fb54d1a362b6..292b8870350b 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -14,37 +14,43 @@ function Push-AuditLogTenant { $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } if ($Configuration) { - $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.defaultDomainName -ReadyToProcess - foreach ($Search in $LogSearches) { - $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.defaultDomainName)' and RowKey eq '$($Search.id)'" - $SearchEntity.Status = 'Processing' - Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force - try { - # Test the audit log rules against the search results - $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.defaultDomainName -SearchId $Search.id + try { + Write-Information "Audit Logs: Processing $($Item.defaultDomainName)" + $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.defaultDomainName -ReadyToProcess + Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) + foreach ($Search in $LogSearches) { + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.defaultDomainName)' and RowKey eq '$($Search.id)'" + $SearchEntity.Status = 'Processing' + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + try { + # Test the audit log rules against the search results + $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.defaultDomainName -SearchId $Search.id - $SearchEntity.CippStatus = 'Completed' - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs - $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs - } catch { - $SearchEntity.CippStatus = 'Failed' - $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $_.InvocationInfo.PositionMessage - } - Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force - $DataToProcess = ($AuditLogTest).DataToProcess - Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items" - if ($DataToProcess) { - foreach ($AuditLog in $DataToProcess) { - Write-Information "Processing $($AuditLog.operation)" - $Webhook = @{ - Data = $AuditLog - CIPPURL = [string]$CIPPURL - TenantFilter = $Item.defaultDomainName + $SearchEntity.CippStatus = 'Completed' + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs + $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs + } catch { + $SearchEntity.CippStatus = 'Failed' + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $_.InvocationInfo.PositionMessage + } + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + $DataToProcess = ($AuditLogTest).DataToProcess + Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items" + if ($DataToProcess) { + foreach ($AuditLog in $DataToProcess) { + Write-Information "Processing $($AuditLog.operation)" + $Webhook = @{ + Data = $AuditLog + CIPPURL = [string]$CIPPURL + TenantFilter = $Item.defaultDomainName + } + Invoke-CippWebhookProcessing @Webhook } - Invoke-CippWebhookProcessing @Webhook } } + } catch { + Write-Information ( 'Audit Logs: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) } } } From c986c60755393c4d7f48db15664111a44df37bfe Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 23:19:04 -0400 Subject: [PATCH 04/31] Update Push-AuditLogTenant.ps1 --- .../Webhooks/Push-AuditLogTenant.ps1 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 292b8870350b..8caf349a773a 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -3,8 +3,10 @@ function Push-AuditLogTenant { $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' $ConfigTable = Get-CippTable -TableName 'WebhookRules' - $TenantFilter = $Item.defaultDomainName + $Tenant = Get-Tenants -TenantFilter $Item.customerId -IncludeErrors + $TenantFilter = $Tenant.defaultDomainName + Write-Information "Audit Logs: Processing $($TenantFilter)" # Query CIPPURL for linking $CIPPURL = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'" | Select-Object -First 1 -ExpandProperty CIPPURL @@ -15,16 +17,15 @@ function Push-AuditLogTenant { $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } if ($Configuration) { try { - Write-Information "Audit Logs: Processing $($Item.defaultDomainName)" - $LogSearches = Get-CippAuditLogSearches -TenantFilter $Item.defaultDomainName -ReadyToProcess + $LogSearches = Get-CippAuditLogSearches -TenantFilter $TenantFilter -ReadyToProcess Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) foreach ($Search in $LogSearches) { - $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($Item.defaultDomainName)' and RowKey eq '$($Search.id)'" + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" $SearchEntity.Status = 'Processing' Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force try { # Test the audit log rules against the search results - $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $Item.defaultDomainName -SearchId $Search.id + $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id $SearchEntity.CippStatus = 'Completed' $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) @@ -43,7 +44,7 @@ function Push-AuditLogTenant { $Webhook = @{ Data = $AuditLog CIPPURL = [string]$CIPPURL - TenantFilter = $Item.defaultDomainName + TenantFilter = $TenantFilter } Invoke-CippWebhookProcessing @Webhook } From 6e12e1579b35410cb8c173cd601bd6965ad86926 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 23:36:59 -0400 Subject: [PATCH 05/31] audit logs --- .../Webhooks/Push-AuditLogTenant.ps1 | 4 +- .../Start-AuditLogOrchestrator.ps1 | 49 ++++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 8caf349a773a..15cfad3d4e89 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -3,8 +3,8 @@ function Push-AuditLogTenant { $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' $ConfigTable = Get-CippTable -TableName 'WebhookRules' - $Tenant = Get-Tenants -TenantFilter $Item.customerId -IncludeErrors - $TenantFilter = $Tenant.defaultDomainName + #$Tenant = Get-Tenants -TenantFilter $Item.customerId -IncludeErrors + $TenantFilter = $Item.TenantFilter Write-Information "Audit Logs: Processing $($TenantFilter)" # Query CIPPURL for linking diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 2aa307705e5a..4abf7965e775 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -6,34 +6,35 @@ function Start-AuditLogOrchestrator { [CmdletBinding(SupportsShouldProcess = $true)] param() try { - #$webhookTable = Get-CIPPTable -tablename webhookTable - #$Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } - #if (($Webhooks | Measure-Object).Count -eq 0) { - # Write-Information 'No webhook subscriptions found. Exiting.' - # return - #} - $StartTime = (Get-Date).AddMinutes(-15) - $EndTime = Get-Date + $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' + $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" + if (($AuditLogSearches | Measure-Object).Count -gt 0) { + Write-Information 'No audit log searches available' + } else { + #$webhookTable = Get-CIPPTable -tablename webhookTable + #$Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } + #if (($Webhooks | Measure-Object).Count -eq 0) { + # Write-Information 'No webhook subscriptions found. Exiting.' + # return + #} - $TenantList = Get-Tenants -IncludeErrors - $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($TenantList | Measure-Object).Count + $StartTime = (Get-Date).AddMinutes(-15) + $EndTime = Get-Date - #$Batch = $Webhooks | Sort-Object -Property PartitionKey -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } }, @{Name = 'StartTime'; Expression = { $StartTime } }, @{Name = 'EndTime'; Expression = { $EndTime } } - $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' - QueueFunction = @{ - FunctionName = 'GetTenants' - TenantParams = @{ - IncludeErrors = $true - } - QueueId = $Queue.RowKey - DurableFunction = 'AuditLogTenant' + $TenantList = Get-Tenants -IncludeErrors + $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count + + $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } } + + $InputObject = [PSCustomObject]@{ + OrchestratorName = 'AuditLogs' + Batch = @( $Batch ) + SkipLog = $true + } + if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { + Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) } - SkipLog = $true - } - if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { - Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) } foreach ($Tenant in $TenantList) { From 90adeed3595b784cb28afa8145031ee96bf0aa1e Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 23:47:38 -0400 Subject: [PATCH 06/31] audit log tweaks --- .../Public/AuditLogs/New-CippAuditLogSearch.ps1 | 2 +- .../Start-AuditLogOrchestrator.ps1 | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 index 39fa70846ce5..6e48cc1c0f84 100644 --- a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 @@ -157,7 +157,7 @@ function New-CippAuditLogSearch { if ($PSCmdlet.ShouldProcess('Create a new audit log search for tenant ' + $TenantFilter)) { $Query = New-GraphPOSTRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -body ($SearchParams | ConvertTo-Json -Compress) -tenantid $TenantFilter -AsApp $true - if ($ProcessLogs.IsPresent) { + if ($ProcessLogs.IsPresent -and $Query.id) { $Entity = [PSCustomObject]@{ PartitionKey = [string]'Search' RowKey = [string]$Query.id diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 4abf7965e775..e41cd2106817 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -9,19 +9,11 @@ function Start-AuditLogOrchestrator { $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" - if (($AuditLogSearches | Measure-Object).Count -gt 0) { + $StartTime = (Get-Date).AddMinutes(-15) + $EndTime = Get-Date + if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' } else { - #$webhookTable = Get-CIPPTable -tablename webhookTable - #$Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } - #if (($Webhooks | Measure-Object).Count -eq 0) { - # Write-Information 'No webhook subscriptions found. Exiting.' - # return - #} - - $StartTime = (Get-Date).AddMinutes(-15) - $EndTime = Get-Date - $TenantList = Get-Tenants -IncludeErrors $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count @@ -29,7 +21,7 @@ function Start-AuditLogOrchestrator { $InputObject = [PSCustomObject]@{ OrchestratorName = 'AuditLogs' - Batch = @( $Batch ) + Batch = @($Batch) SkipLog = $true } if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { @@ -37,6 +29,7 @@ function Start-AuditLogOrchestrator { } } + Write-Information 'Audit Logs: Creating new searches' foreach ($Tenant in $TenantList) { try { $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs From aa01d28238b96bc71e6981b44747b9cb3039be8e Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 00:01:12 -0400 Subject: [PATCH 07/31] Update Push-AuditLogTenant.ps1 --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 15cfad3d4e89..0807058f2e06 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -21,7 +21,7 @@ function Push-AuditLogTenant { Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) foreach ($Search in $LogSearches) { $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" - $SearchEntity.Status = 'Processing' + $SearchEntity.CippStatus = 'Processing' Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force try { # Test the audit log rules against the search results From c6eaf49b89b2e35805a74357a82bbcfbf52cbab0 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 08:36:55 -0400 Subject: [PATCH 08/31] Update Push-AuditLogTenant.ps1 --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 0807058f2e06..91b504e13500 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -20,7 +20,7 @@ function Push-AuditLogTenant { $LogSearches = Get-CippAuditLogSearches -TenantFilter $TenantFilter -ReadyToProcess Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) foreach ($Search in $LogSearches) { - $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "PartitionKey eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "Tenant eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" $SearchEntity.CippStatus = 'Processing' Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force try { From 92f17cb2818e6a7c60b82a00d9496920fee50030 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 08:54:43 -0400 Subject: [PATCH 09/31] audit logs --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 2 +- .../CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 91b504e13500..b9b6b74dc29e 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -33,7 +33,7 @@ function Push-AuditLogTenant { $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs } catch { $SearchEntity.CippStatus = 'Failed' - $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $_.InvocationInfo.PositionMessage + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value (Get-CippException -Exception $_ | ConvertTo-Json) } Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force $DataToProcess = ($AuditLogTest).DataToProcess diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index eaa7c4e68b9a..72f50cd75aea 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -34,10 +34,10 @@ function Test-CIPPAuditLogRules { LogType = $_.Type } } - $AuditLogQuery = @{ - TenantFilter = $TenantFilter - ContentUri = $ContentUri - } + #$AuditLogQuery = @{ + # TenantFilter = $TenantFilter + # ContentUri = $ContentUri + #} #Write-Information 'Getting data from Office 365 Management Activity API' #$Data = Get-CIPPAuditLogContent @AuditLogQuery Write-Information 'Getting audit records from Graph API' From 820d4a49ba2d9806c8bb56173e0827d55d8a910f Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 08:59:32 -0400 Subject: [PATCH 10/31] Update Push-AuditLogTenant.ps1 --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index b9b6b74dc29e..ed1c0388a693 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -33,6 +33,7 @@ function Push-AuditLogTenant { $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs } catch { $SearchEntity.CippStatus = 'Failed' + Write-Information "Error processing audit log rules: $($_.Exception.Message)" $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value (Get-CippException -Exception $_ | ConvertTo-Json) } Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force From 51e3bf732755ad32130a8e7dabeaba59b3a11ca9 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 09:03:01 -0400 Subject: [PATCH 11/31] audit logs fixes --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 2 +- Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index ed1c0388a693..a1dbc795f45f 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -34,7 +34,7 @@ function Push-AuditLogTenant { } catch { $SearchEntity.CippStatus = 'Failed' Write-Information "Error processing audit log rules: $($_.Exception.Message)" - $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value (Get-CippException -Exception $_ | ConvertTo-Json) + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -TypeName String -Value (Get-CippException -Exception $_ | ConvertTo-Json) } Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force $DataToProcess = ($AuditLogTest).DataToProcess diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index 72f50cd75aea..3288712f18a4 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -41,7 +41,7 @@ function Test-CIPPAuditLogRules { #Write-Information 'Getting data from Office 365 Management Activity API' #$Data = Get-CIPPAuditLogContent @AuditLogQuery Write-Information 'Getting audit records from Graph API' - $Data = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -SearchId $SearchId + $Data = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId $LogCount = ($Data | Measure-Object).Count Write-Information "Logs to process: $LogCount" $Results.TotalLogs = $LogCount From 609bcfb2f206f1d24eca7862f01b610a889c0270 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 09:06:05 -0400 Subject: [PATCH 12/31] Fix api role permissions --- .../HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 | 2 +- .../HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 index e31239f1bf02..ab7474163cdc 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 @@ -5,7 +5,7 @@ Function Invoke-AddSite { .FUNCTIONALITY Entrypoint .ROLE - Teams.Group.ReadWrite + Sharepoint.Site.ReadWrite #> [CmdletBinding()] param($Request, $TriggerMetadata) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 index 79ac24fbc508..97f779f10c76 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 @@ -5,7 +5,7 @@ Function Invoke-AddSiteBulk { .FUNCTIONALITY Entrypoint .ROLE - Teams.Group.ReadWrite + Sharepoint.Site.ReadWrite #> [CmdletBinding()] param($Request, $TriggerMetadata) From cbda16f378a3c548e5146f565a44687ced406ed2 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 09:10:35 -0400 Subject: [PATCH 13/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index e41cd2106817..3e2b1cba8b07 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -9,8 +9,12 @@ function Start-AuditLogOrchestrator { $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" - $StartTime = (Get-Date).AddMinutes(-15) - $EndTime = Get-Date + + # Round time down to nearest minute + $Now = Get-Date + $StartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddMinutes(-15) + $EndTime = $Now.AddSeconds(-$Now.Seconds) + if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' } else { From ce39b5673bbc35b0263b51af9edd37e9a599a816 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 09:16:41 -0400 Subject: [PATCH 14/31] Update Push-AuditLogTenant.ps1 --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index a1dbc795f45f..b22dabbdcf3d 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -28,7 +28,7 @@ function Push-AuditLogTenant { $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id $SearchEntity.CippStatus = 'Completed' - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value [string](ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -TypeName String -Value (ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs } catch { From 540e90c43eb529b55aea7dfe065d824373d672ff Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 09:41:18 -0400 Subject: [PATCH 15/31] Update Push-AuditLogTenant.ps1 --- .../Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index b22dabbdcf3d..a2a36fd518a9 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -28,13 +28,15 @@ function Push-AuditLogTenant { $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id $SearchEntity.CippStatus = 'Completed' - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -TypeName String -Value (ConvertTo-Json -Compress -Depth 10 -InputObject $AuditLogTest.MatchedRules) + $MatchedRules = [string](ConvertTo-Json -Compress -InputObject $AuditLogTest.MatchedRules) + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value $MatchedRules $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs } catch { $SearchEntity.CippStatus = 'Failed' Write-Information "Error processing audit log rules: $($_.Exception.Message)" - $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -TypeName String -Value (Get-CippException -Exception $_ | ConvertTo-Json) + $Exception = [string](ConvertTo-Json -Compress -InputObject (Get-CippException -Exception $_)) + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $Exception } Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force $DataToProcess = ($AuditLogTest).DataToProcess From 74a2de57e10c7d9f28d6bdd5d13b0bcc9e166b93 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 10:07:30 -0400 Subject: [PATCH 16/31] Fix issues with graph requests missing data Only use -ComplexFilter when $filter is present --- .../Public/GraphRequests/Get-GraphRequestList.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 b/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 index b8d84bf0052d..43d7617833c1 100644 --- a/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 +++ b/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 @@ -95,11 +95,12 @@ function Get-GraphRequestList { $Count = 0 if ($TenantFilter -ne 'AllTenants') { $GraphRequest = @{ - uri = $GraphQuery.ToString() - tenantid = $TenantFilter - ComplexFilter = $true + uri = $GraphQuery.ToString() + tenantid = $TenantFilter + } + if ($Parameters.'$filter') { + $GraphRequest.ComplexFilter = $true } - if ($NoPagination.IsPresent) { $GraphRequest.noPagination = $NoPagination.IsPresent } From f603d0f957e6f7227daba4b2709af7ac5832ab4e Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 12:08:47 -0400 Subject: [PATCH 17/31] Add sharepoint admin url endpoint --- .../Invoke-ListSharepointAdminUrl.ps1 | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 new file mode 100644 index 000000000000..ef0e1e5c690e --- /dev/null +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 @@ -0,0 +1,50 @@ +function Invoke-ListSharepointAdminUrl { + <# + .FUNCTIONALITY + Entrypoint + .ROLE + CIPP.Core.Read + #> + [CmdletBinding()] + param( + $Request, + $TriggerMetadata + ) + + if ($Request.Query.TenantFilter) { + $TenantFilter = $Request.Query.TenantFilter + + $Tenant = Get-Tenants -TenantFilter $TenantFilter + + if ($Tenant.SharepointAdminUrl) { + $AdminUrl = $Tenant.SharepointAdminUrl + } else { + $tenantName = (New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/sites/root' -asApp $true -tenantid $TenantFilter).id.Split('.')[0] + $AdminUrl = "https://$($tenantName)-admin.sharepoint.com" + $Tenant | Add-Member -MemberType NoteProperty -Name SharepointAdminUrl -Value $AdminUrl + $Table = Get-CIPPTable -TableName 'Tenants' + Add-CIPPAzDataTableEntity @Table -Entity $Tenant -Force + } + + if ($Request.Query.ReturnUrl) { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::OK + Body = @{ + AdminUrl = $AdminUrl + } + }) + } else { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::Found + Headers = @{ + Location = $AdminUrl + } + }) + } + } else { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::BadRequest + Body = 'TenantFilter is required' + }) + } +} From 28b7254829227edb4bfcdd89153843817906f587 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 16:52:30 -0400 Subject: [PATCH 18/31] Create Get-CippLastAuditLogSearch.ps1 --- .../AuditLogs/Get-CippLastAuditLogSearch.ps1 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 new file mode 100644 index 000000000000..5706a4ac3e4d --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 @@ -0,0 +1,19 @@ +function Get-CippLastAuditLogSearch { + <# + .SYNOPSIS + Get the last audit log search + .DESCRIPTION + Query the Graph API for the last audit log search. + .PARAMETER TenantFilter + The tenant to filter on. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter + ) + + $Table = Get-CIPPTable -TableName AuditLogSearches + $LastHour = (Get-Date).AddHours(-1).ToString('yyyy-MM-ddTHH:mm:ssZ') + $LastSearch = Get-AzDataTableEntity @Table -Filter "Tenant eq '$TenantFilter' and Timestamp ge datetime'$LastHour'" | Sort-Object Timestamp -Descending | Select-Object -First 1 + return $LastSearch +} From 2583b6b0ba422f0f15ee77480aad4d2122d40ac7 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 16:52:37 -0400 Subject: [PATCH 19/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 3e2b1cba8b07..403394e8a61d 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -12,7 +12,7 @@ function Start-AuditLogOrchestrator { # Round time down to nearest minute $Now = Get-Date - $StartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddMinutes(-15) + $DefaultStartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddMinutes(-15) $EndTime = $Now.AddSeconds(-$Now.Seconds) if (($AuditLogSearches | Measure-Object).Count -eq 0) { @@ -36,6 +36,12 @@ function Start-AuditLogOrchestrator { Write-Information 'Audit Logs: Creating new searches' foreach ($Tenant in $TenantList) { try { + $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName + if ($LastSearch) { + $StartTime = $LastSearch.EndTime + } else { + $StartTime = $DefaultStartTime + } $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs } catch { } From a99bf175b3fcd60554d3fcaa181324fac206efaf Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 19:52:11 -0400 Subject: [PATCH 20/31] tweak timing --- CIPPTimers.json | 2 +- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CIPPTimers.json b/CIPPTimers.json index aed1c3313710..16122b18c6bb 100644 --- a/CIPPTimers.json +++ b/CIPPTimers.json @@ -17,7 +17,7 @@ { "Command": "Start-AuditLogOrchestrator", "Description": "Orchestrator to process audit logs", - "Cron": "0 */15 * * * *", + "Cron": "0 */30 * * * *", "Priority": 2, "RunOnProcessor": true, "PreferredProcessor": "auditlog" diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 403394e8a61d..5b6640416564 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -12,8 +12,8 @@ function Start-AuditLogOrchestrator { # Round time down to nearest minute $Now = Get-Date - $DefaultStartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddMinutes(-15) - $EndTime = $Now.AddSeconds(-$Now.Seconds) + $DefaultStartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddHours(-1) + $EndTime = $Now.AddMinutes(-30).AddSeconds(-$Now.Seconds) if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' @@ -37,7 +37,7 @@ function Start-AuditLogOrchestrator { foreach ($Tenant in $TenantList) { try { $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName - if ($LastSearch) { + if ($LastSearch -and $StartTime -gt $DefaultStartTime) { $StartTime = $LastSearch.EndTime } else { $StartTime = $DefaultStartTime From 3c1f85263421c8193bc692ece7d9e99f2ad22fef Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 30 Sep 2024 22:53:57 -0400 Subject: [PATCH 21/31] audit logs tweak timing Update Start-AuditLogOrchestrator.ps1 Create Get-CippLastAuditLogSearch.ps1 Add sharepoint admin url endpoint Fix issues with graph requests missing data Only use -ComplexFilter when $filter is present Update Push-AuditLogTenant.ps1 Update Push-AuditLogTenant.ps1 Update Start-AuditLogOrchestrator.ps1 Fix api role permissions audit logs fixes Update Push-AuditLogTenant.ps1 audit logs Update Push-AuditLogTenant.ps1 Update Push-AuditLogTenant.ps1 audit log tweaks audit logs Update Push-AuditLogTenant.ps1 try / catch New audit logs --- CIPPTimers.json | 2 +- .../Get-CippAuditLogSearchResults.ps1 | 23 ++++ .../AuditLogs/Get-CippAuditLogSearches.ps1 | 23 ++++ .../AuditLogs/Get-CippLastAuditLogSearch.ps1 | 19 ++++ .../AuditLogs/New-CippAuditLogSearch.ps1 | 24 ++++- .../Webhooks/Push-AuditLogTenant.ps1 | 100 +++++++----------- .../Teams-Sharepoint/Invoke-AddSite.ps1 | 2 +- .../Teams-Sharepoint/Invoke-AddSiteBulk.ps1 | 2 +- .../Invoke-ListSharepointAdminUrl.ps1 | 50 +++++++++ .../Start-AuditLogOrchestrator.ps1 | 55 ++++++---- .../GraphRequests/Get-GraphRequestList.ps1 | 9 +- .../Webhooks/Test-CIPPAuditLogRules.ps1 | 21 ++-- 12 files changed, 233 insertions(+), 97 deletions(-) create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 create mode 100644 Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 create mode 100644 Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 diff --git a/CIPPTimers.json b/CIPPTimers.json index aed1c3313710..16122b18c6bb 100644 --- a/CIPPTimers.json +++ b/CIPPTimers.json @@ -17,7 +17,7 @@ { "Command": "Start-AuditLogOrchestrator", "Description": "Orchestrator to process audit logs", - "Cron": "0 */15 * * * *", + "Cron": "0 */30 * * * *", "Priority": 2, "RunOnProcessor": true, "PreferredProcessor": "auditlog" diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 new file mode 100644 index 000000000000..5dedbd706609 --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 @@ -0,0 +1,23 @@ +function Get-CippAuditLogSearchResults { + <# + .SYNOPSIS + Get the results of an audit log search + .DESCRIPTION + Get the results of an audit log search from the Graph API + .PARAMETER TenantFilter + The tenant to filter on. + .PARAMETER QueryId + The ID of the query to get the results for. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter, + [Parameter(ValueFromPipelineByPropertyName = $true, Mandatory = $true)] + [Alias('id')] + [string]$QueryId + ) + + process { + New-GraphGetRequest -uri ('https://graph.microsoft.com/beta/security/auditLog/queries/{0}/records' -f $QueryId) -AsApp $true -tenantid $TenantFilter + } +} diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 new file mode 100644 index 000000000000..8c2aa9e9a7ff --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 @@ -0,0 +1,23 @@ +function Get-CippAuditLogSearches { + <# + .SYNOPSIS + Get the available audit log searches + .DESCRIPTION + Query the Graph API for available audit log searches. + .PARAMETER TenantFilter + The tenant to filter on. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter, + [Parameter()] + [switch]$ReadyToProcess + ) + $Queries = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -AsApp $true -tenantid $TenantFilter + if ($ReadyToProcess.IsPresent) { + $AuditLogSearchesTable = Get-CippTable -TableName 'AuditLogSearches' + $PendingQueries = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "Tenant eq '$TenantFilter' and CippStatus eq 'Pending'" + $Queries = $Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' } + } + return $Queries +} diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 new file mode 100644 index 000000000000..5706a4ac3e4d --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippLastAuditLogSearch.ps1 @@ -0,0 +1,19 @@ +function Get-CippLastAuditLogSearch { + <# + .SYNOPSIS + Get the last audit log search + .DESCRIPTION + Query the Graph API for the last audit log search. + .PARAMETER TenantFilter + The tenant to filter on. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter + ) + + $Table = Get-CIPPTable -TableName AuditLogSearches + $LastHour = (Get-Date).AddHours(-1).ToString('yyyy-MM-ddTHH:mm:ssZ') + $LastSearch = Get-AzDataTableEntity @Table -Filter "Tenant eq '$TenantFilter' and Timestamp ge datetime'$LastHour'" | Sort-Object Timestamp -Descending | Select-Object -First 1 + return $LastSearch +} diff --git a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 index e28c6b742cac..6e48cc1c0f84 100644 --- a/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/New-CippAuditLogSearch.ps1 @@ -28,6 +28,8 @@ function New-CippAuditLogSearch { The object IDs to filter on. .PARAMETER AdministrativeUnitFilters The administrative units to filter on. + .PARAMETER ProcessLogs + Store the search in the CIPP AuditLogSearches table for alert processing. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -117,7 +119,9 @@ function New-CippAuditLogSearch { [Parameter()] [string[]]$ObjectIdFilters, [Parameter()] - [string[]]$AdministrativeUnitFilters + [string[]]$AdministrativeUnitFilters, + [Parameter()] + [switch]$ProcessLogs ) $SearchParams = @{ @@ -151,6 +155,22 @@ function New-CippAuditLogSearch { } if ($PSCmdlet.ShouldProcess('Create a new audit log search for tenant ' + $TenantFilter)) { - New-GraphPOSTRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -body ($SearchParams | ConvertTo-Json -Compress) -tenantid $TenantFilter -AsApp $true + $Query = New-GraphPOSTRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -body ($SearchParams | ConvertTo-Json -Compress) -tenantid $TenantFilter -AsApp $true + + if ($ProcessLogs.IsPresent -and $Query.id) { + $Entity = [PSCustomObject]@{ + PartitionKey = [string]'Search' + RowKey = [string]$Query.id + Tenant = [string]$TenantFilter + DisplayName = [string]$DisplayName + StartTime = [datetime]$StartTime.ToUniversalTime() + EndTime = [datetime]$EndTime.ToUniversalTime() + Query = [string]($Query | ConvertTo-Json -Compress) + CippStatus = [string]'Pending' + } + $Table = Get-CIPPTable -TableName 'AuditLogSearches' + Add-CIPPAzDataTableEntity @Table -Entity $Entity -Force | Out-Null + } + return $Query } } diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 index 8525ff065e34..a2a36fd518a9 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 @@ -1,78 +1,60 @@ function Push-AuditLogTenant { Param($Item) - # Get Table contexts - $AuditBundleTable = Get-CippTable -tablename 'AuditLogBundles' $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' - $WebhookTable = Get-CippTable -tablename 'webhookTable' $ConfigTable = Get-CippTable -TableName 'WebhookRules' + #$Tenant = Get-Tenants -TenantFilter $Item.customerId -IncludeErrors + $TenantFilter = $Item.TenantFilter + Write-Information "Audit Logs: Processing $($TenantFilter)" # Query CIPPURL for linking $CIPPURL = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'" | Select-Object -First 1 -ExpandProperty CIPPURL - # Get all webhooks for the tenant - $Webhooks = Get-CIPPAzDataTableEntity @WebhookTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and Version eq '3'" | Where-Object { $_.Resource -match '^Audit' } - # Get webhook rules $ConfigEntries = Get-CIPPAzDataTableEntity @ConfigTable + $LogSearchesTable = Get-CippTable -TableName 'AuditLogSearches' - # Date filter for existing bundles - $LastHour = (Get-Date).AddHours(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss') - - $NewBundles = [System.Collections.Generic.List[object]]::new() - foreach ($Webhook in $Webhooks) { - # only process webhooks that are configured in the webhookrules table - $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } - if ($Configuration.Type -notcontains $Webhook.Resource) { - continue - } - - $TenantFilter = $Webhook.PartitionKey - $LogType = $Webhook.Resource - Write-Information "Querying for $LogType on $TenantFilter" - $ContentBundleQuery = @{ - TenantFilter = $TenantFilter - ContentType = $LogType - StartTime = $Item.StartTime - EndTime = $Item.EndTime - } + $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } + if ($Configuration) { try { - $LogBundles = Get-CIPPAuditLogContentBundles @ContentBundleQuery - $ExistingBundles = Get-CIPPAzDataTableEntity @AuditBundleTable -Filter "PartitionKey eq '$($Item.TenantFilter)' and ContentType eq '$LogType' and Timestamp ge datetime'$($LastHour)'" - - foreach ($Bundle in $LogBundles) { - if ($ExistingBundles.RowKey -notcontains $Bundle.contentId) { - $NewBundles.Add([PSCustomObject]@{ - PartitionKey = $TenantFilter - RowKey = $Bundle.contentId - DefaultDomainName = $TenantFilter - ContentType = $Bundle.contentType - ContentUri = $Bundle.contentUri - ContentCreated = $Bundle.contentCreated - ContentExpiration = $Bundle.contentExpiration - CIPPURL = [string]$CIPPURL - ProcessingStatus = 'Pending' - MatchedRules = '' - MatchedLogs = 0 - }) + $LogSearches = Get-CippAuditLogSearches -TenantFilter $TenantFilter -ReadyToProcess + Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) + foreach ($Search in $LogSearches) { + $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "Tenant eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" + $SearchEntity.CippStatus = 'Processing' + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + try { + # Test the audit log rules against the search results + $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id + + $SearchEntity.CippStatus = 'Completed' + $MatchedRules = [string](ConvertTo-Json -Compress -InputObject $AuditLogTest.MatchedRules) + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value $MatchedRules + $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs + $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs + } catch { + $SearchEntity.CippStatus = 'Failed' + Write-Information "Error processing audit log rules: $($_.Exception.Message)" + $Exception = [string](ConvertTo-Json -Compress -InputObject (Get-CippException -Exception $_)) + $SearchEntity | Add-Member -MemberType NoteProperty -Name Error -Value $Exception + } + Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force + $DataToProcess = ($AuditLogTest).DataToProcess + Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items" + if ($DataToProcess) { + foreach ($AuditLog in $DataToProcess) { + Write-Information "Processing $($AuditLog.operation)" + $Webhook = @{ + Data = $AuditLog + CIPPURL = [string]$CIPPURL + TenantFilter = $TenantFilter + } + Invoke-CippWebhookProcessing @Webhook + } } } } catch { - Write-Information "Could not get audit log content bundles for $TenantFilter - $LogType, $($_.Exception.Message)" - } - } - - if (($NewBundles | Measure-Object).Count -gt 0) { - Add-CIPPAzDataTableEntity @AuditBundleTable -Entity $NewBundles -Force - Write-Information ($NewBundles | ConvertTo-Json -Depth 5 -Compress) - - $Batch = $NewBundles | Select-Object @{Name = 'ContentId'; Expression = { $_.RowKey } }, @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogBundleProcessing' } } - $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' - Batch = @($Batch) - SkipLog = $true + Write-Information ( 'Audit Logs: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) } - $InstanceId = Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) - Write-Host "Started orchestration with ID = '$InstanceId'" } } diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 index e31239f1bf02..ab7474163cdc 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSite.ps1 @@ -5,7 +5,7 @@ Function Invoke-AddSite { .FUNCTIONALITY Entrypoint .ROLE - Teams.Group.ReadWrite + Sharepoint.Site.ReadWrite #> [CmdletBinding()] param($Request, $TriggerMetadata) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 index 79ac24fbc508..97f779f10c76 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-AddSiteBulk.ps1 @@ -5,7 +5,7 @@ Function Invoke-AddSiteBulk { .FUNCTIONALITY Entrypoint .ROLE - Teams.Group.ReadWrite + Sharepoint.Site.ReadWrite #> [CmdletBinding()] param($Request, $TriggerMetadata) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 new file mode 100644 index 000000000000..ef0e1e5c690e --- /dev/null +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Teams-Sharepoint/Invoke-ListSharepointAdminUrl.ps1 @@ -0,0 +1,50 @@ +function Invoke-ListSharepointAdminUrl { + <# + .FUNCTIONALITY + Entrypoint + .ROLE + CIPP.Core.Read + #> + [CmdletBinding()] + param( + $Request, + $TriggerMetadata + ) + + if ($Request.Query.TenantFilter) { + $TenantFilter = $Request.Query.TenantFilter + + $Tenant = Get-Tenants -TenantFilter $TenantFilter + + if ($Tenant.SharepointAdminUrl) { + $AdminUrl = $Tenant.SharepointAdminUrl + } else { + $tenantName = (New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/sites/root' -asApp $true -tenantid $TenantFilter).id.Split('.')[0] + $AdminUrl = "https://$($tenantName)-admin.sharepoint.com" + $Tenant | Add-Member -MemberType NoteProperty -Name SharepointAdminUrl -Value $AdminUrl + $Table = Get-CIPPTable -TableName 'Tenants' + Add-CIPPAzDataTableEntity @Table -Entity $Tenant -Force + } + + if ($Request.Query.ReturnUrl) { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::OK + Body = @{ + AdminUrl = $AdminUrl + } + }) + } else { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::Found + Headers = @{ + Location = $AdminUrl + } + }) + } + } else { + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ + StatusCode = [HttpStatusCode]::BadRequest + Body = 'TenantFilter is required' + }) + } +} diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index b17c0c00f3ad..5b6640416564 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -6,29 +6,48 @@ function Start-AuditLogOrchestrator { [CmdletBinding(SupportsShouldProcess = $true)] param() try { - $webhookTable = Get-CIPPTable -tablename webhookTable - $Webhooks = Get-CIPPAzDataTableEntity @webhookTable -Filter "Version eq '3'" | Where-Object { $_.Resource -match '^Audit' -and $_.Status -ne 'Disabled' } - if (($Webhooks | Measure-Object).Count -eq 0) { - Write-Information 'No webhook subscriptions found. Exiting.' - return - } - $StartTime = (Get-Date).AddMinutes(-30) - $EndTime = Get-Date + $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' + $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" + + # Round time down to nearest minute + $Now = Get-Date + $DefaultStartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddHours(-1) + $EndTime = $Now.AddMinutes(-30).AddSeconds(-$Now.Seconds) - $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($Webhooks | Sort-Object -Property PartitionKey -Unique | Measure-Object).Count + if (($AuditLogSearches | Measure-Object).Count -eq 0) { + Write-Information 'No audit log searches available' + } else { + $TenantList = Get-Tenants -IncludeErrors + $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count - $Batch = $Webhooks | Sort-Object -Property PartitionKey -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.PartitionKey } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } }, @{Name = 'StartTime'; Expression = { $StartTime } }, @{Name = 'EndTime'; Expression = { $EndTime } } - $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' - Batch = @($Batch) - SkipLog = $true + $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } } + + $InputObject = [PSCustomObject]@{ + OrchestratorName = 'AuditLogs' + Batch = @($Batch) + SkipLog = $true + } + if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { + Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) + } } - if ($PSCmdlet.ShouldProcess('Start-AuditLogPolling', 'Starting Audit Log Polling')) { - Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) + + Write-Information 'Audit Logs: Creating new searches' + foreach ($Tenant in $TenantList) { + try { + $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName + if ($LastSearch -and $StartTime -gt $DefaultStartTime) { + $StartTime = $LastSearch.EndTime + } else { + $StartTime = $DefaultStartTime + } + $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs + } catch { + } } } catch { - Write-LogMessage -API 'Webhooks' -message 'Error processing webhooks' -sev Error -LogData (Get-CippException -Exception $_) - Write-Information ( 'Webhook error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + Write-LogMessage -API 'Audit Logs' -message 'Error processing audit logs' -sev Error -LogData (Get-CippException -Exception $_) + Write-Information ( 'Audit logs error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) } } diff --git a/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 b/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 index b8d84bf0052d..43d7617833c1 100644 --- a/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 +++ b/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 @@ -95,11 +95,12 @@ function Get-GraphRequestList { $Count = 0 if ($TenantFilter -ne 'AllTenants') { $GraphRequest = @{ - uri = $GraphQuery.ToString() - tenantid = $TenantFilter - ComplexFilter = $true + uri = $GraphQuery.ToString() + tenantid = $TenantFilter + } + if ($Parameters.'$filter') { + $GraphRequest.ComplexFilter = $true } - if ($NoPagination.IsPresent) { $GraphRequest.noPagination = $NoPagination.IsPresent } diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index 33ef01e6589c..3288712f18a4 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -4,10 +4,7 @@ function Test-CIPPAuditLogRules { [Parameter(Mandatory = $true)] $TenantFilter, [Parameter(Mandatory = $true)] - $ContentUri, - [Parameter(Mandatory = $true)] - [ValidateSet('Audit.AzureActiveDirectory', 'Audit.Exchange')] - $LogType + $SearchId ) $Results = [PSCustomObject]@{ @@ -37,17 +34,19 @@ function Test-CIPPAuditLogRules { LogType = $_.Type } } - $AuditLogQuery = @{ - TenantFilter = $TenantFilter - ContentUri = $ContentUri - } - Write-Information 'Getting data from Office 365 Management Activity API' - $Data = Get-CIPPAuditLogContent @AuditLogQuery + #$AuditLogQuery = @{ + # TenantFilter = $TenantFilter + # ContentUri = $ContentUri + #} + #Write-Information 'Getting data from Office 365 Management Activity API' + #$Data = Get-CIPPAuditLogContent @AuditLogQuery + Write-Information 'Getting audit records from Graph API' + $Data = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId $LogCount = ($Data | Measure-Object).Count Write-Information "Logs to process: $LogCount" $Results.TotalLogs = $LogCount if ($LogCount -gt 0) { - $PreProcessedData = $Data | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties -ErrorAction SilentlyContinue + $PreProcessedData = $Data.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties -ErrorAction SilentlyContinue $LocationTable = Get-CIPPTable -TableName 'knownlocationdb' $ProcessedData = foreach ($Data in $PreProcessedData) { try { From f8b88e53b949ed2282065538e106dfb41bb843fe Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 20:57:47 -0400 Subject: [PATCH 22/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 5b6640416564..c64e475ebd78 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -12,8 +12,8 @@ function Start-AuditLogOrchestrator { # Round time down to nearest minute $Now = Get-Date - $DefaultStartTime = (Get-Date).AddSeconds(-$Now.Seconds).AddHours(-1) - $EndTime = $Now.AddMinutes(-30).AddSeconds(-$Now.Seconds) + $DefaultStartTime = $Now.AddSeconds(-$Now.Seconds).AddHours(-1) + $EndTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' @@ -37,7 +37,7 @@ function Start-AuditLogOrchestrator { foreach ($Tenant in $TenantList) { try { $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName - if ($LastSearch -and $StartTime -gt $DefaultStartTime) { + if ($LastSearch -and $LastSearch.EndTime -le $DefaultStartTime) { $StartTime = $LastSearch.EndTime } else { $StartTime = $DefaultStartTime From d0778b2033f7614f87825cf91d31f641e23b3b11 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 21:01:43 -0400 Subject: [PATCH 23/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index c64e475ebd78..33d1332e18a4 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -44,6 +44,7 @@ function Start-AuditLogOrchestrator { } $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs } catch { + Write-Information "Error creating audit log search $($Tenant.defaultDomainName) - $($_.Exception.Message)" } } } catch { From dd1c64957110e871fc55a9664e6aba8c3cd86055 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 22:03:39 -0400 Subject: [PATCH 24/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 33d1332e18a4..9ccac0a74a64 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -42,7 +42,8 @@ function Start-AuditLogOrchestrator { } else { $StartTime = $DefaultStartTime } - $null = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs + $NewSearch = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs + Write-Information "Created audit log search $($Tenant.defaultDomainName) - $($NewSearch|ConvertTo-Json -Depth 5 -Compress)" } catch { Write-Information "Error creating audit log search $($Tenant.defaultDomainName) - $($_.Exception.Message)" } From 30b6f85428fea35febcff62230b564bdf330e72a Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 1 Oct 2024 22:14:31 -0400 Subject: [PATCH 25/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 9ccac0a74a64..29555267ad28 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -6,10 +6,9 @@ function Start-AuditLogOrchestrator { [CmdletBinding(SupportsShouldProcess = $true)] param() try { - $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" - + $TenantList = Get-Tenants -IncludeErrors # Round time down to nearest minute $Now = Get-Date $DefaultStartTime = $Now.AddSeconds(-$Now.Seconds).AddHours(-1) @@ -18,9 +17,7 @@ function Start-AuditLogOrchestrator { if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' } else { - $TenantList = Get-Tenants -IncludeErrors $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count - $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } } $InputObject = [PSCustomObject]@{ @@ -43,7 +40,7 @@ function Start-AuditLogOrchestrator { $StartTime = $DefaultStartTime } $NewSearch = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs - Write-Information "Created audit log search $($Tenant.defaultDomainName) - $($NewSearch|ConvertTo-Json -Depth 5 -Compress)" + Write-Information "Created audit log search $($Tenant.defaultDomainName) - $($NewSearch.displayName)" } catch { Write-Information "Error creating audit log search $($Tenant.defaultDomainName) - $($_.Exception.Message)" } From 499a20660fae3fb27f932712d65fb82ba5de215e Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 10:31:29 -0400 Subject: [PATCH 26/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 29555267ad28..2d43c1a22b10 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -35,7 +35,7 @@ function Start-AuditLogOrchestrator { try { $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName if ($LastSearch -and $LastSearch.EndTime -le $DefaultStartTime) { - $StartTime = $LastSearch.EndTime + $StartTime = $LastSearch.EndTime.DateTime } else { $StartTime = $DefaultStartTime } From 593cf41a15d8b04405e58531e591aee47ac3e187 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 13:59:07 -0400 Subject: [PATCH 27/31] Update Start-AuditLogOrchestrator.ps1 --- .../Start-AuditLogOrchestrator.ps1 | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 2d43c1a22b10..0f777d758398 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -11,8 +11,8 @@ function Start-AuditLogOrchestrator { $TenantList = Get-Tenants -IncludeErrors # Round time down to nearest minute $Now = Get-Date - $DefaultStartTime = $Now.AddSeconds(-$Now.Seconds).AddHours(-1) - $EndTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) + $DefaultStartTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) + $EndTime = $Now.AddSeconds(-$Now.Seconds) if (($AuditLogSearches | Measure-Object).Count -eq 0) { Write-Information 'No audit log searches available' @@ -33,12 +33,6 @@ function Start-AuditLogOrchestrator { Write-Information 'Audit Logs: Creating new searches' foreach ($Tenant in $TenantList) { try { - $LastSearch = Get-CippLastAuditLogSearch -TenantFilter $Tenant.defaultDomainName - if ($LastSearch -and $LastSearch.EndTime -le $DefaultStartTime) { - $StartTime = $LastSearch.EndTime.DateTime - } else { - $StartTime = $DefaultStartTime - } $NewSearch = New-CippAuditLogSearch -TenantFilter $Tenant.defaultDomainName -StartTime $StartTime -EndTime $EndTime -ProcessLogs Write-Information "Created audit log search $($Tenant.defaultDomainName) - $($NewSearch.displayName)" } catch { From 35e932015cb79e9ee5ba751682858612c6223e60 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 14:01:10 -0400 Subject: [PATCH 28/31] update timer --- CIPPTimers.json | 5 +++-- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CIPPTimers.json b/CIPPTimers.json index 16122b18c6bb..e8763e7cf3d9 100644 --- a/CIPPTimers.json +++ b/CIPPTimers.json @@ -17,10 +17,11 @@ { "Command": "Start-AuditLogOrchestrator", "Description": "Orchestrator to process audit logs", - "Cron": "0 */30 * * * *", + "Cron": "0 */15 * * * *", "Priority": 2, "RunOnProcessor": true, - "PreferredProcessor": "auditlog" + "PreferredProcessor": "auditlog", + "IsSystem": true }, { "Command": "Start-WebhookOrchestrator", diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 0f777d758398..452f0594f8bd 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -11,7 +11,7 @@ function Start-AuditLogOrchestrator { $TenantList = Get-Tenants -IncludeErrors # Round time down to nearest minute $Now = Get-Date - $DefaultStartTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) + $StartTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) $EndTime = $Now.AddSeconds(-$Now.Seconds) if (($AuditLogSearches | Measure-Object).Count -eq 0) { From 7f9e95241d2d02f8f866c6ed7d96a72336eaec76 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 14:49:57 -0400 Subject: [PATCH 29/31] Update Get-CIPPAlertQuotaUsed.ps1 --- Modules/CIPPCore/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1 b/Modules/CIPPCore/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1 index 84e1e0864b87..1e61ffaa01b6 100644 --- a/Modules/CIPPCore/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1 +++ b/Modules/CIPPCore/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1 @@ -18,7 +18,9 @@ function Get-CIPPAlertQuotaUsed { } $OverQuota = $AlertData | ForEach-Object { if ($_.StorageUsedInBytes -eq 0 -or $_.prohibitSendReceiveQuotaInBytes -eq 0) { return } - $PercentLeft = [math]::round(($_.storageUsedInBytes / $_.prohibitSendReceiveQuotaInBytes) * 100) + try { + $PercentLeft = [math]::round(($_.storageUsedInBytes / $_.prohibitSendReceiveQuotaInBytes) * 100) + } catch { $PercentLeft = 100 } try { if ([int]$InputValue -gt 0) { $Value = [int]$InputValue @@ -34,4 +36,4 @@ function Get-CIPPAlertQuotaUsed { } Write-AlertTrace -cmdletName $MyInvocation.MyCommand -tenantFilter $TenantFilter -data $OverQuota -} \ No newline at end of file +} From 891d989c745ba2900ef84db19e8ddbfe3303d14b Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 14:55:45 -0400 Subject: [PATCH 30/31] Update Start-AuditLogOrchestrator.ps1 --- .../Orchestrator Functions/Start-AuditLogOrchestrator.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 452f0594f8bd..4790f78afc9c 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -11,7 +11,7 @@ function Start-AuditLogOrchestrator { $TenantList = Get-Tenants -IncludeErrors # Round time down to nearest minute $Now = Get-Date - $StartTime = $Now.AddSeconds(-$Now.Seconds).AddMinutes(-30) + $StartTime = ($Now.AddSeconds(-$Now.Seconds)).AddMinutes(-30) $EndTime = $Now.AddSeconds(-$Now.Seconds) if (($AuditLogSearches | Measure-Object).Count -eq 0) { From a915505d99d113ea96481e777754ea7607821fe1 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Wed, 2 Oct 2024 15:28:05 -0400 Subject: [PATCH 31/31] Update Get-CippAuditLogSearchResults.ps1 --- .../CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 index 5dedbd706609..4113c17dc8fb 100644 --- a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 @@ -18,6 +18,6 @@ function Get-CippAuditLogSearchResults { ) process { - New-GraphGetRequest -uri ('https://graph.microsoft.com/beta/security/auditLog/queries/{0}/records' -f $QueryId) -AsApp $true -tenantid $TenantFilter + New-GraphGetRequest -uri ('https://graph.microsoft.com/beta/security/auditLog/queries/{0}/records?$top=999' -f $QueryId) -AsApp $true -tenantid $TenantFilter } }