From bb6325cf448892c3f22a140861cf788ea7e7abdc Mon Sep 17 00:00:00 2001 From: KelvinTegelaar Date: Fri, 4 Oct 2024 19:49:55 +0200 Subject: [PATCH] revert --- .../Webhooks/Test-CIPPAuditLogRules.ps1 | 267 ++++++++++++------ 1 file changed, 179 insertions(+), 88 deletions(-) diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index c0ccc91c3443..33f22fc0c953 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -1,101 +1,192 @@ -function Start-AuditLogOrchestrator { - <# - .SYNOPSIS - Start the Audit Log Polling Orchestrator - #> - [CmdletBinding(SupportsShouldProcess = $true)] - param() - try { - $AuditLogSearchesTable = Get-CIPPTable -TableName 'AuditLogSearches' - $AuditLogSearches = Get-CIPPAzDataTableEntity @AuditLogSearchesTable -Filter "CippStatus eq 'Pending'" +function Test-CIPPAuditLogRules { + [CmdletBinding()] + Param( + [Parameter(Mandatory = $true)] + $TenantFilter, + [Parameter(Mandatory = $true)] + $SearchId + ) - $ConfigTable = Get-CippTable -TableName 'WebhookRules' - $ConfigEntries = Get-CIPPAzDataTableEntity @ConfigTable + $Results = [PSCustomObject]@{ + TotalLogs = 0 + MatchedLogs = 0 + MatchedRules = @() + DataToProcess = @() + } - $TenantList = Get-Tenants -IncludeErrors - # Round time down to nearest minute - $Now = Get-Date - $StartTime = ($Now.AddSeconds(-$Now.Seconds)).AddHours(-1) - $EndTime = $Now.AddSeconds(-$Now.Seconds) + $ExtendedPropertiesIgnoreList = @( + 'OAuth2:Authorize' + 'OAuth2:Token' + 'SAS:EndAuth' + 'SAS:ProcessAuth' + 'deviceAuth:ReprocessTls' + 'Consent:Set' + ) - if (($AuditLogSearches | Measure-Object).Count -eq 0) { - Write-Information 'No audit log searches available' - } else { - $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count - $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } } + $TrustedIPTable = Get-CIPPTable -TableName 'trustedIps' + $ConfigTable = Get-CIPPTable -TableName 'WebhookRules' + $ConfigEntries = Get-CIPPAzDataTableEntity @ConfigTable + $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } | ForEach-Object { + [pscustomobject]@{ + Tenants = ($_.Tenants | ConvertFrom-Json).fullValue + Conditions = $_.Conditions + Actions = $_.Actions + LogType = $_.Type + } + } + #write-warning 'Getting audit records from Graph API' + $SearchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId + $LogCount = ($SearchResults | Measure-Object).Count + $RunGuid = New-Guid + Write-Warning "Logs to process: $LogCount - RunGuid: $($RunGuid) - $($TenantFilter)" + $Results.TotalLogs = $LogCount + if ($LogCount -gt 0) { + $LocationTable = Get-CIPPTable -TableName 'knownlocationdb' + $ProcessedData = foreach ($AuditRecord in $SearchResults) { + $RootProperties = $AuditRecord | Select-Object * -ExcludeProperty auditData + $Data = $AuditRecord.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties, AuditRecord -ErrorAction SilentlyContinue + try { + if ($Data.ExtendedProperties) { + $Data.CIPPExtendedProperties = ($Data.ExtendedProperties | ConvertTo-Json) + $Data.ExtendedProperties | ForEach-Object { + if ($_.Value -in $ExtendedPropertiesIgnoreList) { + #write-warning "No need to process this operation as its in our ignore list. Some extended information: $($data.operation):$($_.Value) - $($TenantFilter)" + continue + } + $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue + } + } + if ($Data.DeviceProperties) { + $Data.CIPPDeviceProperties = ($Data.DeviceProperties | ConvertTo-Json) + $Data.DeviceProperties | ForEach-Object { $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue } + } + if ($Data.parameters) { + $Data.CIPPParameters = ($Data.parameters | ConvertTo-Json) + $Data.parameters | ForEach-Object { $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue } + } + if ($Data.ModifiedProperties) { + $Data.CIPPModifiedProperties = ($Data.ModifiedProperties | ConvertTo-Json) + try { + $Data.ModifiedProperties | ForEach-Object { $Data | Add-Member -NotePropertyName "$($_.Name)" -NotePropertyValue "$($_.NewValue)" -Force -ErrorAction SilentlyContinue } + } catch { + ##write-warning ($Data.ModifiedProperties | ConvertTo-Json -Depth 10) + } + try { + $Data.ModifiedProperties | ForEach-Object { $Data | Add-Member -NotePropertyName $("Previous_Value_$($_.Name)") -NotePropertyValue "$($_.OldValue)" -Force -ErrorAction SilentlyContinue } + } catch { + ##write-warning ($Data.ModifiedProperties | ConvertTo-Json -Depth 10) + } + } - $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' - Batch = @($Batch) - SkipLog = $true - } - if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { - Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) + if ($Data.clientip) { + if ($Data.clientip -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+$') { + $Data.clientip = $Data.clientip -replace ':\d+$', '' # Remove the port number if present + } + # Check if IP is on trusted IP list + $TrustedIP = Get-CIPPAzDataTableEntity @TrustedIPTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$($Data.clientip)' and state eq 'Trusted'" + if ($TrustedIP) { + #write-warning "IP $($Data.clientip) is trusted" + $Trusted = $true + } + if (!$Trusted) { + $Location = Get-CIPPAzDataTableEntity @LocationTable -Filter "RowKey eq '$($Data.clientIp)'" | Select-Object -Last 1 + if ($Location) { + $Country = $Location.CountryOrRegion + $City = $Location.City + $Proxy = $Location.Proxy + $hosting = $Location.Hosting + $ASName = $Location.ASName + } else { + try { + $Location = Get-CIPPGeoIPLocation -IP $Data.clientip + } catch { + #write-warning "Unable to get IP location for $($Data.clientip): $($_.Exception.Message)" + } + $Country = if ($Location.CountryCode) { $Location.CountryCode } else { 'Unknown' } + $City = if ($Location.City) { $Location.City } else { 'Unknown' } + $Proxy = if ($Location.Proxy -ne $null) { $Location.Proxy } else { 'Unknown' } + $hosting = if ($Location.Hosting -ne $null) { $Location.Hosting } else { 'Unknown' } + $ASName = if ($Location.ASName) { $Location.ASName } else { 'Unknown' } + $IP = $Data.ClientIP + $LocationInfo = @{ + RowKey = [string]$Data.clientip + PartitionKey = [string]$Data.id + Tenant = [string]$TenantFilter + CountryOrRegion = "$Country" + City = "$City" + Proxy = "$Proxy" + Hosting = "$hosting" + ASName = "$ASName" + } + try { + $null = Add-CIPPAzDataTableEntity @LocationTable -Entity $LocationInfo -Force + } catch { + #write-warning "Failed to add location info for $($Data.clientip) to cache: $($_.Exception.Message)" + + } + } + $Data.CIPPGeoLocation = $Country + $Data.CIPPBadRepIP = $Proxy + $Data.CIPPHostedIP = $hosting + $Data.CIPPIPDetected = $IP + $Data.CIPPLocationInfo = ($Location | ConvertTo-Json) + $Data.AuditRecord = ($RootProperties | ConvertTo-Json) + } + } + $Data | Select-Object * -ExcludeProperty ExtendedProperties, DeviceProperties, parameters + } catch { + #write-warning "Audit log: Error processing data: $($_.Exception.Message)`r`n$($_.InvocationInfo.PositionMessage)" + Write-LogMessage -API 'Webhooks' -message 'Error Processing Audit Log Data' -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter } } + #write-warning "Processed Data: $(($ProcessedData | Measure-Object).Count) - This should be higher than 0 in many cases, because the where object has not run yet." + #write-warning "Creating filters - $(($ProcessedData.operation | Sort-Object -Unique) -join ',') - $($TenantFilter)" - Write-Information 'Audit Logs: Creating new searches' - foreach ($Tenant in $TenantList) { - $Configuration = $ConfigEntries | Where-Object { ($_.Tenants -match $TenantFilter -or $_.Tenants -match 'AllTenants') } - if ($Configuration) { - $ServiceFilters = $Configuration | Select-Object -Property type | Sort-Object -Property type -Unique | ForEach-Object { $_.type.split('.')[1] } - try { - $LogSearch = @{ - StartTime = $StartTime - EndTime = $EndTime - ServiceFilters = $ServiceFilters - TenantFilter = $Tenant.defaultDomainName - ProcessLogs = $true - RecordTypeFilters = @( - 'exchangeAdmin', 'azureActiveDirectory', 'azureActiveDirectoryAccountLogon', 'dataCenterSecurityCmdlet', - 'complianceDLPSharePoint', 'complianceDLPExchange', 'azureActiveDirectoryStsLogon', 'skypeForBusinessPSTNUsage', - 'skypeForBusinessUsersBlocked', 'securityComplianceCenterEOPCmdlet', 'microsoftFlow', 'aeD', 'microsoftStream', - 'threatFinder', 'project', 'dataGovernance', 'securityComplianceAlerts', 'threatIntelligenceUrl', - 'securityComplianceInsights', 'mipLabel', 'workplaceAnalytics', 'powerAppsApp', 'powerAppsPlan', - 'threatIntelligenceAtpContent', 'labelContentExplorer', 'hygieneEvent', - 'dataInsightsRestApiAudit', 'informationBarrierPolicyApplication', 'microsoftTeamsAdmin', 'hrSignal', - 'informationWorkerProtection', 'campaign', 'dlpEndpoint', 'airInvestigation', 'quarantine', 'microsoftForms', - 'applicationAudit', 'complianceSupervisionExchange', 'customerKeyServiceEncryption', 'officeNative', - 'mipAutoLabelSharePointItem', 'mipAutoLabelSharePointPolicyLocation', 'secureScore', - 'mipAutoLabelExchangeItem', 'cortanaBriefing', 'search', 'wdatpAlerts', 'powerPlatformAdminDlp', - 'powerPlatformAdminEnvironment', 'mdatpAudit', 'sensitivityLabelPolicyMatch', 'sensitivityLabelAction', - 'sensitivityLabeledFileAction', 'attackSim', 'airManualInvestigation', 'securityComplianceRBAC', 'userTraining', - 'airAdminActionInvestigation', 'mstic', 'physicalBadgingSignal', 'aipDiscover', 'aipSensitivityLabelAction', - 'aipProtectionAction', 'aipFileDeleted', 'aipHeartBeat', 'mcasAlerts', 'onPremisesFileShareScannerDlp', - 'onPremisesSharePointScannerDlp', 'exchangeSearch', 'privacyDataMinimization', 'labelAnalyticsAggregate', - 'myAnalyticsSettings', 'securityComplianceUserChange', 'complianceDLPExchangeClassification', - 'complianceDLPEndpoint', 'mipExactDataMatch', 'msdeResponseActions', 'msdeGeneralSettings', 'msdeIndicatorsSettings', - 'ms365DCustomDetection', 'msdeRolesSettings', 'mapgAlerts', 'mapgPolicy', 'mapgRemediation', - 'privacyRemediationAction', 'privacyDigestEmail', 'mipAutoLabelSimulationProgress', 'mipAutoLabelSimulationCompletion', - 'mipAutoLabelProgressFeedback', 'dlpSensitiveInformationType', 'mipAutoLabelSimulationStatistics', - 'largeContentMetadata', 'microsoft365Group', 'cdpMlInferencingResult', 'filteringMailMetadata', - 'cdpClassificationMailItem', 'cdpClassificationDocument', 'officeScriptsRunAction', 'filteringPostMailDeliveryAction', - 'cdpUnifiedFeedback', 'tenantAllowBlockList', 'consumptionResource', 'healthcareSignal', 'dlpImportResult', - 'cdpCompliancePolicyExecution', 'multiStageDisposition', 'privacyDataMatch', 'filteringDocMetadata', - 'filteringEmailFeatures', 'powerBIDlp', 'filteringUrlInfo', 'filteringAttachmentInfo', 'coreReportingSettings', - 'complianceConnector', 'powerPlatformLockboxResourceAccessRequest', 'powerPlatformLockboxResourceCommand', - 'cdpPredictiveCodingLabel', 'cdpCompliancePolicyUserFeedback', 'webpageActivityEndpoint', 'omePortal', - 'cmImprovementActionChange', 'filteringUrlClick', 'mipLabelAnalyticsAuditRecord', 'filteringEntityEvent', - 'filteringRuleHits', 'filteringMailSubmission', 'labelExplorer', 'microsoftManagedServicePlatform', - 'powerPlatformServiceActivity', 'scorePlatformGenericAuditRecord', 'filteringTimeTravelDocMetadata', 'alert', - 'alertStatus', 'alertIncident', 'incidentStatus', 'case', 'caseInvestigation', 'recordsManagement', - 'privacyRemediation', 'dataShareOperation', 'cdpDlpSensitive', 'ehrConnector', 'filteringMailGradingResult', - 'microsoftTodoAudit', 'timeTravelFilteringDocMetadata', 'microsoftDefenderForIdentityAudit', - 'supervisoryReviewDayXInsight', 'defenderExpertsforXDRAdmin', 'cdpEdgeBlockedMessage', 'hostedRpa', - 'cdpContentExplorerAggregateRecord', 'cdpHygieneAttachmentInfo', 'cdpHygieneSummary', 'cdpPostMailDeliveryAction', - 'cdpEmailFeatures', 'cdpHygieneUrlInfo', 'cdpUrlClick', 'cdpPackageManagerHygieneEvent', 'filteringDocScan', - 'timeTravelFilteringDocScan', 'mapgOnboard' - ) + $Where = $Configuration | ForEach-Object { + $conditions = $_.Conditions | ConvertFrom-Json | Where-Object { $_.Input.value -ne '' } + $actions = $_.Actions + $conditionStrings = [System.Collections.Generic.List[string]]::new() + $CIPPClause = [System.Collections.Generic.List[string]]::new() + foreach ($condition in $conditions) { + $value = if ($condition.Input.value -is [array]) { + $arrayAsString = $condition.Input.value | ForEach-Object { + "'$_'" } - $NewSearch = New-CippAuditLogSearch @LogSearch - Write-Information "Created audit log search $($Tenant.defaultDomainName) - $($NewSearch.displayName)" - } catch { - Write-Information "Error creating audit log search $($Tenant.defaultDomainName) - $($_.Exception.Message)" + "@($($arrayAsString -join ', '))" + } else { "'$($condition.Input.value)'" } + + $conditionStrings.Add("`$(`$_.$($condition.Property.label)) -$($condition.Operator.value) $value") + $CIPPClause.Add("$($condition.Property.label) is $($condition.Operator.label) $value") + } + $finalCondition = $conditionStrings -join ' -AND ' + + [PSCustomObject]@{ + clause = $finalCondition + expectedAction = $actions + CIPPClause = $CIPPClause + } + + } + + $MatchedRules = [System.Collections.Generic.List[string]]::new() + $DataToProcess = foreach ($clause in $Where) { + #write-warning "Webhook: Processing clause: $($clause.clause)" + $ReturnedData = $ProcessedData | Where-Object { Invoke-Expression $clause.clause } + if ($ReturnedData) { + #write-warning "Webhook: There is matching data: $(($ReturnedData.operation | Select-Object -Unique) -join ', ')" + $ReturnedData = foreach ($item in $ReturnedData) { + $item.CIPPAction = $clause.expectedAction + $item.CIPPClause = $clause.CIPPClause -join ' and ' + $MatchedRules.Add($clause.CIPPClause -join ' and ') + $item } } + $ReturnedData } - } catch { - Write-LogMessage -API 'Audit Logs' -message 'Error processing audit logs' -sev Error -LogData (Get-CippException -Exception $_) - Write-Information ( 'Audit logs error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + $Results.MatchedRules = @($MatchedRules | Select-Object -Unique) + $Results.MatchedLogs = ($DataToProcess | Measure-Object).Count + $Results.DataToProcess = $DataToProcess } + Write-Warning "Finished - RunGuid: $($RunGuid) - $($TenantFilter)" + $Results }