diff --git a/src/CopyAzureContainer/Scripts/Functions.ps1 b/src/CopyAzureContainer/Scripts/Functions.ps1 index 41c296a7b..b7dc6f5c4 100644 --- a/src/CopyAzureContainer/Scripts/Functions.ps1 +++ b/src/CopyAzureContainer/Scripts/Functions.ps1 @@ -61,34 +61,3 @@ Function Install-AzCopy Remove-Item $bootstrap -Recurse -Force } } - -Function Uninstall-NuGetService() { - Param ([string]$ServiceName) - - if (Get-Service $ServiceName -ErrorAction SilentlyContinue) - { - Write-Host Removing service $ServiceName... - Stop-Service $ServiceName -Force - sc.exe delete $ServiceName - Write-Host Removed service $ServiceName. - } else { - Write-Host Skipping removal of service $ServiceName - no such service exists. - } -} - -Function Install-NuGetService() { - Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) - - Write-Host Installing service $ServiceName... - - $installService = "nssm install $ServiceName $ScriptToRun" - cmd /C $installService - - Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic - sc.exe failure $ServiceName reset= 30 actions= restart/5000 - - # Run service - net start $ServiceName - - Write-Host Installed service $ServiceName. -} \ No newline at end of file diff --git a/src/CopyAzureContainer/Scripts/backupv3storage.cmd b/src/CopyAzureContainer/Scripts/backupv3storage.cmd index 6884f4794..2dfb2fbd5 100644 --- a/src/CopyAzureContainer/Scripts/backupv3storage.cmd +++ b/src/CopyAzureContainer/Scripts/backupv3storage.cmd @@ -9,8 +9,9 @@ title #{Jobs.backupv3storage.Title} start /w CopyAzureContainer.exe ^ -SourceContainerInfo_lucene #{Jobs.common.v3.Storage.Primary.Name}:#{Jobs.common.v3.Storage.Primary.Key}:#{Jobs.catalog2lucenev3reg2.LuceneContainer} ^ -SourceContainerInfo_catalog #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.feed2catalogv3.StorageContainer} ^ - -SourceContainerInfo_registration #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg1.StorageContainer} ^ - -SourceContainerInfo_registrationgz #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg1.StorageContainerCompressed} ^ + -SourceContainerInfo_registration #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainer} ^ + -SourceContainerInfo_registrationgz #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainerCompressed} ^ + -SourceContainerInfo_registrationsemver2 #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainerSemVer2} ^ -DestStorageAccountName #{Jobs.backupv3storage.destStorageAccountName} ^ -DestStorageKeyValue #{Jobs.backupv3storage.destStorageKeyValue} ^ -BackupDays #{Jobs.backupv3storage.backupDays} ^ diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index 23903ffbe..00047d83e 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -123,5 +123,10 @@ + + + + + \ No newline at end of file diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.nuspec b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.nuspec index 3c461cab3..768585b2c 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.nuspec +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.nuspec @@ -12,8 +12,8 @@ - - + + diff --git a/src/NuGet.SupportRequests.Notifications/Scripts/Functions.ps1 b/src/NuGet.SupportRequests.Notifications/Scripts/Functions.ps1 new file mode 100644 index 000000000..b6286275b --- /dev/null +++ b/src/NuGet.SupportRequests.Notifications/Scripts/Functions.ps1 @@ -0,0 +1,31 @@ +Function Install-DailyTask +{ + $trigger = New-ScheduledTaskTrigger -DaysInterval 1 -At "12pm" -Daily + + Install-NuGetScheduledTask $trigger "Support Request Daily Notification" "OnCallDailyNotification.cmd" +} + +Function Install-WeeklyTask +{ + $trigger = New-ScheduledTaskTrigger -Weekly -WeeksInterval 1 -DaysOfWeek Monday -At "12pm" + + Install-NuGetScheduledTask $trigger "Support Requests Weekly Notification" "WeeklySummaryNotification.cmd" +} + +Function Install-NuGetScheduledTask +{ + param($Trigger, $Name, $Command) + + #Action to run as + $cmdExe = [system.environment]::getenvironmentvariable("ComSpec") + $action = New-ScheduledTaskAction -Execute $cmdExe -Argument "/c $PSScriptRoot\$Command" -WorkingDirectory $PSScriptRoot + + #Configure when to stop the task and how long it can run for. In this example it does not stop on idle and uses the maximum possible duration by setting a timelimit of 0 + $settings = New-ScheduledTaskSettingsSet -DontStopOnIdleEnd -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances IgnoreNew + + #Configure the principal to use for the scheduled task and the level to run as + $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel "Highest" + + #Register the new scheduled task + Register-ScheduledTask -TaskName $Name -Action $action -Trigger $Trigger -Principal $principal -Settings $settings -Force +} diff --git a/src/NuGet.SupportRequests.Notifications/Scripts/PostDeploy.ps1 b/src/NuGet.SupportRequests.Notifications/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..f6737ecaf --- /dev/null +++ b/src/NuGet.SupportRequests.Notifications/Scripts/PostDeploy.ps1 @@ -0,0 +1,10 @@ +. .\Functions.ps1 + +## The daily task is broken right now due to Pin +# Write-Host Register the daily scheduled task +# Install-DailyTask +# Write-Host Registered the daily scheduled task + +Write-Host Register the weekly scheduled task +Install-WeeklyTask +Write-Host Registered the weekly scheduled task diff --git a/src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs b/src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs index 56a4311f1..42fb5bcfe 100644 --- a/src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs +++ b/src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs @@ -105,12 +105,14 @@ public static CdnLogEntry ParseLogEntryFromLine(int lineNumber, string line, Act // small margin of error caused by non-200 HTTP status codes. if (entry.CacheStatusCode != null) { - // Format: cache status + "/" + HTTP status code - // Example: "TCP_MISS/504" + // Previously, we were not correctly converting logs from China CDN to the format used by Global CDN, so we must support both formats. + // Global format: cache status + "/" + HTTP status code + // Global example: "TCP_MISS/504" + // China format: HTTP status code + // China example: "504" var slashIndex = entry.CacheStatusCode.LastIndexOf('/'); uint httpStatusCode; - if (slashIndex >= 0 - && slashIndex + 1 < entry.CacheStatusCode.Length + if (slashIndex + 1 < entry.CacheStatusCode.Length && uint.TryParse(entry.CacheStatusCode.Substring(slashIndex + 1), out httpStatusCode) && (httpStatusCode < 200 || httpStatusCode >= 300)) { diff --git a/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs b/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs index d41560ae7..5ec5fc0c1 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs @@ -47,7 +47,7 @@ public override OutputLogLine TransformRawLogLine(string line) string.IsNullOrEmpty(line) || line.Trim().StartsWith("c-ip", ignoreCase: true, culture: System.Globalization.CultureInfo.InvariantCulture)) { - //is the header + // Ignore empty lines or the header return null; } @@ -59,11 +59,9 @@ public override OutputLogLine TransformRawLogLine(string line) DateTime dt = DateTime.Parse(timestamp, CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal); string timeStamp2 = ToUnixTimeStamp(dt); - //ignore 400 error codes - if(segments[5] == "400") - { - return null; - } + // Global status code format: cache status + "/" + HTTP status code + // China status code format: HTTP status code + var scstatus = segments[(int)ChinaLogHeaderFields.hitmiss] + "/" + segments[(int)ChinaLogHeaderFields.scstatus]; return new OutputLogLine(timestamp: timeStamp2, timetaken: notAvailableInt, @@ -71,7 +69,7 @@ public override OutputLogLine TransformRawLogLine(string line) filesize: notAvailableInt, sip: segments[(int)ChinaLogHeaderFields.sip], sport: notAvailableInt, - scstatus: segments[(int)ChinaLogHeaderFields.scstatus], + scstatus: scstatus, scbytes: segments[(int)ChinaLogHeaderFields.scbytes], csmethod: segments[(int)ChinaLogHeaderFields.csmethod], csuristem: segments[(int)ChinaLogHeaderFields.csuristem], diff --git a/src/Stats.Warehouse/Programmability/Stored Procedures/dbo.DownloadReportLast6Weeks.sql b/src/Stats.Warehouse/Programmability/Stored Procedures/dbo.DownloadReportLast6Weeks.sql index 81d5514b1..7ad2de370 100644 --- a/src/Stats.Warehouse/Programmability/Stored Procedures/dbo.DownloadReportLast6Weeks.sql +++ b/src/Stats.Warehouse/Programmability/Stored Procedures/dbo.DownloadReportLast6Weeks.sql @@ -19,31 +19,39 @@ BEGIN AND [Year] = @MinYear DECLARE @Cursor DATETIME = (SELECT ISNULL(MAX([Position]), @ReportGenerationTime) FROM [dbo].[Cursors] (NOLOCK) WHERE [Name] = 'GetDirtyPackageId') - - SELECT TOP 6 D.[Year], + DECLARE @MaxDate DATE = DATEADD(DAY, 42, @MinDate); + + WITH WeekLookup AS + ( + -- If we just take all the rows between @MinDate and @MaxDate from Dimension_Date table + -- we might end up the days from the week that contains 1st of January to have two + -- different week numbers: + -- 12/30/2018 -> 53rd of 2018 + -- 12/31/2018 -> 53rd of 2018 + -- 1/1/2019 -> 1st of 2019 + -- 1/2/2019 -> 1st of 2019 + -- etc. + -- which results in the wrong grouping when group by [WeekOfYear] and [Year] is done: + -- the single week gets split into two portions, one for the previous year and another for + -- the new one with aggregations calculated separately for each of the portions. + -- This CTE works around the issue by making sure that all days of the week map to + -- the same [WeekOfYear] and [Year], specifically to that of the first day of that week. + SELECT d.[Id], dd.[WeekOfYear], dd.[Year] + FROM [dbo].[Dimension_Date] AS d WITH(NOLOCK) + CROSS APPLY + ( + SELECT TOP(1) [WeekOfYear], [Year] + FROM [dbo].[Dimension_Date] AS d2 WITH(NOLOCK) + WHERE d2.[Date] <= d.[Date] AND d2.[DayOfWeek] = 1 + ORDER BY d2.[Date] DESC + ) AS dd + WHERE d.[Date] >= @MinDate AND d.[Date] < @MaxDate AND d.[Date] <= @Cursor + ) + SELECT D.[Year], D.[WeekOfYear], - SUM(ISNULL(Facts.[DownloadCount], 0)) 'Downloads' + SUM(ISNULL(Facts.[DownloadCount], 0)) AS [Downloads] FROM [dbo].[Fact_Download] AS Facts (NOLOCK) - - INNER JOIN [dbo].[Dimension_Date] AS D (NOLOCK) - ON D.[Id] = Facts.[Dimension_Date_Id] - - WHERE D.[Date] IS NOT NULL - AND ISNULL(D.[Date], CONVERT(DATE, '1900-01-01')) >= - DATETIMEFROMPARTS( - DATEPART(year, @MinDate), - DATEPART(month, @MinDate), - DATEPART(day, @MinDate), - 0, 0, 0, 0) - AND ISNULL(D.[Date], CONVERT(DATE, DATEADD(day, 1, @ReportGenerationTime))) < - DATETIMEFROMPARTS( - DATEPART(year, @ReportGenerationTime), - DATEPART(month, @ReportGenerationTime), - DATEPART(day, @ReportGenerationTime), - 0, 0, 0, 0) - AND Facts.[Timestamp] <= @Cursor - - GROUP BY D.[Year], D.[WeekOfYear] - ORDER BY [Year], [WeekOfYear] - + INNER JOIN WeekLookup AS D ON D.Id = Facts.Dimension_Date_Id + GROUP BY D.[Year], D.[WeekOfYear] + ORDER BY [Year], [WeekOfYear] END \ No newline at end of file diff --git a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs index c1515aea1..cc2c894f4 100644 --- a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs +++ b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs @@ -9,11 +9,10 @@ namespace Tests.Stats.CollectAzureChinaCDNLogs public class ChinaCollectorTests { [Theory] - [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")] - [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,400,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", null)] + [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 TCP_MISS/200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")] [InlineData("c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip", null)] - [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")] - [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")] + [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")] + [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")] public void TransformRawLogLine(string input, string expectedOutput) { var collector = new ChinaStatsCollector(); diff --git a/tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs b/tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs index f7a71211d..1c9545c5d 100644 --- a/tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs +++ b/tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs @@ -22,6 +22,11 @@ public class TheParseLogEntryFromLineMethod [InlineData("SOMETHING_ELSE/404")] [InlineData("TCP_MISS/504")] [InlineData("TCP_MISS/604")] + [InlineData("0")] + [InlineData("304")] + [InlineData("400")] + [InlineData("404")] + [InlineData("500")] public void IgnoresNon200HttpStatusCodes(string status) { // Arrange @@ -43,7 +48,6 @@ public void IgnoresNon200HttpStatusCodes(string status) [InlineData("TCP_MISS/")] [InlineData("TCP_MISS")] [InlineData("200")] - [InlineData("500")] public void DoesNotIgnore200LevelAndUnrecognizedHttpStatusCodes(string status) { // Arrange