Merge pull request #716 from NuGet/dev

[ReleasePrep][2019.02.04]RI of dev into master

src/CopyAzureContainer/Scripts/Functions.ps1

@@ -61,34 +61,3 @@ Function Install-AzCopy
         Remove-Item $bootstrap -Recurse -Force
     }
 }
-
-Function Uninstall-NuGetService() {
-	Param ([string]$ServiceName)
-
-	if (Get-Service $ServiceName -ErrorAction SilentlyContinue)
-	{
-		Write-Host Removing service $ServiceName...
-		Stop-Service $ServiceName -Force
-		sc.exe delete $ServiceName 
-		Write-Host Removed service $ServiceName.
-	} else {
-		Write-Host Skipping removal of service $ServiceName - no such service exists.
-	}
-}
-
-Function Install-NuGetService() {
-	Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun)
-
-	Write-Host Installing service $ServiceName...
-
-	$installService = "nssm install $ServiceName $ScriptToRun"
-	cmd /C $installService
-
-	Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic
-	sc.exe failure $ServiceName reset= 30 actions= restart/5000 
-
-	# Run service
-	net start $ServiceName
-
-	Write-Host Installed service $ServiceName.
-}

src/CopyAzureContainer/Scripts/backupv3storage.cmd

@@ -9,8 +9,9 @@ title #{Jobs.backupv3storage.Title}
 start /w CopyAzureContainer.exe ^
     -SourceContainerInfo_lucene #{Jobs.common.v3.Storage.Primary.Name}:#{Jobs.common.v3.Storage.Primary.Key}:#{Jobs.catalog2lucenev3reg2.LuceneContainer} ^
     -SourceContainerInfo_catalog #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.feed2catalogv3.StorageContainer} ^
-    -SourceContainerInfo_registration #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg1.StorageContainer} ^
-    -SourceContainerInfo_registrationgz #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg1.StorageContainerCompressed} ^
+    -SourceContainerInfo_registration #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainer} ^
+    -SourceContainerInfo_registrationgz #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainerCompressed} ^
+    -SourceContainerInfo_registrationsemver2 #{Jobs.common.v3.c2r.StorageAccountName}:#{Jobs.common.v3.c2r.StorageAccountKey}:#{Jobs.catalog2registrationv3reg3.StorageContainerSemVer2} ^
     -DestStorageAccountName #{Jobs.backupv3storage.destStorageAccountName} ^
     -DestStorageKeyValue #{Jobs.backupv3storage.destStorageKeyValue} ^
     -BackupDays #{Jobs.backupv3storage.backupDays} ^

src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj

@@ -123,5 +123,10 @@
   </PropertyGroup>
   <Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
   <Import Project="$(SignPath)\sign.microbuild.targets" Condition="Exists('$(SignPath)\sign.microbuild.targets')" />
+  <ItemGroup>
+    <PowerShellScriptsToNotSign Include="PreDeploy.ps1"/>
+    <ExecutablesToNotSign Include="nssm.exe"/>
+  </ItemGroup>
+  <Import Project="$(SignPath)\sign.scripts.targets" Condition="Exists('$(SignPath)\sign.scripts.targets')" />
   <Import Project="..\..\sign.thirdparty.targets" />
 </Project>

src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.nuspec

@@ -12,8 +12,8 @@
   <files>
     <file src="bin\$configuration$\*.*" target="bin"/>
 
-    <file src="Scripts\OnCallDailyNotification.cmd" />
-    <file src="Scripts\WeeklySummaryNotification.cmd" />
+    <file src="Scripts\*.cmd" />
+    <file src="Scripts\*.ps1" />
 
     <file src="Settings\*.json" target="bin" />
   </files>

src/NuGet.SupportRequests.Notifications/Scripts/Functions.ps1

@@ -0,0 +1,31 @@
+Function Install-DailyTask
+{
+    $trigger = New-ScheduledTaskTrigger -DaysInterval 1 -At "12pm" -Daily
+
+    Install-NuGetScheduledTask $trigger "Support Request Daily Notification" "OnCallDailyNotification.cmd"
+}
+
+Function Install-WeeklyTask
+{
+    $trigger = New-ScheduledTaskTrigger -Weekly -WeeksInterval 1 -DaysOfWeek Monday -At "12pm"
+
+    Install-NuGetScheduledTask $trigger "Support Requests Weekly Notification" "WeeklySummaryNotification.cmd"
+}
+
+Function Install-NuGetScheduledTask
+{
+    param($Trigger, $Name, $Command)
+
+    #Action to run as
+    $cmdExe = [system.environment]::getenvironmentvariable("ComSpec")
+    $action = New-ScheduledTaskAction -Execute $cmdExe -Argument "/c $PSScriptRoot\$Command" -WorkingDirectory $PSScriptRoot
+
+    #Configure when to stop the task and how long it can run for. In this example it does not stop on idle and uses the maximum possible duration by setting a timelimit of 0
+    $settings = New-ScheduledTaskSettingsSet -DontStopOnIdleEnd -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances IgnoreNew
+
+    #Configure the principal to use for the scheduled task and the level to run as
+    $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel "Highest"
+
+    #Register the new scheduled task
+    Register-ScheduledTask -TaskName $Name -Action $action -Trigger $Trigger -Principal $principal -Settings $settings -Force
+}

src/NuGet.SupportRequests.Notifications/Scripts/PostDeploy.ps1

@@ -0,0 +1,10 @@
+. .\Functions.ps1
+
+## The daily task is broken right now due to Pin
+# Write-Host Register the daily scheduled task
+# Install-DailyTask
+# Write-Host Registered the daily scheduled task 
+
+Write-Host Register the weekly scheduled task
+Install-WeeklyTask
+Write-Host Registered the weekly scheduled task 

src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs

@@ -105,12 +105,14 @@ public static CdnLogEntry ParseLogEntryFromLine(int lineNumber, string line, Act
             // small margin of error caused by non-200 HTTP status codes.
             if (entry.CacheStatusCode != null)
             {
-                // Format: cache status + "/" + HTTP status code
-                // Example: "TCP_MISS/504"
+                // Previously, we were not correctly converting logs from China CDN to the format used by Global CDN, so we must support both formats.
+                // Global format: cache status + "/" + HTTP status code
+                // Global example: "TCP_MISS/504"
+                // China format: HTTP status code
+                // China example: "504"
                 var slashIndex = entry.CacheStatusCode.LastIndexOf('/');
                 uint httpStatusCode;
-                if (slashIndex >= 0
-                    && slashIndex + 1 < entry.CacheStatusCode.Length
+                if (slashIndex + 1 < entry.CacheStatusCode.Length
                     && uint.TryParse(entry.CacheStatusCode.Substring(slashIndex + 1), out httpStatusCode)
                     && (httpStatusCode < 200 || httpStatusCode >= 300))
                 {

src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs

@@ -47,7 +47,7 @@ public override OutputLogLine TransformRawLogLine(string line)
                 string.IsNullOrEmpty(line) || 
                 line.Trim().StartsWith("c-ip", ignoreCase: true, culture: System.Globalization.CultureInfo.InvariantCulture))
             {
-                //is the header
+                // Ignore empty lines or the header
                 return null;
             }
 
@@ -59,19 +59,17 @@ public override OutputLogLine TransformRawLogLine(string line)
             DateTime dt = DateTime.Parse(timestamp, CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal);
             string timeStamp2 = ToUnixTimeStamp(dt);
 
-            //ignore 400 error codes
-            if(segments[5] == "400")
-            {
-                return null;
-            }
+            // Global status code format: cache status + "/" + HTTP status code
+            // China status code format: HTTP status code
+            var scstatus = segments[(int)ChinaLogHeaderFields.hitmiss] + "/" + segments[(int)ChinaLogHeaderFields.scstatus];
 
             return new OutputLogLine(timestamp: timeStamp2,
                 timetaken: notAvailableInt,
                 cip:segments[(int)ChinaLogHeaderFields.cip],
                 filesize: notAvailableInt,
                 sip: segments[(int)ChinaLogHeaderFields.sip],
                 sport: notAvailableInt,
-                scstatus: segments[(int)ChinaLogHeaderFields.scstatus],
+                scstatus: scstatus,
                 scbytes: segments[(int)ChinaLogHeaderFields.scbytes],
                 csmethod: segments[(int)ChinaLogHeaderFields.csmethod],
                 csuristem: segments[(int)ChinaLogHeaderFields.csuristem],

src/Stats.Warehouse/Programmability/Stored Procedures/dbo.DownloadReportLast6Weeks.sql

@@ -19,31 +19,39 @@ BEGIN
 		AND	[Year] = @MinYear
 
 	DECLARE @Cursor DATETIME = (SELECT ISNULL(MAX([Position]), @ReportGenerationTime) FROM [dbo].[Cursors] (NOLOCK) WHERE [Name] = 'GetDirtyPackageId')
-
-	SELECT	TOP 6 D.[Year],
+	DECLARE @MaxDate DATE = DATEADD(DAY, 42, @MinDate);
+
+	WITH WeekLookup AS 
+	(
+		-- If we just take all the rows between @MinDate and @MaxDate from Dimension_Date table
+		-- we might end up the days from the week that contains 1st of January to have two
+		-- different week numbers:
+		-- 12/30/2018 -> 53rd of 2018
+		-- 12/31/2018 -> 53rd of 2018
+		-- 1/1/2019 -> 1st of 2019
+		-- 1/2/2019 -> 1st of 2019
+		-- etc.
+		-- which results in the wrong grouping when group by [WeekOfYear] and [Year] is done:
+		-- the single week gets split into two portions, one for the previous year and another for
+		-- the new one with aggregations calculated separately for each of the portions.
+		-- This CTE works around the issue by making sure that all days of the week map to
+		-- the same [WeekOfYear] and [Year], specifically to that of the first day of that week.
+		SELECT d.[Id], dd.[WeekOfYear], dd.[Year]
+		FROM [dbo].[Dimension_Date] AS d WITH(NOLOCK)
+		CROSS APPLY
+		(
+			SELECT TOP(1) [WeekOfYear], [Year]
+			FROM [dbo].[Dimension_Date] AS d2 WITH(NOLOCK)
+			WHERE d2.[Date] <= d.[Date] AND d2.[DayOfWeek] = 1
+			ORDER BY d2.[Date] DESC
+		) AS dd
+		WHERE d.[Date] >= @MinDate AND d.[Date] < @MaxDate AND d.[Date] <= @Cursor
+	)
+	SELECT	D.[Year],
 			D.[WeekOfYear],
-			SUM(ISNULL(Facts.[DownloadCount], 0)) 'Downloads'
+			SUM(ISNULL(Facts.[DownloadCount], 0)) AS [Downloads]
 	FROM	[dbo].[Fact_Download] AS Facts (NOLOCK)
-
-	INNER JOIN	[dbo].[Dimension_Date] AS D (NOLOCK)
-	ON			D.[Id] = Facts.[Dimension_Date_Id]
-
-	WHERE	D.[Date] IS NOT NULL
-			AND ISNULL(D.[Date], CONVERT(DATE, '1900-01-01')) >=
-				DATETIMEFROMPARTS(
-					DATEPART(year, @MinDate),
-					DATEPART(month, @MinDate),
-					DATEPART(day, @MinDate),
-					0, 0, 0, 0)
-			AND ISNULL(D.[Date], CONVERT(DATE, DATEADD(day, 1, @ReportGenerationTime))) <
-				DATETIMEFROMPARTS(
-					DATEPART(year, @ReportGenerationTime),
-					DATEPART(month, @ReportGenerationTime),
-					DATEPART(day, @ReportGenerationTime),
-					0, 0, 0, 0)
-			AND Facts.[Timestamp] <= @Cursor
-
-	GROUP BY	D.[Year], D.[WeekOfYear]
-	ORDER BY	[Year], [WeekOfYear]
-
+	INNER JOIN WeekLookup AS D ON D.Id = Facts.Dimension_Date_Id
+	GROUP BY D.[Year], D.[WeekOfYear]
+	ORDER BY [Year], [WeekOfYear]
 END

tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs

@@ -9,11 +9,10 @@ namespace Tests.Stats.CollectAzureChinaCDNLogs
     public class ChinaCollectorTests
     {
         [Theory]
-        [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")]
-        [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,400,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", null)]
+        [InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 TCP_MISS/200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")]
         [InlineData("c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip", null)]
-        [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")]
-        [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")]
+        [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")]
+        [InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")]
         public void TransformRawLogLine(string input, string expectedOutput)
         {
             var collector  = new ChinaStatsCollector();

tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs

@@ -22,6 +22,11 @@ public class TheParseLogEntryFromLineMethod
             [InlineData("SOMETHING_ELSE/404")]
             [InlineData("TCP_MISS/504")]
             [InlineData("TCP_MISS/604")]
+            [InlineData("0")]
+            [InlineData("304")]
+            [InlineData("400")]
+            [InlineData("404")]
+            [InlineData("500")]
             public void IgnoresNon200HttpStatusCodes(string status)
             {
                 // Arrange
@@ -43,7 +48,6 @@ public void IgnoresNon200HttpStatusCodes(string status)
             [InlineData("TCP_MISS/")]
             [InlineData("TCP_MISS")]
             [InlineData("200")]
-            [InlineData("500")]
             public void DoesNotIgnore200LevelAndUnrecognizedHttpStatusCodes(string status)
             {
                 // Arrange