diff --git a/.gitignore b/.gitignore
index 992ec7f2e..c7bdc0656 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,8 @@ tools/
.nuget/credprovider
.nuget/.marker.v*
nuget.exe
+functionaltests.*.xml
+AssemblyInfo.g.cs
# Roslyn cache directories
*.ide/
@@ -206,3 +208,8 @@ AssemblyInfo.*.cs
/tests/Validation.Common.Tests/Validation.Common.Tests.nuget.props
/tests/Validation.Common.Tests/Validation.Common.Tests.nuget.targets
/tests/Validation.Helper.Tests/Validation.Helper.Tests.nuget.targets
+/tests/packages
+/tests/CatalogMetadataTests/CatalogMetadataTests.nuget.props
+*.lock.json
+/src/V3PerPackage/Settings.json
+artifacts/*
\ No newline at end of file
diff --git a/.nuget/packages.config b/.nuget/packages.config
index bb7daf160..eaca3ffaf 100644
--- a/.nuget/packages.config
+++ b/.nuget/packages.config
@@ -1,6 +1,8 @@
+
+
\ No newline at end of file
diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln
index 4ba1d9ec4..7ebeda871 100644
--- a/NuGet.Jobs.sln
+++ b/NuGet.Jobs.sln
@@ -3,6 +3,12 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29722.177
MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Jobs", "Jobs", "{C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Services", "Services", "{97E23323-BA7A-48F0-A578-858B82B6D8FB}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Packages", "Packages", "{5DE01C58-D5F7-482F-8256-A8333064384C}"
+EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common", "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj", "{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{05F997A5-CD3A-4C60-9CCB-D630542EDA48}"
@@ -21,6 +27,46 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
Local.testsettings = Local.testsettings
EndProjectSection
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Metadata.Catalog", "src\Catalog\NuGet.Services.Metadata.Catalog.csproj", "{E97F23B8-ECB0-4AFA-B00C-015C39395FEF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogTests", "tests\CatalogTests\CatalogTests.csproj", "{4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ng", "src\Ng\Ng.csproj", "{5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NgTests", "tests\NgTests\NgTests.csproj", "{05C1C78A-9966-4922-9065-A099023E7366}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.ApplicationInsights.Owin", "src\NuGet.ApplicationInsights.Owin\NuGet.ApplicationInsights.Owin.csproj", "{717E9A81-75C5-418E-92ED-18CAC55BC345}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Metadata.Catalog.Monitoring", "src\NuGet.Services.Metadata.Catalog.Monitoring\NuGet.Services.Metadata.Catalog.Monitoring.csproj", "{1745A383-D0BE-484B-81EB-27B20F6AC6C5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogMetadataTests", "tests\CatalogMetadataTests\CatalogMetadataTests.csproj", "{34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.AzureSearch", "src\NuGet.Services.AzureSearch\NuGet.Services.AzureSearch.csproj", "{1A53FE3D-8041-4773-942F-D73AEF5B82B2}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.AzureSearch.Tests", "tests\NuGet.Services.AzureSearch.Tests\NuGet.Services.AzureSearch.Tests.csproj", "{6A9C3802-A2A2-49CF-87BD-C1303533B846}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Db2AzureSearch", "src\NuGet.Jobs.Db2AzureSearch\NuGet.Jobs.Db2AzureSearch.csproj", "{209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2AzureSearch", "src\NuGet.Jobs.Catalog2AzureSearch\NuGet.Jobs.Catalog2AzureSearch.csproj", "{F591130F-181A-4C53-8025-4390F46BD51D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog", "src\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj", "{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog.Tests", "tests\NuGet.Protocol.Catalog.Tests\NuGet.Protocol.Catalog.Tests.csproj", "{1F3BC053-796C-4A35-88F4-955A0F142197}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.SearchService", "src\NuGet.Services.SearchService\NuGet.Services.SearchService.csproj", "{DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.SearchService.Tests", "tests\NuGet.Services.SearchService.Tests\NuGet.Services.SearchService.Tests.csproj", "{F009209D-A663-45E1-87E8-158569A0F097}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Auxiliary2AzureSearch", "src\NuGet.Jobs.Auxiliary2AzureSearch\NuGet.Jobs.Auxiliary2AzureSearch.csproj", "{7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2Registration", "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.csproj", "{5ABE8807-2209-4948-9FC5-1980A507C47A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3", "src\NuGet.Services.V3\NuGet.Services.V3.csproj", "{C3F9A738-9759-4B2B-A50D-6507B28A659B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2Registration.Tests", "tests\NuGet.Jobs.Catalog2Registration.Tests\NuGet.Jobs.Catalog2Registration.Tests.csproj", "{296703A3-67BA-4876-8C1D-ACE13DF901EF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3.Tests", "tests\NuGet.Services.V3.Tests\NuGet.Services.V3.Tests.csproj", "{CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}"
+EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Stats.CollectAzureCdnLogs", "src\Stats.CollectAzureCdnLogs\Stats.CollectAzureCdnLogs.csproj", "{664CA8BB-BCF5-432C-AF68-9F97D308E623}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Stats.CollectAzureCdnLogs", "tests\Tests.Stats.CollectAzureCdnLogs\Tests.Stats.CollectAzureCdnLogs.csproj", "{578BCF9F-673B-4F18-8095-3B69A3D946DD}"
@@ -49,8 +95,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Gallery", "Gallery", "{8872
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gallery.CredentialExpiration", "src\Gallery.CredentialExpiration\Gallery.CredentialExpiration.csproj", "{FA8C7905-985F-4919-AAA9-4B9A252F4977}"
EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SupportRequests", "SupportRequests", "{BEC3DF4D-9A04-42C8-8B4F-D42750202B4D}"
-EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.SupportRequests.Notifications", "src\NuGet.SupportRequests.Notifications\NuGet.SupportRequests.Notifications.csproj", "{12719498-B87E-4E92-8C2B-30046393CF85}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gallery.Maintenance", "src\Gallery.Maintenance\Gallery.Maintenance.csproj", "{EFF021CA-1BF4-4C09-BFB8-D314EAAD24D2}"
@@ -145,6 +189,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Ad-Hoc Tools", "Ad-Hoc Tool
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SplitLargeFiles.Tests", "tests\SplitLargeFiles.Tests\SplitLargeFiles.Tests.csproj", "{DD70035C-1BAB-4EFF-B270-EF45F7473C22}"
EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "V3", "V3", "{0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -159,6 +205,86 @@ Global
{A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Release|Any CPU.Build.0 = Release|Any CPU
+ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Release|Any CPU.Build.0 = Release|Any CPU
+ {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Release|Any CPU.Build.0 = Release|Any CPU
+ {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|Any CPU.Build.0 = Release|Any CPU
+ {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {05C1C78A-9966-4922-9065-A099023E7366}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {05C1C78A-9966-4922-9065-A099023E7366}.Release|Any CPU.Build.0 = Release|Any CPU
+ {717E9A81-75C5-418E-92ED-18CAC55BC345}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {717E9A81-75C5-418E-92ED-18CAC55BC345}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {717E9A81-75C5-418E-92ED-18CAC55BC345}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {717E9A81-75C5-418E-92ED-18CAC55BC345}.Release|Any CPU.Build.0 = Release|Any CPU
+ {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Release|Any CPU.Build.0 = Release|Any CPU
+ {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Release|Any CPU.Build.0 = Release|Any CPU
+ {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Release|Any CPU.Build.0 = Release|Any CPU
+ {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Release|Any CPU.Build.0 = Release|Any CPU
+ {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Release|Any CPU.Build.0 = Release|Any CPU
+ {F591130F-181A-4C53-8025-4390F46BD51D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {F591130F-181A-4C53-8025-4390F46BD51D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {F591130F-181A-4C53-8025-4390F46BD51D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {F591130F-181A-4C53-8025-4390F46BD51D}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.Build.0 = Release|Any CPU
+ {1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.Build.0 = Release|Any CPU
+ {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Release|Any CPU.Build.0 = Release|Any CPU
+ {F009209D-A663-45E1-87E8-158569A0F097}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {F009209D-A663-45E1-87E8-158569A0F097}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {F009209D-A663-45E1-87E8-158569A0F097}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {F009209D-A663-45E1-87E8-158569A0F097}.Release|Any CPU.Build.0 = Release|Any CPU
+ {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|Any CPU.Build.0 = Release|Any CPU
+ {5ABE8807-2209-4948-9FC5-1980A507C47A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {5ABE8807-2209-4948-9FC5-1980A507C47A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {5ABE8807-2209-4948-9FC5-1980A507C47A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {5ABE8807-2209-4948-9FC5-1980A507C47A}.Release|Any CPU.Build.0 = Release|Any CPU
+ {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.Build.0 = Release|Any CPU
+ {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Release|Any CPU.Build.0 = Release|Any CPU
+ {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.Build.0 = Release|Any CPU
{664CA8BB-BCF5-432C-AF68-9F97D308E623}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{664CA8BB-BCF5-432C-AF68-9F97D308E623}.Debug|Any CPU.Build.0 = Debug|Any CPU
{664CA8BB-BCF5-432C-AF68-9F97D308E623}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -390,8 +516,31 @@ Global
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
+ {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6}
+ {97E23323-BA7A-48F0-A578-858B82B6D8FB} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6}
+ {5DE01C58-D5F7-482F-8256-A8333064384C} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6}
{A08F0185-4DA3-4D5B-BF0C-B178596D5789} = {FA5644B5-4F08-43F6-86B3-039374312A47}
{6078A997-EDEF-4F84-9E9C-163EB1E7454D} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+ {05C1C78A-9966-4922-9065-A099023E7366} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {717E9A81-75C5-418E-92ED-18CAC55BC345} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {1745A383-D0BE-484B-81EB-27B20F6AC6C5} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {1A53FE3D-8041-4773-942F-D73AEF5B82B2} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {6A9C3802-A2A2-49CF-87BD-C1303533B846} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+ {F591130F-181A-4C53-8025-4390F46BD51D} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {1F3BC053-796C-4A35-88F4-955A0F142197} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7} = {97E23323-BA7A-48F0-A578-858B82B6D8FB}
+ {F009209D-A663-45E1-87E8-158569A0F097} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+ {5ABE8807-2209-4948-9FC5-1980A507C47A} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+ {C3F9A738-9759-4B2B-A50D-6507B28A659B} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+ {296703A3-67BA-4876-8C1D-ACE13DF901EF} = {6A776396-02B1-475D-A104-26940ADB04AB}
+ {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} = {6A776396-02B1-475D-A104-26940ADB04AB}
{664CA8BB-BCF5-432C-AF68-9F97D308E623} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
{578BCF9F-673B-4F18-8095-3B69A3D946DD} = {6A776396-02B1-475D-A104-26940ADB04AB}
{F72C31A7-424D-48C6-924C-EBFD4BE0918B} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
@@ -403,10 +552,10 @@ Global
{B5C01B7A-933D-483E-AF07-6AA266B0EB49} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
{3E0A20C8-C6D2-4762-955D-C7BF35C2C9A7} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
{FA8C7905-985F-4919-AAA9-4B9A252F4977} = {88725659-D5F8-49F9-9B7E-D87C5B9917D7}
- {12719498-B87E-4E92-8C2B-30046393CF85} = {BEC3DF4D-9A04-42C8-8B4F-D42750202B4D}
+ {12719498-B87E-4E92-8C2B-30046393CF85} = {FA5644B5-4F08-43F6-86B3-039374312A47}
{EFF021CA-1BF4-4C09-BFB8-D314EAAD24D2} = {88725659-D5F8-49F9-9B7E-D87C5B9917D7}
{A07F7D0C-F269-43D5-A812-3ABC47090885} = {FA5644B5-4F08-43F6-86B3-039374312A47}
- {BC9EA7CE-AD21-4D17-B581-F8ED8CBD7191} = {FA5644B5-4F08-43F6-86B3-039374312A47}
+ {BC9EA7CE-AD21-4D17-B581-F8ED8CBD7191} = {97E23323-BA7A-48F0-A578-858B82B6D8FB}
{147A757D-864B-4C74-B8CF-14DFF9793605} = {6A776396-02B1-475D-A104-26940ADB04AB}
{FC0CEF12-D501-46D1-B1BF-D4134BD8D478} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
{0C887292-C5AB-4107-946C-A53B18A38D22} = {6A776396-02B1-475D-A104-26940ADB04AB}
diff --git a/README.md b/README.md
index 507081ce7..a8cd75730 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,8 @@
NuGet.Jobs
-==============
+==========
+
+This repo contains nuget.org's implementation of the [NuGet V3 API](https://docs.microsoft.com/en-us/nuget/api/overview)
+as well as many other back-end jobs for the operation of nuget.org.
1. Each job would be an exe with 2 main classes Program and Job
2. Program.Main should simply do the following and nothing more
@@ -20,6 +23,14 @@ NuGet.Jobs
7. Also, add settings.job file to mark the job as singleton, if the job will be run as a webjob, and it be a continuously running singleton
+## Feedback
+
+If you're having trouble with the NuGet.org Website, file a bug on the [NuGet Gallery Issue Tracker](https://github.com/nuget/NuGetGallery/issues).
+
+If you're having trouble with the NuGet client tools (the Visual Studio extension, NuGet.exe command line tool, etc.), file a bug on [NuGet Home](https://github.com/nuget/home/issues).
+
+Check out the [contributing](http://docs.nuget.org/contribute) page to see the best places to log issues and start discussions. The [NuGet Home](https://github.com/NuGet/Home) repo provides an overview of the different NuGet projects available.
+
Open Source Code of Conduct
===================
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
diff --git a/build.ps1 b/build.ps1
index 29bbcf539..65fc9b4e4 100644
--- a/build.ps1
+++ b/build.ps1
@@ -87,25 +87,36 @@ Invoke-BuildStep 'Clearing artifacts' { Clear-Artifacts } `
Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
$versionMetadata =
- "$PSScriptRoot\src\CopyAzureContainer\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\NuGetCDNRedirect\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\NuGet.Services.Validation.Orchestrator\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\NuGet.Services.Revalidate\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\StatusAggregator\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\Stats.CDNLogsSanitizer\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\NuGet.Jobs.GitHubIndexer\Properties\AssemblyInfo.g.cs",
- "$PSScriptRoot\src\SplitLargeFiles\Properties\AssemblyInfo.g.cs"
+ "src\CopyAzureContainer\Properties\AssemblyInfo.g.cs",
+ "src\NuGetCDNRedirect\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Services.Validation.Orchestrator\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Services.Revalidate\Properties\AssemblyInfo.g.cs",
+ "src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs",
+ "src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs",
+ "src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs",
+ "src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs",
+ "src\Validation.Common.Job\Properties\AssemblyInfo.g.cs",
+ "src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs",
+ "src\PackageLagMonitor\Properties\AssemblyInfo.g.cs",
+ "src\StatusAggregator\Properties\AssemblyInfo.g.cs",
+ "src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs",
+ "src\Stats.CDNLogsSanitizer\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Jobs.GitHubIndexer\Properties\AssemblyInfo.g.cs",
+ "src\SplitLargeFiles\Properties\AssemblyInfo.g.cs",
+ "src\Catalog\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.ApplicationInsights.Owin\Properties\AssemblyInfo.g.cs",
+ "src\Ng\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Services.Metadata.Catalog.Monitoring\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Protocol.Catalog\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Services.AzureSearch\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Jobs.Db2AzureSearch\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Jobs.Catalog2AzureSearch\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Services.SearchService\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Jobs.Auxiliary2AzureSearch\Properties\AssemblyInfo.g.cs",
+ "src\NuGet.Jobs.Catalog2Registration\Properties\AssemblyInfo.g.cs"
$versionMetadata | ForEach-Object {
- Set-VersionInfo -Path $_ -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA
+ Set-VersionInfo -Path (Join-Path $PSScriptRoot $_) -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA
}
} `
-ev +BuildErrors
@@ -122,6 +133,12 @@ Invoke-BuildStep 'Building solution' {
-args $Configuration, $BuildNumber, (Join-Path $PSScriptRoot "NuGet.Jobs.sln"), $SkipRestore `
-ev +BuildErrors
+Invoke-BuildStep 'Building functional test solution' {
+ $SolutionPath = Join-Path $PSScriptRoot "tests\NuGetServicesMetadata.FunctionalTests.sln"
+ Build-Solution $Configuration $BuildNumber -MSBuildVersion "$msBuildVersion" $SolutionPath -SkipRestore:$SkipRestore `
+ } `
+ -ev +BuildErrors
+
Invoke-BuildStep 'Signing the binaries' {
Sign-Binaries -Configuration $Configuration -BuildNumber $BuildNumber -MSBuildVersion "15" `
} `
@@ -135,40 +152,57 @@ Invoke-BuildStep 'Creating artifacts' {
# We need symbols published for those, too. All other packages are deployment ones and
# don't need to be shared, hence no need for symbols for them
$CsprojProjects =
- "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj",
- "src/Validation.Common.Job/Validation.Common.Job.csproj",
- "src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj",
- "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj"
+ "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj",
+ "src\Validation.Common.Job\Validation.Common.Job.csproj",
+ "src\Validation.ScanAndSign.Core\Validation.ScanAndSign.Core.csproj",
+ "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj",
+ "src\Catalog\NuGet.Services.Metadata.Catalog.csproj",
+ "src\NuGet.ApplicationInsights.Owin\NuGet.ApplicationInsights.Owin.csproj",
+ "src\NuGet.Services.Metadata.Catalog.Monitoring\NuGet.Services.Metadata.Catalog.Monitoring.csproj",
+ "src\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj",
+ "src\NuGet.Services.AzureSearch\NuGet.Services.AzureSearch.csproj"
$CsprojProjects | ForEach-Object {
New-ProjectPackage (Join-Path $PSScriptRoot $_) -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -Symbols
}
$NuspecProjects = `
- "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", `
- "src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj", `
- "src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj", `
- "src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj", `
- "src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj", `
- "src/Gallery.Maintenance/Gallery.Maintenance.nuspec", `
- "src/ArchivePackages/ArchivePackages.csproj", `
- "src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj", `
- "src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj", `
- "src/CopyAzureContainer/CopyAzureContainer.csproj", `
- "src/NuGet.Services.Validation.Orchestrator/Validation.Orchestrator.nuspec", `
- "src/NuGet.Services.Validation.Orchestrator/Validation.SymbolsOrchestrator.nuspec", `
- "src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj", `
- "src/Stats.CollectAzureChinaCDNLogs/Stats.CollectAzureChinaCDNLogs.csproj", `
- "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", `
- "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
- "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", `
- "src/PackageLagMonitor/Monitoring.PackageLag.csproj", `
- "src/StatusAggregator/StatusAggregator.csproj", `
- "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj", `
- "src/Validation.Symbols/Validation.Symbols.Job.csproj", `
- "src/Stats.CDNLogsSanitizer/Stats.CDNLogsSanitizer.csproj", `
- "src/NuGet.Jobs.GitHubIndexer/NuGet.Jobs.GitHubIndexer.nuspec", `
- "src/SplitLargeFiles/SplitLargeFiles.nuspec"
+ "src\Stats.CollectAzureCdnLogs\Stats.CollectAzureCdnLogs.csproj", `
+ "src\Stats.AggregateCdnDownloadsInGallery\Stats.AggregateCdnDownloadsInGallery.csproj", `
+ "src\Stats.ImportAzureCdnStatistics\Stats.ImportAzureCdnStatistics.csproj", `
+ "src\Stats.CreateAzureCdnWarehouseReports\Stats.CreateAzureCdnWarehouseReports.csproj", `
+ "src\Gallery.CredentialExpiration\Gallery.CredentialExpiration.csproj", `
+ "src\Gallery.Maintenance\Gallery.Maintenance.nuspec", `
+ "src\ArchivePackages\ArchivePackages.csproj", `
+ "src\Stats.RollUpDownloadFacts\Stats.RollUpDownloadFacts.csproj", `
+ "src\NuGet.SupportRequests.Notifications\NuGet.SupportRequests.Notifications.csproj", `
+ "src\CopyAzureContainer\CopyAzureContainer.csproj", `
+ "src\NuGet.Services.Validation.Orchestrator\Validation.Orchestrator.nuspec", `
+ "src\NuGet.Services.Validation.Orchestrator\Validation.SymbolsOrchestrator.nuspec", `
+ "src\NuGet.Services.Revalidate\NuGet.Services.Revalidate.csproj", `
+ "src\Stats.CollectAzureChinaCDNLogs\Stats.CollectAzureChinaCDNLogs.csproj", `
+ "src\Validation.PackageSigning.ProcessSignature\Validation.PackageSigning.ProcessSignature.csproj", `
+ "src\Validation.PackageSigning.ValidateCertificate\Validation.PackageSigning.ValidateCertificate.csproj", `
+ "src\Validation.PackageSigning.RevalidateCertificate\Validation.PackageSigning.RevalidateCertificate.csproj", `
+ "src\PackageLagMonitor\Monitoring.PackageLag.csproj", `
+ "src\StatusAggregator\StatusAggregator.csproj", `
+ "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", `
+ "src\Validation.Symbols\Validation.Symbols.Job.csproj", `
+ "src\Stats.CDNLogsSanitizer\Stats.CDNLogsSanitizer.csproj", `
+ "src\NuGet.Jobs.GitHubIndexer\NuGet.Jobs.GitHubIndexer.nuspec", `
+ "src\SplitLargeFiles\SplitLargeFiles.nuspec", `
+ "src\Ng\Catalog2Dnx.nuspec", `
+ "src\Ng\Catalog2icon.nuspec", `
+ "src\Ng\Catalog2Monitoring.nuspec", `
+ "src\Ng\Db2Catalog.nuspec", `
+ "src\Ng\Db2Monitoring.nuspec", `
+ "src\Ng\Monitoring2Monitoring.nuspec", `
+ "src\Ng\MonitoringProcessor.nuspec", `
+ "src\Ng\Ng.Operations.nuspec", `
+ "src\NuGet.Jobs.Db2AzureSearch\NuGet.Jobs.Db2AzureSearch.nuspec", `
+ "src\NuGet.Jobs.Catalog2AzureSearch\NuGet.Jobs.Catalog2AzureSearch.nuspec", `
+ "src\NuGet.Jobs.Auxiliary2AzureSearch\NuGet.Jobs.Auxiliary2AzureSearch.nuspec", `
+ "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.nuspec"
Foreach ($Project in $NuspecProjects) {
New-Package (Join-Path $PSScriptRoot "$Project") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "$msBuildVersion"
diff --git a/docs/Azure-Search-indexes.md b/docs/Azure-Search-indexes.md
new file mode 100644
index 000000000..f4dc7a86e
--- /dev/null
+++ b/docs/Azure-Search-indexes.md
@@ -0,0 +1,111 @@
+# Azure Search indexes
+
+**Subsystem: Search 🔎**
+
+The search subsystem heavily depends on Azure Search for storing package metadata and performing package queries. Within
+a single Azure Search resource, there can be multiple indexes. An index is simply a collection of documents with a
+common schema. For the NuGet search subsystem, there are two indexes expected in each Azure Search resource:
+
+- [`search-XXX`](#search-index) - this is the "search" index which contains documents for *discovery* queries
+- [`hijack-XXX`](#hijack-index) - this is the "hijack" index which contains documents for *metadata lookup* queries
+
+## Search index
+
+The search index is designed to fulfill queries for package discovery. This is likely the scenario you would think about
+first when you imagine how package search would work. It's optimized for searching package metadata field by one or more
+keywords and has a scoring profile that returns the most relevant package first.
+
+This index has up to four documents per package ID. Each of the four ID-specific documents represents a different view
+of available package versions. There are two factors for filtering in and out package versions: whether or not to
+consider prerelease versions and whether or not to consider SemVer 2.0.0 versions.
+
+This may seem is a little strange at first, so it's best to consider an example. Consider a package
+[`BaseTestPackage.SearchFilters`](https://www.nuget.org/packages/BaseTestPackage.SearchFilters) that has four versions:
+
+- `1.1.0` - stable, SemVer 1.0.0
+- `1.2.0-beta`, prerelease, SemVer 1.0.0
+- `1.3.0+metadata`, stable, SemVer 2.0.0 (due to build metadata)
+- `1.4.0-delta.4`, prerelease, SemVer 2.0.0 (due to a dot in the prerelease label)
+
+As mentioned before there are up to four documents per package ID. In the case of the example package
+`BaseTestPackage.SearchFilters`, there will be four documents, each with a different set of versions included in the
+document.
+
+- Stable + SemVer 1.0.0: contains only `1.1.0` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters))
+- Stable/Prerelease + SemVer 1.0.0: contains `1.1.0` and `1.2.0-beta` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&prerelease=true))
+- Stable + SemVer 2.0.0: contains `1.1.0` and `1.3.0+metadata` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&semVerLevel=2.0.0))
+- Stable/Prerelease + SemVer 2.0.0: contains all versions ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&prerelease=true&semVerLevel=2.0.0))
+
+The four "flavors" of search documents per ID are referred to as **search filters**.
+
+The documents in the search index are identified (via the `key` property) by a unique string with the following format:
+
+```
+{sanitized lowercase ID}-{base64 lowercase ID}-{search filter}
+```
+
+The `sanitized lowercase ID` removes all characters from the package ID that are not acceptable for Azure Search
+document keys, like dots and non-ASCII word characters (like Chinese characters). This component of the document key is
+included for readability purposes only.
+
+The `base64 lowercase ID` is the base64 encoding of the package ID's bytes, encoded with UTF-8. This string is
+guaranteed to be a 1:1 mapping with the lowercase package ID and is included for uniqueness. The
+`HttpServerUtility.UrlTokenEncode` API is used for base64 encoding.
+
+The `search filter` has one of four values:
+
+- `Default` - Stable + SemVer 1.0.0
+- `IncludePrerelease` - Stable/Prerelease + SemVer 1.0.0
+- `IncludeSemVer2` - Stable + SemVer 2.0.0
+- `IncludePrereleaseAndSemVer2` - Stable/Prerelease + SemVer 2.0.0
+
+For the package ID `BaseTestPackage.SearchFilters`, the Stable + 1.0.0 document key would be:
+
+```
+basetestpackage_searchfilters-YmFzZXRlc3RwYWNrYWdlLnNlYXJjaGZpbHRlcnM1-Default
+```
+
+Each document contains a variety of metadata fields originating from the latest version in the application version list
+as well as a field listing all versions. See the
+[`NuGet.Services.AzureSearch.SearchDocument.Full`](../src/NuGet.Services.AzureSearch/Models/SearchDocument.cs) class and
+its inherited members for a full list of the fields.
+
+Unlisted package versions do not appear in the search index at all.
+
+## Hijack index
+
+The hijack index is used by the gallery to fulfill specific metadata lookup operations. For example, if a
+customer is looking for metadata about all versions of the package ID `Newtonsoft.Json`, in certain cases the gallery
+will query the search service for this metadata and the search service will use the hijack index to fetch the
+data.
+
+This index has one document for every version of every package ID, whether it is unlisted or not. The search service
+uses this index to find all versions of a package via the `ignoreFilter=true` parameter including,
+
+- unlisted packages ([example query](https://azuresearch-usnc.nuget.org/search/query?q=packageid:BaseTestPackage.Unlisted&ignoreFilter=true))
+- multiple versions of a single ID ([example query](https://azuresearch-usnc.nuget.org/search/query?q=packageid:BaseTestPackage.SearchFilters&ignoreFilter=true&semVerLevel=2.0.0))
+
+The documents in the hijack index are identified (via the `key` property) by a unique string with the following format:
+
+```
+{sanitized ID/version}-{base64 ID/version}
+```
+
+The `sanitized ID/version` removes all characters from the `{lowercase package ID}/{lowercase, normalized version}`
+that are not acceptable for Azure Search document keys, like dots and non-ASCII word characters (like Chinese
+characters). This component of the document key is included for readability purposes only.
+
+The `base64 ID/version` is the base64 encoding of the previously mentioned concatenation of ID and version, encoded
+with UTF-8. This string is guaranteed to be a 1:1 mapping with the lowercase package ID and version and is included
+for uniqueness. The `HttpServerUtility.UrlTokenEncode` API is used for base64 encoding.
+
+For the package ID `BaseTestPackage.SearchFilters` and version `1.3.0+metadata`, the document key would be:
+
+```
+basetestpackage_searchfilters_1_3_0-YmFzZXRlc3RwYWNrYWdlLnNlYXJjaGZpbHRlcnMvMS4zLjA1
+```
+
+Each document contains a variety of metadata fields originating from the latest version in the application version list
+as well as a field listing all versions. See the
+[`NuGet.Services.AzureSearch.HijackDocument.Full`](../src/NuGet.Services.AzureSearch/Models/HijackDocument.cs) class and
+its inherited members for a full list of the fields.
diff --git a/docs/Search-auxiliary-files.md b/docs/Search-auxiliary-files.md
new file mode 100644
index 000000000..1aaad3c57
--- /dev/null
+++ b/docs/Search-auxiliary-files.md
@@ -0,0 +1,169 @@
+# Search auxiliary files
+
+**Subsystem: Search 🔎**
+
+Aside from metadata stored in the [Azure Search indexes](Azure-Search-indexes.md), there is data stored in Azure Blob
+Storage for bookkeeping and performance reasons. These data files are called **auxiliary files**. The data files
+mentioned here are those explicitly managed by the search subsystem. Other data files exist (manually created,
+created by the statistics subsystem, etc.). Those will not be covered here but are mentioned in the job-specific
+documentation that uses them as input.
+
+Each search auxiliary file is copied to the individual region that a [search service](../src/NuGet.Services.SearchService/README.md)
+is deployed. For nuget.org, we run search in four regions, so there are four copies of each of these files.
+
+The search auxiliary files are:
+
+ - [`downloads/downloads.v2.json`](#download-count-data) - total download count for every package version
+ - [`owners/owners.v2.json` and change history](#package-ownership-data) - owners for every package ID
+ - [`verified-packages/verified-packages.v1.json`](#verified-packages-data) - package IDs that are verified
+ - [`popularity-transfers/popularity-transfers.v1.json`](#popularity-transfer-data) - popularity transfers between package IDs
+
+## Download count data
+
+The `downloads/downloads.v2.json` file has the total download count for all package versions. The total download count
+for a package ID as a whole can be calculated simply by adding all version download counts.
+
+The downloads data file looks like this:
+
+```json
+{
+ "Newtonsoft.Json": {
+ "8.0.3": 10508321,
+ "9.0.1": 55801938
+ },
+ "NuGet.Versioning": {
+ "5.6.0-preview.3.6558": 988,
+ "5.6.0": 10224
+ }
+}
+```
+
+The package ID and version keys are not guaranteed to have the original (author-intended) casing and should be treated
+in a case insensitive manner. The version keys will always be normalized via [standard `NuGetVersion` normalization rules](https://docs.microsoft.com/en-us/nuget/concepts/package-versioning#normalized-version-numbers)
+(e.g. no build metadata will appear, no leading zeroes, etc.).
+
+If a package ID or version does not exist in the data file, this only indicates that there is no download count data and
+does not imply that the package ID or version does not exist on the package source. It is possible for package IDs or
+versions that do not exist (perhaps due to deletion) to exist in the data file.
+
+The order of the IDs and versions in the file is undefined.
+
+This file has a "v2" in the file name because it is the second version of this data. The "v1" format is still produced
+by the statistics subsystem and has a less friendly data format.
+
+The class for reading and writing this file to Blob Storage is [`DownloadDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadDataClient.cs).
+
+## Package ownership data
+
+The `owners/owners.v2.json` file contains the owner information about all package IDs. Each time this file is updated,
+the set of package IDs that changed is written to a "change history" file with a path pattern like
+`owners/changes/TIMESTAMP.json`.
+
+The class for reading and writing these files to Blob Storage is [`OwnerDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/OwnerDataClient.cs).
+
+### `owners/owners.v2.json`
+
+The owners data file looks like this:
+
+```json
+{
+ "Newtonsoft.Json": [
+ "dotnetfoundation",
+ "jamesnk",
+ "newtonsoft"
+ ],
+ "NuGet.Versioning": [
+ "Microsoft",
+ "nuget"
+ ]
+}
+```
+
+The package ID key is not guaranteed to have the original (author-intended) casing and should be treated
+in a case insensitive manner. The owner values will have the same casing that is shown on NuGetGallery but should be
+treated in a case insensitive manner.
+
+If a package ID does not exist in the data file, this indicates that the package ID has no owners (a possible but
+relatively rare scenario for NuGetGallery). It is possible for a package ID with no versions to appear in this file.
+
+The order of the IDs and owner usernames in the file is case insensitive ascending lexicographical order.
+
+This file has a "v2" in the file name because it is the second version of this data. The "v1" format was deprecated when
+nuget.org moved from a Lucene-based search service to Azure Search. The "v1" format had a less friendly data format.
+
+### Change history
+
+The change history files do not contain owner usernames for GDPR reasons but mention all of the package IDs that had
+ownership changes since the last time that the `owners.v2.json` file was generated. If a package ID is not mentioned in
+a file, that means that there were no ownership changes in the time window. An ownership change is defined as one or
+more owners being added or removed from the set of owners for that package ID.
+
+Each change history data file has a file name with timestamp format `yyyy-MM-dd-HH-mm-ss-FFFFFFF` (UTC) and a file
+extension of `.json`.
+
+The files look like this:
+
+```json
+[
+ "Newtonsoft.Json",
+ "NuGet.Versioning"
+]
+```
+
+By processing the files in order of their timestamp file name, a rough log of ownership changes can be produced. These
+files are currently not read by any job and are produced for future investigative purposes.
+
+The package ID key is not guaranteed to have the original (author-intended) casing and should be treated
+in a case insensitive manner.
+
+The order of the package IDs in the file is undefined.
+
+## Verified packages data
+
+The `verified-packages/verified-packages.v1.json` data file contains all package IDs that are considered verified by the [prefix reservation feature](https://docs.microsoft.com/en-us/nuget/nuget-org/id-prefix-reservation). This essentially defines the verified checkmark icon in the search UIs.
+
+The data file looks like this:
+
+```json
+[
+ "Newtonsoft.Json",
+ "NuGet.Versioning"
+]
+```
+
+If a package ID is in the file, then it is verified. The package ID is not guaranteed to have the original
+(author-intended) casing and should be treated in a case insensitive manner.
+
+The order of the package IDs is undefined.
+
+The class for reading and writing this file to Blob Storage is [`VerifiedPackagesDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/VerifiedPackagesDataClient.cs).
+
+## Popularity transfer data
+
+The `popularity-transfers/popularity-transfers.v1.json` data file has a mapping of all package IDs that have
+transferred their popularity to one or more other packages.
+
+The data file looks like this:
+
+```json
+{
+ "OldPackageA": [
+ "NewPackage1",
+ "NewPackage2"
+ ],
+ "OldPackageB": [
+ "NewPackage3"
+ ]
+}
+```
+
+For each key-value pair, the package ID key has its popularity transferred to the package ID values. The implementation
+of the popularity transfer is out of scope for the data file format. Package IDs that do not appear as a key in this
+file do not have their popularity transferred.
+
+The package ID keys and values are not guaranteed to have the original (author-intended) casing and should be treated
+in a case insensitive manner.
+
+The order of the package ID keys and values is case insensitive ascending lexicographical order.
+
+The class for reading and writing this file to Blob Storage is [`PopularityTransferDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferDataClient.cs).
diff --git a/docs/Search-version-list-resource.md b/docs/Search-version-list-resource.md
new file mode 100644
index 000000000..3370b93f8
--- /dev/null
+++ b/docs/Search-version-list-resource.md
@@ -0,0 +1,49 @@
+# Search version list resource
+
+**Subsystem: Search 🔎**
+
+The version list resource is bookkeeping used to update the search subsytem. Effectively, this resource is a mapping of package IDs to their versions. It is initially created by the [Db2AzureSearch](../src/NuGet.Jobs.Db2AzureSearch) tool and is kept up-to-date by the [Catalog2AzureSearch](../src/NuGet.Jobs.Catalog2AzureSearch) job.
+
+## Purpose
+
+The [search index](./Azure-Search-indexes.md#Search-index) stores up to 4 documents per package ID. These documents store the metadata for the latest listed version across these pivots:
+
+1. Prerelease - Includes packages that aren't stable
+1. SemVer - Includes packages that require SemVer 2.0.0 support
+
+When packages are created, modified, or deleted we must decide which Azure Search documents must be updated. Furthermore, we need to decide which package version is the latest listed version across both prerelease and semver pivots. How do we do this? Using the search version list resource!
+
+## Content
+
+Version lists are stored in the Azure Blob Storage container for [search auxiliary files](Search-auxiliary-files.md), in the `version-lists` folder. There is a JSON blob for each package ID where the blob's name is the package ID, lower-cased.
+
+Say package `Foo.Bar` has four versions:
+
+* `1.0.0` - This version is unlisted and should be hidden from search results
+* `2.0.0` - This version is listed
+* `3.0.0` - This version is listed is requires SemVer 2.0.0 support
+* `4.0.0-prerelease` This version is listed and is prerelease
+
+`Foo.Bar` would have a blob named `version-lists/foo.bar.json` with content:
+
+```json
+{
+ "VersionProperties": {
+ "1.0.0": {},
+ "2.0.0": {
+ "Listed": true
+ },
+ "3.0.0": {
+ "Listed": true,
+ "SemVer2": true
+ },
+ "4.0.0": {
+ "Listed": true
+ },
+ }
+}
+```
+
+Notice that properties with `false` values are omitted. An unlisted version that does not require SemVer 2.0.0 does not have any properties.
+
+The order of versions within the `VersionProperties` object is undefined.
\ No newline at end of file
diff --git a/sign.thirdparty.props b/sign.thirdparty.props
index 927874066..3df4ce762 100644
--- a/sign.thirdparty.props
+++ b/sign.thirdparty.props
@@ -5,9 +5,13 @@
+
+
+
+
@@ -23,7 +27,9 @@
+
+
diff --git a/src/Catalog/AggregateCursor.cs b/src/Catalog/AggregateCursor.cs
new file mode 100644
index 000000000..6573f2fb7
--- /dev/null
+++ b/src/Catalog/AggregateCursor.cs
@@ -0,0 +1,35 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// A that returns the least value for from a set of s.
+ ///
+ public class AggregateCursor : ReadCursor
+ {
+ public AggregateCursor(IEnumerable innerCursors)
+ {
+ if (innerCursors == null || !innerCursors.Any())
+ {
+ throw new ArgumentException("Must supply at least one cursor!", nameof(innerCursors));
+ }
+
+ InnerCursors = innerCursors.ToList();
+ }
+
+ public IEnumerable InnerCursors { get; private set; }
+
+ public override async Task LoadAsync(CancellationToken cancellationToken)
+ {
+ await Task.WhenAll(InnerCursors.Select(c => c.LoadAsync(cancellationToken)));
+ Value = InnerCursors.Min(c => c.Value);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/AppendOnlyCatalogItem.cs b/src/Catalog/AppendOnlyCatalogItem.cs
new file mode 100644
index 000000000..3bd5a2c9b
--- /dev/null
+++ b/src/Catalog/AppendOnlyCatalogItem.cs
@@ -0,0 +1,34 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using System;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public abstract class AppendOnlyCatalogItem : CatalogItem
+ {
+ public Uri GetBaseAddress()
+ {
+ return new Uri(BaseAddress, "data/" + MakeTimeStampPathComponent(TimeStamp));
+ }
+
+ protected virtual string GetItemIdentity()
+ {
+ return string.Empty;
+ }
+
+ public string GetRelativeAddress()
+ {
+ return GetItemIdentity() + ".json";
+ }
+
+ public override Uri GetItemAddress()
+ {
+ return new Uri(GetBaseAddress(), GetRelativeAddress());
+ }
+
+ protected static string MakeTimeStampPathComponent(DateTime timeStamp)
+ {
+ return string.Format("{0:0000}.{1:00}.{2:00}.{3:00}.{4:00}.{5:00}/", timeStamp.Year, timeStamp.Month, timeStamp.Day, timeStamp.Hour, timeStamp.Minute, timeStamp.Second);
+ }
+ }
+}
diff --git a/src/Catalog/AppendOnlyCatalogWriter.cs b/src/Catalog/AppendOnlyCatalogWriter.cs
new file mode 100644
index 000000000..39e5327a5
--- /dev/null
+++ b/src/Catalog/AppendOnlyCatalogWriter.cs
@@ -0,0 +1,144 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class AppendOnlyCatalogWriter : CatalogWriterBase
+ {
+ private readonly ITelemetryService _telemetryService;
+ private readonly bool _append;
+ private bool _first;
+
+ public AppendOnlyCatalogWriter(
+ IStorage storage,
+ ITelemetryService telemetryService,
+ int maxPageSize = 1000,
+ bool append = true,
+ ICatalogGraphPersistence catalogGraphPersistence = null,
+ CatalogContext context = null)
+ : base(storage, catalogGraphPersistence, context)
+ {
+ _telemetryService = telemetryService;
+ _append = append;
+ _first = true;
+ MaxPageSize = maxPageSize;
+ }
+
+ public int MaxPageSize
+ {
+ get;
+ private set;
+ }
+
+ protected override Uri[] GetAdditionalRootType()
+ {
+ return new Uri[] { Schema.DataTypes.AppendOnlyCatalog, Schema.DataTypes.Permalink };
+ }
+
+ protected override async Task SaveRoot(
+ Guid commitId,
+ DateTime commitTimeStamp,
+ IDictionary pageEntries,
+ IGraph commitMetadata,
+ CancellationToken cancellationToken)
+ {
+ var stopwatch = Stopwatch.StartNew();
+ await base.SaveRoot(commitId, commitTimeStamp, pageEntries, commitMetadata, cancellationToken);
+ _telemetryService.TrackCatalogIndexWriteDuration(stopwatch.Elapsed, RootUri);
+ }
+
+ protected override async Task> SavePages(Guid commitId, DateTime commitTimeStamp, IDictionary itemEntries, CancellationToken cancellationToken)
+ {
+ IDictionary pageEntries;
+ if (_first && !_append)
+ {
+ pageEntries = new Dictionary();
+ _first = false;
+ }
+ else
+ {
+ pageEntries = await LoadIndexResource(RootUri, cancellationToken);
+ }
+
+ bool isExistingPage;
+ Uri pageUri = GetPageUri(pageEntries, itemEntries.Count, out isExistingPage);
+
+ var items = new Dictionary(itemEntries);
+
+ if (isExistingPage)
+ {
+ IDictionary existingItemEntries = await LoadIndexResource(pageUri, cancellationToken);
+ foreach (var entry in existingItemEntries)
+ {
+ items.Add(entry.Key, entry.Value);
+ }
+ }
+
+ await SaveIndexResource(pageUri, Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, items, RootUri, null, null, cancellationToken);
+
+ pageEntries[pageUri.AbsoluteUri] = new CatalogItemSummary(Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, items.Count);
+
+ return pageEntries;
+ }
+
+ private Uri GetPageUri(IDictionary currentPageEntries, int newItemCount, out bool isExistingPage)
+ {
+ Tuple latest = ExtractLatest(currentPageEntries);
+ int nextPageNumber = latest.Item1 + 1;
+ Uri latestUri = latest.Item2;
+ int latestCount = latest.Item3;
+
+ isExistingPage = false;
+
+ if (latestUri == null)
+ {
+ return CreatePageUri(Storage.BaseAddress, "page0");
+ }
+
+ if (latestCount + newItemCount > MaxPageSize)
+ {
+ return CreatePageUri(Storage.BaseAddress, string.Format("page{0}", nextPageNumber));
+ }
+
+ isExistingPage = true;
+
+ return latestUri;
+ }
+
+ private static Tuple ExtractLatest(IDictionary currentPageEntries)
+ {
+ int maxPageNumber = -1;
+ Uri latestUri = null;
+ int latestCount = 0;
+
+ foreach (KeyValuePair entry in currentPageEntries)
+ {
+ int first = entry.Key.IndexOf("page") + 4;
+ int last = first;
+ while (last < entry.Key.Length && char.IsNumber(entry.Key, last))
+ {
+ last++;
+ }
+ string s = entry.Key.Substring(first, last - first);
+ int pageNumber = int.Parse(s);
+
+ if (pageNumber > maxPageNumber)
+ {
+ maxPageNumber = pageNumber;
+ latestUri = new Uri(entry.Key);
+ latestCount = entry.Value.Count.Value;
+ }
+ }
+
+ return new Tuple(maxPageNumber, latestUri, latestCount);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/BatchProcessingException.cs b/src/Catalog/BatchProcessingException.cs
new file mode 100644
index 000000000..cdc135cf1
--- /dev/null
+++ b/src/Catalog/BatchProcessingException.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public sealed class BatchProcessingException : Exception
+ {
+ public BatchProcessingException(Exception inner)
+ : base(Strings.BatchProcessingFailure, inner ?? throw new ArgumentNullException(nameof(inner)))
+ {
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogCommit.cs b/src/Catalog/CatalogCommit.cs
new file mode 100644
index 000000000..366eefd98
--- /dev/null
+++ b/src/Catalog/CatalogCommit.cs
@@ -0,0 +1,51 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Globalization;
+using Newtonsoft.Json.Linq;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// Represents a single catalog commit.
+ ///
+ public sealed class CatalogCommit : IComparable
+ {
+ private CatalogCommit(DateTime commitTimeStamp, Uri uri)
+ {
+ CommitTimeStamp = commitTimeStamp;
+ Uri = uri;
+ }
+
+ public DateTime CommitTimeStamp { get; }
+ public Uri Uri { get; }
+
+ public int CompareTo(object obj)
+ {
+ var other = obj as CatalogCommit;
+
+ if (ReferenceEquals(other, null))
+ {
+ throw new ArgumentException(
+ string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommit)),
+ nameof(obj));
+ }
+
+ return CommitTimeStamp.CompareTo(other.CommitTimeStamp);
+ }
+
+ public static CatalogCommit Create(JObject commit)
+ {
+ if (commit == null)
+ {
+ throw new ArgumentNullException(nameof(commit));
+ }
+
+ var commitTimeStamp = Utils.Deserialize(commit, "commitTimeStamp");
+ var uri = Utils.Deserialize(commit, "@id");
+
+ return new CatalogCommit(commitTimeStamp, uri);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogCommitItem.cs b/src/Catalog/CatalogCommitItem.cs
new file mode 100644
index 000000000..4e0a1f505
--- /dev/null
+++ b/src/Catalog/CatalogCommitItem.cs
@@ -0,0 +1,126 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Linq;
+using Newtonsoft.Json.Linq;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// Represents a single item in a catalog commit.
+ ///
+ [DebuggerDisplay("{DebuggerDisplay,nq}")]
+ public sealed class CatalogCommitItem : IComparable
+ {
+ private string DebuggerDisplay =>
+ $"Catalog item {PackageIdentity?.Id} {PackageIdentity?.Version.ToNormalizedString()}" +
+ $"{(IsPackageDetails ? " (" + nameof(Schema.DataTypes.PackageDetails) + ")" : string.Empty)}" +
+ $"{(IsPackageDelete ? " (" + nameof(Schema.DataTypes.PackageDelete) + ")" : string.Empty)}";
+
+ private const string _typeKeyword = "@type";
+
+ public CatalogCommitItem(
+ Uri uri,
+ string commitId,
+ DateTime commitTimeStamp,
+ IReadOnlyList types,
+ IReadOnlyList typeUris,
+ PackageIdentity packageIdentity)
+ {
+ Uri = uri;
+ CommitId = commitId;
+ CommitTimeStamp = commitTimeStamp;
+ PackageIdentity = packageIdentity;
+ Types = types;
+ TypeUris = typeUris;
+
+ IsPackageDetails = HasTypeUri(Schema.DataTypes.PackageDetails);
+ IsPackageDelete = HasTypeUri(Schema.DataTypes.PackageDelete);
+ }
+
+ public Uri Uri { get; }
+ public DateTime CommitTimeStamp { get; }
+ public string CommitId { get; }
+ public PackageIdentity PackageIdentity { get; }
+ public IReadOnlyList Types { get; }
+ public IReadOnlyList TypeUris { get; }
+
+ public bool IsPackageDetails { get; }
+ public bool IsPackageDelete { get; }
+
+ public bool HasTypeUri(Uri typeUri)
+ {
+ return TypeUris.Any(x => x.IsAbsoluteUri && x.AbsoluteUri == typeUri.AbsoluteUri);
+ }
+
+ public int CompareTo(object obj)
+ {
+ var other = obj as CatalogCommitItem;
+
+ if (ReferenceEquals(other, null))
+ {
+ throw new ArgumentException(
+ string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommitItem)),
+ nameof(obj));
+ }
+
+ return CommitTimeStamp.CompareTo(other.CommitTimeStamp);
+ }
+
+ public static CatalogCommitItem Create(JObject context, JObject commitItem)
+ {
+ if (context == null)
+ {
+ throw new ArgumentNullException(nameof(context));
+ }
+
+ if (commitItem == null)
+ {
+ throw new ArgumentNullException(nameof(commitItem));
+ }
+
+ var commitTimeStamp = Utils.Deserialize(commitItem, "commitTimeStamp");
+ var commitId = Utils.Deserialize(commitItem, "commitId");
+ var idUri = Utils.Deserialize(commitItem, "@id");
+ var packageId = Utils.Deserialize(commitItem, "nuget:id");
+ var packageVersion = Utils.Deserialize(commitItem, "nuget:version");
+ var packageIdentity = new PackageIdentity(packageId, new NuGetVersion(packageVersion));
+ var types = GetTypes(commitItem).ToArray();
+
+ if (!types.Any())
+ {
+ throw new ArgumentException(
+ string.Format(CultureInfo.InvariantCulture, Strings.NonEmptyPropertyValueRequired, _typeKeyword),
+ nameof(commitItem));
+ }
+
+ var typeUris = types.Select(type => Utils.Expand(context, type)).ToArray();
+
+ return new CatalogCommitItem(idUri, commitId, commitTimeStamp, types, typeUris, packageIdentity);
+ }
+
+ private static IEnumerable GetTypes(JObject commitItem)
+ {
+ if (commitItem.TryGetValue(_typeKeyword, out var value))
+ {
+ if (value is JArray)
+ {
+ foreach (JToken typeToken in ((JArray)value).Values())
+ {
+ yield return typeToken.ToString();
+ }
+ }
+ else
+ {
+ yield return value.ToString();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogCommitItemBatch.cs b/src/Catalog/CatalogCommitItemBatch.cs
new file mode 100644
index 000000000..3fc3a28d3
--- /dev/null
+++ b/src/Catalog/CatalogCommitItemBatch.cs
@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// Represents a group of .
+ /// Items may span multiple commits but are grouped on common criteria (e.g.: lower-cased package ID).
+ ///
+ public sealed class CatalogCommitItemBatch
+ {
+ ///
+ /// Initializes a instance.
+ ///
+ /// An enumerable of . Items may span multiple commits.
+ /// A unique key for all items in a batch. This is used for parallelization and may be
+ /// null if parallelization is not used.
+ /// The commit timestamp to use for this batch. If null, the minimum of
+ /// all item commit timestamps will be used.
+ /// Thrown if is either null or empty.
+ public CatalogCommitItemBatch(
+ IEnumerable items,
+ string key = null,
+ DateTime? commitTimestamp = null)
+ {
+ if (items == null || !items.Any())
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(items));
+ }
+
+ var list = items.ToList();
+
+ CommitTimeStamp = commitTimestamp ?? list.Min(item => item.CommitTimeStamp);
+ Key = key;
+
+ list.Sort();
+
+ Items = list;
+ }
+
+ public DateTime CommitTimeStamp { get; }
+ public IReadOnlyList Items { get; }
+ public string Key { get; }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogCommitItemBatchTask.cs b/src/Catalog/CatalogCommitItemBatchTask.cs
new file mode 100644
index 000000000..3d8e58401
--- /dev/null
+++ b/src/Catalog/CatalogCommitItemBatchTask.cs
@@ -0,0 +1,68 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// Represents an asynchrononous task associated with catalog changes for a specific
+ /// and potentially spanning multiple commits.
+ ///
+ public sealed class CatalogCommitItemBatchTask : IEquatable
+ {
+ ///
+ /// Initializes a instance.
+ ///
+ /// A .
+ /// A tracking completion of
+ /// processing.
+ /// Thrown if is null.
+ /// Thrown if is null.
+ /// Thrown if is null.
+ public CatalogCommitItemBatchTask(CatalogCommitItemBatch batch, Task task)
+ {
+ if (batch == null)
+ {
+ throw new ArgumentNullException(nameof(batch));
+ }
+
+ if (batch.Key == null)
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNull, $"{nameof(batch)}.{nameof(batch.Key)}");
+ }
+
+ if (task == null)
+ {
+ throw new ArgumentNullException(nameof(task));
+ }
+
+ Batch = batch;
+ Task = task;
+ }
+
+ public CatalogCommitItemBatch Batch { get; }
+ public Task Task { get; }
+
+ public override int GetHashCode()
+ {
+ return Batch.Key.GetHashCode();
+ }
+
+ public override bool Equals(object obj)
+ {
+ return Equals(obj as CatalogCommitItemBatchTask);
+ }
+
+ public bool Equals(CatalogCommitItemBatchTask other)
+ {
+ if (ReferenceEquals(other, null))
+ {
+ return false;
+ }
+
+ return string.Equals(Batch.Key, other.Batch.Key);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogCommitUtilities.cs b/src/Catalog/CatalogCommitUtilities.cs
new file mode 100644
index 000000000..1597ec8ff
--- /dev/null
+++ b/src/Catalog/CatalogCommitUtilities.cs
@@ -0,0 +1,323 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using ExceptionUtilities = NuGet.Common.ExceptionUtilities;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class CatalogCommitUtilities
+ {
+ private static readonly EventId _eventId = new EventId(id: 0);
+
+ ///
+ /// Creates an enumerable of instances.
+ ///
+ ///
+ /// A instance contains only the latest commit for each package identity.
+ ///
+ /// An enumerable of .
+ /// A function that returns a key for a .
+ /// An enumerable of with no ordering guarantee.
+ /// Thrown if is null.
+ /// Thrown if is null.
+ public static IEnumerable CreateCommitItemBatches(
+ IEnumerable catalogItems,
+ GetCatalogCommitItemKey getCatalogCommitItemKey)
+ {
+ if (catalogItems == null)
+ {
+ throw new ArgumentNullException(nameof(catalogItems));
+ }
+
+ if (getCatalogCommitItemKey == null)
+ {
+ throw new ArgumentNullException(nameof(getCatalogCommitItemKey));
+ }
+
+ var catalogItemsGroups = catalogItems
+ .GroupBy(catalogItem => getCatalogCommitItemKey(catalogItem));
+
+ var batches = new List();
+
+ foreach (var catalogItemsGroup in catalogItemsGroups)
+ {
+ var catalogItemsWithOnlyLatestCommitForEachPackageIdentity = catalogItemsGroup
+ .GroupBy(commitItem => new
+ {
+ PackageId = commitItem.PackageIdentity.Id.ToLowerInvariant(),
+ PackageVersion = commitItem.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant()
+ })
+ .Select(group => group.OrderBy(item => item.CommitTimeStamp).Last())
+ .ToArray();
+ var minCommitTimeStamp = catalogItemsWithOnlyLatestCommitForEachPackageIdentity
+ .Select(catalogItem => catalogItem.CommitTimeStamp)
+ .Min();
+
+ batches.Add(
+ new CatalogCommitItemBatch(
+ catalogItemsWithOnlyLatestCommitForEachPackageIdentity,
+ catalogItemsGroup.Key));
+ }
+
+ // Assert only after skipping older commits for each package identity to reduce the likelihood
+ // of unnecessary failures.
+ AssertNotMoreThanOneCommitIdPerCommitTimeStamp(batches, nameof(catalogItems));
+
+ return batches;
+ }
+
+ public static void StartProcessingBatchesIfNoFailures(
+ CollectorHttpClient client,
+ JToken context,
+ List unprocessedBatches,
+ List processingBatches,
+ int maxConcurrentBatches,
+ ProcessCommitItemBatchAsync processCommitItemBatchAsync,
+ CancellationToken cancellationToken)
+ {
+ if (client == null)
+ {
+ throw new ArgumentNullException(nameof(client));
+ }
+
+ if (context == null)
+ {
+ throw new ArgumentNullException(nameof(context));
+ }
+
+ if (unprocessedBatches == null)
+ {
+ throw new ArgumentNullException(nameof(unprocessedBatches));
+ }
+
+ if (processingBatches == null)
+ {
+ throw new ArgumentNullException(nameof(processingBatches));
+ }
+
+ if (maxConcurrentBatches < 1)
+ {
+ throw new ArgumentOutOfRangeException(
+ nameof(maxConcurrentBatches),
+ maxConcurrentBatches,
+ string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
+ }
+
+ if (processCommitItemBatchAsync == null)
+ {
+ throw new ArgumentNullException(nameof(processCommitItemBatchAsync));
+ }
+
+ var hasAnyBatchFailed = processingBatches.Any(batch => batch.Task.IsFaulted || batch.Task.IsCanceled);
+
+ if (hasAnyBatchFailed)
+ {
+ return;
+ }
+
+ var batchesToEnqueue = Math.Min(
+ maxConcurrentBatches - processingBatches.Count(batch => !batch.Task.IsCompleted),
+ unprocessedBatches.Count);
+
+ for (var i = 0; i < batchesToEnqueue; ++i)
+ {
+ var batch = unprocessedBatches[0];
+
+ unprocessedBatches.RemoveAt(0);
+
+ var task = processCommitItemBatchAsync(
+ client,
+ context,
+ batch.Key,
+ batch,
+ lastBatch: null,
+ cancellationToken: cancellationToken);
+ var batchTask = new CatalogCommitItemBatchTask(batch, task);
+
+ processingBatches.Add(batchTask);
+ }
+ }
+
+ internal static async Task ProcessCatalogCommitsAsync(
+ CollectorHttpClient client,
+ ReadWriteCursor front,
+ ReadCursor back,
+ FetchCatalogCommitsAsync fetchCatalogCommitsAsync,
+ CreateCommitItemBatchesAsync createCommitItemBatchesAsync,
+ ProcessCommitItemBatchAsync processCommitItemBatchAsync,
+ int maxConcurrentBatches,
+ ILogger logger,
+ CancellationToken cancellationToken)
+ {
+ var rootItems = await fetchCatalogCommitsAsync(client, front, back, cancellationToken);
+
+ var hasAnyBatchFailed = false;
+ var hasAnyBatchBeenProcessed = false;
+
+ foreach (CatalogCommit rootItem in rootItems)
+ {
+ JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
+ var context = (JObject)page["@context"];
+ CatalogCommitItemBatch[] batches = await CreateBatchesForAllAvailableItemsInPageAsync(
+ front,
+ back,
+ page,
+ context,
+ createCommitItemBatchesAsync);
+
+ if (!batches.Any())
+ {
+ continue;
+ }
+
+ hasAnyBatchBeenProcessed = true;
+
+ DateTime maxCommitTimeStamp = GetMaxCommitTimeStamp(batches);
+ var unprocessedBatches = batches.ToList();
+ var processingBatches = new List();
+ var exceptions = new List();
+
+ StartProcessingBatchesIfNoFailures(
+ client,
+ context,
+ unprocessedBatches,
+ processingBatches,
+ maxConcurrentBatches,
+ processCommitItemBatchAsync,
+ cancellationToken);
+
+ while (processingBatches.Any())
+ {
+ var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted)
+ .Select(batch => batch.Task)
+ .DefaultIfEmpty(Task.CompletedTask);
+
+ await Task.WhenAny(activeTasks);
+
+ for (var i = 0; i < processingBatches.Count; ++i)
+ {
+ var batch = processingBatches[i];
+
+ if (batch.Task.IsFaulted || batch.Task.IsCanceled)
+ {
+ hasAnyBatchFailed = true;
+
+ if (batch.Task.Exception != null)
+ {
+ var exception = ExceptionUtilities.Unwrap(batch.Task.Exception);
+
+ exceptions.Add(exception);
+ }
+ }
+
+ if (batch.Task.IsCompleted)
+ {
+ processingBatches.RemoveAt(i);
+ --i;
+ }
+ }
+
+ if (!hasAnyBatchFailed)
+ {
+ StartProcessingBatchesIfNoFailures(
+ client,
+ context,
+ unprocessedBatches,
+ processingBatches,
+ maxConcurrentBatches,
+ processCommitItemBatchAsync,
+ cancellationToken);
+ }
+ }
+
+ if (hasAnyBatchFailed)
+ {
+ foreach (var exception in exceptions)
+ {
+ logger.LogError(_eventId, exception, Strings.BatchProcessingFailure);
+ }
+
+ var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions);
+
+ throw new BatchProcessingException(innerException);
+ }
+
+ front.Value = maxCommitTimeStamp;
+
+ await front.SaveAsync(cancellationToken);
+
+ Trace.TraceInformation($"{nameof(CatalogCommitUtilities)}.{nameof(ProcessCatalogCommitsAsync)} " +
+ $"{nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front);
+ }
+
+ return hasAnyBatchBeenProcessed;
+ }
+
+ public static string GetPackageIdKey(CatalogCommitItem item)
+ {
+ if (item == null)
+ {
+ throw new ArgumentNullException(nameof(item));
+ }
+
+ return item.PackageIdentity.Id.ToLowerInvariant();
+ }
+
+ private static async Task CreateBatchesForAllAvailableItemsInPageAsync(
+ ReadWriteCursor front,
+ ReadCursor back,
+ JObject page,
+ JObject context,
+ CreateCommitItemBatchesAsync createCommitItemBatchesAsync)
+ {
+ IEnumerable commitItems = page["items"]
+ .Select(item => CatalogCommitItem.Create(context, (JObject)item))
+ .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value);
+
+ IEnumerable batches = await createCommitItemBatchesAsync(commitItems);
+
+ return batches
+ .OrderBy(batch => batch.CommitTimeStamp)
+ .ToArray();
+ }
+
+ private static DateTime GetMaxCommitTimeStamp(CatalogCommitItemBatch[] batches)
+ {
+ return batches.SelectMany(batch => batch.Items)
+ .Select(item => item.CommitTimeStamp)
+ .Max();
+ }
+
+ private static void AssertNotMoreThanOneCommitIdPerCommitTimeStamp(
+ IEnumerable batches,
+ string parameterName)
+ {
+ var commitsWithSameTimeStampButDifferentCommitIds = batches
+ .SelectMany(batch => batch.Items)
+ .GroupBy(commitItem => commitItem.CommitTimeStamp)
+ .Where(group => group.Select(item => item.CommitId).Distinct().Count() > 1);
+
+ if (commitsWithSameTimeStampButDifferentCommitIds.Any())
+ {
+ var commits = commitsWithSameTimeStampButDifferentCommitIds.SelectMany(group => group)
+ .Select(commit => $"{{ CommitId = {commit.CommitId}, CommitTimeStamp = {commit.CommitTimeStamp.ToString("O")} }}");
+
+ throw new ArgumentException(
+ string.Format(
+ CultureInfo.InvariantCulture,
+ Strings.MultipleCommitIdsForSameCommitTimeStamp,
+ string.Join(", ", commits)),
+ parameterName);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogContext.cs b/src/Catalog/CatalogContext.cs
new file mode 100644
index 000000000..7f764c99f
--- /dev/null
+++ b/src/Catalog/CatalogContext.cs
@@ -0,0 +1,33 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CatalogContext
+ {
+ ConcurrentDictionary _jsonLdContext;
+
+ public CatalogContext()
+ {
+ _jsonLdContext = new ConcurrentDictionary();
+ }
+
+ public JObject GetJsonLdContext(string name, Uri type)
+ {
+ return _jsonLdContext.GetOrAdd(name + "#" + type.ToString(), (key) =>
+ {
+ using (JsonReader jsonReader = new JsonTextReader(new StreamReader(Utils.GetResourceStream(name))))
+ {
+ JObject obj = JObject.Load(jsonReader);
+ obj["@type"] = type.ToString();
+ return obj;
+ }
+ });
+ }
+ }
+}
diff --git a/src/Catalog/CatalogIndexEntry.cs b/src/Catalog/CatalogIndexEntry.cs
new file mode 100644
index 000000000..48112b8dc
--- /dev/null
+++ b/src/Catalog/CatalogIndexEntry.cs
@@ -0,0 +1,143 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Newtonsoft.Json;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public sealed class CatalogIndexEntry : IComparable
+ {
+ [JsonConstructor]
+ private CatalogIndexEntry()
+ {
+ Types = Enumerable.Empty();
+ }
+
+ public CatalogIndexEntry(
+ Uri uri,
+ string type,
+ string commitId,
+ DateTime commitTs,
+ PackageIdentity packageIdentity)
+ {
+ if (string.IsNullOrWhiteSpace(type))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullEmptyOrWhitespace, nameof(type));
+ }
+
+ Initialize(uri, new[] { type }, commitId, commitTs, packageIdentity);
+ }
+
+ public CatalogIndexEntry(
+ Uri uri,
+ IReadOnlyList types,
+ string commitId,
+ DateTime commitTs,
+ PackageIdentity packageIdentity)
+ {
+ Initialize(uri, types, commitId, commitTs, packageIdentity);
+ }
+
+ [JsonProperty("@id")]
+ [JsonRequired]
+ public Uri Uri { get; private set; }
+
+ [JsonProperty("@type")]
+ [JsonRequired]
+ [JsonConverter(typeof(CatalogTypeConverter))]
+ public IEnumerable Types { get; private set; }
+
+ [JsonProperty("commitId")]
+ [JsonRequired]
+ public string CommitId { get; private set; }
+
+ [JsonProperty("commitTimeStamp")]
+ [JsonRequired]
+ public DateTime CommitTimeStamp { get; private set; }
+
+ [JsonProperty("nuget:id")]
+ [JsonRequired]
+ public string Id { get; private set; }
+
+ [JsonProperty("nuget:version")]
+ [JsonRequired]
+ public NuGetVersion Version { get; private set; }
+
+ [JsonIgnore]
+ public bool IsDelete
+ {
+ get
+ {
+ return Types.Any(type => type == "nuget:PackageDelete");
+ }
+ }
+
+ public int CompareTo(CatalogIndexEntry other)
+ {
+ if (other == null)
+ {
+ throw new ArgumentNullException(nameof(other));
+ }
+
+ return CommitTimeStamp.CompareTo(other.CommitTimeStamp);
+ }
+
+ public static CatalogIndexEntry Create(CatalogCommitItem commitItem)
+ {
+ if (commitItem == null)
+ {
+ throw new ArgumentNullException(nameof(commitItem));
+ }
+
+ return new CatalogIndexEntry(
+ commitItem.Uri,
+ commitItem.Types,
+ commitItem.CommitId,
+ commitItem.CommitTimeStamp,
+ commitItem.PackageIdentity);
+ }
+
+ private void Initialize(
+ Uri uri,
+ IReadOnlyList types,
+ string commitId,
+ DateTime commitTs,
+ PackageIdentity packageIdentity)
+ {
+ Uri = uri ?? throw new ArgumentNullException(nameof(uri));
+
+ if (types == null || !types.Any())
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(types));
+ }
+
+ if (types.Any(type => string.IsNullOrWhiteSpace(type)))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullEmptyOrWhitespace, nameof(types));
+ }
+
+ Types = types;
+
+ if (string.IsNullOrWhiteSpace(commitId))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(commitId));
+ }
+
+ CommitId = commitId;
+ CommitTimeStamp = commitTs;
+
+ if (packageIdentity == null)
+ {
+ throw new ArgumentNullException(nameof(packageIdentity));
+ }
+
+ Id = packageIdentity.Id;
+ Version = packageIdentity.Version;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogIndexReader.cs b/src/Catalog/CatalogIndexReader.cs
new file mode 100644
index 000000000..e33b5a4a9
--- /dev/null
+++ b/src/Catalog/CatalogIndexReader.cs
@@ -0,0 +1,105 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CatalogIndexReader
+ {
+ private readonly Uri _indexUri;
+ private readonly CollectorHttpClient _httpClient;
+ private readonly ITelemetryService _telemetryService;
+ private JObject _context;
+
+ public CatalogIndexReader(Uri indexUri, CollectorHttpClient httpClient, ITelemetryService telemetryService)
+ {
+ _indexUri = indexUri;
+ _httpClient = httpClient;
+ _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+ }
+
+ public async Task> GetEntries()
+ {
+ var stopwatch = Stopwatch.StartNew();
+ JObject index = await _httpClient.GetJObjectAsync(_indexUri);
+ _telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, _indexUri);
+
+ // save the context used on the index
+ JToken context = null;
+ if (index.TryGetValue("@context", out context))
+ {
+ _context = context as JObject;
+ }
+
+ List> pages = new List>();
+
+ foreach (var item in index["items"])
+ {
+ pages.Add(new Tuple(DateTime.Parse(item["commitTimeStamp"].ToString()), new Uri(item["@id"].ToString())));
+ }
+
+ return await GetEntriesAsync(pages.Select(p => p.Item2));
+ }
+
+ private async Task> GetEntriesAsync(IEnumerable pageUris)
+ {
+ var pageUriBag = new ConcurrentBag(pageUris);
+ var entries = new ConcurrentBag();
+ var interner = new StringInterner();
+
+ var tasks = Enumerable
+ .Range(0, ServicePointManager.DefaultConnectionLimit)
+ .Select(i => ProcessPageUris(pageUriBag, entries, interner))
+ .ToList();
+
+ await Task.WhenAll(tasks);
+
+ return entries;
+ }
+
+ private async Task ProcessPageUris(ConcurrentBag pageUriBag, ConcurrentBag entries, StringInterner interner)
+ {
+ await Task.Yield();
+ Uri pageUri;
+ while (pageUriBag.TryTake(out pageUri))
+ {
+ var json = await _httpClient.GetJObjectAsync(pageUri);
+
+ foreach (var item in json["items"])
+ {
+ // This string is unique.
+ var id = item["@id"].ToString();
+
+ // These strings should be shared.
+ var type = interner.Intern(item["@type"].ToString());
+ var commitId = interner.Intern(item["commitId"].ToString());
+ var nugetId = interner.Intern(item["nuget:id"].ToString());
+ var nugetVersion = interner.Intern(item["nuget:version"].ToString());
+ var packageIdentity = new PackageIdentity(nugetId, NuGetVersion.Parse(nugetVersion));
+
+ // No string is directly operated on here.
+ var commitTimeStamp = item["commitTimeStamp"].ToObject();
+
+ var entry = new CatalogIndexEntry(
+ new Uri(id),
+ type,
+ commitId,
+ commitTimeStamp,
+ packageIdentity);
+
+ entries.Add(entry);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogItem.cs b/src/Catalog/CatalogItem.cs
new file mode 100644
index 000000000..51a9ddad4
--- /dev/null
+++ b/src/Catalog/CatalogItem.cs
@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public abstract class CatalogItem
+ {
+ public DateTime TimeStamp { get; set; }
+
+ public Guid CommitId { get; set; }
+
+ public Uri BaseAddress { get; set; }
+
+ public abstract Uri GetItemType();
+
+ public abstract Uri GetItemAddress();
+
+ public virtual StorageContent CreateContent(CatalogContext context)
+ {
+ return null;
+ }
+
+ public virtual IGraph CreatePageContent(CatalogContext context)
+ {
+ return null;
+ }
+
+ ///
+ /// Create the core graph used in CreateContent(context)
+ ///
+ public virtual IGraph CreateContentGraph(CatalogContext context)
+ {
+ return null;
+ }
+ }
+}
diff --git a/src/Catalog/CatalogItemSummary.cs b/src/Catalog/CatalogItemSummary.cs
new file mode 100644
index 000000000..3dbbef9f7
--- /dev/null
+++ b/src/Catalog/CatalogItemSummary.cs
@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using System;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CatalogItemSummary
+ {
+ public CatalogItemSummary(Uri type, Guid commitId, DateTime commitTimeStamp, int? count = null, IGraph content = null)
+ {
+ Type = type;
+ CommitId = commitId;
+ CommitTimeStamp = commitTimeStamp;
+ Count = count;
+ Content = content;
+ }
+
+ public Uri Type { get; private set; }
+ public Guid CommitId { get; private set; }
+ public DateTime CommitTimeStamp { get; private set; }
+ public int? Count { get; private set; }
+ public IGraph Content { get; private set; }
+ }
+}
diff --git a/src/Catalog/CatalogTypeConverter.cs b/src/Catalog/CatalogTypeConverter.cs
new file mode 100644
index 000000000..c2e433924
--- /dev/null
+++ b/src/Catalog/CatalogTypeConverter.cs
@@ -0,0 +1,53 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Newtonsoft.Json;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CatalogTypeConverter : JsonConverter
+ {
+ public override bool CanConvert(Type objectType)
+ {
+ return objectType == typeof(IEnumerable);
+ }
+
+ public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+ {
+ object type = null;
+ if (reader.TokenType == JsonToken.StartArray)
+ {
+ // If the type is stored in an array, use the last value and then discard the rest.
+ do
+ {
+ reader.Read();
+ if (reader.TokenType == JsonToken.String)
+ {
+ type = reader.Value;
+ }
+ } while (reader.TokenType != JsonToken.EndArray);
+ }
+ else
+ {
+ type = reader.Value;
+ }
+
+ if (type == null)
+ {
+ throw new InvalidDataException("Failed to parse the type of a catalog entry!");
+ }
+
+ return new string[] { type.ToString() };
+ }
+
+ public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+ {
+ var types = value as IEnumerable;
+ serializer.Serialize(writer, types.First());
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CatalogWriterBase.cs b/src/Catalog/CatalogWriterBase.cs
new file mode 100644
index 000000000..a5a8bead9
--- /dev/null
+++ b/src/Catalog/CatalogWriterBase.cs
@@ -0,0 +1,381 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public abstract class CatalogWriterBase : IDisposable
+ {
+ protected List _batch;
+ protected bool _open;
+
+ public CatalogWriterBase(IStorage storage, ICatalogGraphPersistence graphPersistence = null, CatalogContext context = null)
+ {
+ Options.InternUris = false;
+
+ Storage = storage;
+ GraphPersistence = graphPersistence;
+ Context = context ?? new CatalogContext();
+
+ _batch = new List();
+ _open = true;
+
+ RootUri = Storage.ResolveUri("index.json");
+ }
+
+ public void Dispose()
+ {
+ _batch.Clear();
+ _open = false;
+ }
+
+ public IStorage Storage { get; private set; }
+
+ public ICatalogGraphPersistence GraphPersistence { get; private set; }
+
+ public Uri RootUri { get; private set; }
+
+ public CatalogContext Context { get; private set; }
+
+ public int Count { get { return _batch.Count; } }
+
+ public void Add(CatalogItem item)
+ {
+ if (!_open)
+ {
+ throw new ObjectDisposedException(GetType().Name);
+ }
+
+ _batch.Add(item);
+ }
+ public Task> Commit(IGraph commitMetadata, CancellationToken cancellationToken)
+ {
+ return Commit(DateTime.UtcNow, commitMetadata, cancellationToken);
+ }
+
+ public virtual async Task> Commit(DateTime commitTimeStamp, IGraph commitMetadata, CancellationToken cancellationToken)
+ {
+ if (!_open)
+ {
+ throw new ObjectDisposedException(GetType().Name);
+ }
+
+ if (_batch.Count == 0)
+ {
+ return Enumerable.Empty();
+ }
+
+ // the commitId is only used for tracing and trouble shooting
+
+ Guid commitId = Guid.NewGuid();
+
+ // save items
+
+ IDictionary newItemEntries = await SaveItems(commitId, commitTimeStamp, cancellationToken);
+
+ // save index pages - this is abstract as the derived class determines the index pagination
+
+ IDictionary pageEntries = await SavePages(commitId, commitTimeStamp, newItemEntries, cancellationToken);
+
+ // save index root
+
+ await SaveRoot(commitId, commitTimeStamp, pageEntries, commitMetadata, cancellationToken);
+
+ _batch.Clear();
+
+ return newItemEntries.Keys.Select(s => new Uri(s));
+ }
+
+ private async Task> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken)
+ {
+ ConcurrentDictionary pageItems = new ConcurrentDictionary();
+
+ int batchIndex = 0;
+
+ var saveTasks = new List();
+ foreach (CatalogItem item in _batch)
+ {
+ ResourceSaveOperation saveOperationForItem = null;
+
+ try
+ {
+ item.TimeStamp = commitTimeStamp;
+ item.CommitId = commitId;
+ item.BaseAddress = Storage.BaseAddress;
+
+ saveOperationForItem = CreateSaveOperationForItem(Storage, Context, item, cancellationToken);
+ if (saveOperationForItem.SaveTask != null)
+ {
+ saveTasks.Add(saveOperationForItem.SaveTask);
+ }
+
+ IGraph pageContent = item.CreatePageContent(Context);
+
+ if (!pageItems.TryAdd(saveOperationForItem.ResourceUri.AbsoluteUri, new CatalogItemSummary(item.GetItemType(), commitId, commitTimeStamp, null, pageContent)))
+ {
+ throw new Exception("Duplicate page: " + saveOperationForItem.ResourceUri.AbsoluteUri);
+ }
+
+ batchIndex++;
+ }
+ catch (Exception e)
+ {
+ string msg = (saveOperationForItem == null || saveOperationForItem.ResourceUri == null)
+ ? string.Format("batch index: {0}", batchIndex)
+ : string.Format("batch index: {0} resourceUri: {1}", batchIndex, saveOperationForItem.ResourceUri);
+
+ throw new Exception(msg, e);
+ }
+ }
+
+ await Task.WhenAll(saveTasks);
+
+ return pageItems;
+ }
+
+ protected virtual ResourceSaveOperation CreateSaveOperationForItem(IStorage storage, CatalogContext context, CatalogItem item, CancellationToken cancellationToken)
+ {
+ // This method decides what to do with the item.
+ // Standard method of operation: if content == null, don't do a thing. Else, write content.
+
+ var content = item.CreateContent(Context); // note: always do this first
+ var resourceUri = item.GetItemAddress();
+
+ var operation = new ResourceSaveOperation();
+ operation.ResourceUri = resourceUri;
+
+ if (content != null)
+ {
+ operation.SaveTask = storage.SaveAsync(resourceUri, content, cancellationToken);
+ }
+
+ return operation;
+ }
+
+ protected virtual async Task SaveRoot(Guid commitId, DateTime commitTimeStamp, IDictionary pageEntries, IGraph commitMetadata, CancellationToken cancellationToken)
+ {
+ await SaveIndexResource(RootUri, Schema.DataTypes.CatalogRoot, commitId, commitTimeStamp, pageEntries, null, commitMetadata, GetAdditionalRootType(), cancellationToken);
+ }
+
+ protected virtual Uri[] GetAdditionalRootType()
+ {
+ return null;
+ }
+
+ protected abstract Task> SavePages(Guid commitId, DateTime commitTimeStamp, IDictionary itemEntries, CancellationToken cancellationToken);
+
+ protected virtual StorageContent CreateIndexContent(IGraph graph, Uri type)
+ {
+ JObject frame = Context.GetJsonLdContext("context.Container.json", type);
+ return new JTokenStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store");
+ }
+
+ protected async Task SaveIndexResource(Uri resourceUri, Uri typeUri, Guid commitId, DateTime commitTimeStamp, IDictionary entries, Uri parent, IGraph extra, Uri[] additionalResourceTypes, CancellationToken cancellationToken)
+ {
+ IGraph graph = new Graph();
+
+ INode resourceNode = graph.CreateUriNode(resourceUri);
+ INode itemPredicate = graph.CreateUriNode(Schema.Predicates.CatalogItem);
+ INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type);
+ INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp);
+ INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId);
+ INode countPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCount);
+
+ graph.Assert(resourceNode, typePredicate, graph.CreateUriNode(typeUri));
+ graph.Assert(resourceNode, commitIdPredicate, graph.CreateLiteralNode(commitId.ToString()));
+ graph.Assert(resourceNode, timeStampPredicate, graph.CreateLiteralNode(commitTimeStamp.ToString("O"), Schema.DataTypes.DateTime));
+ graph.Assert(resourceNode, countPredicate, graph.CreateLiteralNode(entries.Count.ToString(), Schema.DataTypes.Integer));
+
+ foreach (KeyValuePair itemEntry in entries)
+ {
+ INode itemNode = graph.CreateUriNode(new Uri(itemEntry.Key));
+
+ graph.Assert(resourceNode, itemPredicate, itemNode);
+ graph.Assert(itemNode, typePredicate, graph.CreateUriNode(itemEntry.Value.Type));
+ graph.Assert(itemNode, commitIdPredicate, graph.CreateLiteralNode(itemEntry.Value.CommitId.ToString()));
+ graph.Assert(itemNode, timeStampPredicate, graph.CreateLiteralNode(itemEntry.Value.CommitTimeStamp.ToString("O"), Schema.DataTypes.DateTime));
+
+ if (itemEntry.Value.Count != null)
+ {
+ graph.Assert(itemNode, countPredicate, graph.CreateLiteralNode(itemEntry.Value.Count.ToString(), Schema.DataTypes.Integer));
+ }
+
+ if (itemEntry.Value.Content != null)
+ {
+ graph.Merge(itemEntry.Value.Content, true);
+ }
+ }
+
+ if (parent != null)
+ {
+ graph.Assert(resourceNode, graph.CreateUriNode(Schema.Predicates.CatalogParent), graph.CreateUriNode(parent));
+ }
+
+ if (extra != null)
+ {
+ graph.Merge(extra, true);
+ }
+
+ if (additionalResourceTypes != null)
+ {
+ foreach (Uri resourceType in additionalResourceTypes)
+ {
+ graph.Assert(resourceNode, typePredicate, graph.CreateUriNode(resourceType));
+ }
+ }
+
+ await SaveGraph(resourceUri, graph, typeUri, cancellationToken);
+ }
+
+ protected async Task> LoadIndexResource(Uri resourceUri, CancellationToken cancellationToken)
+ {
+ IDictionary entries = new Dictionary();
+
+ IGraph graph = await LoadGraph(resourceUri, cancellationToken);
+
+ if (graph == null)
+ {
+ return entries;
+ }
+
+ INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type);
+ INode itemPredicate = graph.CreateUriNode(Schema.Predicates.CatalogItem);
+ INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp);
+ INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId);
+ INode countPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCount);
+
+ CheckScheme(resourceUri, graph);
+
+ INode resourceNode = graph.CreateUriNode(resourceUri);
+
+ foreach (IUriNode itemNode in graph.GetTriplesWithSubjectPredicate(resourceNode, itemPredicate).Select((t) => t.Object))
+ {
+ Triple typeTriple = graph.GetTriplesWithSubjectPredicate(itemNode, typePredicate).First();
+ Uri type = ((IUriNode)typeTriple.Object).Uri;
+
+ Triple commitIdTriple = graph.GetTriplesWithSubjectPredicate(itemNode, commitIdPredicate).First();
+ Guid commitId = Guid.Parse(((ILiteralNode)commitIdTriple.Object).Value);
+
+ Triple timeStampTriple = graph.GetTriplesWithSubjectPredicate(itemNode, timeStampPredicate).First();
+ DateTime timeStamp = DateTime.Parse(((ILiteralNode)timeStampTriple.Object).Value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
+
+ Triple countTriple = graph.GetTriplesWithSubjectPredicate(itemNode, countPredicate).FirstOrDefault();
+ int? count = (countTriple != null) ? int.Parse(((ILiteralNode)countTriple.Object).Value) : (int?)null;
+
+ IGraph itemContent = null;
+ INode itemContentSubjectNode = null;
+ foreach (Triple itemContentTriple in graph.GetTriplesWithSubject(itemNode))
+ {
+ if (itemContentTriple.Predicate.Equals(typePredicate))
+ {
+ continue;
+ }
+ if (itemContentTriple.Predicate.Equals(timeStampPredicate))
+ {
+ continue;
+ }
+ if (itemContentTriple.Predicate.Equals(commitIdPredicate))
+ {
+ continue;
+ }
+ if (itemContentTriple.Predicate.Equals(countPredicate))
+ {
+ continue;
+ }
+ if (itemContentTriple.Predicate.Equals(itemPredicate))
+ {
+ continue;
+ }
+
+ if (itemContent == null)
+ {
+ itemContent = new Graph();
+ itemContentSubjectNode = itemContentTriple.Subject.CopyNode(itemContent, false);
+ }
+
+ INode itemContentPredicateNode = itemContentTriple.Predicate.CopyNode(itemContent, false);
+ INode itemContentObjectNode = itemContentTriple.Object.CopyNode(itemContent, false);
+
+ itemContent.Assert(itemContentSubjectNode, itemContentPredicateNode, itemContentObjectNode);
+
+ if (itemContentTriple.Object is IUriNode)
+ {
+ Utils.CopyCatalogContentGraph(itemContentTriple.Object, graph, itemContent);
+ }
+ }
+
+ entries.Add(itemNode.Uri.AbsoluteUri, new CatalogItemSummary(type, commitId, timeStamp, count, itemContent));
+ }
+
+ return entries;
+ }
+
+ private async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken)
+ {
+ if (GraphPersistence != null)
+ {
+ await GraphPersistence.SaveGraph(resourceUri, graph, typeUri, cancellationToken);
+ }
+ else
+ {
+ await Storage.SaveAsync(resourceUri, CreateIndexContent(graph, typeUri), cancellationToken);
+ }
+ }
+
+ private async Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken)
+ {
+ if (GraphPersistence != null)
+ {
+ return await GraphPersistence.LoadGraph(resourceUri, cancellationToken);
+ }
+ else
+ {
+ return Utils.CreateGraph(resourceUri, await Storage.LoadStringAsync(resourceUri, cancellationToken));
+ }
+ }
+
+ protected Uri CreatePageUri(Uri baseAddress, string relativeAddress)
+ {
+ if (GraphPersistence != null)
+ {
+ return GraphPersistence.CreatePageUri(baseAddress, relativeAddress);
+ }
+ else
+ {
+ return new Uri(baseAddress, relativeAddress + ".json");
+ }
+ }
+
+ private void CheckScheme(Uri resourceUri, IGraph graph)
+ {
+ INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type);
+
+ Triple catalogRoot = graph.GetTriplesWithPredicateObject(typePredicate, graph.CreateUriNode(Schema.DataTypes.CatalogRoot)).FirstOrDefault();
+ if (catalogRoot != null)
+ {
+ if (((UriNode)catalogRoot.Subject).Uri.Scheme != resourceUri.Scheme)
+ {
+ throw new ArgumentException("the resource scheme does not match the existing catalog");
+ }
+ }
+ Triple catalogPage = graph.GetTriplesWithPredicateObject(typePredicate, graph.CreateUriNode(Schema.DataTypes.CatalogPage)).FirstOrDefault();
+ if (catalogPage != null)
+ {
+ if (((UriNode)catalogPage.Subject).Uri.Scheme != resourceUri.Scheme)
+ {
+ throw new ArgumentException("the resource scheme does not match the existing catalog");
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CloudBlobStorageExtensions.cs b/src/Catalog/CloudBlobStorageExtensions.cs
new file mode 100644
index 000000000..f40a891d3
--- /dev/null
+++ b/src/Catalog/CloudBlobStorageExtensions.cs
@@ -0,0 +1,37 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ internal static class CloudBlobStorageExtensions
+ {
+ public static async Task> ListBlobsAsync(
+ this CloudBlobDirectory directory, CancellationToken cancellationToken)
+ {
+ var items = new List();
+ BlobContinuationToken continuationToken = null;
+ do
+ {
+ var segment = await directory.ListBlobsSegmentedAsync(
+ useFlatBlobListing: true,
+ blobListingDetails: BlobListingDetails.None,
+ maxResults: null,
+ currentToken: continuationToken,
+ options: null,
+ operationContext: null,
+ cancellationToken: cancellationToken);
+
+ continuationToken = segment.ContinuationToken;
+ items.AddRange(segment.Results);
+ }
+ while (continuationToken != null && !cancellationToken.IsCancellationRequested);
+
+ return items;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CollectorBase.cs b/src/Catalog/CollectorBase.cs
new file mode 100644
index 000000000..952431a81
--- /dev/null
+++ b/src/Catalog/CollectorBase.cs
@@ -0,0 +1,79 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public abstract class CollectorBase
+ {
+ protected readonly ITelemetryService _telemetryService;
+ private readonly Func _handlerFunc;
+ private readonly IHttpRetryStrategy _httpRetryStrategy;
+ private readonly TimeSpan? _httpClientTimeout;
+
+ public CollectorBase(
+ Uri index,
+ ITelemetryService telemetryService,
+ Func handlerFunc = null,
+ TimeSpan? httpClientTimeout = null,
+ IHttpRetryStrategy httpRetryStrategy = null)
+ {
+ _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+ _handlerFunc = handlerFunc;
+ _httpClientTimeout = httpClientTimeout;
+ _httpRetryStrategy = httpRetryStrategy;
+ Index = index ?? throw new ArgumentNullException(nameof(index));
+ }
+
+ public Uri Index { get; }
+
+ public async Task RunAsync(CancellationToken cancellationToken)
+ {
+ return await RunAsync(MemoryCursor.CreateMin(), MemoryCursor.CreateMax(), cancellationToken);
+ }
+
+ public async Task RunAsync(DateTime front, DateTime back, CancellationToken cancellationToken)
+ {
+ return await RunAsync(new MemoryCursor(front), new MemoryCursor(back), cancellationToken);
+ }
+
+ public async Task RunAsync(ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
+ {
+ await Task.WhenAll(front.LoadAsync(cancellationToken), back.LoadAsync(cancellationToken));
+
+ Trace.TraceInformation("Run ( {0} , {1} )", front, back);
+
+ bool result = false;
+
+ HttpMessageHandler handler = null;
+
+ if (_handlerFunc != null)
+ {
+ handler = _handlerFunc();
+ }
+
+ using (CollectorHttpClient client = new CollectorHttpClient(handler, _httpRetryStrategy))
+ {
+ if (_httpClientTimeout.HasValue)
+ {
+ client.Timeout = _httpClientTimeout.Value;
+ }
+
+ result = await FetchAsync(client, front, back, cancellationToken);
+ }
+
+ return result;
+ }
+
+ protected abstract Task FetchAsync(
+ CollectorHttpClient client,
+ ReadWriteCursor front,
+ ReadCursor back,
+ CancellationToken cancellationToken);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CollectorHttpClient.cs b/src/Catalog/CollectorHttpClient.cs
new file mode 100644
index 000000000..6305a333d
--- /dev/null
+++ b/src/Catalog/CollectorHttpClient.cs
@@ -0,0 +1,110 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CollectorHttpClient : HttpClient
+ {
+ private int _requestCount;
+ private readonly IHttpRetryStrategy _retryStrategy;
+
+ public CollectorHttpClient()
+ : this(new WebRequestHandler { AllowPipelining = true })
+ {
+ }
+
+ public CollectorHttpClient(HttpMessageHandler handler, IHttpRetryStrategy retryStrategy = null)
+ : base(handler ?? new WebRequestHandler { AllowPipelining = true })
+ {
+ _requestCount = 0;
+ _retryStrategy = retryStrategy ?? new RetryWithExponentialBackoff();
+ }
+
+ public int RequestCount
+ {
+ get { return _requestCount; }
+ }
+
+ protected void InReqCount()
+ {
+ Interlocked.Increment(ref _requestCount);
+ }
+
+ public virtual Task GetJObjectAsync(Uri address)
+ {
+ return GetJObjectAsync(address, CancellationToken.None);
+ }
+
+ public virtual async Task GetJObjectAsync(Uri address, CancellationToken token)
+ {
+ InReqCount();
+
+ var json = await GetStringAsync(address, token);
+
+ try
+ {
+ return ParseJObject(json);
+ }
+ catch (Exception e)
+ {
+ throw new Exception($"{nameof(GetJObjectAsync)}({address})", e);
+ }
+ }
+
+ private static JObject ParseJObject(string json)
+ {
+ using (var reader = new JsonTextReader(new StringReader(json)))
+ {
+ reader.DateParseHandling = DateParseHandling.DateTimeOffset; // make sure we always preserve timezone info
+
+ return JObject.Load(reader);
+ }
+ }
+
+ public virtual Task GetGraphAsync(Uri address)
+ {
+ return GetGraphAsync(address, readOnly: false, token: CancellationToken.None);
+ }
+
+ public virtual Task GetGraphAsync(Uri address, bool readOnly, CancellationToken token)
+ {
+ var task = GetJObjectAsync(address, token);
+
+ return task.ContinueWith((t) =>
+ {
+ try
+ {
+ return Utils.CreateGraph(t.Result, readOnly);
+ }
+ catch (Exception e)
+ {
+ throw new Exception($"{nameof(GetGraphAsync)}({address})", e);
+ }
+ }, token);
+ }
+
+ public virtual async Task GetStringAsync(Uri address, CancellationToken token)
+ {
+ try
+ {
+ using (var httpResponse = await _retryStrategy.SendAsync(this, address, token))
+ {
+ return await httpResponse.Content.ReadAsStringAsync();
+ }
+ }
+ catch (Exception e)
+ {
+ throw new Exception($"{nameof(GetStringAsync)}({address})", e);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CommitCollector.cs b/src/Catalog/CommitCollector.cs
new file mode 100644
index 000000000..a120bbe45
--- /dev/null
+++ b/src/Catalog/CommitCollector.cs
@@ -0,0 +1,199 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public abstract class CommitCollector : CollectorBase
+ {
+ public CommitCollector(
+ Uri index,
+ ITelemetryService telemetryService,
+ Func handlerFunc = null,
+ TimeSpan? httpClientTimeout = null,
+ IHttpRetryStrategy httpRetryStrategy = null)
+ : base(index, telemetryService, handlerFunc, httpClientTimeout, httpRetryStrategy)
+ {
+ }
+
+ protected override async Task FetchAsync(
+ CollectorHttpClient client,
+ ReadWriteCursor front,
+ ReadCursor back,
+ CancellationToken cancellationToken)
+ {
+ var commits = await FetchCatalogCommitsAsync(client, front, back, cancellationToken);
+
+ bool acceptNextBatch = false;
+
+ foreach (CatalogCommit commit in commits)
+ {
+ JObject page = await client.GetJObjectAsync(commit.Uri, cancellationToken);
+
+ JToken context = null;
+ page.TryGetValue("@context", out context);
+
+ var batches = await CreateBatchesAsync(page["items"]
+ .Select(item => CatalogCommitItem.Create((JObject)context, (JObject)item))
+ .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
+
+ var orderedBatches = batches
+ .OrderBy(batch => batch.CommitTimeStamp)
+ .ToList();
+
+ var lastBatch = orderedBatches.LastOrDefault();
+ DateTime? previousCommitTimeStamp = null;
+
+ foreach (var batch in orderedBatches)
+ {
+ // If the commit timestamp has changed from the previous batch, commit. This is important because if
+ // two batches have the same commit timestamp but processing the second fails, we should not
+ // progress the cursor forward.
+ if (previousCommitTimeStamp.HasValue && previousCommitTimeStamp != batch.CommitTimeStamp)
+ {
+ front.Value = previousCommitTimeStamp.Value;
+ await front.SaveAsync(cancellationToken);
+ Trace.TraceInformation("CommitCatalog.Fetch front.Value saved since timestamp changed from previous: {0}", front);
+ }
+
+ using (_telemetryService.TrackDuration(TelemetryConstants.ProcessBatchSeconds, new Dictionary()
+ {
+ { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
+ }))
+ {
+ acceptNextBatch = await OnProcessBatchAsync(
+ client,
+ batch.Items,
+ context,
+ batch.CommitTimeStamp,
+ batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
+ cancellationToken);
+ }
+
+ // If this is the last batch, commit the cursor.
+ if (ReferenceEquals(batch, lastBatch))
+ {
+ front.Value = batch.CommitTimeStamp;
+ await front.SaveAsync(cancellationToken);
+ Trace.TraceInformation("CommitCatalog.Fetch front.Value saved due to last batch: {0}", front);
+ }
+
+ previousCommitTimeStamp = batch.CommitTimeStamp;
+
+ Trace.TraceInformation("CommitCatalog.Fetch front.Value is: {0}", front);
+
+ if (!acceptNextBatch)
+ {
+ break;
+ }
+ }
+
+ if (!acceptNextBatch)
+ {
+ break;
+ }
+ }
+
+ return acceptNextBatch;
+ }
+
+ protected async Task> FetchCatalogCommitsAsync(
+ CollectorHttpClient client,
+ ReadCursor front,
+ ReadCursor back,
+ CancellationToken cancellationToken)
+ {
+ JObject root;
+
+ using (_telemetryService.TrackDuration(
+ TelemetryConstants.CatalogIndexReadDurationSeconds,
+ new Dictionary() { { TelemetryConstants.Uri, Index.AbsoluteUri } }))
+ {
+ root = await client.GetJObjectAsync(Index, cancellationToken);
+ }
+
+ var commits = root["items"].Select(item => CatalogCommit.Create((JObject)item));
+ return GetCommitsInRange(commits, front.Value, back.Value);
+ }
+
+
+ public static IEnumerable GetCommitsInRange(
+ IEnumerable commits,
+ DateTimeOffset minCommitTimestamp,
+ DateTimeOffset maxCommitTimestamp)
+ {
+ // Only consider pages that have a (latest) commit timestamp greater than the minimum bound. If a page has
+ // a commit timestamp greater than the minimum bound, then there is at least one item with a commit
+ // timestamp greater than the minimum bound. Sort the pages by commit timestamp so that they are
+ // in chronological order.
+ var upperRange = commits
+ .Where(x => x.CommitTimeStamp > minCommitTimestamp)
+ .OrderBy(x => x.CommitTimeStamp);
+
+ // Take pages from the sorted list until the (latest) commit timestamp goes past the maximum commit
+ // timestamp. This essentially LINQ's TakeWhile plus one more element. Because the maximum bound is
+ // inclusive, we need to yield any page that has a (latest) commit timestamp that is less than or
+ // equal to the maximum bound.
+ //
+ // Consider the following pages (bounded by square brackets, labeled P-0 ... P-N) containing commits
+ // (C-0 ... C-N). The front cursor (exclusive minimum bound) is marked by the letter "F" and the back
+ // cursor (inclusive upper bound) is marked by the letter "B".
+ //
+ // ---- P-0 ---- ---- P-1 ---- ---- P-2 ---- ----- P-3 -----
+ // [ C-0, C-1, C-2 ] [ C-3, C-4, C-5 ] [ C-6, C-7, C-8 ] [ C-9, C-10, C-11 ]
+ // | | | | | |
+ // Scenario #1: F | | | B |
+ // | | | |
+ // P-0, P-1, and P-2 should be downloaded and C-2 to C-7 should be processed. Note that P-3 should not
+ // even be considered because P-2 is the first page with a maximum commit timestamp greater than "B".
+ // | | | |
+ // Scenario #2: | F | B
+ // | |
+ // P-1 and P-2 should be downloaded and C-4 to C-8 should be processed. The concept of a timestamp-based
+ // cursor requires that commit timestamps strictly increase. Additionally, our catalog implementation
+ // never allows a commit to be split across multiple pages. In other words, if C-8 is at the end of P-2
+ // the consumer of the catalog can assume that P-3 only has commits later than C-8 and therefore P-3 need
+ // not be considered. |
+ // | |
+ // Scenario #3: F B
+ //
+ // P-1 and P-2 should be downloaded and C-3 to C-6 should be processed. Because "F" (an exclusive bound)
+ // is pointing to the latest commit timestamp in P-0, that page can be completely ignored.
+ foreach (var page in upperRange)
+ {
+ yield return page;
+
+ if (page.CommitTimeStamp >= maxCommitTimestamp)
+ {
+ break;
+ }
+ }
+ }
+
+ protected virtual Task> CreateBatchesAsync(IEnumerable catalogItems)
+ {
+ var batches = catalogItems
+ .GroupBy(item => item.CommitTimeStamp)
+ .OrderBy(group => group.Key)
+ .Select(group => new CatalogCommitItemBatch(group));
+
+ return Task.FromResult(batches);
+ }
+
+ protected abstract Task OnProcessBatchAsync(
+ CollectorHttpClient client,
+ IEnumerable items,
+ JToken context,
+ DateTime commitTimeStamp,
+ bool isLastBatch,
+ CancellationToken cancellationToken);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CommitMetadata.cs b/src/Catalog/CommitMetadata.cs
new file mode 100644
index 000000000..687827dfe
--- /dev/null
+++ b/src/Catalog/CommitMetadata.cs
@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class CommitMetadata
+ {
+ public CommitMetadata()
+ {
+ }
+
+ public CommitMetadata(DateTime? lastCreated, DateTime? lastEdited, DateTime? lastDeleted)
+ {
+ LastCreated = lastCreated;
+ LastEdited = lastEdited;
+ LastDeleted = lastDeleted;
+ }
+
+ public DateTime? LastCreated { get; set; }
+ public DateTime? LastEdited { get; set; }
+ public DateTime? LastDeleted { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Constants.cs b/src/Catalog/Constants.cs
new file mode 100644
index 000000000..72e42dace
--- /dev/null
+++ b/src/Catalog/Constants.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class Constants
+ {
+ public static readonly DateTime DateTimeMinValueUtc = DateTimeOffset.MinValue.UtcDateTime;
+ public const int MaxPageSize = 550;
+ public const string Sha512 = "SHA512";
+ public static readonly DateTime UnpublishedDate = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/CreateCommitItemBatchesAsync.cs b/src/Catalog/CreateCommitItemBatchesAsync.cs
new file mode 100644
index 000000000..d95efee77
--- /dev/null
+++ b/src/Catalog/CreateCommitItemBatchesAsync.cs
@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ internal delegate Task> CreateCommitItemBatchesAsync(
+ IEnumerable catalogItems);
+}
\ No newline at end of file
diff --git a/src/Catalog/DeleteCatalogItem.cs b/src/Catalog/DeleteCatalogItem.cs
new file mode 100644
index 000000000..ca7c2cc7c
--- /dev/null
+++ b/src/Catalog/DeleteCatalogItem.cs
@@ -0,0 +1,118 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using NuGet.Versioning;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class DeleteCatalogItem : AppendOnlyCatalogItem
+ {
+ private string _id;
+ private string _version;
+ private readonly DateTime _published;
+
+ public DeleteCatalogItem(string id, string version, DateTime published)
+ {
+ _id = id;
+ _version = TryNormalize(version);
+ _published = published;
+ }
+
+ private string TryNormalize(string version)
+ {
+ SemanticVersion semVer;
+ if (SemanticVersion.TryParse(version, out semVer))
+ {
+ return semVer.ToNormalizedString();
+ }
+ return version;
+ }
+
+ public override Uri GetItemType()
+ {
+ return Schema.DataTypes.PackageDelete;
+ }
+
+ protected override string GetItemIdentity()
+ {
+ return (_id + "." + _version).ToLowerInvariant();
+ }
+
+ public override StorageContent CreateContent(CatalogContext context)
+ {
+ using (IGraph graph = new Graph())
+ {
+ INode entry = graph.CreateUriNode(GetItemAddress());
+
+ // catalog infrastructure fields
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(GetItemType()));
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.Permalink));
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp), graph.CreateLiteralNode(TimeStamp.ToString("O"), Schema.DataTypes.DateTime));
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.CatalogCommitId), graph.CreateLiteralNode(CommitId.ToString()));
+
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Published), graph.CreateLiteralNode(_published.ToString("O"), Schema.DataTypes.DateTime));
+
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Id), graph.CreateLiteralNode(_id));
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.OriginalId), graph.CreateLiteralNode(_id));
+ graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Version), graph.CreateLiteralNode(_version));
+
+ SetIdVersionFromGraph(graph);
+
+ // create JSON content
+ JObject frame = context.GetJsonLdContext("context.Catalog.json", GetItemType());
+ StorageContent content = new StringStorageContent(Utils.CreateArrangedJson(graph, frame), "application/json", "no-store");
+
+ return content;
+ }
+ }
+
+ public override IGraph CreatePageContent(CatalogContext context)
+ {
+ var resourceUri = new Uri(GetBaseAddress() + GetRelativeAddress());
+
+ var graph = new Graph();
+
+ var subject = graph.CreateUriNode(resourceUri);
+
+ var idPredicate = graph.CreateUriNode(Schema.Predicates.Id);
+ var versionPredicate = graph.CreateUriNode(Schema.Predicates.Version);
+
+ if (_id != null)
+ {
+ graph.Assert(subject, idPredicate, graph.CreateLiteralNode(_id));
+ }
+
+ if (_version != null)
+ {
+ graph.Assert(subject, versionPredicate, graph.CreateLiteralNode(_version));
+ }
+
+ return graph;
+ }
+
+ private void SetIdVersionFromGraph(IGraph graph)
+ {
+ var resource = graph.GetTriplesWithPredicateObject(
+ graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(GetItemType())).First();
+
+ var id = graph.GetTriplesWithSubjectPredicate(
+ resource.Subject, graph.CreateUriNode(Schema.Predicates.Id)).FirstOrDefault();
+ if (id != null)
+ {
+ _id = ((ILiteralNode)id.Object).Value;
+ }
+
+ var version = graph.GetTriplesWithSubjectPredicate(
+ resource.Subject, graph.CreateUriNode(Schema.Predicates.Version)).FirstOrDefault();
+ if (version != null)
+ {
+ _version = ((ILiteralNode)version.Object).Value;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs
new file mode 100644
index 000000000..2f06bf2e1
--- /dev/null
+++ b/src/Catalog/Dnx/DnxCatalogCollector.cs
@@ -0,0 +1,564 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using NuGet.Protocol.Catalog;
+using NuGet.Services.Metadata.Catalog.Helpers;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+ public class DnxCatalogCollector : CommitCollector
+ {
+ private readonly StorageFactory _storageFactory;
+ private readonly IAzureStorage _sourceStorage;
+ private readonly DnxMaker _dnxMaker;
+ private readonly Func _catalogClientFactory;
+ private readonly ILogger _logger;
+ private readonly int _maxConcurrentBatches;
+ private readonly int _maxConcurrentCommitItemsWithinBatch;
+ private readonly Uri _contentBaseAddress;
+
+ public DnxCatalogCollector(
+ Uri index,
+ StorageFactory storageFactory,
+ IAzureStorage preferredPackageSourceStorage,
+ Uri contentBaseAddress,
+ ITelemetryService telemetryService,
+ ILogger logger,
+ int maxDegreeOfParallelism,
+ Func catalogClientFactory,
+ Func handlerFunc = null,
+ TimeSpan? httpClientTimeout = null)
+ : base(index, telemetryService, handlerFunc, httpClientTimeout)
+ {
+ _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
+ _sourceStorage = preferredPackageSourceStorage;
+ _contentBaseAddress = contentBaseAddress;
+ _dnxMaker = new DnxMaker(storageFactory, telemetryService, logger);
+ _catalogClientFactory = catalogClientFactory ?? throw new ArgumentNullException(nameof(catalogClientFactory));
+ _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+ if (maxDegreeOfParallelism < 1)
+ {
+ throw new ArgumentOutOfRangeException(
+ nameof(maxDegreeOfParallelism),
+ string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
+ }
+
+ // Find two factors which are close or equal to each other.
+ var squareRoot = Math.Sqrt(maxDegreeOfParallelism);
+
+ // If the max degree of parallelism is a perfect square, great.
+ // Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch.
+ _maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot));
+ _maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches);
+
+ ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch;
+ }
+
+ protected override Task> CreateBatchesAsync(
+ IEnumerable catalogItems)
+ {
+ var batches = CatalogCommitUtilities.CreateCommitItemBatches(
+ catalogItems,
+ CatalogCommitUtilities.GetPackageIdKey);
+
+ return Task.FromResult(batches);
+ }
+
+ protected override Task FetchAsync(
+ CollectorHttpClient client,
+ ReadWriteCursor front,
+ ReadCursor back,
+ CancellationToken cancellationToken)
+ {
+ return CatalogCommitUtilities.ProcessCatalogCommitsAsync(
+ client,
+ front,
+ back,
+ FetchCatalogCommitsAsync,
+ CreateBatchesAsync,
+ ProcessBatchAsync,
+ _maxConcurrentBatches,
+ _logger,
+ cancellationToken);
+ }
+
+ protected override async Task OnProcessBatchAsync(
+ CollectorHttpClient client,
+ IEnumerable items,
+ JToken context,
+ DateTime commitTimeStamp,
+ bool isLastBatch,
+ CancellationToken cancellationToken)
+ {
+ var catalogEntries = items.Select(item => CatalogEntry.Create(item))
+ .ToList();
+
+ // Sanity check: a single catalog batch should not contain multiple entries for the same package identity.
+ AssertNoMultipleEntriesForSamePackageIdentity(commitTimeStamp, catalogEntries);
+
+ // Process .nupkg/.nuspec adds and deletes.
+ var processedCatalogEntries = await ProcessCatalogEntriesAsync(client, catalogEntries, cancellationToken);
+
+ // Update the package version index with adds and deletes.
+ await UpdatePackageVersionIndexAsync(processedCatalogEntries, cancellationToken);
+
+ return true;
+ }
+
+ private async Task> ProcessCatalogEntriesAsync(
+ CollectorHttpClient client,
+ IEnumerable catalogEntries,
+ CancellationToken cancellationToken)
+ {
+ var processedCatalogEntries = new ConcurrentBag();
+
+ await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntry =>
+ {
+ var packageId = catalogEntry.PackageId;
+ var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion;
+
+ if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri)
+ {
+ var telemetryProperties = GetTelemetryProperties(catalogEntry);
+
+ using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, telemetryProperties))
+ {
+ var packageFileName = PackageUtility.GetPackageFileName(
+ packageId,
+ normalizedPackageVersion);
+ var sourceUri = new Uri(_contentBaseAddress, packageFileName);
+ var destinationStorage = _storageFactory.Create(packageId);
+ var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg(
+ packageId,
+ normalizedPackageVersion);
+ var destinationUri = destinationStorage.GetUri(destinationRelativeUri);
+
+ var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri);
+ var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync(
+ destinationStorage,
+ packageId,
+ normalizedPackageVersion,
+ cancellationToken);
+ var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri);
+
+ if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent)
+ {
+ _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion);
+
+ return;
+ }
+
+ if ((isNupkgSynchronized && areRequiredPropertiesPresent)
+ || await ProcessPackageDetailsAsync(
+ client,
+ packageId,
+ normalizedPackageVersion,
+ sourceUri,
+ catalogEntry.Uri,
+ telemetryProperties,
+ cancellationToken))
+ {
+ processedCatalogEntries.Add(catalogEntry);
+ }
+ }
+ }
+ else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri)
+ {
+ var properties = GetTelemetryProperties(catalogEntry);
+
+ using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties))
+ {
+ await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken);
+
+ processedCatalogEntries.Add(catalogEntry);
+ }
+ }
+ });
+
+ return processedCatalogEntries;
+ }
+
+ private async Task AreRequiredPropertiesPresentAsync(Storage destinationStorage, Uri destinationUri)
+ {
+ var azureStorage = destinationStorage as IAzureStorage;
+
+ if (azureStorage == null)
+ {
+ return true;
+ }
+
+ return await azureStorage.HasPropertiesAsync(
+ destinationUri,
+ DnxConstants.ApplicationOctetStreamContentType,
+ DnxConstants.DefaultCacheControl);
+ }
+
+ private async Task UpdatePackageVersionIndexAsync(
+ IEnumerable catalogEntries,
+ CancellationToken cancellationToken)
+ {
+ var catalogEntryGroups = catalogEntries.GroupBy(catalogEntry => catalogEntry.PackageId);
+
+ await catalogEntryGroups.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntryGroup =>
+ {
+ var packageId = catalogEntryGroup.Key;
+ var properties = new Dictionary()
+ {
+ { TelemetryConstants.Id, packageId },
+ { TelemetryConstants.BatchItemCount, catalogEntryGroup.Count().ToString() }
+ };
+
+ using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageVersionIndexSeconds, properties))
+ {
+ await _dnxMaker.UpdatePackageVersionIndexAsync(packageId, versions =>
+ {
+ foreach (var catalogEntry in catalogEntryGroup)
+ {
+ if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri)
+ {
+ versions.Add(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion));
+ }
+ else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri)
+ {
+ versions.Remove(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion));
+ }
+ }
+ }, cancellationToken);
+ }
+
+ foreach (var catalogEntry in catalogEntryGroup)
+ {
+ _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.NormalizedPackageVersion);
+ }
+ });
+ }
+
+ private async Task ProcessPackageDetailsAsync(
+ HttpClient client,
+ string packageId,
+ string normalizedPackageVersion,
+ Uri sourceUri,
+ Uri catalogLeafUri,
+ Dictionary telemetryProperties,
+ CancellationToken cancellationToken)
+ {
+ var catalogClient = _catalogClientFactory(client);
+ var catalogLeaf = await catalogClient.GetPackageDetailsLeafAsync(catalogLeafUri.AbsoluteUri);
+
+ if (await ProcessPackageDetailsViaStorageAsync(
+ packageId,
+ normalizedPackageVersion,
+ catalogLeaf,
+ telemetryProperties,
+ cancellationToken))
+ {
+ return true;
+ }
+
+ _telemetryService.TrackMetric(
+ TelemetryConstants.UsePackageSourceFallback,
+ metric: 1,
+ properties: GetTelemetryProperties(packageId, normalizedPackageVersion));
+
+ return await ProcessPackageDetailsViaHttpAsync(
+ client,
+ packageId,
+ normalizedPackageVersion,
+ sourceUri,
+ catalogLeaf,
+ telemetryProperties,
+ cancellationToken);
+ }
+
+ private async Task ProcessPackageDetailsViaStorageAsync(
+ string packageId,
+ string normalizedPackageVersion,
+ PackageDetailsCatalogLeaf catalogLeaf,
+ Dictionary telemetryProperties,
+ CancellationToken cancellationToken)
+ {
+ if (_sourceStorage == null)
+ {
+ return false;
+ }
+
+ var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion);
+ var sourceUri = _sourceStorage.ResolveUri(packageFileName);
+
+ var sourceBlob = await _sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri);
+
+ if (await sourceBlob.ExistsAsync(cancellationToken))
+ {
+ // It's possible (though unlikely) that the blob may change between reads. Reading a blob with a
+ // single GET request returns the whole blob in a consistent state, but we're reading the blob many
+ // different times. To detect the blob changing between reads, we check the ETag again later.
+ // If the ETag's differ, we'll fall back to using a single HTTP GET request.
+ var token1 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken);
+
+ telemetryProperties[TelemetryConstants.SizeInBytes] = sourceBlob.Length.ToString();
+
+ var nuspec = await GetNuspecAsync(sourceBlob, packageId, cancellationToken);
+
+ if (string.IsNullOrEmpty(nuspec))
+ {
+ _logger.LogWarning(
+ "No .nuspec available for {Id}/{Version}. Falling back to HTTP processing.",
+ packageId,
+ normalizedPackageVersion);
+ }
+ else
+ {
+ await _dnxMaker.AddPackageAsync(
+ _sourceStorage,
+ nuspec,
+ packageId,
+ normalizedPackageVersion,
+ catalogLeaf.IconFile,
+ cancellationToken);
+
+ var token2 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken);
+
+ if (token1 == token2)
+ {
+ _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", packageId, normalizedPackageVersion);
+
+ return true;
+ }
+ else
+ {
+ _telemetryService.TrackMetric(
+ TelemetryConstants.BlobModified,
+ metric: 1,
+ properties: GetTelemetryProperties(packageId, normalizedPackageVersion));
+ }
+ }
+ }
+ else
+ {
+ _telemetryService.TrackMetric(
+ TelemetryConstants.NonExistentBlob,
+ metric: 1,
+ properties: GetTelemetryProperties(packageId, normalizedPackageVersion));
+ }
+
+ return false;
+ }
+
+ private async Task ProcessPackageDetailsViaHttpAsync(
+ HttpClient client,
+ string id,
+ string version,
+ Uri sourceUri,
+ PackageDetailsCatalogLeaf catalogLeaf,
+ Dictionary telemetryProperties,
+ CancellationToken cancellationToken)
+ {
+ var packageDownloader = new PackageDownloader(client, _logger);
+ var requestUri = Utilities.GetNugetCacheBustingUri(sourceUri);
+
+ using (var stream = await packageDownloader.DownloadAsync(requestUri, cancellationToken))
+ {
+ if (stream == null)
+ {
+ _logger.LogWarning("Package {Id}/{Version} not found.", id, version);
+
+ return false;
+ }
+
+ telemetryProperties[TelemetryConstants.SizeInBytes] = stream.Length.ToString();
+
+ var nuspec = GetNuspec(stream, id);
+
+ if (nuspec == null)
+ {
+ _logger.LogWarning("No .nuspec available for {Id}/{Version}. Skipping.", id, version);
+
+ return false;
+ }
+
+ stream.Position = 0;
+
+ await _dnxMaker.AddPackageAsync(
+ stream,
+ nuspec,
+ id,
+ version,
+ catalogLeaf.IconFile,
+ cancellationToken);
+ }
+
+ _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", id, version);
+
+ return true;
+ }
+
+ private async Task ProcessPackageDeleteAsync(
+ string packageId,
+ string normalizedPackageVersion,
+ CancellationToken cancellationToken)
+ {
+ await _dnxMaker.UpdatePackageVersionIndexAsync(
+ packageId,
+ versions => versions.Remove(NuGetVersion.Parse(normalizedPackageVersion)),
+ cancellationToken);
+
+ await _dnxMaker.DeletePackageAsync(packageId, normalizedPackageVersion, cancellationToken);
+
+ _logger.LogInformation("Commit delete: {Id}/{Version}", packageId, normalizedPackageVersion);
+ }
+
+ private static void AssertNoMultipleEntriesForSamePackageIdentity(
+ DateTime commitTimeStamp,
+ IEnumerable catalogEntries)
+ {
+ var catalogEntriesForSamePackageIdentity = catalogEntries.GroupBy(
+ catalogEntry => new
+ {
+ catalogEntry.PackageId,
+ catalogEntry.NormalizedPackageVersion
+ })
+ .Where(group => group.Count() > 1)
+ .Select(group => $"{group.Key.PackageId} {group.Key.NormalizedPackageVersion}");
+
+ if (catalogEntriesForSamePackageIdentity.Any())
+ {
+ var packageIdentities = string.Join(", ", catalogEntriesForSamePackageIdentity);
+
+ throw new InvalidOperationException($"The catalog batch {commitTimeStamp} contains multiple entries for the same package identity. Package(s): {packageIdentities}");
+ }
+ }
+
+ private async Task GetNuspecAsync(
+ ICloudBlockBlob sourceBlob,
+ string packageId,
+ CancellationToken cancellationToken)
+ {
+ using (var stream = await sourceBlob.GetStreamAsync(cancellationToken))
+ {
+ return GetNuspec(stream, packageId);
+ }
+ }
+
+ private static string GetNuspec(Stream stream, string id)
+ {
+ string name = $"{id}.nuspec";
+
+ using (var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true))
+ {
+ // first look for a nuspec file named as the package id
+ foreach (ZipArchiveEntry entry in archive.Entries)
+ {
+ if (entry.FullName.Equals(name, StringComparison.InvariantCultureIgnoreCase))
+ {
+ using (TextReader reader = new StreamReader(entry.Open()))
+ {
+ return reader.ReadToEnd();
+ }
+ }
+ }
+ // failing that, just return the first file that appears to be a nuspec
+ foreach (ZipArchiveEntry entry in archive.Entries)
+ {
+ if (entry.FullName.EndsWith(".nuspec", StringComparison.InvariantCultureIgnoreCase))
+ {
+ using (TextReader reader = new StreamReader(entry.Open()))
+ {
+ return reader.ReadToEnd();
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ private static Dictionary GetTelemetryProperties(CatalogEntry catalogEntry)
+ {
+ return GetTelemetryProperties(catalogEntry.PackageId, catalogEntry.NormalizedPackageVersion);
+ }
+
+ private static Dictionary GetTelemetryProperties(string packageId, string normalizedPackageVersion)
+ {
+ return new Dictionary()
+ {
+ { TelemetryConstants.Id, packageId },
+ { TelemetryConstants.Version, normalizedPackageVersion }
+ };
+ }
+
+ private async Task ProcessBatchAsync(
+ CollectorHttpClient client,
+ JToken context,
+ string packageId,
+ CatalogCommitItemBatch batch,
+ CatalogCommitItemBatch lastBatch,
+ CancellationToken cancellationToken)
+ {
+ await Task.Yield();
+
+ using (_telemetryService.TrackDuration(
+ TelemetryConstants.ProcessBatchSeconds,
+ new Dictionary()
+ {
+ { TelemetryConstants.Id, packageId },
+ { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
+ }))
+ {
+ await OnProcessBatchAsync(
+ client,
+ batch.Items,
+ context,
+ batch.CommitTimeStamp,
+ isLastBatch: false,
+ cancellationToken: cancellationToken);
+ }
+ }
+
+ private sealed class CatalogEntry
+ {
+ internal DateTime CommitTimeStamp { get; }
+ internal string PackageId { get; }
+ internal string NormalizedPackageVersion { get; }
+ internal Uri Type { get; }
+ internal Uri Uri { get; }
+
+ private CatalogEntry(DateTime commitTimeStamp, string packageId, string normalizedPackageVersion, Uri type, Uri uri)
+ {
+ CommitTimeStamp = commitTimeStamp;
+ PackageId = packageId;
+ NormalizedPackageVersion = normalizedPackageVersion;
+ Type = type;
+ Uri = uri;
+ }
+
+ internal static CatalogEntry Create(CatalogCommitItem item)
+ {
+ var typeUri = item.TypeUris.Single(uri =>
+ uri.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri ||
+ uri.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri);
+
+ return new CatalogEntry(
+ item.CommitTimeStamp,
+ item.PackageIdentity.Id.ToLowerInvariant(),
+ item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(),
+ typeUri,
+ item.Uri);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Dnx/DnxConstants.cs b/src/Catalog/Dnx/DnxConstants.cs
new file mode 100644
index 000000000..385f520a7
--- /dev/null
+++ b/src/Catalog/Dnx/DnxConstants.cs
@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+ internal static class DnxConstants
+ {
+ internal const string ApplicationOctetStreamContentType = "application/octet-stream";
+ internal const string DefaultCacheControl = "max-age=120";
+
+ internal static readonly IReadOnlyDictionary RequiredBlobProperties = new Dictionary()
+ {
+ { StorageConstants.CacheControl, DefaultCacheControl },
+ { StorageConstants.ContentType, ApplicationOctetStreamContentType }
+ };
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Dnx/DnxEntry.cs b/src/Catalog/Dnx/DnxEntry.cs
new file mode 100644
index 000000000..9c8eac3a4
--- /dev/null
+++ b/src/Catalog/Dnx/DnxEntry.cs
@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+ public sealed class DnxEntry
+ {
+ public Uri Nupkg { get; }
+ public Uri Nuspec { get; }
+
+ internal DnxEntry(Uri nupkg, Uri nuspec)
+ {
+ Nupkg = nupkg;
+ Nuspec = nuspec;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Dnx/DnxMaker.cs b/src/Catalog/Dnx/DnxMaker.cs
new file mode 100644
index 000000000..d4ddaabb7
--- /dev/null
+++ b/src/Catalog/Dnx/DnxMaker.cs
@@ -0,0 +1,482 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Common;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Metadata.Catalog.Helpers;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using NuGet.Versioning;
+
+using ILogger = Microsoft.Extensions.Logging.ILogger;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+ public class DnxMaker
+ {
+ private readonly StorageFactory _storageFactory;
+ private readonly ITelemetryService _telemetryService;
+ private readonly ILogger _logger;
+
+ public DnxMaker(StorageFactory storageFactory, ITelemetryService telemetryService, ILogger logger)
+ {
+ _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
+ _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+ _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+ }
+
+ public async Task AddPackageAsync(
+ Stream nupkgStream,
+ string nuspec,
+ string packageId,
+ string normalizedPackageVersion,
+ string iconFilename,
+ CancellationToken cancellationToken)
+ {
+ if (nupkgStream == null)
+ {
+ throw new ArgumentNullException(nameof(nupkgStream));
+ }
+
+ if (!nupkgStream.CanSeek)
+ {
+ throw new ArgumentException($"{nameof(nupkgStream)} must be seekable stream", nameof(nupkgStream));
+ }
+
+ if (string.IsNullOrEmpty(nuspec))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec));
+ }
+
+ if (string.IsNullOrEmpty(packageId))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId));
+ }
+
+ if (string.IsNullOrEmpty(normalizedPackageVersion))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var storage = _storageFactory.Create(packageId);
+ var nuspecUri = await SaveNuspecAsync(storage, packageId, normalizedPackageVersion, nuspec, cancellationToken);
+ nupkgStream.Seek(0, SeekOrigin.Begin);
+ if (!string.IsNullOrWhiteSpace(iconFilename))
+ {
+ await CopyIconFromNupkgStreamAsync(nupkgStream, iconFilename, storage, packageId, normalizedPackageVersion, cancellationToken);
+ }
+ else
+ {
+ _logger.LogInformation("Package {PackageId} {PackageVersion} doesn't have an icon file specified in fallback to package stream case.",
+ packageId,
+ normalizedPackageVersion);
+ }
+ var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, packageId, normalizedPackageVersion, cancellationToken);
+
+ return new DnxEntry(nupkgUri, nuspecUri);
+ }
+
+ public async Task AddPackageAsync(
+ IAzureStorage sourceStorage,
+ string nuspec,
+ string packageId,
+ string normalizedPackageVersion,
+ string iconFilename,
+ CancellationToken cancellationToken)
+ {
+ if (sourceStorage == null)
+ {
+ throw new ArgumentNullException(nameof(sourceStorage));
+ }
+
+ if (string.IsNullOrEmpty(nuspec))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec));
+ }
+
+ if (string.IsNullOrEmpty(packageId))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId));
+ }
+
+ if (string.IsNullOrEmpty(normalizedPackageVersion))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var destinationStorage = _storageFactory.Create(packageId);
+ var nuspecUri = await SaveNuspecAsync(destinationStorage, packageId, normalizedPackageVersion, nuspec, cancellationToken);
+ if (!string.IsNullOrWhiteSpace(iconFilename))
+ {
+ await CopyIconFromAzureStorageIfExistAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, iconFilename, cancellationToken);
+ }
+ else
+ {
+ _logger.LogInformation("Package {PackageId} {PackageVersion} doesn't have icon file specified in Azure Storage stream case",
+ packageId,
+ normalizedPackageVersion);
+ }
+ var nupkgUri = await CopyNupkgAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, cancellationToken);
+
+ return new DnxEntry(nupkgUri, nuspecUri);
+ }
+
+ public async Task DeletePackageAsync(string id, string version, CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrEmpty(id))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id));
+ }
+
+ if (string.IsNullOrEmpty(version))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var storage = _storageFactory.Create(id);
+ var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version);
+
+ await DeleteNuspecAsync(storage, id, normalizedVersion, cancellationToken);
+ await DeleteIconAsync(storage, id, normalizedVersion, cancellationToken);
+ await DeleteNupkgAsync(storage, id, normalizedVersion, cancellationToken);
+ }
+
+ public async Task HasPackageInIndexAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+ {
+ if (storage == null)
+ {
+ throw new ArgumentNullException(nameof(storage));
+ }
+
+ if (string.IsNullOrEmpty(id))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id));
+ }
+
+ if (string.IsNullOrEmpty(version))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var versionsContext = await GetVersionsAsync(storage, cancellationToken);
+ var parsedVersion = NuGetVersion.Parse(version);
+
+ return versionsContext.Versions.Contains(parsedVersion);
+ }
+
+ private async Task SaveNuspecAsync(Storage storage, string id, string version, string nuspec, CancellationToken cancellationToken)
+ {
+ var relativeAddress = GetRelativeAddressNuspec(id, version);
+ var nuspecUri = new Uri(storage.BaseAddress, relativeAddress);
+ var content = new StringStorageContent(nuspec, "text/xml", DnxConstants.DefaultCacheControl);
+
+ await storage.SaveAsync(nuspecUri, content, cancellationToken);
+
+ return nuspecUri;
+ }
+
+ public async Task UpdatePackageVersionIndexAsync(string id, Action> updateAction, CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrEmpty(id))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id));
+ }
+
+ if (updateAction == null)
+ {
+ throw new ArgumentNullException(nameof(updateAction));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var storage = _storageFactory.Create(id);
+ var versionsContext = await GetVersionsAsync(storage, cancellationToken);
+ var relativeAddress = versionsContext.RelativeAddress;
+ var resourceUri = versionsContext.ResourceUri;
+ var versions = versionsContext.Versions;
+
+ updateAction(versions);
+ var result = new List(versions);
+
+ if (result.Any())
+ {
+ // Store versions (sorted)
+ result.Sort();
+
+ await storage.SaveAsync(resourceUri, CreateContent(result.Select(version => version.ToNormalizedString())), cancellationToken);
+ }
+ else
+ {
+ // Remove versions file if no versions are present
+ if (storage.Exists(relativeAddress))
+ {
+ await storage.DeleteAsync(resourceUri, cancellationToken);
+ }
+ }
+ }
+
+ private async Task GetVersionsAsync(Storage storage, CancellationToken cancellationToken)
+ {
+ var relativeAddress = "index.json";
+ var resourceUri = new Uri(storage.BaseAddress, relativeAddress);
+ var versions = GetVersions(await storage.LoadStringAsync(resourceUri, cancellationToken));
+
+ return new VersionsResult(relativeAddress, resourceUri, versions);
+ }
+
+ private static HashSet GetVersions(string json)
+ {
+ var result = new HashSet();
+ if (json != null)
+ {
+ JObject obj = JObject.Parse(json);
+
+ JArray versions = obj["versions"] as JArray;
+
+ if (versions != null)
+ {
+ foreach (JToken version in versions)
+ {
+ result.Add(NuGetVersion.Parse(version.ToString()));
+ }
+ }
+ }
+ return result;
+ }
+
+ private StorageContent CreateContent(IEnumerable versions)
+ {
+ JObject obj = new JObject { { "versions", new JArray(versions) } };
+ return new StringStorageContent(obj.ToString(), "application/json", "no-store");
+ }
+
+ private async Task SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken)
+ {
+ Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version));
+ var content = new StreamStorageContent(
+ nupkgStream,
+ DnxConstants.ApplicationOctetStreamContentType,
+ DnxConstants.DefaultCacheControl);
+
+ await storage.SaveAsync(nupkgUri, content, cancellationToken);
+
+ return nupkgUri;
+ }
+
+ private async Task CopyNupkgAsync(
+ IStorage sourceStorage,
+ Storage destinationStorage,
+ string id, string version, CancellationToken cancellationToken)
+ {
+ var packageFileName = PackageUtility.GetPackageFileName(id, version);
+ var sourceUri = sourceStorage.ResolveUri(packageFileName);
+ var destinationRelativeUri = GetRelativeAddressNupkg(id, version);
+ var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri);
+
+ await sourceStorage.CopyAsync(
+ sourceUri,
+ destinationStorage,
+ destinationUri,
+ DnxConstants.RequiredBlobProperties,
+ cancellationToken);
+
+ return destinationUri;
+ }
+
+ private async Task CopyIconFromAzureStorageIfExistAsync(
+ IAzureStorage sourceStorage,
+ Storage destinationStorage,
+ string packageId,
+ string normalizedPackageVersion,
+ string iconFilename,
+ CancellationToken cancellationToken)
+ {
+ using (var packageStream = await GetPackageStreamAsync(sourceStorage, packageId, normalizedPackageVersion, cancellationToken))
+ {
+ await CopyIconAsync(
+ packageStream,
+ iconFilename,
+ destinationStorage,
+ packageId,
+ normalizedPackageVersion,
+ cancellationToken);
+ }
+ }
+
+ private async Task CopyIconFromNupkgStreamAsync(
+ Stream nupkgStream,
+ string iconFilename,
+ Storage destinationStorage,
+ string packageId,
+ string normalizedPackageVersion,
+ CancellationToken cancellationToken)
+ {
+ await CopyIconAsync(
+ nupkgStream,
+ iconFilename,
+ destinationStorage,
+ packageId,
+ normalizedPackageVersion,
+ cancellationToken);
+ }
+
+ private async Task CopyIconAsync(
+ Stream packageStream,
+ string iconFilename,
+ Storage destinationStorage,
+ string packageId,
+ string normalizedPackageVersion,
+ CancellationToken cancellationToken)
+ {
+ _logger.LogInformation("Processing icon {IconFilename} for the package {PackageId} {PackageVersion}",
+ iconFilename,
+ packageId,
+ normalizedPackageVersion);
+
+ var iconPath = PathUtility.StripLeadingDirectorySeparators(iconFilename);
+
+ var destinationRelativeUri = GetRelativeAddressIcon(packageId, normalizedPackageVersion);
+ var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri);
+
+ await ExtractAndStoreIconAsync(
+ packageStream,
+ iconPath,
+ destinationStorage,
+ destinationUri,
+ cancellationToken,
+ packageId,
+ normalizedPackageVersion);
+ }
+
+ private async Task ExtractAndStoreIconAsync(
+ Stream packageStream,
+ string iconPath,
+ Storage destinationStorage,
+ Uri destinationUri,
+ CancellationToken cancellationToken,
+ string packageId,
+ string normalizedPackageVersion)
+ {
+ using (var zipArchive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true))
+ {
+ var iconEntry = zipArchive.Entries.FirstOrDefault(e => e.FullName.Equals(iconPath, StringComparison.InvariantCultureIgnoreCase));
+ if (iconEntry != null)
+ {
+ using (var iconStream = iconEntry.Open())
+ {
+ _logger.LogInformation("Extracting icon to the destination storage {DestinationUri}", destinationUri);
+ // TODO: align the mime type determination with Gallery https://github.com/nuget/nugetgallery/issues/7061
+ var iconContent = new StreamStorageContent(iconStream, string.Empty, DnxConstants.DefaultCacheControl);
+ await destinationStorage.SaveAsync(destinationUri, iconContent, cancellationToken);
+ _logger.LogInformation("Done");
+ }
+ }
+ else
+ {
+ _telemetryService.TrackIconExtractionFailure(packageId, normalizedPackageVersion);
+ _logger.LogWarning("Zip archive entry {IconPath} does not exist", iconPath);
+ }
+ }
+ }
+
+ private async Task GetPackageStreamAsync(
+ IAzureStorage sourceStorage,
+ string packageId,
+ string normalizedPackageVersion,
+ CancellationToken cancellationToken)
+ {
+ var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion);
+ var sourceUri = sourceStorage.ResolveUri(packageFileName);
+ var packageSourceBlob = await sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri);
+ return await packageSourceBlob.GetStreamAsync(cancellationToken);
+ }
+
+ private async Task DeleteNuspecAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+ {
+ string relativeAddress = GetRelativeAddressNuspec(id, version);
+ Uri nuspecUri = new Uri(storage.BaseAddress, relativeAddress);
+ if (storage.Exists(relativeAddress))
+ {
+ await storage.DeleteAsync(nuspecUri, cancellationToken);
+ }
+ }
+
+ private async Task DeleteNupkgAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+ {
+ string relativeAddress = GetRelativeAddressNupkg(id, version);
+ Uri nupkgUri = new Uri(storage.BaseAddress, relativeAddress);
+ if (storage.Exists(relativeAddress))
+ {
+ await storage.DeleteAsync(nupkgUri, cancellationToken);
+ }
+ }
+
+ private async Task DeleteIconAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+ {
+ string relativeAddress = GetRelativeAddressIcon(id, version);
+ Uri iconUri = new Uri(storage.BaseAddress, relativeAddress);
+ if (storage.Exists(relativeAddress))
+ {
+ await storage.DeleteAsync(iconUri, cancellationToken);
+ }
+ }
+
+ private static string GetRelativeAddressNuspec(string id, string version)
+ {
+ return $"{NuGetVersion.Parse(version).ToNormalizedString()}/{id}.nuspec";
+ }
+
+ public static string GetRelativeAddressNupkg(string id, string version)
+ {
+ if (string.IsNullOrEmpty(id))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id));
+ }
+
+ if (string.IsNullOrEmpty(version))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version));
+ }
+
+ var normalizedVersion = NuGetVersion.Parse(version).ToNormalizedString();
+
+ return $"{normalizedVersion}/{id}.{normalizedVersion}.nupkg";
+ }
+
+ private static string GetRelativeAddressIcon(string id, string version)
+ {
+ var normalizedVersion = NuGetVersion.Parse(version).ToNormalizedString();
+
+ return $"{normalizedVersion}/icon";
+ }
+
+ private class VersionsResult
+ {
+ public VersionsResult(string relativeAddress, Uri resourceUri, HashSet versions)
+ {
+ RelativeAddress = relativeAddress;
+ ResourceUri = resourceUri;
+ Versions = versions;
+ }
+
+ public string RelativeAddress { get; }
+ public Uri ResourceUri { get; }
+ public HashSet Versions { get; }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/DurableCursor.cs b/src/Catalog/DurableCursor.cs
new file mode 100644
index 000000000..a0a2a611b
--- /dev/null
+++ b/src/Catalog/DurableCursor.cs
@@ -0,0 +1,46 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class DurableCursor : ReadWriteCursor
+ {
+ Uri _address;
+ Storage _storage;
+ DateTime _defaultValue;
+
+ public DurableCursor(Uri address, Storage storage, DateTime defaultValue)
+ {
+ _address = address;
+ _storage = storage;
+ _defaultValue = defaultValue;
+ }
+
+ public override async Task SaveAsync(CancellationToken cancellationToken)
+ {
+ JObject obj = new JObject { { "value", Value.ToString("O") } };
+ StorageContent content = new StringStorageContent(obj.ToString(), "application/json", "no-store");
+ await _storage.SaveAsync(_address, content, cancellationToken);
+ }
+
+ public override async Task LoadAsync(CancellationToken cancellationToken)
+ {
+ string json = await _storage.LoadStringAsync(_address, cancellationToken);
+
+ if (json == null)
+ {
+ Value = _defaultValue;
+ return;
+ }
+
+ JObject obj = JObject.Parse(json);
+ Value = obj["value"].ToObject();
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Extensions/DateTimeExtensions.cs b/src/Catalog/Extensions/DateTimeExtensions.cs
new file mode 100644
index 000000000..0cf0a8e88
--- /dev/null
+++ b/src/Catalog/Extensions/DateTimeExtensions.cs
@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace System
+{
+ public static class DateTimeExtensions
+ {
+ public static DateTime ForceUtc(this DateTime date)
+ {
+ if (date.Kind != DateTimeKind.Utc)
+ {
+ date = new DateTime(date.Ticks, DateTimeKind.Utc);
+ }
+
+ return date;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Extensions/DbDataReaderExtensions.cs b/src/Catalog/Extensions/DbDataReaderExtensions.cs
new file mode 100644
index 000000000..cd58b7fd0
--- /dev/null
+++ b/src/Catalog/Extensions/DbDataReaderExtensions.cs
@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace System.Data.Common
+{
+ public static class DbDataReaderExtensions
+ {
+ public static int? ReadInt32OrNull(this DbDataReader dataReader, string columnName)
+ {
+ return ReadColumnOrNull(dataReader, columnName, (r, o) => r.GetInt32(o), (int?)null);
+ }
+
+ public static string ReadStringOrNull(this DbDataReader dataReader, string columnName)
+ {
+ return ReadColumnOrNull(dataReader, columnName, (r, o) => r.GetString(o), nullValue: null);
+ }
+
+ private static T ReadColumnOrNull(DbDataReader dataReader, string columnName, Func provideValue, T nullValue)
+ {
+ if (dataReader == null)
+ {
+ throw new ArgumentNullException(nameof(dataReader));
+ }
+
+ if (columnName == null)
+ {
+ throw new ArgumentNullException(nameof(columnName));
+ }
+
+ if (provideValue == null)
+ {
+ throw new ArgumentNullException(nameof(provideValue));
+ }
+
+ try
+ {
+ var ordinal = dataReader.GetOrdinal(columnName);
+
+ if (!dataReader.IsDBNull(ordinal))
+ {
+ return provideValue(dataReader, ordinal);
+ }
+
+ return nullValue;
+
+ }
+ catch (IndexOutOfRangeException)
+ {
+ // Thrown by DbDataReader.GetOrdinal(string) when the column does not exist.
+ // The exception can be swallowed as the intention of this method is to return null instead.
+ return nullValue;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Extensions/IDataRecordExtensions.cs b/src/Catalog/Extensions/IDataRecordExtensions.cs
new file mode 100644
index 000000000..acc86f9e2
--- /dev/null
+++ b/src/Catalog/Extensions/IDataRecordExtensions.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace System.Data
+{
+ ///
+ /// Extension methods that make working with more convenient.
+ ///
+ public static class IDataRecordExtensions
+ {
+ public static DateTime ReadNullableUtcDateTime(this IDataRecord dataRecord, string columnName)
+ {
+ if (dataRecord == null)
+ {
+ throw new ArgumentNullException(nameof(dataRecord));
+ }
+
+ if (columnName == null)
+ {
+ throw new ArgumentNullException(nameof(columnName));
+ }
+
+ return (dataRecord[columnName] == DBNull.Value ? DateTime.MinValue : ReadDateTime(dataRecord, columnName)).ForceUtc();
+ }
+
+ public static DateTime ReadDateTime(this IDataRecord dataRecord, string columnName)
+ {
+ if (dataRecord == null)
+ {
+ throw new ArgumentNullException(nameof(dataRecord));
+ }
+
+ if (columnName == null)
+ {
+ throw new ArgumentNullException(nameof(columnName));
+ }
+
+ return dataRecord.GetDateTime(dataRecord.GetOrdinal(columnName));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/FetchCatalogCommitsAsync.cs b/src/Catalog/FetchCatalogCommitsAsync.cs
new file mode 100644
index 000000000..cd6164346
--- /dev/null
+++ b/src/Catalog/FetchCatalogCommitsAsync.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ internal delegate Task> FetchCatalogCommitsAsync(
+ CollectorHttpClient client,
+ ReadCursor front,
+ ReadCursor back,
+ CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/src/Catalog/FixPackageHashHandler.cs b/src/Catalog/FixPackageHashHandler.cs
new file mode 100644
index 000000000..11f1cdeaa
--- /dev/null
+++ b/src/Catalog/FixPackageHashHandler.cs
@@ -0,0 +1,74 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net.Http;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// Adds the Content MD5 property to package blobs that are missing it.
+ ///
+ public class FixPackageHashHandler : IPackagesContainerHandler
+ {
+ private readonly HttpClient _httpClient;
+ private readonly ITelemetryService _telemetryService;
+ private readonly ILogger _logger;
+
+ public FixPackageHashHandler(
+ HttpClient httpClient,
+ ITelemetryService telemetryService,
+ ILogger logger)
+ {
+ _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+ _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+ _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+ }
+
+ public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob)
+ {
+ await blob.FetchAttributesAsync(CancellationToken.None);
+
+ // Skip the package if it has a Content MD5 hash
+ if (blob.ContentMD5 != null)
+ {
+ _telemetryService.TrackPackageAlreadyHasHash(packageEntry.Id, packageEntry.Version);
+ return;
+ }
+
+ // Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK
+ // occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470
+ string hash;
+ using (var hashAlgorithm = MD5.Create())
+ using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri))
+ {
+ var hashBytes = hashAlgorithm.ComputeHash(packageStream);
+
+ hash = Convert.ToBase64String(hashBytes);
+ }
+
+ blob.ContentMD5 = hash;
+
+ var condition = AccessCondition.GenerateIfMatchCondition(blob.ETag);
+ await blob.SetPropertiesAsync(
+ condition,
+ options: null,
+ operationContext: null);
+
+ _telemetryService.TrackPackageHashFixed(packageEntry.Id, packageEntry.Version);
+
+ _logger.LogWarning(
+ "Updated package {PackageId} {PackageVersion}, set hash to '{Hash}' using ETag {ETag}",
+ packageEntry.Id,
+ packageEntry.Version,
+ hash,
+ blob.ETag);
+ }
+ }
+}
diff --git a/src/Catalog/FlatContainerPackagePathProvider.cs b/src/Catalog/FlatContainerPackagePathProvider.cs
new file mode 100644
index 000000000..b583694ac
--- /dev/null
+++ b/src/Catalog/FlatContainerPackagePathProvider.cs
@@ -0,0 +1,69 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using NuGet.Services.Metadata.Catalog.Helpers;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class FlatContainerPackagePathProvider
+ {
+ private readonly string _container;
+
+ public FlatContainerPackagePathProvider(string container)
+ {
+ _container = container;
+ }
+
+ public string GetPackagePath(string id, string version)
+ {
+ if (id == null)
+ {
+ throw new ArgumentNullException(nameof(id));
+ }
+
+ if (version == null)
+ {
+ throw new ArgumentNullException(nameof(version));
+ }
+
+ var idLowerCase = id.ToLowerInvariant();
+ var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant();
+ var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase);
+
+ return $"{_container}/{idLowerCase}/{versionLowerCase}/{packageFileName}";
+ }
+
+ public string GetIconPath(string id, string version)
+ {
+ return GetIconPath(id, version, normalize: true);
+ }
+
+ public string GetIconPath(string id, string version, bool normalize)
+ {
+ if (id == null)
+ {
+ throw new ArgumentNullException(nameof(id));
+ }
+
+ if (version == null)
+ {
+ throw new ArgumentNullException(nameof(version));
+ }
+
+ var idLowerCase = id.ToLowerInvariant();
+
+ string versionLowerCase;
+ if (normalize)
+ {
+ versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant();
+ }
+ else
+ {
+ versionLowerCase = version.ToLowerInvariant();
+ }
+
+ return $"{_container}/{idLowerCase}/{versionLowerCase}/icon";
+ }
+ }
+}
diff --git a/src/Catalog/GetCatalogCommitItemKey.cs b/src/Catalog/GetCatalogCommitItemKey.cs
new file mode 100644
index 000000000..d28bf0659
--- /dev/null
+++ b/src/Catalog/GetCatalogCommitItemKey.cs
@@ -0,0 +1,7 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public delegate string GetCatalogCommitItemKey(CatalogCommitItem item);
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/AsyncExtensions.cs b/src/Catalog/Helpers/AsyncExtensions.cs
new file mode 100644
index 000000000..c2e1b277b
--- /dev/null
+++ b/src/Catalog/Helpers/AsyncExtensions.cs
@@ -0,0 +1,47 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public static class AsyncExtensions
+ {
+ public static Task ForEachAsync(this IEnumerable enumerable, int maxDegreeOfParallelism, Func func)
+ {
+ if (enumerable == null)
+ {
+ throw new ArgumentNullException(nameof(enumerable));
+ }
+
+ if (maxDegreeOfParallelism < 1)
+ {
+ throw new ArgumentOutOfRangeException(
+ nameof(maxDegreeOfParallelism),
+ string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
+ }
+
+ if (func == null)
+ {
+ throw new ArgumentNullException(nameof(func));
+ }
+
+ return Task.WhenAll(
+ from partition in Partitioner.Create(enumerable).GetPartitions(maxDegreeOfParallelism)
+ select Task.Run(async delegate
+ {
+ using (partition)
+ {
+ while (partition.MoveNext())
+ {
+ await func(partition.Current);
+ }
+ }
+ }));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/CatalogProperties.cs b/src/Catalog/Helpers/CatalogProperties.cs
new file mode 100644
index 000000000..acef7ed86
--- /dev/null
+++ b/src/Catalog/Helpers/CatalogProperties.cs
@@ -0,0 +1,89 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public sealed class CatalogProperties
+ {
+ public DateTime? LastCreated { get; }
+ public DateTime? LastDeleted { get; }
+ public DateTime? LastEdited { get; }
+
+ public CatalogProperties(DateTime? lastCreated, DateTime? lastDeleted, DateTime? lastEdited)
+ {
+ LastCreated = lastCreated;
+ LastDeleted = lastDeleted;
+ LastEdited = lastEdited;
+ }
+
+ ///
+ /// Asynchronously reads and returns top-level metadata from the catalog's index.json.
+ ///
+ /// The metadata values include "nuget:lastCreated", "nuget:lastDeleted", and "nuget:lastEdited",
+ /// which are the timestamps of the catalog cursor.
+ ///
+ ///
+ ///
+ /// A task that represents the asynchronous operation.
+ /// The task result () returns a .
+ /// Thrown if is null.
+ /// Thrown if
+ /// is cancelled.
+ public static async Task ReadAsync(
+ IStorage storage,
+ ITelemetryService telemetryService,
+ CancellationToken cancellationToken)
+ {
+ if (storage == null)
+ {
+ throw new ArgumentNullException(nameof(storage));
+ }
+
+ if (telemetryService == null)
+ {
+ throw new ArgumentNullException(nameof(telemetryService));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ DateTime? lastCreated = null;
+ DateTime? lastDeleted = null;
+ DateTime? lastEdited = null;
+
+ var stopwatch = Stopwatch.StartNew();
+ var indexUri = storage.ResolveUri("index.json");
+ var json = await storage.LoadStringAsync(indexUri, cancellationToken);
+
+ if (json != null)
+ {
+ var obj = JObject.Parse(json);
+ telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, indexUri);
+ JToken token;
+
+ if (obj.TryGetValue("nuget:lastCreated", out token))
+ {
+ lastCreated = token.ToObject().ToUniversalTime();
+ }
+
+ if (obj.TryGetValue("nuget:lastDeleted", out token))
+ {
+ lastDeleted = token.ToObject().ToUniversalTime();
+ }
+
+ if (obj.TryGetValue("nuget:lastEdited", out token))
+ {
+ lastEdited = token.ToObject().ToUniversalTime();
+ }
+ }
+
+ return new CatalogProperties(lastCreated, lastDeleted, lastEdited);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/CatalogWriterHelper.cs b/src/Catalog/Helpers/CatalogWriterHelper.cs
new file mode 100644
index 000000000..9ab1dece6
--- /dev/null
+++ b/src/Catalog/Helpers/CatalogWriterHelper.cs
@@ -0,0 +1,164 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Helper methods for writing to the catalog.
+ ///
+ public static class CatalogWriterHelper
+ {
+ ///
+ /// Asynchronously writes package metadata to the catalog.
+ ///
+ /// A package catalog item creator.
+ /// Packages to download metadata for.
+ /// Storage.
+ /// The catalog's last created datetime.
+ /// The catalog's last edited datetime.
+ /// The catalog's last deleted datetime.
+ /// The maximum degree of parallelism for package processing.
+ /// true to include created packages; otherwise, false.
+ /// true to update the created cursor from the last edited cursor;
+ /// otherwise, false.
+ /// A cancellation token.
+ /// A telemetry service.
+ /// A logger.
+ /// A task that represents the asynchronous operation.
+ /// The task result () returns the latest
+ /// that was processed.
+ public static async Task WritePackageDetailsToCatalogAsync(
+ IPackageCatalogItemCreator packageCatalogItemCreator,
+ SortedList> packages,
+ IStorage storage,
+ DateTime lastCreated,
+ DateTime lastEdited,
+ DateTime lastDeleted,
+ int maxDegreeOfParallelism,
+ bool? createdPackages,
+ bool updateCreatedFromEdited,
+ CancellationToken cancellationToken,
+ ITelemetryService telemetryService,
+ ILogger logger)
+ {
+ if (packageCatalogItemCreator == null)
+ {
+ throw new ArgumentNullException(nameof(packageCatalogItemCreator));
+ }
+
+ if (packages == null)
+ {
+ throw new ArgumentNullException(nameof(packages));
+ }
+
+ if (storage == null)
+ {
+ throw new ArgumentNullException(nameof(storage));
+ }
+
+ if (maxDegreeOfParallelism < 1)
+ {
+ throw new ArgumentOutOfRangeException(
+ nameof(maxDegreeOfParallelism),
+ string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
+ }
+
+ if (telemetryService == null)
+ {
+ throw new ArgumentNullException(nameof(telemetryService));
+ }
+
+ if (logger == null)
+ {
+ throw new ArgumentNullException(nameof(logger));
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var writer = new AppendOnlyCatalogWriter(storage, telemetryService, Constants.MaxPageSize);
+
+ var lastDate = DetermineLastDate(lastCreated, lastEdited, createdPackages);
+
+ if (packages.Count == 0)
+ {
+ return lastDate;
+ }
+
+ // Flatten the sorted list.
+ var workItems = packages.SelectMany(
+ pair => pair.Value.Select(
+ details => new PackageWorkItem(pair.Key, details)))
+ .ToArray();
+
+ await workItems.ForEachAsync(maxDegreeOfParallelism, async workItem =>
+ {
+ workItem.PackageCatalogItem = await packageCatalogItemCreator.CreateAsync(
+ workItem.FeedPackageDetails,
+ workItem.Timestamp,
+ cancellationToken);
+ });
+
+ lastDate = packages.Last().Key;
+
+ // AppendOnlyCatalogWriter.Add(...) is not thread-safe, so add them all at once on one thread.
+ foreach (var workItem in workItems.Where(workItem => workItem.PackageCatalogItem != null))
+ {
+ writer.Add(workItem.PackageCatalogItem);
+
+ logger?.LogInformation("Add metadata from: {PackageDetailsContentUri}", workItem.FeedPackageDetails.ContentUri);
+ }
+
+ if (createdPackages.HasValue)
+ {
+ lastEdited = !createdPackages.Value ? lastDate : lastEdited;
+
+ if (updateCreatedFromEdited)
+ {
+ lastCreated = lastEdited;
+ }
+ else
+ {
+ lastCreated = createdPackages.Value ? lastDate : lastCreated;
+ }
+ }
+
+ var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted));
+
+ await writer.Commit(commitMetadata, cancellationToken);
+
+ logger?.LogInformation("COMMIT metadata to catalog.");
+
+ return lastDate;
+ }
+
+ private static DateTime DetermineLastDate(DateTime lastCreated, DateTime lastEdited, bool? createdPackages)
+ {
+ if (!createdPackages.HasValue)
+ {
+ return DateTime.MinValue;
+ }
+ return createdPackages.Value ? lastCreated : lastEdited;
+ }
+
+ private sealed class PackageWorkItem
+ {
+ internal DateTime Timestamp { get; }
+ internal FeedPackageDetails FeedPackageDetails { get; }
+ internal PackageCatalogItem PackageCatalogItem { get; set; }
+
+ internal PackageWorkItem(DateTime timestamp, FeedPackageDetails feedPackageDetails)
+ {
+ Timestamp = timestamp;
+ FeedPackageDetails = feedPackageDetails;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/Db2CatalogCursor.cs b/src/Catalog/Helpers/Db2CatalogCursor.cs
new file mode 100644
index 000000000..a2704472b
--- /dev/null
+++ b/src/Catalog/Helpers/Db2CatalogCursor.cs
@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Data.SqlTypes;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public sealed class Db2CatalogCursor
+ {
+ private Db2CatalogCursor(string columnName, DateTime cursorValue, int top)
+ {
+ ColumnName = columnName ?? throw new ArgumentNullException(nameof(columnName));
+ CursorValue = cursorValue < SqlDateTime.MinValue.Value ? SqlDateTime.MinValue.Value : cursorValue;
+
+ if (top <= 0)
+ {
+ throw new ArgumentOutOfRangeException("Argument value must be a positive non-zero integer.", nameof(top));
+ }
+
+ Top = top;
+ }
+
+ public string ColumnName { get; }
+ public DateTime CursorValue { get; }
+ public int Top { get; }
+
+ public static Db2CatalogCursor ByCreated(DateTime since, int top) => new Db2CatalogCursor(Db2CatalogProjectionColumnNames.Created, since, top);
+ public static Db2CatalogCursor ByLastEdited(DateTime since, int top) => new Db2CatalogCursor(Db2CatalogProjectionColumnNames.LastEdited, since, top);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/Db2CatalogProjection.cs b/src/Catalog/Helpers/Db2CatalogProjection.cs
new file mode 100644
index 000000000..42521c5b8
--- /dev/null
+++ b/src/Catalog/Helpers/Db2CatalogProjection.cs
@@ -0,0 +1,134 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.Common;
+using System.Linq;
+using System.Threading.Tasks;
+using NuGet.Protocol.Plugins;
+using NuGet.Services.Entities;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Utility class to project s retrieved by db2catalog
+ /// into a format that is consumable by the catalog writer.
+ ///
+ public class Db2CatalogProjection
+ {
+ public const string AlternatePackageVersionWildCard = "*";
+
+ private readonly PackageContentUriBuilder _packageContentUriBuilder;
+
+ public Db2CatalogProjection(PackageContentUriBuilder packageContentUriBuilder)
+ {
+ _packageContentUriBuilder = packageContentUriBuilder ?? throw new ArgumentNullException(nameof(packageContentUriBuilder));
+ }
+
+ public string ReadPackageVersionKeyFromDataReader(DbDataReader dataReader) =>
+ dataReader[Db2CatalogProjectionColumnNames.Key].ToString();
+
+ ///
+ /// Note that this method will read details from current and end by reading next, closing reader (to communicate state) when end reached.
+ ///
+ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader dataReader)
+ {
+ if (dataReader == null)
+ {
+ throw new ArgumentNullException(nameof(dataReader));
+ }
+
+ var packageId = dataReader[Db2CatalogProjectionColumnNames.PackageId].ToString();
+ var normalizedPackageVersion = dataReader[Db2CatalogProjectionColumnNames.NormalizedVersion].ToString();
+ var fullPackageVersion = dataReader[Db2CatalogProjectionColumnNames.FullVersion].ToString();
+ var listed = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.Listed));
+ var hideLicenseReport = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.HideLicenseReport));
+
+ var packageContentUri = _packageContentUriBuilder.Build(packageId, normalizedPackageVersion);
+ var deprecationInfo = ReadDeprecationInfoFromDataReader(dataReader);
+
+ return new FeedPackageDetails(
+ packageContentUri,
+ dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Created).ForceUtc(),
+ dataReader.ReadNullableUtcDateTime(Db2CatalogProjectionColumnNames.LastEdited),
+ listed ? dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Published).ForceUtc() : Constants.UnpublishedDate,
+ packageId,
+ normalizedPackageVersion,
+ fullPackageVersion,
+ hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseNames]?.ToString(),
+ hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseReportUrl]?.ToString(),
+ deprecationInfo,
+ dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.RequiresLicenseAcceptance)));
+ }
+
+ public PackageVulnerabilityItem ReadPackageVulnerabilityFromDataReader(DbDataReader dataReader)
+ {
+ var gitHubDatabaseKey = dataReader[Db2CatalogProjectionColumnNames.VulnerabilityGitHubDatabaseKey].ToString();
+ var advisoryUrl = dataReader[Db2CatalogProjectionColumnNames.VulnerabilityAdvisoryUrl].ToString();
+ var severity = dataReader[Db2CatalogProjectionColumnNames.VulnerabilitySeverity].ToString();
+
+ if (string.IsNullOrEmpty(gitHubDatabaseKey) || string.IsNullOrEmpty(advisoryUrl) || string.IsNullOrEmpty(severity))
+ {
+ return null;
+ }
+
+ return new PackageVulnerabilityItem(gitHubDatabaseKey: gitHubDatabaseKey, advisoryUrl: advisoryUrl, severity: severity);
+ }
+
+ public PackageDeprecationItem ReadDeprecationInfoFromDataReader(DbDataReader dataReader)
+ {
+ if (dataReader == null)
+ {
+ throw new ArgumentNullException(nameof(dataReader));
+ }
+
+ var deprecationReasons = new List();
+ var deprecationStatusValue = dataReader.ReadInt32OrNull(Db2CatalogProjectionColumnNames.DeprecationStatus);
+
+ if (!deprecationStatusValue.HasValue)
+ {
+ return null;
+ }
+
+ var deprecationStatus = (PackageDeprecationStatus)deprecationStatusValue.Value;
+
+ foreach (var deprecationStatusFlag in Enum.GetValues(typeof(PackageDeprecationStatus)).Cast())
+ {
+ if (deprecationStatusFlag == PackageDeprecationStatus.NotDeprecated)
+ {
+ continue;
+ }
+
+ if (deprecationStatus.HasFlag(deprecationStatusFlag))
+ {
+ deprecationReasons.Add(deprecationStatusFlag.ToString());
+ }
+ }
+
+ var alternatePackageId = dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.AlternatePackageId);
+ string alternatePackageVersion = null;
+ if (alternatePackageId != null)
+ {
+ alternatePackageVersion = dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.AlternatePackageVersion);
+
+ if (alternatePackageVersion == null)
+ {
+ alternatePackageVersion = AlternatePackageVersionWildCard;
+ }
+ else
+ {
+ // The alternate package version range is defined by a minimum version (lower-bound), or higher (no upper-bound).
+ alternatePackageVersion = $"[{alternatePackageVersion}, )";
+ }
+ }
+
+ return new PackageDeprecationItem(
+ deprecationReasons,
+ dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.DeprecationMessage),
+ alternatePackageId,
+ alternatePackageVersion);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs
new file mode 100644
index 000000000..f321f11ca
--- /dev/null
+++ b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs
@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Defines the column names for db2catalog SQL queries.
+ ///
+ public static class Db2CatalogProjectionColumnNames
+ {
+ public const string Key = "Key";
+ public const string PackageId = "Id";
+ public const string NormalizedVersion = "NormalizedVersion";
+ public const string FullVersion = "Version";
+ public const string Listed = "Listed";
+ public const string HideLicenseReport = "HideLicenseReport";
+ public const string Created = "Created";
+ public const string LastEdited = "LastEdited";
+ public const string Published = "Published";
+ public const string LicenseNames = "LicenseNames";
+ public const string LicenseReportUrl = "LicenseReportUrl";
+ public const string AlternatePackageId = "AlternatePackageId";
+ public const string AlternatePackageVersion = "AlternatePackageVersion";
+ public const string DeprecationStatus = "DeprecationStatus";
+ public const string DeprecationMessage = "DeprecationMessage";
+ public const string RequiresLicenseAcceptance = "RequiresLicenseAcceptance";
+ public const string VulnerabilityGitHubDatabaseKey = "VulnerabilityGitHubDatabaseKey";
+ public const string VulnerabilityAdvisoryUrl = "VulnerabilityAdvisoryUrl";
+ public const string VulnerabilitySeverity = "VulnerabilitySeverity";
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/DeletionAuditEntry.cs b/src/Catalog/Helpers/DeletionAuditEntry.cs
new file mode 100644
index 000000000..0d70e264e
--- /dev/null
+++ b/src/Catalog/Helpers/DeletionAuditEntry.cs
@@ -0,0 +1,261 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NuGet.Packaging.Core;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Represents the information in the audit entry for a deletion.
+ ///
+ public class DeletionAuditEntry
+ {
+ ///
+ /// Over time, we have had multiple file names for audit records for deletes.
+ ///
+ public readonly static IList FileNameSuffixes = new List
+ {
+ "-Deleted.audit.v1.json",
+ "-deleted.audit.v1.json",
+ "-softdeleted.audit.v1.json",
+ "-softdelete.audit.v1.json",
+ "-delete.audit.v1.json"
+ };
+
+ ///
+ /// Asynchronously creates a from a
+ /// and an .
+ ///
+ /// The through which
+ /// can be accessed.
+ /// A to the record to build a
+ /// from.
+ /// A cancellation token.
+ /// A logger.
+ /// A task that represents the asynchronous operation.
+ /// The task result () returns a
+ /// instance.
+ public static async Task CreateAsync(
+ IStorage auditingStorage,
+ Uri uri,
+ CancellationToken cancellationToken,
+ ILogger logger)
+ {
+ try
+ {
+ return new DeletionAuditEntry(uri, await auditingStorage.LoadStringAsync(uri, cancellationToken));
+ }
+ catch (JsonReaderException)
+ {
+ logger.LogWarning("Audit record at {AuditRecordUri} contains invalid JSON.", uri);
+ }
+ catch (NullReferenceException)
+ {
+ logger.LogWarning("Audit record at {AuditRecordUri} does not contain required JSON properties to perform a package delete.", uri);
+ }
+ catch (ArgumentException)
+ {
+ logger.LogWarning("Audit record at {AuditRecordUri} has no contents.", uri);
+ }
+
+ return null;
+ }
+
+ private DeletionAuditEntry(Uri uri, string contents)
+ {
+ if (string.IsNullOrEmpty(contents))
+ {
+ throw new ArgumentException($"{nameof(contents)} must not be null or empty!", nameof(contents));
+ }
+
+ Uri = uri;
+ Record = JObject.Parse(contents);
+ InitValues();
+ }
+
+ [JsonConstructor]
+ public DeletionAuditEntry()
+ {
+ }
+
+ ///
+ /// Constructor for testing.
+ ///
+ public DeletionAuditEntry(Uri uri, JObject record, string id, string version, DateTime? timestamp)
+ {
+ Uri = uri;
+ Record = record;
+ PackageId = id;
+ PackageVersion = version;
+ TimestampUtc = timestamp;
+ }
+
+ ///
+ /// The for the audit entry.
+ ///
+ [JsonProperty("uri")]
+ public Uri Uri { get; private set; }
+
+ ///
+ /// The entire contents of the audit entry file.
+ ///
+ [JsonProperty("record")]
+ public JObject Record { get; private set; }
+
+ ///
+ /// The id of the package being audited.
+ ///
+ [JsonProperty("id")]
+ public string PackageId { get; private set; }
+
+ ///
+ /// The version of the package being audited.
+ ///
+ [JsonProperty("version")]
+ public string PackageVersion { get; private set; }
+
+ ///
+ /// The the package was deleted.
+ ///
+ [JsonProperty("timestamp")]
+ public DateTime? TimestampUtc { get; private set; }
+
+ private const string RecordPart = "Record";
+ private const string ActorPart = "Actor";
+
+ private JObject GetPart(string partName)
+ {
+ return (JObject)Record?.GetValue(partName, StringComparison.OrdinalIgnoreCase);
+ }
+
+ private void InitValues()
+ {
+ PackageId = GetPart(RecordPart).GetValue("Id", StringComparison.OrdinalIgnoreCase).ToString();
+ PackageVersion = GetPart(RecordPart).GetValue("Version", StringComparison.OrdinalIgnoreCase).ToString();
+ TimestampUtc =
+ GetPart(ActorPart).GetValue("TimestampUtc", StringComparison.OrdinalIgnoreCase).Value();
+ }
+
+ ///
+ /// Fetches s.
+ ///
+ /// The to fetch audit records from.
+ /// A that can be used to cancel the task.
+ /// If specified, will only fetch s that represent operations on this package.
+ /// If specified, will only fetch s that are newer than this (non-inclusive).
+ /// If specified, will only fetch s that are older than this (non-inclusive).
+ /// An to log messages to.
+ /// An containing the relevant s.
+ public static Task> GetAsync(
+ StorageFactory auditingStorageFactory,
+ CancellationToken cancellationToken,
+ PackageIdentity package = null,
+ DateTime? minTime = null,
+ DateTime? maxTime = null,
+ ILogger logger = null)
+ {
+ Storage storage = auditingStorageFactory.Create(package != null ? GetAuditRecordPrefixFromPackage(package) : null);
+ return GetAsync(storage, cancellationToken, minTime, maxTime, logger);
+ }
+
+ ///
+ /// Asynchronously fetches s.
+ ///
+ /// The to fetch audit records from.
+ /// A that can be used to cancel
+ /// the task.
+ /// If specified, will only fetch s that are newer than
+ /// this (non-inclusive).
+ /// If specified, will only fetch s that are older than
+ /// this (non-inclusive).
+ /// An to log messages to.
+ /// A task that represents the asynchronous operation.
+ /// The task result () returns an
+ /// containing the relevant
+ /// s.
+ public static async Task> GetAsync(
+ IStorage auditingStorage,
+ CancellationToken cancellationToken,
+ DateTime? minTime = null,
+ DateTime? maxTime = null,
+ ILogger logger = null)
+ {
+ Func filterAuditRecord = (record) =>
+ {
+ if (!IsPackageDelete(record))
+ {
+ return false;
+ }
+
+ // We can't do anything if the last modified time is not available.
+ if (record.LastModifiedUtc == null)
+ {
+ logger?.LogWarning("Could not get date for filename in filterAuditRecord. Uri: {AuditRecordUri}", record.Uri);
+ return false;
+ }
+
+ var recordTimestamp = record.LastModifiedUtc.Value;
+ if (minTime != null && recordTimestamp < minTime.Value)
+ {
+ return false;
+ }
+
+ if (maxTime != null && recordTimestamp > maxTime.Value)
+ {
+ return false;
+ }
+
+ return true;
+ };
+
+ // Get all audit blobs (based on their filename which starts with a date that can be parsed).
+ /// Filter on the and fields provided.
+ var auditRecords =
+ (await auditingStorage.ListAsync(cancellationToken)).Where(filterAuditRecord);
+
+ return
+ (await Task.WhenAll(
+ auditRecords.Select(record => DeletionAuditEntry.CreateAsync(auditingStorage, record.Uri, cancellationToken, logger))))
+ // Filter out null records.
+ .Where(entry => entry?.Record != null);
+ }
+
+ ///
+ /// Returns the prefix of the audit record, which contains the id and version of the package being audited.
+ ///
+ private static string GetAuditRecordPrefix(Uri uri)
+ {
+ var parts = uri.PathAndQuery.Split('/');
+ return string.Join("/", parts.Where(p => !string.IsNullOrEmpty(p)).ToList().GetRange(0, parts.Length - 2).ToArray());
+ }
+
+ private static string GetAuditRecordPrefixFromPackage(PackageIdentity package)
+ {
+ return $"{package.Id.ToLowerInvariant()}/{package.Version.ToNormalizedString().ToLowerInvariant()}";
+ }
+
+ ///
+ /// Returns the file name of the audit record, which contains the the record was made as well as the type of record it is.
+ ///
+ private static string GetAuditRecordFileName(Uri uri)
+ {
+ var parts = uri.PathAndQuery.Split('/');
+ return parts.Length > 0 ? parts[parts.Length - 1] : null;
+ }
+
+ private static bool IsPackageDelete(StorageListItem auditRecord)
+ {
+ var fileName = GetAuditRecordFileName(auditRecord.Uri);
+ return FileNameSuffixes.Any(suffix => fileName.EndsWith(suffix));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/FeedHelpers.cs b/src/Catalog/Helpers/FeedHelpers.cs
new file mode 100644
index 000000000..28be296c2
--- /dev/null
+++ b/src/Catalog/Helpers/FeedHelpers.cs
@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net.Http;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Helper methods for accessing and parsing the gallery's V2 feed.
+ ///
+ public static class FeedHelpers
+ {
+ ///
+ /// Creates an HttpClient for reading the feed.
+ ///
+ public static HttpClient CreateHttpClient(Func handlerFunc)
+ {
+ var handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };
+ return new HttpClient(handler);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/FeedPackageDetails.cs b/src/Catalog/Helpers/FeedPackageDetails.cs
new file mode 100644
index 000000000..e165d72a5
--- /dev/null
+++ b/src/Catalog/Helpers/FeedPackageDetails.cs
@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public sealed class FeedPackageDetails
+ {
+ public Uri ContentUri { get; }
+ public DateTime CreatedDate { get; }
+ public DateTime LastEditedDate { get; }
+ public DateTime PublishedDate { get; }
+ public string PackageId { get; }
+ public string PackageNormalizedVersion { get; }
+ public string PackageFullVersion { get; }
+ public string LicenseNames { get; }
+ public string LicenseReportUrl { get; }
+ public bool RequiresLicenseAcceptance { get; }
+ public PackageDeprecationItem DeprecationInfo { get; }
+ public IList VulnerabilityInfo { get; private set; }
+
+ public bool HasDeprecationInfo => DeprecationInfo != null;
+
+ public FeedPackageDetails(
+ Uri contentUri,
+ DateTime createdDate,
+ DateTime lastEditedDate,
+ DateTime publishedDate,
+ string packageId,
+ string packageNormalizedVersion,
+ string packageFullVersion)
+ : this(
+ contentUri,
+ createdDate,
+ lastEditedDate,
+ publishedDate,
+ packageId,
+ packageNormalizedVersion,
+ packageFullVersion,
+ licenseNames: null,
+ licenseReportUrl: null,
+ deprecationInfo: null,
+ requiresLicenseAcceptance: false)
+ {
+ }
+
+ public FeedPackageDetails(
+ Uri contentUri,
+ DateTime createdDate,
+ DateTime lastEditedDate,
+ DateTime publishedDate,
+ string packageId,
+ string packageNormalizedVersion,
+ string packageFullVersion,
+ string licenseNames,
+ string licenseReportUrl,
+ PackageDeprecationItem deprecationInfo,
+ bool requiresLicenseAcceptance)
+ {
+ ContentUri = contentUri;
+ CreatedDate = createdDate;
+ LastEditedDate = lastEditedDate;
+ PublishedDate = publishedDate;
+ PackageId = packageId;
+ PackageNormalizedVersion = packageNormalizedVersion;
+ PackageFullVersion = packageFullVersion;
+ LicenseNames = licenseNames;
+ LicenseReportUrl = licenseReportUrl;
+ DeprecationInfo = deprecationInfo;
+ RequiresLicenseAcceptance = requiresLicenseAcceptance;
+ }
+
+ public void AddVulnerability(PackageVulnerabilityItem vulnerability)
+ {
+ VulnerabilityInfo = VulnerabilityInfo ?? new List();
+ VulnerabilityInfo.Add(vulnerability);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/FeedPackageIdentity.cs b/src/Catalog/Helpers/FeedPackageIdentity.cs
new file mode 100644
index 000000000..fac5cd5f4
--- /dev/null
+++ b/src/Catalog/Helpers/FeedPackageIdentity.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Newtonsoft.Json;
+using NuGet.Packaging.Core;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public class FeedPackageIdentity : IEquatable
+ {
+ [JsonProperty("id")]
+ public string Id { get; private set; }
+
+ [JsonProperty("version")]
+ public string Version { get; private set; }
+
+ [JsonConstructor]
+ public FeedPackageIdentity(string id, string version)
+ {
+ Id = id;
+ Version = version;
+ }
+
+ public FeedPackageIdentity(PackageIdentity package)
+ {
+ Id = package.Id;
+ Version = package.Version.ToFullString();
+ }
+
+ public bool Equals(FeedPackageIdentity other)
+ {
+ return Id.ToLowerInvariant() == other.Id.ToLowerInvariant() && Version.ToLowerInvariant() == other.Version.ToLowerInvariant();
+ }
+
+ public override int GetHashCode()
+ {
+ return Tuple.Create(Id.ToLowerInvariant(), Version.ToLowerInvariant()).GetHashCode();
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/GalleryDatabaseQueryService.cs b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs
new file mode 100644
index 000000000..b82898ae1
--- /dev/null
+++ b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs
@@ -0,0 +1,287 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.Common;
+using System.Data.SqlClient;
+using System.Linq;
+using System.Threading.Tasks;
+using NuGet.Services.Entities;
+using NuGet.Services.Sql;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Utility class for all SQL queries invoked by Db2Catalog.
+ ///
+ public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService
+ {
+ private const string CursorParameterName = "Cursor";
+ private const string PackageIdParameterName = "PackageId";
+ private const string PackageVersionParameterName = "PackageVersion";
+
+ // insertions are:
+ // {0} - inner SELECT type
+ // {1} - additional WHERE conditions and ORDER BY if needed for inner SELECT's TOP n
+ // {2} - outer ORDER BY if needed
+ private static readonly string Db2CatalogSqlSubQuery = @"SELECT
+ P_EXT.*,
+ PV.[GitHubDatabaseKey] AS '" + Db2CatalogProjectionColumnNames.VulnerabilityGitHubDatabaseKey + @"',
+ PV.[AdvisoryUrl] AS '" + Db2CatalogProjectionColumnNames.VulnerabilityAdvisoryUrl + @"',
+ PV.[Severity] AS '" + Db2CatalogProjectionColumnNames.VulnerabilitySeverity + @"'
+ FROM
+ ({0} P.[Key],
+ PR.[Id],
+ P.[NormalizedVersion],
+ P.[Version],
+ P.[Created],
+ P.[LastEdited],
+ P.[Published],
+ P.[Listed],
+ P.[HideLicenseReport],
+ P.[LicenseNames],
+ P.[LicenseReportUrl],
+ P.[RequiresLicenseAcceptance],
+ PD.[Status] AS '" + Db2CatalogProjectionColumnNames.DeprecationStatus + @"',
+ APR.[Id] AS '" + Db2CatalogProjectionColumnNames.AlternatePackageId + @"',
+ AP.[NormalizedVersion] AS '" + Db2CatalogProjectionColumnNames.AlternatePackageVersion + @"',
+ PD.[CustomMessage] AS '" + Db2CatalogProjectionColumnNames.DeprecationMessage + @"'
+ FROM [dbo].[Packages] AS P
+ INNER JOIN [dbo].[PackageRegistrations] AS PR ON P.[PackageRegistrationKey] = PR.[Key]
+ LEFT JOIN [dbo].[PackageDeprecations] AS PD ON PD.[PackageKey] = P.[Key]
+ LEFT JOIN [dbo].[Packages] AS AP ON AP.[Key] = PD.[AlternatePackageKey]
+ LEFT JOIN [dbo].[PackageRegistrations] AS APR ON APR.[Key] = ISNULL(AP.[PackageRegistrationKey], PD.[AlternatePackageRegistrationKey])
+ WHERE P.[PackageStatusKey] = 0
+ {1}
+ ) AS P_EXT
+ LEFT JOIN [dbo].[VulnerablePackageVersionRangePackages] AS VPVRP ON VPVRP.[Package_Key] = P_EXT.[Key]
+ LEFT JOIN [dbo].[VulnerablePackageVersionRanges] AS VPVR ON VPVR.[Key] = VPVRP.[VulnerablePackageVersionRange_Key]
+ LEFT JOIN [dbo].[PackageVulnerabilities] AS PV ON PV.[Key] = VPVR.[VulnerabilityKey]
+ {2}";
+
+ private readonly ISqlConnectionFactory _connectionFactory;
+ private readonly Db2CatalogProjection _db2catalogProjection;
+ private readonly ITelemetryService _telemetryService;
+ private readonly int _commandTimeout;
+
+ public GalleryDatabaseQueryService(
+ ISqlConnectionFactory connectionFactory,
+ PackageContentUriBuilder packageContentUriBuilder,
+ ITelemetryService telemetryService,
+ int commandTimeout)
+ {
+ _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
+ _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+ _db2catalogProjection = new Db2CatalogProjection(packageContentUriBuilder);
+ _commandTimeout = commandTimeout;
+ }
+
+ public Task>> GetPackagesCreatedSince(DateTime since, int top)
+ {
+ return GetPackagesInOrder(
+ package => package.CreatedDate,
+ Db2CatalogCursor.ByCreated(since, top));
+ }
+
+ public Task>> GetPackagesEditedSince(DateTime since, int top)
+ {
+ return GetPackagesInOrder(
+ package => package.LastEditedDate,
+ Db2CatalogCursor.ByLastEdited(since, top));
+ }
+
+ public async Task GetPackageOrNull(string id, string version)
+ {
+ if (id == null)
+ {
+ throw new ArgumentNullException(nameof(id));
+ }
+
+ if (version == null)
+ {
+ throw new ArgumentNullException(nameof(version));
+ }
+
+ var packages = new List();
+ var packageQuery = BuildGetPackageSqlQuery();
+
+ using (var sqlConnection = await _connectionFactory.OpenAsync())
+ {
+ using (var packagesCommand = new SqlCommand(packageQuery, sqlConnection)
+ {
+ CommandTimeout = _commandTimeout
+ })
+ {
+ packagesCommand.Parameters.AddWithValue(PackageIdParameterName, id);
+ packagesCommand.Parameters.AddWithValue(PackageVersionParameterName, version);
+
+ using (_telemetryService.TrackGetPackageQueryDuration(id, version))
+ {
+ return (await ReadPackagesAsync(packagesCommand)).SingleOrDefault();
+ }
+ }
+ }
+ }
+
+ ///
+ /// Returns a from the gallery database.
+ ///
+ /// The field to sort the on.
+ private async Task>> GetPackagesInOrder(
+ Func keyDateFunc,
+ Db2CatalogCursor cursor)
+ {
+ var allPackages = await GetPackages(cursor);
+
+ return OrderPackagesByKeyDate(allPackages, keyDateFunc);
+ }
+
+ ///
+ /// Returns a of packages.
+ ///
+ /// The field to sort the on.
+ internal static SortedList> OrderPackagesByKeyDate(
+ IReadOnlyCollection packages,
+ Func keyDateFunc)
+ {
+ var result = new SortedList>();
+
+ foreach (var package in packages)
+ {
+ var packageKeyDate = keyDateFunc(package);
+ if (!result.TryGetValue(packageKeyDate, out IList packagesWithSameKeyDate))
+ {
+ packagesWithSameKeyDate = new List();
+ result.Add(packageKeyDate, packagesWithSameKeyDate);
+ }
+
+ packagesWithSameKeyDate.Add(package);
+ }
+
+ var packagesCount = 0;
+ var filteredResult = new SortedList>();
+ foreach (var keyDate in result.Keys)
+ {
+ if (result.TryGetValue(keyDate, out IList packagesForKeyDate))
+ {
+ if (packagesCount > 0 && packagesCount + packagesForKeyDate.Count > Constants.MaxPageSize)
+ {
+ break;
+ }
+
+ packagesCount += packagesForKeyDate.Count;
+ filteredResult.Add(keyDate, packagesForKeyDate);
+ }
+ }
+
+ return filteredResult;
+ }
+
+ ///
+ /// Builds the SQL query string for db2catalog.
+ ///
+ /// The to be used.
+ /// The SQL query string for the db2catalog job, build from the the provided .
+ internal static string BuildDb2CatalogSqlQuery(Db2CatalogCursor cursor)
+ {
+ // We need to provide an inner ORDER BY to support the TOP clause
+ return string.Format(Db2CatalogSqlSubQuery,
+ $@"SELECT TOP {cursor.Top} WITH TIES ",
+ $@"AND P.[{cursor.ColumnName}] > @{CursorParameterName}
+ ORDER BY P.[{cursor.ColumnName}]",
+ $@"ORDER BY P_EXT.[{cursor.ColumnName}], P_EXT.[{Db2CatalogProjectionColumnNames.Key}]");
+ }
+
+ ///
+ /// Builds the parameterized SQL query for retrieving package details given an id and version.
+ ///
+ internal static string BuildGetPackageSqlQuery()
+ {
+ return string.Format(Db2CatalogSqlSubQuery,
+ "SELECT ",
+ "AND PR.[Id] = @PackageId AND P.[NormalizedVersion] = @PackageVersion",
+ "");
+ }
+
+ ///
+ /// Asynchronously gets a from the gallery database.
+ ///
+ /// Defines the cursor to be used.
+ /// A task that represents the asynchronous operation.
+ /// The task result () returns an
+ /// .
+ private async Task> GetPackages(Db2CatalogCursor cursor)
+ {
+ using (var sqlConnection = await _connectionFactory.OpenAsync())
+ {
+ return await GetPackageDetailsAsync(sqlConnection, cursor);
+ }
+ }
+
+ private async Task> GetPackageDetailsAsync(
+ SqlConnection sqlConnection,
+ Db2CatalogCursor cursor)
+ {
+ var packageQuery = BuildDb2CatalogSqlQuery(cursor);
+
+ using (var packagesCommand = new SqlCommand(packageQuery, sqlConnection)
+ {
+ CommandTimeout = _commandTimeout
+ })
+ {
+ packagesCommand.Parameters.AddWithValue(CursorParameterName, cursor.CursorValue);
+
+ using (_telemetryService.TrackGetPackageDetailsQueryDuration(cursor))
+ {
+ return await ReadPackagesAsync(packagesCommand);
+ }
+ }
+ }
+
+ private async Task> ReadPackagesAsync(SqlCommand packagesCommand)
+ {
+ var packages = new List();
+
+ using (var packagesReader = await packagesCommand.ExecuteReaderAsync())
+ {
+ // query has been ordered by package/version key, so we can use control break logic here to read it efficiently
+ // - we break on key change to find the end of a group of rows for which we must accumulate vulnerability data
+ if (await packagesReader.ReadAsync()) // priming read
+ {
+ var key = _db2catalogProjection.ReadPackageVersionKeyFromDataReader(packagesReader);
+ var readEnded = false;
+ do
+ {
+ var package = _db2catalogProjection.ReadFeedPackageDetailsFromDataReader(packagesReader);
+ packages.Add(package);
+
+ // loop through all rows with the same key, so we cover all vulnerabilities for package/version
+ var thisKey = key;
+ do
+ {
+ var vulnerability = _db2catalogProjection.ReadPackageVulnerabilityFromDataReader(packagesReader);
+ if (vulnerability != null)
+ {
+ package.AddVulnerability(vulnerability);
+ }
+
+ // read next, prime for control break
+ if (await packagesReader.ReadAsync())
+ {
+ key = _db2catalogProjection.ReadPackageVersionKeyFromDataReader(packagesReader);
+ }
+ else
+ {
+ readEnded = true;
+ }
+ } while (!readEnded && key == thisKey);
+ } while (!readEnded);
+ }
+ }
+
+ return packages;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/GraphLoading.cs b/src/Catalog/Helpers/GraphLoading.cs
new file mode 100644
index 000000000..9f5441fd8
--- /dev/null
+++ b/src/Catalog/Helpers/GraphLoading.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class GraphLoading
+ {
+ static async Task Load(Uri uri)
+ {
+ HttpClient client = new HttpClient();
+ string json = await client.GetStringAsync(uri);
+ return Utils.CreateGraph(json);
+ }
+
+ public static async Task Load(Uri root, IDictionary rules)
+ {
+ ISet resourceList = new HashSet();
+ resourceList.Add(root);
+
+ IGraph graph = await Load(root);
+
+ bool dirty = true;
+
+ while (dirty)
+ {
+ dirty = false;
+
+ foreach (KeyValuePair rule in rules)
+ {
+ foreach (Triple t1 in graph.GetTriplesWithPredicateObject(
+ graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")),
+ graph.CreateUriNode(new Uri(rule.Key))))
+ {
+ foreach (Triple t2 in graph.GetTriplesWithSubjectPredicate(
+ t1.Subject,
+ graph.CreateUriNode(new Uri(rule.Value))))
+ {
+ Uri next = ((IUriNode)t2.Object).Uri;
+
+ if (!resourceList.Contains(next))
+ {
+ IGraph nextGraph = await Load(next);
+ graph.Merge(nextGraph, true);
+
+ resourceList.Add(next);
+
+ dirty = true;
+ }
+ }
+ }
+ }
+ }
+
+ return graph;
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/GraphSplitting.cs b/src/Catalog/Helpers/GraphSplitting.cs
new file mode 100644
index 000000000..93289f9d3
--- /dev/null
+++ b/src/Catalog/Helpers/GraphSplitting.cs
@@ -0,0 +1,197 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class GraphSplitting
+ {
+ public static Uri GetPackageRegistrationUri(IGraph graph)
+ {
+ return ((IUriNode)graph.GetTriplesWithPredicateObject(
+ graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")),
+ graph.CreateUriNode(new Uri("http://schema.nuget.org/schema#PackageRegistration")))
+ .First().Subject).Uri;
+ }
+ public static IList GetResources(IGraph graph)
+ {
+ IList resources = new List();
+
+ INode rdfType = graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"));
+
+ Uri[] types = new Uri[]
+ {
+ //new Uri("http://schema.nuget.org/schema#PackageRegistration"),
+ new Uri("http://schema.nuget.org/schema#PackageList")
+ };
+
+ foreach (Uri type in types)
+ {
+ foreach (Triple triple in graph.GetTriplesWithPredicateObject(rdfType, graph.CreateUriNode(type)))
+ {
+ resources.Add(((IUriNode)triple.Subject).Uri);
+ }
+ }
+
+ return resources;
+ }
+
+ public static IGraph ReplaceResourceUris(IGraph original, IDictionary replacements)
+ {
+ IGraph modified = new Graph();
+ foreach (Triple triple in original.Triples)
+ {
+ Uri subjectUri;
+ if (!replacements.TryGetValue(triple.Subject.ToString(), out subjectUri))
+ {
+ subjectUri = ((IUriNode)triple.Subject).Uri;
+ }
+
+ INode subjectNode = modified.CreateUriNode(subjectUri);
+ INode predicateNode = triple.Predicate.CopyNode(modified);
+
+ INode objectNode;
+ if (triple.Object is IUriNode)
+ {
+ Uri objectUri;
+ if (!replacements.TryGetValue(triple.Object.ToString(), out objectUri))
+ {
+ objectUri = ((IUriNode)triple.Object).Uri;
+ }
+ objectNode = modified.CreateUriNode(objectUri);
+ }
+ else
+ {
+ objectNode = triple.Object.CopyNode(modified);
+ }
+
+ modified.Assert(subjectNode, predicateNode, objectNode);
+ }
+
+ return modified;
+ }
+ public static void Collect(IGraph source, INode subject, IGraph destination, ISet exclude)
+ {
+ foreach (Triple triple in source.GetTriplesWithSubject(subject))
+ {
+ destination.Assert(triple.CopyTriple(destination));
+
+ if (triple.Object is IUriNode && !exclude.Contains(((IUriNode)triple.Object).Uri.ToString()))
+ {
+ Collect(source, triple.Object, destination, exclude);
+ }
+ }
+ }
+
+ static Uri RebaseUri(Uri nodeUri, Uri sourceUri, Uri destinationUri)
+ {
+ if (nodeUri == sourceUri && nodeUri.ToString() != sourceUri.ToString())
+ {
+ return new Uri(destinationUri.ToString() + nodeUri.Fragment);
+ }
+ return nodeUri;
+ }
+ public static void Rebase(IGraph source, IGraph destination, Uri sourceUri, Uri destinationUri)
+ {
+ Uri modifiedDestinationUri = new Uri(destinationUri.ToString().Replace('#', '/'));
+
+ foreach (Triple triple in source.Triples)
+ {
+ Uri subjectUri;
+ if (triple.Subject.ToString() == destinationUri.ToString())
+ {
+ subjectUri = modifiedDestinationUri;
+ }
+ else
+ {
+ subjectUri = RebaseUri(((IUriNode)triple.Subject).Uri, sourceUri, modifiedDestinationUri);
+ }
+
+ INode subjectNode = destination.CreateUriNode(subjectUri);
+ INode predicateNode = triple.Predicate.CopyNode(destination);
+
+ INode objectNode;
+ if (triple.Object is IUriNode)
+ {
+ Uri objectUri = RebaseUri(((IUriNode)triple.Object).Uri, sourceUri, modifiedDestinationUri);
+ objectNode = destination.CreateUriNode(objectUri);
+ }
+ else
+ {
+ objectNode = triple.Object.CopyNode(destination);
+ }
+
+ destination.Assert(subjectNode, predicateNode, objectNode);
+ }
+ }
+
+ static IDictionary CreateReplacements(Uri originalUri, IList resources)
+ {
+ IDictionary replacements = new Dictionary();
+
+ foreach (Uri resource in resources)
+ {
+ if (resource == originalUri)
+ {
+ string oldUri = resource.ToString();
+ string newUri = oldUri.Replace(".json#", "/");
+
+ if (!newUri.EndsWith(".json"))
+ {
+ newUri += ".json";
+ }
+
+ replacements.Add(oldUri, new Uri(newUri));
+ }
+ }
+
+ return replacements;
+ }
+
+ public static IDictionary Split(Uri originalUri, IGraph originalGraph)
+ {
+ IList resources = GraphSplitting.GetResources(originalGraph);
+
+ IDictionary replacements = CreateReplacements(originalUri, resources);
+
+ ISet exclude = new HashSet();
+ exclude.Add(originalUri.ToString());
+
+ IGraph modified = GraphSplitting.ReplaceResourceUris(originalGraph, replacements);
+
+ IDictionary graphs = new Dictionary();
+
+ IGraph parent = new Graph();
+
+ foreach (Triple triple in modified.Triples)
+ {
+ triple.CopyTriple(parent);
+ parent.Assert(triple);
+ }
+
+ foreach (Uri uri in replacements.Values)
+ {
+ INode subject = modified.CreateUriNode(uri);
+
+ IGraph cutGraph = new Graph();
+ GraphSplitting.Collect(modified, subject, cutGraph, exclude);
+
+ foreach (Triple triple in cutGraph.Triples)
+ {
+ triple.CopyTriple(parent);
+ parent.Retract(triple);
+ }
+
+ IGraph rebasedGraph = new Graph();
+ GraphSplitting.Rebase(cutGraph, rebasedGraph, originalUri, uri);
+
+ graphs.Add(uri, rebasedGraph);
+ }
+
+ graphs.Add(originalUri, parent);
+
+ return graphs;
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs b/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs
new file mode 100644
index 000000000..cd204a348
--- /dev/null
+++ b/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs
@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public interface IGalleryDatabaseQueryService
+ {
+ Task GetPackageOrNull(string id, string version);
+ Task>> GetPackagesCreatedSince(DateTime since, int top);
+ Task>> GetPackagesEditedSince(DateTime since, int top);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/JsonSort.cs b/src/Catalog/Helpers/JsonSort.cs
new file mode 100644
index 000000000..0e147bebf
--- /dev/null
+++ b/src/Catalog/Helpers/JsonSort.cs
@@ -0,0 +1,110 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// This is a hacky attempt at organizing compacted json into a more visually appealing form.
+ ///
+ public class JsonSort : IComparer
+ {
+ ///
+ /// Order the json so arrays are at the bottom and single properties are at the top
+ ///
+ public static JObject OrderJson(JObject json)
+ {
+ JObject ordered = new JObject();
+
+ var children = json.Children().ToList();
+
+ children.Sort(new JsonSort());
+
+ foreach (var child in children)
+ {
+ ordered.Add(child);
+ }
+
+ return ordered;
+ }
+
+ public int Compare(JToken x, JToken y)
+ {
+ JProperty xProp = x as JProperty;
+ JProperty yProp = y as JProperty;
+
+ if (xProp != null && yProp == null)
+ {
+ return -1;
+ }
+
+ if (xProp == null && yProp != null)
+ {
+ return 1;
+ }
+
+ if (xProp != null && yProp != null)
+ {
+ if (xProp.Name.Equals("@id"))
+ {
+ return -1;
+ }
+
+ if (yProp.Name.Equals("@id"))
+ {
+ return 1;
+ }
+
+ if (xProp.Name.Equals("@type"))
+ {
+ return -1;
+ }
+
+ if (yProp.Name.Equals("@type"))
+ {
+ return 1;
+ }
+
+ if (xProp.Name.Equals("@context"))
+ {
+ return 1;
+ }
+
+ if (yProp.Name.Equals("@context"))
+ {
+ return -1;
+ }
+
+ JArray xValArray = xProp.Value as JArray;
+ JArray yValArray = yProp.Value as JArray;
+
+ if (xValArray == null && yValArray != null)
+ {
+ return -1;
+ }
+
+ if (xValArray != null && yValArray == null)
+ {
+ return 1;
+ }
+
+ if (xProp.Name.StartsWith("@") && !yProp.Name.StartsWith("@"))
+ {
+ return 1;
+ }
+
+ if (!xProp.Name.StartsWith("@") && yProp.Name.StartsWith("@"))
+ {
+ return -1;
+ }
+
+ return StringComparer.OrdinalIgnoreCase.Compare(xProp.Name, yProp.Name);
+ }
+
+ return 0;
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/LicenseHelper.cs b/src/Catalog/Helpers/LicenseHelper.cs
new file mode 100644
index 000000000..04566c93c
--- /dev/null
+++ b/src/Catalog/Helpers/LicenseHelper.cs
@@ -0,0 +1,30 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class LicenseHelper
+ {
+ ///
+ /// Generate the license url given package info and gallery base url.
+ ///
+ /// package Id
+ /// package version
+ /// url of gallery base address
+ /// The url of license in gallery
+ public static string GetGalleryLicenseUrl(string packageId, string packageVersion, Uri galleryBaseAddress)
+ {
+ if (galleryBaseAddress == null || string.IsNullOrWhiteSpace(packageId) || string.IsNullOrWhiteSpace(packageVersion))
+ {
+ return null;
+ }
+
+ var uriBuilder = new UriBuilder(galleryBaseAddress);
+ uriBuilder.Path = string.Join("/", new string[] { "packages", packageId, packageVersion, "license" });
+
+ return uriBuilder.Uri.AbsoluteUri;
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/NuGetVersionUtility.cs b/src/Catalog/Helpers/NuGetVersionUtility.cs
new file mode 100644
index 000000000..70722a886
--- /dev/null
+++ b/src/Catalog/Helpers/NuGetVersionUtility.cs
@@ -0,0 +1,43 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public static class NuGetVersionUtility
+ {
+ public static string NormalizeVersion(string version)
+ {
+ NuGetVersion parsedVersion;
+ if (!NuGetVersion.TryParse(version, out parsedVersion))
+ {
+ return version;
+ }
+
+ return parsedVersion.ToNormalizedString();
+ }
+
+ public static string NormalizeVersionRange(string versionRange, string defaultValue)
+ {
+ VersionRange parsedVersionRange;
+ if (!VersionRange.TryParse(versionRange, out parsedVersionRange))
+ {
+ return defaultValue;
+ }
+
+ return parsedVersionRange.ToNormalizedString();
+ }
+
+ public static string GetFullVersionString(string version)
+ {
+ NuGetVersion parsedVersion;
+ if (!NuGetVersion.TryParse(version, out parsedVersion))
+ {
+ return version;
+ }
+
+ return parsedVersion.ToFullString();
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/PackageContentUriBuilder.cs b/src/Catalog/Helpers/PackageContentUriBuilder.cs
new file mode 100644
index 000000000..b82e372ec
--- /dev/null
+++ b/src/Catalog/Helpers/PackageContentUriBuilder.cs
@@ -0,0 +1,56 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Utility class to validate package content URL format strings.
+ ///
+ public class PackageContentUriBuilder
+ {
+ public const string IdLowerPlaceholderString = "{id-lower}";
+ public const string VersionLowerPlaceholderString = "{version-lower}";
+
+ private const string NuPkgExtension = ".nupkg";
+ private readonly string _packageContentUrlFormat;
+
+ public PackageContentUriBuilder(string packageContentUrlFormat)
+ {
+ _packageContentUrlFormat = packageContentUrlFormat ?? throw new ArgumentNullException(nameof(packageContentUrlFormat));
+
+ if (!_packageContentUrlFormat.Contains(IdLowerPlaceholderString) || !_packageContentUrlFormat.Contains(VersionLowerPlaceholderString))
+ {
+ throw new ArgumentException(
+ $"The package content URL format must contain the following placeholders to be valid: {IdLowerPlaceholderString}, {VersionLowerPlaceholderString}. " +
+ "(e.g. https://storageaccountname.blob.core.windows.net/packages/{id-lower}.{version-lower}.nupkg)",
+ nameof(packageContentUrlFormat));
+ }
+ else if (!_packageContentUrlFormat.EndsWith(NuPkgExtension))
+ {
+ throw new ArgumentException(
+ $"The package content URL format must point to files with the {NuPkgExtension} extension.",
+ nameof(packageContentUrlFormat));
+ }
+ }
+
+ public Uri Build(string packageId, string normalizedPackageVersion)
+ {
+ if (packageId == null)
+ {
+ throw new ArgumentNullException(nameof(packageId));
+ }
+
+ if (normalizedPackageVersion == null)
+ {
+ throw new ArgumentNullException(nameof(normalizedPackageVersion));
+ }
+
+ return new Uri(
+ _packageContentUrlFormat
+ .Replace(IdLowerPlaceholderString, packageId.ToLowerInvariant())
+ .Replace(VersionLowerPlaceholderString, normalizedPackageVersion.ToLowerInvariant()));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/PackageUtility.cs b/src/Catalog/Helpers/PackageUtility.cs
new file mode 100644
index 000000000..22cf8dd8c
--- /dev/null
+++ b/src/Catalog/Helpers/PackageUtility.cs
@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public static class PackageUtility
+ {
+ public static string GetPackageFileName(string packageId, string packageVersion)
+ {
+ if (string.IsNullOrEmpty(packageId))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId));
+ }
+
+ if (string.IsNullOrEmpty(packageVersion))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageVersion));
+ }
+
+ return $"{packageId}.{packageVersion}.nupkg";
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/ParallelAsync.cs b/src/Catalog/Helpers/ParallelAsync.cs
new file mode 100644
index 000000000..6dd851bfb
--- /dev/null
+++ b/src/Catalog/Helpers/ParallelAsync.cs
@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public static class ParallelAsync
+ {
+ ///
+ /// Creates a number of tasks specified by using and then runs them in parallel.
+ ///
+ /// Creates each task to run.
+ /// The number of tasks to create. Defaults to
+ /// A task that completes when all tasks have completed.
+ public static Task Repeat(Func taskFactory, int? degreeOfParallelism = null)
+ {
+ return Task.WhenAll(
+ Enumerable
+ .Repeat(taskFactory, degreeOfParallelism ?? ServicePointManager.DefaultConnectionLimit)
+ .Select(f => f()));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/Retry.cs b/src/Catalog/Helpers/Retry.cs
new file mode 100644
index 000000000..614a5b76d
--- /dev/null
+++ b/src/Catalog/Helpers/Retry.cs
@@ -0,0 +1,84 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ ///
+ /// Can (and probably should) be replaced with Polly library if the project is updated to target .netfx 4.7.2.
+ /// In current state Polly pulls a ton of System.* dependencies which we previously didn't have.
+ ///
+ public class Retry
+ {
+ ///
+ /// Retries async operation if it throws with delays between attempts.
+ ///
+ /// Operation to try.
+ /// Exception predicate. If it returns false, the exception will propagate to the caller.
+ /// Max number of attempts to make.
+ /// Delay after the first failure.
+ /// Delay increment for subsequent attempts.
+ public static async Task IncrementalAsync(
+ Func runLogicAsync,
+ Func shouldRetryOnException,
+ int maxRetries,
+ TimeSpan initialWaitInterval,
+ TimeSpan waitIncrement)
+ {
+ for (int currentRetry = 0; currentRetry < maxRetries; ++currentRetry)
+ {
+ try
+ {
+ await runLogicAsync();
+ return;
+ }
+ catch (Exception e) when (currentRetry < maxRetries - 1 && shouldRetryOnException(e))
+ {
+ await Task.Delay(initialWaitInterval + TimeSpan.FromSeconds(waitIncrement.TotalSeconds * currentRetry));
+ }
+ }
+ }
+
+ ///
+ /// Retries async operation if it throws or returns certain result with delays between attempts.
+ ///
+ /// Attempted operation result type.
+ /// Operation to try.
+ /// Exception predicate. If it returns false, the exception will propagate to the caller.
+ /// Result predicate. If returns true, the result will be discarded and operation retried.
+ /// Max number of attempts to make.
+ /// Delay after the first failure.
+ /// Delay increment for subsequent attempts.
+ /// The result of () call if predicate fails.
+ /// default() if suceeded for all attempts.
+ public static async Task IncrementalAsync(
+ Func> runLogicAsync,
+ Func shouldRetryOnException,
+ Func shouldRetry,
+ int maxRetries,
+ TimeSpan initialWaitInterval,
+ TimeSpan waitIncrement)
+ {
+ var result = default(TResult);
+ for (int currentRetry = 0; currentRetry < maxRetries; ++currentRetry)
+ {
+ try
+ {
+ result = await runLogicAsync();
+ if (!shouldRetry(result))
+ {
+ return result;
+ }
+ }
+ catch (Exception e) when (currentRetry < maxRetries - 1 && shouldRetryOnException(e))
+ {
+ await Task.Delay(initialWaitInterval + TimeSpan.FromSeconds(waitIncrement.TotalSeconds * currentRetry));
+ }
+ }
+
+ return result;
+ }
+ }
+}
diff --git a/src/Catalog/Helpers/UriUtils.cs b/src/Catalog/Helpers/UriUtils.cs
new file mode 100644
index 000000000..f27a368b2
--- /dev/null
+++ b/src/Catalog/Helpers/UriUtils.cs
@@ -0,0 +1,113 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+ public static class UriUtils
+ {
+ private const char QueryStartCharacter = '?';
+ private const char BridgingCharacter = '&';
+
+ private const string Filter = "$filter=";
+ private const string NonhijackableFilter = "true";
+ private const string And = "%20and%20";
+
+ private const string OrderBy = "$orderby=";
+ private const string NonhijackableOrderBy = "Version";
+
+ private const string GetSpecificPackageFormatRegExp =
+ @"Packages\(Id='(?[^']*)',Version='(?[^']*)'\)";
+
+ ///
+ /// The format of a nonhijacked "Packages(Id='...',Version='...')" request.
+ ///
+ ///
+ /// Note that we are using "normalized version" here.
+ /// This is because a hijacked request does a comparison on the normalized version, but the nonhijacked request does not.
+ /// Additionally, we must add the "semVerLevel=2.0.0" or else this will not work for SemVer2 packages.
+ ///
+ private static string GetSpecificPackageNonhijackable =
+ $"Packages?{Filter}{NonhijackableFilter}{And}" +
+ "Id eq '{0}'" + $"{And}" + "NormalizedVersion eq '{1}'&semVerLevel=2.0.0";
+
+ private static IEnumerable HijackableEndpoints = new List
+ {
+ "/Packages",
+ "/Search",
+ "/FindPackagesById"
+ };
+
+ public static Uri GetNonhijackableUri(Uri originalUri)
+ {
+ var nonhijackableUri = originalUri;
+
+ var originalUriString = originalUri.ToString();
+ string nonhijackableUriString = null;
+
+ // Modify the request uri so that it will not be hijacked by the search service.
+
+ // This can be done in two ways:
+ /// 1 - convert the query into a "Packages" query with filter that cannot be hijacked ().
+ /// 2 - specify an orderby that cannot be hijacked ().
+ if (originalUriString.Contains(OrderBy))
+ {
+ // If there is an orderby on the request, simply replace the orderby with a nonhijackable orderby.
+ var orderByStartIndex = originalUriString.IndexOf(OrderBy);
+
+ /// Find the start of the next parameter () or the end of the query.
+ var orderByEndIndex = originalUriString.IndexOf(BridgingCharacter, orderByStartIndex);
+ if (orderByEndIndex == -1)
+ {
+ orderByEndIndex = originalUriString.Length;
+ }
+
+ // Replace the entire orderby with a nonhijackable orderby.
+ var orderByExpression = originalUriString.Substring(orderByStartIndex, orderByEndIndex - orderByStartIndex);
+ nonhijackableUriString = originalUriString.Replace(orderByExpression, OrderBy + NonhijackableOrderBy);
+ }
+ else
+ {
+ // If this is a Packages(Id='...',Version='...') request, rewrite it as a request to Packages() with a filter and add the expression.
+ // Note that Packages() returns a feed and Packages(Id='...',Version='...') returns an entry, but this is fine because the client reads both the same.
+ var getSpecificPackageMatch = Regex.Match(originalUriString, GetSpecificPackageFormatRegExp);
+ if (getSpecificPackageMatch.Success)
+ {
+ nonhijackableUriString =
+ originalUriString.Substring(0, getSpecificPackageMatch.Index) +
+ string.Format(
+ GetSpecificPackageNonhijackable,
+ getSpecificPackageMatch.Groups["Id"].Value,
+ NuGetVersion.Parse(getSpecificPackageMatch.Groups["Version"].Value).ToNormalizedString());
+ }
+ else
+ {
+ // If this is a request to a hijackable endpoint without an orderby, add the orderby to the request.
+ if (HijackableEndpoints.Any(endpoint => originalUriString.Contains(endpoint)))
+ {
+ var bridgingCharacter = BridgingCharacter;
+
+ if (!originalUriString.Contains(QueryStartCharacter))
+ {
+ bridgingCharacter = QueryStartCharacter;
+ }
+
+ nonhijackableUriString = $"{originalUri}{bridgingCharacter}{OrderBy}{NonhijackableOrderBy}";
+ }
+ }
+ }
+
+ if (nonhijackableUriString != null)
+ {
+ nonhijackableUri = new Uri(nonhijackableUriString);
+ }
+
+ return nonhijackableUri;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/Utils.cs b/src/Catalog/Helpers/Utils.cs
new file mode 100644
index 000000000..a26ad14a5
--- /dev/null
+++ b/src/Catalog/Helpers/Utils.cs
@@ -0,0 +1,447 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Reflection;
+using System.Security.Cryptography;
+using System.Text;
+using System.Xml;
+using System.Xml.Linq;
+using System.Xml.Xsl;
+using JsonLD.Core;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Helpers;
+using NuGet.Services.Metadata.Catalog.JsonLDIntegration;
+using VDS.RDF;
+using VDS.RDF.Parsing;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public static class Utils
+ {
+ private const string XslTransformNuSpec = "xslt.nuspec.xslt";
+ private const string XslTransformNormalizeNuSpecNamespace = "xslt.normalizeNuspecNamespace.xslt";
+
+ private static readonly Lazy XslTransformNuSpecCache = new Lazy(() => SafeLoadXslTransform(XslTransformNuSpec));
+ private static readonly Lazy XslTransformNormalizeNuSpecNamespaceCache = new Lazy(() => SafeLoadXslTransform(XslTransformNormalizeNuSpecNamespace));
+
+ private static readonly char[] TagTrimChars = { ',', ' ', '\t', '|', ';' };
+
+ public static string[] SplitTags(string original)
+ {
+ var fields = original
+ .Split(TagTrimChars)
+ .Select(w => w.Trim(TagTrimChars))
+ .Where(w => w.Length > 0)
+ .ToArray();
+
+ return fields;
+ }
+
+ public static Stream GetResourceStream(string resourceName)
+ {
+ if (string.IsNullOrEmpty(resourceName))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(resourceName));
+ }
+
+ var assembly = Assembly.GetExecutingAssembly();
+
+ string name = assembly.GetName().Name;
+
+ return assembly.GetManifestResourceStream($"{name}.{resourceName}");
+ }
+
+ public static IGraph CreateNuspecGraph(XDocument nuspec, string baseAddress, bool normalizeXml = false)
+ {
+ XsltArgumentList arguments = new XsltArgumentList();
+ arguments.AddParam("base", "", baseAddress);
+ arguments.AddParam("extension", "", ".json");
+
+ arguments.AddExtensionObject("urn:helper", new XsltHelper());
+
+ nuspec = SafeXmlTransform(nuspec.CreateReader(), XslTransformNormalizeNuSpecNamespaceCache.Value);
+ var rdfxml = SafeXmlTransform(nuspec.CreateReader(), XslTransformNuSpecCache.Value, arguments);
+
+ var doc = SafeCreateXmlDocument(rdfxml.CreateReader());
+ if (normalizeXml)
+ {
+ NormalizeXml(doc);
+ }
+
+ RdfXmlParser rdfXmlParser = new RdfXmlParser();
+ IGraph graph = new Graph();
+ rdfXmlParser.Load(graph, doc);
+
+ return graph;
+ }
+
+ private static void NormalizeXml(XmlNode xmlNode)
+ {
+ if (xmlNode.Attributes != null)
+ {
+ foreach (XmlAttribute attribute in xmlNode.Attributes)
+ {
+ attribute.Value = attribute.Value.Normalize(NormalizationForm.FormC);
+ }
+ }
+
+ if (xmlNode.Value != null)
+ {
+ xmlNode.Value = xmlNode.Value.Normalize(NormalizationForm.FormC);
+ return;
+ }
+
+ foreach (XmlNode childNode in xmlNode.ChildNodes)
+ {
+ NormalizeXml(childNode);
+ }
+ }
+
+ internal static XmlDocument SafeCreateXmlDocument(XmlReader reader = null)
+ {
+ // CodeAnalysis / XmlDocument: set the resolver to null or instance
+ var xmlDoc = new XmlDocument();
+ xmlDoc.XmlResolver = null;
+
+ if (reader != null)
+ {
+ xmlDoc.Load(reader);
+ }
+
+ return xmlDoc;
+ }
+
+ private static XDocument SafeXmlTransform(XmlReader reader, XslCompiledTransform transform, XsltArgumentList arguments = null)
+ {
+ XDocument result = new XDocument();
+ using (XmlWriter writer = result.CreateWriter())
+ {
+ if (arguments == null)
+ {
+ arguments = new XsltArgumentList();
+ }
+
+ // CodeAnalysis / XslCompiledTransform.Transform: set resolver property to null or instance
+ transform.Transform(reader, arguments, writer, documentResolver: null);
+ }
+ return result;
+ }
+
+ private static XslCompiledTransform SafeLoadXslTransform(string resourceName)
+ {
+ var transform = new XslCompiledTransform();
+
+ // CodeAnalysis / XmlReader.Create: provide settings instance and set resolver property to null or instance
+ var settings = new XmlReaderSettings();
+ settings.XmlResolver = null;
+
+ var reader = XmlReader.Create(new StreamReader(GetResourceStream(resourceName)), settings);
+
+ // CodeAnalysis / XslCompiledTransform.Load: specify default settings or set resolver property to null or instance
+ transform.Load(reader, XsltSettings.Default, stylesheetResolver: null);
+ return transform;
+ }
+
+ public static XDocument GetNuspec(ZipArchive package)
+ {
+ if (package == null) { return null; }
+
+ foreach (ZipArchiveEntry part in package.Entries)
+ {
+ if (part.FullName.EndsWith(".nuspec") && part.FullName.IndexOf('/') == -1)
+ {
+ XDocument nuspec = XDocument.Load(part.Open());
+ return nuspec;
+ }
+ }
+ return null;
+ }
+
+ public static JToken CreateJson(IGraph graph, JToken frame = null)
+ {
+ System.IO.StringWriter writer = new System.IO.StringWriter();
+ IRdfWriter rdfWriter = new JsonLdWriter();
+ rdfWriter.Save(graph, writer);
+ writer.Flush();
+
+ if (frame == null)
+ {
+ return JToken.Parse(writer.ToString());
+ }
+ else
+ {
+ JToken flattened = JToken.Parse(writer.ToString());
+ JObject framed = JsonLdProcessor.Frame(flattened, frame, new JsonLdOptions());
+ JObject compacted = JsonLdProcessor.Compact(framed, frame["@context"], new JsonLdOptions());
+
+ return JsonSort.OrderJson(compacted);
+ }
+ }
+
+ public static string CreateArrangedJson(IGraph graph, JToken frame = null)
+ {
+ System.IO.StringWriter writer = new System.IO.StringWriter();
+ IRdfWriter rdfWriter = new JsonLdWriter();
+ rdfWriter.Save(graph, writer);
+ writer.Flush();
+
+ if (frame == null)
+ {
+ return writer.ToString();
+ }
+ else
+ {
+ JToken flattened = JToken.Parse(writer.ToString());
+ JObject framed = JsonLdProcessor.Frame(flattened, frame, new JsonLdOptions());
+ JObject compacted = JsonLdProcessor.Compact(framed, frame["@context"], new JsonLdOptions());
+
+ var arranged = JsonSort.OrderJson(compacted);
+
+ return arranged.ToString();
+ }
+ }
+
+ public static IGraph CreateGraph(Uri resourceUri, string json)
+ {
+ if (json == null)
+ {
+ return null;
+ }
+
+ try
+ {
+ JToken compacted = JToken.Parse(json);
+ return CreateGraph(compacted, readOnly: false);
+ }
+ catch (JsonException e)
+ {
+ Trace.TraceError("Exception: failed to parse {0} {1}", resourceUri, e);
+ throw;
+ }
+ }
+
+ public static IGraph CreateGraph(JToken compacted, bool readOnly)
+ {
+ JToken flattened = JsonLdProcessor.Flatten(compacted, new JsonLdOptions());
+
+ IRdfReader rdfReader = new JsonLdReader();
+ IGraph graph = new Graph();
+ rdfReader.Load(graph, new StringReader(flattened.ToString(Newtonsoft.Json.Formatting.None, new Newtonsoft.Json.JsonConverter[0])));
+
+ if (readOnly)
+ {
+ graph = new ReadOnlyGraph(graph);
+ }
+
+ return graph;
+ }
+
+ public static bool IsCatalogNode(INode sourceNode, IGraph source)
+ {
+ Triple rootTriple = source.GetTriplesWithSubjectObject(sourceNode, source.CreateUriNode(Schema.DataTypes.CatalogRoot)).FirstOrDefault();
+ Triple pageTriple = source.GetTriplesWithSubjectObject(sourceNode, source.CreateUriNode(Schema.DataTypes.CatalogPage)).FirstOrDefault();
+
+ return (rootTriple != null || pageTriple != null);
+ }
+
+ public static void CopyCatalogContentGraph(INode sourceNode, IGraph source, IGraph target)
+ {
+ if (IsCatalogNode(sourceNode, source))
+ {
+ return;
+ }
+
+ foreach (Triple triple in source.GetTriplesWithSubject(sourceNode))
+ {
+ if (target.Assert(triple.CopyTriple(target)) && triple.Object is IUriNode)
+ {
+ CopyCatalogContentGraph(triple.Object, source, target);
+ }
+ }
+ }
+
+ public static Uri Expand(JToken context, string term)
+ {
+ if (term.StartsWith("http:", StringComparison.OrdinalIgnoreCase))
+ {
+ return new Uri(term);
+ }
+
+ int indexOf = term.IndexOf(':');
+ if (indexOf > 0)
+ {
+ string ns = term.Substring(0, indexOf);
+ return new Uri(context[ns].ToString() + term.Substring(indexOf + 1));
+ }
+
+ return new Uri(context["@vocab"] + term);
+ }
+
+ // where the property exists on the graph being merged in remove it from the existing graph
+ public static void RemoveExistingProperties(IGraph existingGraph, IGraph graphToMerge, Uri[] properties)
+ {
+ foreach (Uri property in properties)
+ {
+ foreach (Triple t1 in graphToMerge.GetTriplesWithPredicate(graphToMerge.CreateUriNode(property)))
+ {
+ INode subject = t1.Subject.CopyNode(existingGraph);
+ INode predicate = t1.Predicate.CopyNode(existingGraph);
+
+ IList retractList = new List(existingGraph.GetTriplesWithSubjectPredicate(subject, predicate));
+ foreach (Triple t2 in retractList)
+ {
+ existingGraph.Retract(t2);
+ }
+ }
+ }
+ }
+
+ public static string GenerateHash(Stream stream)
+ {
+ stream.Seek(0, SeekOrigin.Begin);
+
+ using (var hashAlgorithm = HashAlgorithm.Create(Constants.Sha512))
+ {
+ return Convert.ToBase64String(hashAlgorithm.ComputeHash(stream));
+ }
+ }
+
+ public static IEnumerable GetEntries(ZipArchive package)
+ {
+ IList result = new List();
+
+ foreach (ZipArchiveEntry entry in package.Entries)
+ {
+ if (entry.FullName.EndsWith("/.rels", StringComparison.OrdinalIgnoreCase))
+ {
+ continue;
+ }
+
+ if (entry.FullName.EndsWith("[Content_Types].xml", StringComparison.OrdinalIgnoreCase))
+ {
+ continue;
+ }
+
+ if (entry.FullName.EndsWith(".psmdcp", StringComparison.OrdinalIgnoreCase))
+ {
+ continue;
+ }
+
+ result.Add(new PackageEntry(entry));
+ }
+
+ return result;
+ }
+
+ public static NupkgMetadata GetNupkgMetadata(Stream stream, string packageHash)
+ {
+ if (stream == null)
+ {
+ throw new ArgumentNullException(nameof(stream));
+ }
+
+ var packageSize = stream.Length;
+
+ packageHash = packageHash ?? GenerateHash(stream);
+
+ stream.Seek(0, SeekOrigin.Begin);
+
+ using (var package = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true))
+ {
+ var nuspec = GetNuspec(package);
+
+ if (nuspec == null)
+ {
+ throw new InvalidDataException("Unable to find nuspec");
+ }
+
+ var entries = GetEntries(package);
+
+ return new NupkgMetadata(nuspec, entries, packageSize, packageHash);
+ }
+ }
+
+ public static PackageCatalogItem CreateCatalogItem(
+ string origin,
+ Stream stream,
+ DateTime createdDate,
+ DateTime? lastEditedDate = null,
+ DateTime? publishedDate = null,
+ string licenseNames = null,
+ string licenseReportUrl = null,
+ string packageHash = null,
+ PackageDeprecationItem deprecationItem = null,
+ IList vulnerabilities = null)
+ {
+ try
+ {
+ NupkgMetadata nupkgMetadata = GetNupkgMetadata(stream, packageHash);
+ return new PackageCatalogItem(
+ nupkgMetadata,
+ createdDate,
+ lastEditedDate,
+ publishedDate,
+ deprecation: deprecationItem,
+ vulnerabilities: vulnerabilities);
+ }
+ catch (InvalidDataException e)
+ {
+ Trace.TraceError("Exception: {0} {1} {2}", origin, e.GetType().Name, e);
+ return null;
+ }
+ catch (Exception e)
+ {
+ throw new Exception(string.Format("Exception processsing {0}", origin), e);
+ }
+ }
+
+ public static void TraceException(Exception e)
+ {
+ if (e is AggregateException)
+ {
+ foreach (Exception ex in ((AggregateException)e).InnerExceptions)
+ {
+ TraceException(ex);
+ }
+ }
+ else
+ {
+ Trace.TraceError("{0} {1}", e.GetType().Name, e.Message);
+ Trace.TraceError("{0}", e.StackTrace);
+
+ if (e.InnerException != null)
+ {
+ TraceException(e.InnerException);
+ }
+ }
+ }
+
+ internal static T Deserialize(JObject jObject, string propertyName)
+ {
+ if (jObject == null)
+ {
+ throw new ArgumentNullException(nameof(jObject));
+ }
+
+ if (string.IsNullOrEmpty(propertyName))
+ {
+ throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(propertyName));
+ }
+
+ if (!jObject.TryGetValue(propertyName, out var value) || value == null)
+ {
+ throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, Strings.PropertyRequired, propertyName));
+ }
+
+ return value.ToObject();
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/Helpers/XsltHelper.cs b/src/Catalog/Helpers/XsltHelper.cs
new file mode 100644
index 000000000..819016491
--- /dev/null
+++ b/src/Catalog/Helpers/XsltHelper.cs
@@ -0,0 +1,68 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Xml;
+using System.Xml.XPath;
+using NuGet.Services.Metadata.Catalog.Helpers;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class XsltHelper
+ {
+ ///
+ /// Default to an empty string if the dependency version range is invalid or missing. This is meant to be a
+ /// predictable signal to the client that they need to handle this invalid version case. The official NuGet
+ /// client treats this as a dependency of any version.
+ ///
+ private static readonly string DefaultVersionRange = string.Empty;
+
+ public XPathNavigator Split(string original)
+ {
+ var fields = Utils.SplitTags(original);
+
+ XmlDocument xmlDoc = Utils.SafeCreateXmlDocument();
+ XmlElement root = xmlDoc.CreateElement("list");
+ xmlDoc.AppendChild(root);
+
+ foreach (string s in fields)
+ {
+ XmlElement element = xmlDoc.CreateElement("item");
+ element.InnerText = s;
+ root.AppendChild(element);
+ }
+
+ return xmlDoc.CreateNavigator();
+ }
+
+ public string LowerCase(string original)
+ {
+ return original.ToLowerInvariant();
+ }
+
+ public string NormalizeVersion(string original)
+ {
+ return NuGetVersionUtility.NormalizeVersion(original);
+ }
+
+ public string GetFullVersionString(string original)
+ {
+ return NuGetVersionUtility.GetFullVersionString(original);
+ }
+
+ public string NormalizeVersionRange(string original)
+ {
+ return NuGetVersionUtility.NormalizeVersionRange(original, DefaultVersionRange);
+ }
+
+ public string IsPrerelease(string original)
+ {
+ NuGetVersion nugetVersion;
+ if (NuGetVersion.TryParse(original, out nugetVersion))
+ {
+ return nugetVersion.IsPrerelease ? "true" : "false";
+ }
+ return "true";
+ }
+ }
+}
diff --git a/src/Catalog/HttpReadCursor.cs b/src/Catalog/HttpReadCursor.cs
new file mode 100644
index 000000000..b7202d8b6
--- /dev/null
+++ b/src/Catalog/HttpReadCursor.cs
@@ -0,0 +1,70 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Helpers;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public class HttpReadCursor : ReadCursor
+ {
+ private readonly Uri _address;
+ private readonly DateTime? _defaultValue;
+ private readonly Func _handlerFunc;
+
+ public HttpReadCursor(Uri address, DateTime defaultValue, Func handlerFunc = null)
+ {
+ _address = address;
+ _defaultValue = defaultValue;
+ _handlerFunc = handlerFunc;
+ }
+
+ public HttpReadCursor(Uri address, Func handlerFunc = null)
+ {
+ _address = address;
+ _defaultValue = null;
+ _handlerFunc = handlerFunc;
+ }
+
+ public override async Task LoadAsync(CancellationToken cancellationToken)
+ {
+ await Retry.IncrementalAsync(
+ async () =>
+ {
+ HttpMessageHandler handler = (_handlerFunc != null) ? _handlerFunc() : new WebRequestHandler { AllowPipelining = true };
+
+ using (HttpClient client = new HttpClient(handler))
+ using (HttpResponseMessage response = await client.GetAsync(_address, cancellationToken))
+ {
+ Trace.TraceInformation("HttpReadCursor.Load {0} {1}", response.StatusCode, _address.AbsoluteUri);
+
+ if (_defaultValue != null && response.StatusCode == HttpStatusCode.NotFound)
+ {
+ Value = _defaultValue.Value;
+ }
+ else
+ {
+ response.EnsureSuccessStatusCode();
+
+ string json = await response.Content.ReadAsStringAsync();
+
+ JObject obj = JObject.Parse(json);
+ Value = obj["value"].ToObject();
+ }
+ }
+
+ Trace.TraceInformation("HttpReadCursor.Load: {0} {1}", this, _address.AbsoluteUri);
+ },
+ ex => ex is HttpRequestException || ex is TaskCanceledException,
+ maxRetries: 5,
+ initialWaitInterval: TimeSpan.Zero,
+ waitIncrement: TimeSpan.FromSeconds(10));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/ICatalogGraphPersistence.cs b/src/Catalog/ICatalogGraphPersistence.cs
new file mode 100644
index 000000000..119b84c6b
--- /dev/null
+++ b/src/Catalog/ICatalogGraphPersistence.cs
@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public interface ICatalogGraphPersistence
+ {
+ Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken);
+ Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken);
+ Uri CreatePageUri(Uri baseAddress, string relativeAddress);
+ }
+}
diff --git a/src/Catalog/ICatalogIndexProcessor.cs b/src/Catalog/ICatalogIndexProcessor.cs
new file mode 100644
index 000000000..76fa46319
--- /dev/null
+++ b/src/Catalog/ICatalogIndexProcessor.cs
@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///
+ /// A processor that runs on entries found on a catalog page.
+ /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page
+ ///
+ public interface ICatalogIndexProcessor
+ {
+ ///
+ /// Process a single entry from a catalog page.
+ ///
+ /// The catalog index entry that should be processed.
+ /// A task that completes once the entry has been processed.
+ Task ProcessCatalogIndexEntryAsync(CatalogIndexEntry catalogEntry);
+ }
+}
diff --git a/src/Catalog/IHttpRetryStrategy.cs b/src/Catalog/IHttpRetryStrategy.cs
new file mode 100644
index 000000000..ea03c3858
--- /dev/null
+++ b/src/Catalog/IHttpRetryStrategy.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public interface IHttpRetryStrategy
+ {
+ Task SendAsync(HttpClient client, Uri address, CancellationToken cancellationToken);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/IPackageCatalogItemCreator.cs b/src/Catalog/IPackageCatalogItemCreator.cs
new file mode 100644
index 000000000..21b247556
--- /dev/null
+++ b/src/Catalog/IPackageCatalogItemCreator.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NuGet.Services.Metadata.Catalog.Helpers;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ public interface IPackageCatalogItemCreator
+ {
+ Task CreateAsync(FeedPackageDetails packageItem, DateTime timestamp, CancellationToken cancellationToken);
+ }
+}
\ No newline at end of file
diff --git a/src/Catalog/IPackagesContainerHandler.cs b/src/Catalog/IPackagesContainerHandler.cs
new file mode 100644
index 000000000..f1a689c45
--- /dev/null
+++ b/src/Catalog/IPackagesContainerHandler.cs
@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+ ///