diff --git a/build.ps1 b/build.ps1 index c45f61e27..1baf81bbb 100644 --- a/build.ps1 +++ b/build.ps1 @@ -127,7 +127,6 @@ Invoke-BuildStep 'Creating artifacts' { "src\Ng\Catalog2Dnx.nuspec", ` "src\Ng\Catalog2icon.nuspec", ` "src\Ng\Catalog2Monitoring.nuspec", ` - "src\Ng\Catalog2Registration.nuspec", ` "src\Ng\Db2Catalog.nuspec", ` "src\Ng\Db2Monitoring.nuspec", ` "src\Ng\Monitoring2Monitoring.nuspec", ` diff --git a/src/Catalog/FileSystemEmulatorHandler.cs b/src/Catalog/FileSystemEmulatorHandler.cs deleted file mode 100644 index 4c466f0b3..000000000 --- a/src/Catalog/FileSystemEmulatorHandler.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace NuGet.Services.Metadata.Catalog -{ - public class FileSystemEmulatorHandler : DelegatingHandler - { - private Uri _rootUrl; - private string _rootNormalized; - - public FileSystemEmulatorHandler() : base() { } - public FileSystemEmulatorHandler(HttpMessageHandler innerHandler) : base(innerHandler) { } - - public string RootFolder { get; set; } - public Uri BaseAddress - { - get - { return _rootUrl; } - set - { - _rootUrl = value; - _rootNormalized = GetNormalizedUrl(_rootUrl); - } - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - // Compare the URL - string requestNormalized = GetNormalizedUrl(request.RequestUri); - if (requestNormalized.StartsWith(_rootNormalized)) - { - var relative = requestNormalized.Substring(_rootNormalized.Length); - return Intercept(relative, request, cancellationToken); - } - else - { - return base.SendAsync(request, cancellationToken); - } - } - - private Task Intercept(string relative, HttpRequestMessage request, CancellationToken cancellationToken) - { - string path = Path.Combine( - RootFolder, - relative.Trim('/').Replace('/', Path.DirectorySeparatorChar)); - if (File.Exists(path)) - { - // The framework will dispose of the original stream after "transferring" it - - string content; - using (StreamReader reader = new StreamReader(path)) - { - content = reader.ReadToEnd(); - } - - HttpContent httpContent = new StringContent(content); - - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = httpContent - }); - } - else - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); - } - } - - private static string GetNormalizedUrl(Uri url) - { - return url.GetComponents( - UriComponents.HostAndPort | - UriComponents.UserInfo | - UriComponents.Path, - UriFormat.SafeUnescaped); - } - } -} diff --git a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs b/src/Catalog/FlatContainerPackagePathProvider.cs similarity index 94% rename from src/Catalog/Registration/FlatContainerPackagePathProvider.cs rename to src/Catalog/FlatContainerPackagePathProvider.cs index 162cd92fe..b583694ac 100644 --- a/src/Catalog/Registration/FlatContainerPackagePathProvider.cs +++ b/src/Catalog/FlatContainerPackagePathProvider.cs @@ -4,9 +4,9 @@ using System; using NuGet.Services.Metadata.Catalog.Helpers; -namespace NuGet.Services.Metadata.Catalog.Registration +namespace NuGet.Services.Metadata.Catalog { - public class FlatContainerPackagePathProvider : IPackagePathProvider + public class FlatContainerPackagePathProvider { private readonly string _container; diff --git a/src/Catalog/Helpers/NuGetVersionUtility.cs b/src/Catalog/Helpers/NuGetVersionUtility.cs index e4224b52b..70722a886 100644 --- a/src/Catalog/Helpers/NuGetVersionUtility.cs +++ b/src/Catalog/Helpers/NuGetVersionUtility.cs @@ -1,11 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.Linq; using NuGet.Versioning; -using VDS.RDF; -using VDS.RDF.Query; namespace NuGet.Services.Metadata.Catalog.Helpers { @@ -43,87 +39,5 @@ public static string GetFullVersionString(string version) return parsedVersion.ToFullString(); } - - public static bool IsVersionSemVer2(string version) - { - NuGetVersion parsedVersion; - if (!NuGetVersion.TryParse(version, out parsedVersion)) - { - return false; - } - - return parsedVersion.IsSemVer2; - } - - public static bool IsVersionRangeSemVer2(string versionRange) - { - VersionRange parsedVersionRange; - if (!VersionRange.TryParse(versionRange, out parsedVersionRange)) - { - return false; - } - - if (parsedVersionRange.HasLowerBound && parsedVersionRange.MinVersion.IsSemVer2) - { - return true; - } - - if (parsedVersionRange.HasUpperBound && parsedVersionRange.MaxVersion.IsSemVer2) - { - return true; - } - - return false; - } - - public static bool IsGraphSemVer2(string version, string resourceUri, IGraph graph) - { - // Is the package version itself SemVer 2.0.0? - if (IsVersionSemVer2(version)) - { - return true; - } - - if (resourceUri == null) - { - throw new ArgumentNullException(nameof(resourceUri)); - } - - if (graph == null) - { - throw new ArgumentNullException(nameof(graph)); - } - - // Is the verbatim version SemVer 2.0.0? - var parsedResourceUri = new Uri(resourceUri); - var verbatimVersion = graph.GetTriplesWithSubjectPredicate( - graph.CreateUriNode(parsedResourceUri), - graph.CreateUriNode(Schema.Predicates.VerbatimVersion)).FirstOrDefault()?.Object as ILiteralNode; - if (verbatimVersion != null && IsVersionSemVer2(verbatimVersion.Value)) - { - return true; - } - - // Are any of the dependency version ranges SemVer 2.0.0? - var sparql = new SparqlParameterizedString - { - CommandText = Utils.GetResource("sparql.SelectDistinctDependencyVersionRanges.rq") - }; - sparql.SetUri("resourceUri", new Uri(resourceUri)); - var query = sparql.ToString(); - - TripleStore store = new TripleStore(); - store.Add(graph, true); - foreach (SparqlResult row in SparqlHelpers.Select(store, query)) - { - var unparsedVersionRange = row["versionRange"].ToString(); - if (IsVersionRangeSemVer2(unparsedVersionRange)) - { - return true; - } - } - - return false; - } } } diff --git a/src/Catalog/Helpers/SparqlHelpers.cs b/src/Catalog/Helpers/SparqlHelpers.cs deleted file mode 100644 index 059fb2075..000000000 --- a/src/Catalog/Helpers/SparqlHelpers.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using VDS.RDF; -using VDS.RDF.Parsing; -using VDS.RDF.Query; -using VDS.RDF.Query.Datasets; - -namespace NuGet.Services.Metadata.Catalog.Helpers -{ - class SparqlHelpers - { - public static IGraph Construct(TripleStore store, string sparql) - { - return (IGraph)Execute(store, sparql); - } - - public static SparqlResultSet Select(TripleStore store, string sparql) - { - return (SparqlResultSet)Execute(store, sparql); - } - - static object Execute(TripleStore store, string sparql) - { - InMemoryDataset ds = new InMemoryDataset(store); - ISparqlQueryProcessor processor = new LeviathanQueryProcessor(ds); - SparqlQueryParser sparqlparser = new SparqlQueryParser(); - SparqlQuery query = sparqlparser.ParseFromString(sparql); - return processor.ProcessQuery(query); - } - } -} diff --git a/src/Catalog/Helpers/Utils.cs b/src/Catalog/Helpers/Utils.cs index 5db22d6ef..a14047311 100644 --- a/src/Catalog/Helpers/Utils.cs +++ b/src/Catalog/Helpers/Utils.cs @@ -2,9 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Diagnostics; using System.Globalization; using System.IO; @@ -23,7 +21,6 @@ using NuGet.Services.Metadata.Catalog.JsonLDIntegration; using VDS.RDF; using VDS.RDF.Parsing; -using VDS.RDF.Writing; namespace NuGet.Services.Metadata.Catalog { @@ -35,8 +32,6 @@ public static class Utils private static readonly Lazy XslTransformNuSpecCache = new Lazy(() => SafeLoadXslTransform(XslTransformNuSpec)); private static readonly Lazy XslTransformNormalizeNuSpecNamespaceCache = new Lazy(() => SafeLoadXslTransform(XslTransformNormalizeNuSpecNamespace)); - private static readonly ConcurrentDictionary _resourceStringCache = new ConcurrentDictionary(); - private static readonly char[] TagTrimChars = { ',', ' ', '\t', '|', ';' }; public static string[] SplitTags(string original) @@ -64,22 +59,6 @@ public static Stream GetResourceStream(string resourceName) return assembly.GetManifestResourceStream($"{name}.{resourceName}"); } - public static string GetResource(string resourceName) - { - if (string.IsNullOrEmpty(resourceName)) - { - throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(resourceName)); - } - - return _resourceStringCache.GetOrAdd(resourceName, _ => - { - using (var reader = new StreamReader(GetResourceStream(resourceName))) - { - return reader.ReadToEnd(); - } - }); - } - public static IGraph CreateNuspecGraph(XDocument nuspec, string baseAddress, bool normalizeXml = false) { XsltArgumentList arguments = new XsltArgumentList(); @@ -171,29 +150,6 @@ private static XslCompiledTransform SafeLoadXslTransform(string resourceName) return transform; } - public static void Dump(IGraph graph, TextWriter writer) - { - CompressingTurtleWriter turtleWriter = new CompressingTurtleWriter(); - turtleWriter.DefaultNamespaces.AddNamespace("nuget", new Uri("http://nuget.org/schema#")); - turtleWriter.DefaultNamespaces.AddNamespace("catalog", new Uri("http://nuget.org/catalog#")); - turtleWriter.PrettyPrintMode = true; - turtleWriter.CompressionLevel = 10; - turtleWriter.Save(graph, writer); - } - - public static void Dump(TripleStore store, TextWriter writer) - { - Dump(store.Graphs.First(), writer); - } - - public static IGraph Load(string name) - { - TurtleParser parser = new TurtleParser(); - IGraph g = new Graph(); - parser.Load(g, new StreamReader(GetResourceStream(name))); - return g; - } - public static XDocument GetNuspec(ZipArchive package) { if (package == null) { return null; } @@ -209,12 +165,6 @@ public static XDocument GetNuspec(ZipArchive package) return null; } - public static ZipArchive GetPackage(Stream stream) - { - ZipArchive package = new ZipArchive(stream); - return package; - } - public static JToken CreateJson(IGraph graph, JToken frame = null) { System.IO.StringWriter writer = new System.IO.StringWriter(); @@ -318,57 +268,6 @@ public static void CopyCatalogContentGraph(INode sourceNode, IGraph source, IGra } } - public static int CountItems(string indexJson) - { - if (indexJson == null) - { - return 0; - } - - JObject index = JObject.Parse(indexJson); - - int total = 0; - foreach (JObject item in index["items"]) - { - total += item["count"].ToObject(); - } - - return total; - } - - public static bool IsType(JToken context, JToken obj, Uri type) - { - JToken objTypeToken; - if (((JObject)obj).TryGetValue("@type", out objTypeToken)) - { - if (objTypeToken is JArray) - { - foreach (JToken typeToken in ((JArray)objTypeToken).Values()) - { - Uri objType = Expand(context, typeToken); - - if (objType.AbsoluteUri == type.AbsoluteUri) - { - return true; - } - } - return false; - } - else - { - Uri objType = Expand(context, objTypeToken); - - return objType.AbsoluteUri == type.AbsoluteUri; - } - } - return false; - } - - public static Uri Expand(JToken context, JToken token) - { - return Expand(context, token.ToString()); - } - public static Uri Expand(JToken context, string term) { if (term.StartsWith("http:", StringComparison.OrdinalIgnoreCase)) @@ -386,63 +285,6 @@ public static Uri Expand(JToken context, string term) return new Uri(context["@vocab"] + term); } - public static string GetNuspecRelativeAddress(XDocument document) - { - XNamespace nuget = XNamespace.Get("http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd"); - - XElement package = document.Element(nuget.GetName("package")); - - if (package == null) - { - throw new ArgumentException("document, missing "); - } - - XElement metadata = package.Element(nuget.GetName("metadata")); - - if (metadata == null) - { - throw new ArgumentException("document, missing "); - } - - string id = metadata.Element(nuget.GetName("id")).Value; - string version = metadata.Element(nuget.GetName("version")).Value; - - if (string.IsNullOrEmpty(id) || string.IsNullOrEmpty(version)) - { - throw new ArgumentException("document, missing or "); - } - - string relativeAddress = id.ToLowerInvariant() + "." + version.ToLowerInvariant() + ".xml"; - - return relativeAddress; - } - - public static IEnumerable> Partition(IEnumerable source, int size) - { - T[] array = null; - int count = 0; - foreach (T item in source) - { - if (array == null) - { - array = new T[size]; - } - array[count] = item; - count++; - if (count == size) - { - yield return new ReadOnlyCollection(array); - array = null; - count = 0; - } - } - if (array != null) - { - Array.Resize(ref array, count); - yield return new ReadOnlyCollection(array); - } - } - // where the property exists on the graph being merged in remove it from the existing graph public static void RemoveExistingProperties(IGraph existingGraph, IGraph graphToMerge, Uri[] properties) { diff --git a/src/Catalog/Helpers/XmlHtmlWriter.cs b/src/Catalog/Helpers/XmlHtmlWriter.cs deleted file mode 100644 index 6f9b1dc20..000000000 --- a/src/Catalog/Helpers/XmlHtmlWriter.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Collections.Generic; -using System.IO; -using System.Text; -using System.Xml; - -namespace NuGet.Services.Metadata.Catalog -{ - class XmlHtmlWriter : XmlTextWriter - { - HashSet fullEndElement = new HashSet(); - string openingElement = ""; - - public XmlHtmlWriter(Stream stream, Encoding en) - : base(stream, en) - { - Init(); - } - - public XmlHtmlWriter(TextWriter writer) - : base(writer) - { - Init(); - } - - void Init() - { - fullEndElement.Add("script"); - fullEndElement.Add("div"); - } - - public override void WriteEndElement() - { - if (fullEndElement.Contains(openingElement)) - { - WriteFullEndElement(); - } - else - { - base.WriteEndElement(); - } - } - - public override void WriteStartElement(string prefix, string localName, string ns) - { - base.WriteStartElement(prefix, localName, ns); - openingElement = localName; - } - } -} \ No newline at end of file diff --git a/src/Catalog/Icons/CatalogLeafDataProcessor.cs b/src/Catalog/Icons/CatalogLeafDataProcessor.cs index 0762b132e..faf16b10a 100644 --- a/src/Catalog/Icons/CatalogLeafDataProcessor.cs +++ b/src/Catalog/Icons/CatalogLeafDataProcessor.cs @@ -7,7 +7,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.DataMovement; using NuGet.Services.Metadata.Catalog.Helpers; using NuGet.Services.Metadata.Catalog.Persistence; diff --git a/src/Catalog/LogEvents.cs b/src/Catalog/LogEvents.cs deleted file mode 100644 index cd465d231..000000000 --- a/src/Catalog/LogEvents.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using Microsoft.Extensions.Logging; - -namespace NuGet.Services.Metadata.Catalog -{ - public static class LogEvents - { - public static EventId ValidationFailed = new EventId(900, "Validation failed!"); - public static EventId ValidationFailedToRun = new EventId(901, "Failed to run validation!"); - public static EventId ValidationFailedToInitialize = new EventId(902, "Failed to initialize validation!"); - } -} diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj index d73f674ae..b36f40d6c 100644 --- a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -149,7 +149,6 @@ - @@ -164,10 +163,9 @@ - + - True @@ -184,36 +182,17 @@ - - - - - - - - - - - - - - - - - - - @@ -222,7 +201,6 @@ - @@ -240,14 +218,11 @@ - - - @@ -261,19 +236,8 @@ - - - - - - - - - - - diff --git a/src/Catalog/Persistence/AggregateStorage.cs b/src/Catalog/Persistence/AggregateStorage.cs deleted file mode 100644 index eb5d30f39..000000000 --- a/src/Catalog/Persistence/AggregateStorage.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace NuGet.Services.Metadata.Catalog.Persistence -{ - public class AggregateStorage : Storage - { - public delegate StorageContent WriteSecondaryStorageContentInterceptor( - Uri primaryStorageBaseUri, - Uri primaryResourceUri, - Uri secondaryStorageBaseUri, - Uri secondaryResourceUri, - StorageContent content); - - private readonly Storage _primaryStorage; - private readonly Storage[] _secondaryStorage; - private readonly WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - - public AggregateStorage(Uri baseAddress, Storage primaryStorage, Storage[] secondaryStorage, - WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) - : base(baseAddress) - { - _primaryStorage = primaryStorage; - _secondaryStorage = secondaryStorage; - _writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor; - - BaseAddress = _primaryStorage.BaseAddress; - } - - protected override Task OnCopyAsync( - Uri sourceUri, - IStorage destinationStorage, - Uri destinationUri, - IReadOnlyDictionary destinationProperties, - CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - - protected override Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) - { - var tasks = new List(); - tasks.Add(_primaryStorage.SaveAsync(resourceUri, content, cancellationToken)); - - foreach (var storage in _secondaryStorage) - { - var secondaryResourceUri = new Uri(resourceUri.ToString() - .Replace(_primaryStorage.BaseAddress.ToString(), storage.BaseAddress.ToString())); - - var secondaryContent = content; - if (_writeSecondaryStorageContentInterceptor != null) - { - secondaryContent = _writeSecondaryStorageContentInterceptor( - _primaryStorage.BaseAddress, - resourceUri, - storage.BaseAddress, - secondaryResourceUri, content); - } - - tasks.Add(storage.SaveAsync(secondaryResourceUri, secondaryContent, cancellationToken)); - } - - return Task.WhenAll(tasks); - } - - protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) - { - return _primaryStorage.LoadAsync(resourceUri, cancellationToken); - } - - protected override Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken) - { - var tasks = new List(); - tasks.Add(_primaryStorage.DeleteAsync(resourceUri, cancellationToken, deleteRequestOptions)); - - foreach (var storage in _secondaryStorage) - { - var secondaryResourceUri = new Uri(resourceUri.ToString() - .Replace(_primaryStorage.BaseAddress.ToString(), storage.BaseAddress.ToString())); - - tasks.Add(storage.DeleteAsync(secondaryResourceUri, cancellationToken, deleteRequestOptions)); - } - - return Task.WhenAll(tasks); - } - - public override bool Exists(string fileName) - { - return _primaryStorage.Exists(fileName); - } - - public override Task> ListAsync(CancellationToken cancellationToken) - { - return _primaryStorage.ListAsync(cancellationToken); - } - - public override Uri GetUri(string name) - { - return this._primaryStorage.GetUri(name); - } - } -} \ No newline at end of file diff --git a/src/Catalog/Persistence/AggregateStorageFactory.cs b/src/Catalog/Persistence/AggregateStorageFactory.cs deleted file mode 100644 index cb82f0a4b..000000000 --- a/src/Catalog/Persistence/AggregateStorageFactory.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Linq; - -namespace NuGet.Services.Metadata.Catalog.Persistence -{ - public class AggregateStorageFactory : StorageFactory - { - private readonly AggregateStorage.WriteSecondaryStorageContentInterceptor _writeSecondaryStorageContentInterceptor; - - public AggregateStorageFactory( - StorageFactory primaryStorageFactory, - StorageFactory[] secondaryStorageFactories, - bool verbose) - : this( - primaryStorageFactory, - secondaryStorageFactories, - writeSecondaryStorageContentInterceptor: null, - verbose: verbose) - { - } - - public AggregateStorageFactory( - StorageFactory primaryStorageFactory, - StorageFactory[] secondaryStorageFactories, - AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor, - bool verbose) - { - PrimaryStorageFactory = primaryStorageFactory; - SecondaryStorageFactories = secondaryStorageFactories; - _writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor; - - BaseAddress = PrimaryStorageFactory.BaseAddress; - DestinationAddress = PrimaryStorageFactory.DestinationAddress; - Verbose = verbose; - } - - public override Storage Create(string name = null) - { - var primaryStorage = PrimaryStorageFactory.Create(name); - var secondaryStorage = SecondaryStorageFactories.Select(f => f.Create(name)).ToArray(); - - return new AggregateStorage( - PrimaryStorageFactory.BaseAddress, - primaryStorage, - secondaryStorage, - _writeSecondaryStorageContentInterceptor); - } - - public StorageFactory PrimaryStorageFactory { get; } - public IEnumerable SecondaryStorageFactories { get; } - } -} \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlobDirectory.cs b/src/Catalog/Persistence/ICloudBlobDirectory.cs index c8bbe31d0..f008f4d94 100644 --- a/src/Catalog/Persistence/ICloudBlobDirectory.cs +++ b/src/Catalog/Persistence/ICloudBlobDirectory.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using System.Collections.Generic; -using System.Collections; using Microsoft.WindowsAzure.Storage.Blob; namespace NuGet.Services.Metadata.Catalog.Persistence diff --git a/src/Catalog/Registration/IPackagePathProvider.cs b/src/Catalog/Registration/IPackagePathProvider.cs deleted file mode 100644 index 4f8d9a315..000000000 --- a/src/Catalog/Registration/IPackagePathProvider.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - /// - /// An interface for providing the relative path of a package or an icon in the base address - /// - public interface IPackagePathProvider - { - string GetIconPath(string id, string version); - string GetIconPath(string id, string version, bool normalize); - string GetPackagePath(string id, string version); - } -} diff --git a/src/Catalog/Registration/IRegistrationPersistence.cs b/src/Catalog/Registration/IRegistrationPersistence.cs deleted file mode 100644 index d73b2ce38..000000000 --- a/src/Catalog/Registration/IRegistrationPersistence.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public interface IRegistrationPersistence - { - Task> Load(CancellationToken cancellationToken); - Task Save(IDictionary registration, CancellationToken cancellationToken); - } -} diff --git a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs b/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs deleted file mode 100644 index f842e1110..000000000 --- a/src/Catalog/Registration/PackagesFolderPackagePathProvider.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using NuGet.Services.Metadata.Catalog.Helpers; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class PackagesFolderPackagePathProvider : IPackagePathProvider - { - public string GetPackagePath(string id, string version) - { - if (id == null) - { - throw new ArgumentNullException(nameof(id)); - } - - if (version == null) - { - throw new ArgumentNullException(nameof(version)); - } - - var idLowerCase = id.ToLowerInvariant(); - var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); - - var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); - - return $"packages/{packageFileName}"; - } - - public string GetIconPath(string id, string version) - { - return GetIconPath(id, version, normalize: true); - } - - public string GetIconPath(string id, string version, bool normalize) - { - if (id == null) - { - throw new ArgumentNullException(nameof(id)); - } - - if (version == null) - { - throw new ArgumentNullException(nameof(version)); - } - - var idLowerCase = id.ToLowerInvariant(); - string versionLowerCase; - if (normalize) - { - versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); - } - else - { - versionLowerCase = version.ToLowerInvariant(); - } - - return $"packages/{idLowerCase}/{versionLowerCase}/icon"; - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RecordingStorage.cs b/src/Catalog/Registration/RecordingStorage.cs deleted file mode 100644 index 076548b70..000000000 --- a/src/Catalog/Registration/RecordingStorage.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RecordingStorage : IStorage - { - IStorage _innerStorage; - - public RecordingStorage(IStorage storage) - { - _innerStorage = storage; - - Loaded = new HashSet(); - Saved = new HashSet(); - } - - public HashSet Loaded { get; private set; } - public HashSet Saved { get; private set; } - - public Task GetOptimisticConcurrencyControlTokenAsync( - Uri resourceUri, - CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - - public Task CopyAsync( - Uri sourceUri, - IStorage destinationStorage, - Uri destinationUri, - IReadOnlyDictionary destinationProperties, - CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - - public Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) - { - Task result = _innerStorage.SaveAsync(resourceUri, content, cancellationToken); - Saved.Add(resourceUri); - return result; - } - - public Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) - { - Task result = _innerStorage.LoadAsync(resourceUri, cancellationToken); - Loaded.Add(resourceUri); - return result; - } - - public Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken, DeleteRequestOptions deleteRequestOptions = null) - { - return _innerStorage.DeleteAsync(resourceUri, cancellationToken, deleteRequestOptions); - } - - public Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) - { - Task result = _innerStorage.LoadStringAsync(resourceUri, cancellationToken); - Loaded.Add(resourceUri); - return result; - } - - public Uri BaseAddress - { - get { return _innerStorage.BaseAddress; } - } - - public Uri ResolveUri(string relativeUri) - { - return _innerStorage.ResolveUri(relativeUri); - } - - public Task> ListAsync(CancellationToken cancellationToken) - { - return _innerStorage.ListAsync(cancellationToken); - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationCatalogEntry.cs b/src/Catalog/Registration/RegistrationCatalogEntry.cs deleted file mode 100644 index 4c377cb5c..000000000 --- a/src/Catalog/Registration/RegistrationCatalogEntry.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - /// - /// A delegate used to determine whether a package should be included in a registration hive. The delegate is - /// important because some registration hives exclude specific packages (typically packages that break older - /// clients). The first example of this difference is SemVer 2.0.0 packages, which should be excluded from the legacy - /// registration hives. - /// - /// The package key. This contains the ID and version of the package. - /// The URI (identifier) of the package in the RDF graph. - /// The RDF graph containing metadata about the package. - /// True if the package should be included in the registration hive. False otherwise. - public delegate bool ShouldIncludeRegistrationPackage(RegistrationEntryKey key, string resourceUri, IGraph graph); - - public class RegistrationCatalogEntry - { - public RegistrationCatalogEntry(string resourceUri, IGraph graph, bool isExistingItem) - { - ResourceUri = resourceUri; - Graph = graph; - IsExistingItem = isExistingItem; - } - - public string ResourceUri { get; set; } - public IGraph Graph { get; set; } - public bool IsExistingItem { get; set; } - - public static KeyValuePair Promote( - string resourceUri, - IGraph graph, - ShouldIncludeRegistrationPackage shouldInclude, - bool isExistingItem) - { - INode subject = graph.CreateUriNode(new Uri(resourceUri)); - var triples = graph.GetTriplesWithSubjectPredicate(subject, graph.CreateUriNode(Schema.Predicates.Version)); - string version = triples.First().Object.ToString(); - - var registrationEntryKey = new RegistrationEntryKey(RegistrationKey.Promote(resourceUri, graph), version); - - RegistrationCatalogEntry registrationCatalogEntry = null; - if (!IsDelete(subject, graph) && shouldInclude(registrationEntryKey, resourceUri, graph)) - { - registrationCatalogEntry = new RegistrationCatalogEntry(resourceUri, graph, isExistingItem); - } - - return new KeyValuePair(registrationEntryKey, registrationCatalogEntry); - } - - static bool IsDelete(INode subject, IGraph graph) - { - return graph.ContainsTriple(new Triple(subject, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.CatalogDelete))) - || graph.ContainsTriple(new Triple(subject, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.PackageDelete))); - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationCollector.cs b/src/Catalog/Registration/RegistrationCollector.cs deleted file mode 100644 index c4fa2dd8e..000000000 --- a/src/Catalog/Registration/RegistrationCollector.cs +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.Metadata.Catalog.Persistence; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationCollector : SortingGraphCollector - { - public const int PartitionSize = 64; - public const int PackageCountThreshold = 128; - - // This is simply the arbitrary limit that I tested. There may be a better value. - public const int DefaultMaxConcurrentBatches = 10; - - private readonly StorageFactory _legacyStorageFactory; - private readonly StorageFactory _semVer2StorageFactory; - private readonly ShouldIncludeRegistrationPackage _shouldIncludeSemVer2ForLegacyStorageFactory; - private readonly RegistrationMakerCatalogItem.PostProcessGraph _postProcessGraphForLegacyStorageFactory; - private readonly bool _forcePackagePathProviderForIcons; - private readonly int _maxConcurrentBatches; - private readonly ILogger _logger; - - public RegistrationCollector( - Uri index, - StorageFactory legacyStorageFactory, - StorageFactory semVer2StorageFactory, - Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons, - ITelemetryService telemetryService, - ILogger logger, - Func handlerFunc = null, - IHttpRetryStrategy httpRetryStrategy = null, - int maxConcurrentBatches = DefaultMaxConcurrentBatches) - : base( - index, - new Uri[] { Schema.DataTypes.PackageDetails, Schema.DataTypes.PackageDelete }, - telemetryService, - handlerFunc, - httpRetryStrategy) - { - _legacyStorageFactory = legacyStorageFactory ?? throw new ArgumentNullException(nameof(legacyStorageFactory)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _semVer2StorageFactory = semVer2StorageFactory; - _shouldIncludeSemVer2ForLegacyStorageFactory = GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_semVer2StorageFactory); - _postProcessGraphForLegacyStorageFactory = GetPostProcessGraphForLegacyStorageFactory(_semVer2StorageFactory); - ContentBaseAddress = contentBaseAddress; - GalleryBaseAddress = galleryBaseAddress; - _forcePackagePathProviderForIcons = forcePackagePathProviderForIcons; - - if (maxConcurrentBatches < 1) - { - throw new ArgumentOutOfRangeException( - nameof(maxConcurrentBatches), - maxConcurrentBatches, - string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); - } - - _maxConcurrentBatches = maxConcurrentBatches; - } - - public Uri ContentBaseAddress { get; } - public Uri GalleryBaseAddress { get; } - - protected override Task> CreateBatchesAsync( - IEnumerable catalogItems) - { - // Grouping batches by commit is slow if it contains - // the same package registration id over and over again. - // This happens when, for example, a package publish "wave" - // occurs. - // - // If one package registration id is part of 20 batches, - // we'll have to process all registration leafs 20 times. - // It would be better to process these leafs only once. - // - // So let's batch by package registration id here, - // ensuring we never write a commit timestamp to the cursor - // that is higher than the last item currently processed. - // - // So, group by id, then make sure the batch key is the - // *lowest* timestamp of all commits in that batch. - // This ensures that on retries, we will retry - // from the correct location (even though we may have - // a little rework). - - var batches = CatalogCommitUtilities.CreateCommitItemBatches(catalogItems, GetKey); - - return Task.FromResult(batches); - } - - protected override string GetKey(CatalogCommitItem item) - { - return CatalogCommitUtilities.GetPackageIdKey(item); - } - - // Summary: - // - // 1. Process one catalog page at a time. - // 2. Within a given catalog page, batch catalog commit entries by lower-cased package ID. - // 3. Process up to `n` batches in parallel. Note that the batches may span multiple catalog commits. - // 4. Cease processing new batches if a failure has been observed. This job will eventually retry - // batches on its next outermost job loop. - // 5. If a failure has been observed, wait for all existing tasks to complete. Avoid task cancellation - // as that could lead to the entirety of a package registration being in an inconsistent state. - // To be fair, a well-timed exception could have the same result, but registration updates have never - // been transactional. Actively cancelling tasks would make an inconsistent registration more likely. - // 6. Update the cursor if and only if all preceding commits and the current (oldest) commit have been - // fully and successfully processed. - protected override Task FetchAsync( - CollectorHttpClient client, - ReadWriteCursor front, - ReadCursor back, - CancellationToken cancellationToken) - { - return CatalogCommitUtilities.ProcessCatalogCommitsAsync( - client, - front, - back, - FetchCatalogCommitsAsync, - CreateBatchesAsync, - ProcessBatchAsync, - _maxConcurrentBatches, - _logger, - cancellationToken); - } - - protected override async Task ProcessGraphsAsync( - KeyValuePair> sortedGraphs, - CancellationToken cancellationToken) - { - var tasks = new List(); - - using (_telemetryService.TrackDuration(TelemetryConstants.ProcessGraphsSeconds, - new Dictionary() - { - { TelemetryConstants.Id, sortedGraphs.Key.ToLowerInvariant() } - })) - { - var legacyTask = RegistrationMaker.ProcessAsync( - registrationKey: new RegistrationKey(sortedGraphs.Key), - newItems: sortedGraphs.Value, - shouldInclude: _shouldIncludeSemVer2ForLegacyStorageFactory, - storageFactory: _legacyStorageFactory, - postProcessGraph: _postProcessGraphForLegacyStorageFactory, - contentBaseAddress: ContentBaseAddress, - galleryBaseAddress: GalleryBaseAddress, - partitionSize: PartitionSize, - packageCountThreshold: PackageCountThreshold, - forcePackagePathProviderForIcons: _forcePackagePathProviderForIcons, - telemetryService: _telemetryService, - cancellationToken: cancellationToken); - tasks.Add(legacyTask); - - if (_semVer2StorageFactory != null) - { - var semVer2Task = RegistrationMaker.ProcessAsync( - registrationKey: new RegistrationKey(sortedGraphs.Key), - newItems: sortedGraphs.Value, - storageFactory: _semVer2StorageFactory, - contentBaseAddress: ContentBaseAddress, - galleryBaseAddress: GalleryBaseAddress, - partitionSize: PartitionSize, - packageCountThreshold: PackageCountThreshold, - forcePackagePathProviderForIcons: _forcePackagePathProviderForIcons, - telemetryService: _telemetryService, - cancellationToken: cancellationToken); - tasks.Add(semVer2Task); - } - - await Task.WhenAll(tasks); - } - } - - public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPackageForLegacyStorageFactory(StorageFactory semVer2StorageFactory) - { - // If SemVer 2.0.0 storage is disabled, put SemVer 2.0.0 registration in the legacy storage factory. In no - // case should a package be completely ignored. That is, if a package is SemVer 2.0.0 but SemVer 2.0.0 - // storage is not enabled, our only choice is to put SemVer 2.0.0 packages in the legacy storage. - if (semVer2StorageFactory == null) - { - return (k, u, g) => true; - } - - return (k, u, g) => !NuGetVersionUtility.IsGraphSemVer2(k.Version, u, g); - } - - public static RegistrationMakerCatalogItem.PostProcessGraph GetPostProcessGraphForLegacyStorageFactory(StorageFactory semVer2StorageFactory) - { - // If SemVer 2.0.0 storage is disabled, put deprecation metadata in the legacy storage. - // A package's deprecation metadata should never be completely ignored. - // If a package contains deprecation metadata but SemVer 2.0.0 storage is not enabled, - // our only choice is to put deprecation metadata in the legacy storage. - if (semVer2StorageFactory == null) - { - return g => g; - } - - return FilterOutDeprecationInformation; - } - - public static RegistrationMakerCatalogItem.PostProcessGraph FilterOutDeprecationInformation = g => - { - var deprecationTriples = g - .GetTriplesWithPredicate(g.CreateUriNode(Schema.Predicates.Deprecation)) - .ToList(); - - g.Retract(deprecationTriples); - return g; - }; - - private async Task ProcessBatchAsync( - CollectorHttpClient client, - JToken context, - string packageId, - CatalogCommitItemBatch batch, - CatalogCommitItemBatch lastBatch, - CancellationToken cancellationToken) - { - await Task.Yield(); - - using (_telemetryService.TrackDuration( - TelemetryConstants.ProcessBatchSeconds, - new Dictionary() - { - { TelemetryConstants.Id, packageId }, - { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } - })) - { - await OnProcessBatchAsync( - client, - batch.Items, - context, - batch.CommitTimeStamp, - isLastBatch: false, - cancellationToken: cancellationToken); - } - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationEntryKey.cs b/src/Catalog/Registration/RegistrationEntryKey.cs deleted file mode 100644 index b38e03d35..000000000 --- a/src/Catalog/Registration/RegistrationEntryKey.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using NuGet.Services.Metadata.Catalog.Helpers; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationEntryKey - { - private readonly string _normalizedVersion; - - public RegistrationEntryKey(RegistrationKey registrationKey, string version) - { - RegistrationKey = registrationKey; - Version = version; - _normalizedVersion = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); - } - - public RegistrationKey RegistrationKey { get; } - public string Version { get; } - - public override string ToString() - { - return RegistrationKey.ToString() + "/" + Version; - } - - public override int GetHashCode() - { - return $"{RegistrationKey}/{_normalizedVersion}".GetHashCode(); - } - - public override bool Equals(object obj) - { - RegistrationEntryKey rhs = obj as RegistrationEntryKey; - - if (rhs == null) - { - return false; - } - - return RegistrationKey.Equals(rhs.RegistrationKey) && - _normalizedVersion == rhs._normalizedVersion; - } - } -} diff --git a/src/Catalog/Registration/RegistrationKey.cs b/src/Catalog/Registration/RegistrationKey.cs deleted file mode 100644 index 4bd615042..000000000 --- a/src/Catalog/Registration/RegistrationKey.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationKey - { - public RegistrationKey(string id) - { - Id = id ?? throw new ArgumentNullException(nameof(id)); - } - - public string Id { get; } - - public override string ToString() - { - return Id.ToLowerInvariant(); - } - - public override int GetHashCode() - { - return ToString().GetHashCode(); - } - - public override bool Equals(object obj) - { - RegistrationKey rhs = obj as RegistrationKey; - - if (rhs == null) - { - return false; - } - - return StringComparer.OrdinalIgnoreCase.Equals(Id, rhs.Id); - } - - public static RegistrationKey Promote(string resourceUri, IGraph graph) - { - INode subject = graph.CreateUriNode(new Uri(resourceUri)); - string id = graph.GetTriplesWithSubjectPredicate(subject, graph.CreateUriNode(Schema.Predicates.Id)).First().Object.ToString(); - - return new RegistrationKey(id); - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationMaker.cs b/src/Catalog/Registration/RegistrationMaker.cs deleted file mode 100644 index 76c17691b..000000000 --- a/src/Catalog/Registration/RegistrationMaker.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using NuGet.Services.Metadata.Catalog.Persistence; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public static class RegistrationMaker - { - public static async Task ProcessAsync( - RegistrationKey registrationKey, - IReadOnlyDictionary newItems, - StorageFactory storageFactory, - Uri contentBaseAddress, - Uri galleryBaseAddress, - int partitionSize, - int packageCountThreshold, - bool forcePackagePathProviderForIcons, - ITelemetryService telemetryService, - CancellationToken cancellationToken) - { - await ProcessAsync( - registrationKey, - newItems, - (k, u, g) => true, - storageFactory, - g => g, - contentBaseAddress, - galleryBaseAddress, - partitionSize, - packageCountThreshold, - forcePackagePathProviderForIcons, - telemetryService, - cancellationToken); - } - - public static async Task ProcessAsync( - RegistrationKey registrationKey, - IReadOnlyDictionary newItems, - ShouldIncludeRegistrationPackage shouldInclude, - StorageFactory storageFactory, - RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph, - Uri contentBaseAddress, - Uri galleryBaseAddress, - int partitionSize, - int packageCountThreshold, - bool forcePackagePathProviderForIcons, - ITelemetryService telemetryService, - CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationMaker.Process: registrationKey = {0} newItems: {1}", registrationKey, newItems.Count); - - IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, postProcessGraph, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons); - - IDictionary existing = await registration.Load(cancellationToken); - - Trace.TraceInformation("RegistrationMaker.Process: existing = {0}", existing.Count); - - IDictionary delta = PromoteRegistrationKey(newItems, shouldInclude); - - Trace.TraceInformation("RegistrationMaker.Process: delta = {0}", delta.Count); - telemetryService.TrackMetric(TelemetryConstants.RegistrationDeltaCount, (uint)delta.Count, new Dictionary() - { - { TelemetryConstants.ContentBaseAddress, contentBaseAddress.AbsoluteUri }, - { TelemetryConstants.GalleryBaseAddress, galleryBaseAddress.AbsoluteUri } - }); - - IDictionary resulting = Apply(existing, delta); - - Trace.TraceInformation("RegistrationMaker.Process: resulting = {0}", resulting.Count); - await registration.Save(resulting, cancellationToken); - } - - private static IDictionary PromoteRegistrationKey( - IReadOnlyDictionary newItems, - ShouldIncludeRegistrationPackage shouldInclude) - { - IDictionary promoted = new Dictionary(); - foreach (var newItem in newItems) - { - var promotedEntry = RegistrationCatalogEntry.Promote( - newItem.Key, - newItem.Value, - shouldInclude, - isExistingItem: false); - - promoted[promotedEntry.Key] = promotedEntry.Value; - } - - return promoted; - } - - private static IDictionary Apply( - IDictionary existing, - IDictionary delta) - { - IDictionary resulting = new Dictionary(); - - foreach (var item in existing) - { - if (delta.ContainsKey(item.Key)) - { - resulting.Add(item.Key, delta[item.Key]); - delta.Remove(item.Key); - } - else - { - resulting.Add(item); - } - } - - foreach (var item in delta) - { - resulting.Add(item); - } - - return resulting; - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationMakerCatalogItem.cs b/src/Catalog/Registration/RegistrationMakerCatalogItem.cs deleted file mode 100644 index 96e405d8a..000000000 --- a/src/Catalog/Registration/RegistrationMakerCatalogItem.cs +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.Metadata.Catalog.Persistence; -using VDS.RDF; -using VDS.RDF.Query; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationMakerCatalogItem : CatalogItem - { - public delegate IGraph PostProcessGraph(IGraph graph); - - private readonly Uri _catalogUri; - private readonly IGraph _catalogItem; - private Uri _itemAddress; - private readonly Uri _packageContentBaseAddress; - private readonly Uri _galleryBaseAddress; - private Uri _packageContentAddress; - private readonly Uri _registrationBaseAddress; - private Uri _registrationAddress; - private DateTime _publishedDate; - private bool _listed; - private readonly PostProcessGraph _postProcessGraph; - private readonly bool _forcePathProviderForIcons; - - // This should be set before class is instantiated - public static IPackagePathProvider PackagePathProvider = null; - - public RegistrationMakerCatalogItem( - Uri catalogUri, - IGraph catalogItem, - Uri registrationBaseAddress, - bool isExistingItem, - PostProcessGraph postProcessGraph, - bool forcePathProviderForIcons, - Uri packageContentBaseAddress = null, - Uri galleryBaseAddress = null) - { - _catalogUri = catalogUri; - _catalogItem = catalogItem; - _packageContentBaseAddress = packageContentBaseAddress; - _galleryBaseAddress = galleryBaseAddress; - _registrationBaseAddress = registrationBaseAddress; - _postProcessGraph = postProcessGraph; - _forcePathProviderForIcons = forcePathProviderForIcons; - - IsExistingItem = isExistingItem; - } - - public override StorageContent CreateContent(CatalogContext context) - { - IGraph graph = new Graph(); - INode subject = graph.CreateUriNode(GetItemAddress()); - - graph.CreateUriNode(_catalogUri); - - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.Package)); - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.Permalink)); - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.CatalogEntry), graph.CreateUriNode(_catalogUri)); - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.Registration), graph.CreateUriNode(GetRegistrationAddress())); - - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.PackageContent), graph.CreateUriNode(GetPackageContentAddress())); - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.Published), graph.CreateLiteralNode(GetPublishedDate().ToString("O"), Schema.DataTypes.DateTime)); - graph.Assert(subject, graph.CreateUriNode(Schema.Predicates.Listed), graph.CreateLiteralNode(_listed.ToString(), Schema.DataTypes.Boolean)); - - JObject frame = context.GetJsonLdContext("context.Package.json", Schema.DataTypes.Package); - return new JTokenStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store"); - } - - public bool IsExistingItem { get; private set; } - - public override Uri GetItemType() - { - return Schema.DataTypes.Package; - } - - public override Uri GetItemAddress() - { - if (_itemAddress == null) - { - INode subject = _catalogItem.CreateUriNode(_catalogUri); - string version = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Version) - .ToLowerInvariant(); - - version = NuGetVersionUtility.NormalizeVersion(version); - - _itemAddress = new Uri(BaseAddress, version + ".json"); - } - - return _itemAddress; - } - - private Uri GetRegistrationAddress() - { - if (_registrationAddress == null) - { - INode subject = _catalogItem.CreateUriNode(_catalogUri); - string id = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Id).ToLowerInvariant(); - string path = string.Format("{0}/index.json", id.ToLowerInvariant()); - _registrationAddress = new Uri(_registrationBaseAddress, path); - } - - return _registrationAddress; - } - - private DateTime GetPublishedDate() - { - if (_publishedDate == default(DateTime)) - { - INode subject = _catalogItem.CreateUriNode(_catalogUri); - var pubTriple = _catalogItem.GetTriplesWithSubjectPredicate(subject, _catalogItem.CreateUriNode(Schema.Predicates.Published)).SingleOrDefault(); - - if (pubTriple != null) - { - if (pubTriple.Object is ILiteralNode node) - { - _publishedDate = DateTime.Parse(node.Value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - } - } - } - - var utcYear = _publishedDate.ToUniversalTime().Year; - if ((utcYear < 1900) || - (utcYear > 1900 && utcYear < 2010)) - { - Trace.TraceWarning($"Package with published date less than 2010 encountered. Catalog URI: '{_catalogUri}'. Published date: '{_publishedDate:O}'"); - } - - _listed = utcYear != 1900; - - return _publishedDate; - } - - private Uri GetPackageContentAddress() - { - if (PackagePathProvider == null) - { - throw new NullReferenceException("PackagePathProvider should not be null"); - } - - if (_packageContentAddress == null) - { - INode subject = _catalogItem.CreateUriNode(_catalogUri); - - Triple packageContentTriple = _catalogItem.GetTriplesWithSubjectPredicate(subject, _catalogItem.CreateUriNode(Schema.Predicates.PackageContent)).FirstOrDefault(); - if (packageContentTriple != null) - { - _packageContentAddress = new Uri(packageContentTriple.Object.ToString()); - } - else - { - string id = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Id).ToLowerInvariant(); - string version = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Version).ToLowerInvariant(); - string path = PackagePathProvider.GetPackagePath(id, version); - _packageContentAddress = new Uri(_packageContentBaseAddress, path); - } - } - - return _packageContentAddress; - } - - private string GetLicenseUrl() - { - INode subject = _catalogItem.CreateUriNode(_catalogUri); - - string packageId = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Id); - string packageVersion = NuGetVersionUtility.NormalizeVersion(GetRequiredObject(_catalogItem, subject, Schema.Predicates.Version)); - string licenseExpression = GetOptionalObject(_catalogItem, subject, Schema.Predicates.LicenseExpression); - string licenseFile = GetOptionalObject(_catalogItem, subject, Schema.Predicates.LicenseFile); - string licenseUrl = GetOptionalObject(_catalogItem, subject, Schema.Predicates.LicenseUrl); - - if (_galleryBaseAddress != null && - !string.IsNullOrWhiteSpace(packageId) && - !string.IsNullOrWhiteSpace(packageVersion) && - (!string.IsNullOrWhiteSpace(licenseExpression) || - !string.IsNullOrWhiteSpace(licenseFile))) - { - return LicenseHelper.GetGalleryLicenseUrl(packageId, packageVersion, _galleryBaseAddress); - } - - if (!string.IsNullOrWhiteSpace(licenseUrl)) - { - return licenseUrl; - } - - return string.Empty; - } - - public override IGraph CreatePageContent(CatalogContext context) - { - try - { - IGraph content; - - using (TripleStore store = new TripleStore()) - { - store.Add(_catalogItem, true); - - SparqlParameterizedString sparql = new SparqlParameterizedString - { - CommandText = Utils.GetResource("sparql.ConstructRegistrationPageContentGraph.rq") - }; - - sparql.SetUri("package", GetItemAddress()); - sparql.SetUri("catalogEntry", _catalogUri); - sparql.SetUri("baseAddress", BaseAddress); - sparql.SetUri("packageContent", GetPackageContentAddress()); - sparql.SetUri("registrationBaseAddress", _registrationBaseAddress); - sparql.SetLiteral("licenseUrl", GetLicenseUrl()); - sparql.SetLiteral("iconUrl", GetIconUrl()); - - content = SparqlHelpers.Construct(store, sparql.ToString()); - } - - return _postProcessGraph(content); - } - catch (Exception e) - { - throw new Exception(string.Format("Exception processing catalog item {0}", _catalogUri), e); - } - } - - private string GetIconUrl() - { - var subject = _catalogItem.CreateUriNode(_catalogUri); - - var packageId = GetRequiredObject(_catalogItem, subject, Schema.Predicates.Id); - var packageVersion = NuGetVersionUtility.NormalizeVersion(GetRequiredObject(_catalogItem, subject, Schema.Predicates.Version)); - var iconUrl = GetOptionalObject(_catalogItem, subject, Schema.Predicates.IconUrl); - var iconFile = GetOptionalObject(_catalogItem, subject, Schema.Predicates.IconFile); - - var shouldUsePathProvider = !string.IsNullOrWhiteSpace(iconFile) || (_forcePathProviderForIcons && !string.IsNullOrWhiteSpace(iconUrl)); - - if (shouldUsePathProvider && !string.IsNullOrWhiteSpace(packageId) && !string.IsNullOrWhiteSpace(packageVersion)) - { - // The embedded icon file case. We assume here that catalog2dnx did its job - // and extracted the icon file to the appropriate location. - string path = PackagePathProvider.GetIconPath(packageId, packageVersion); - return new Uri(_packageContentBaseAddress, path).AbsoluteUri; - } - - return _forcePathProviderForIcons || iconUrl == null ? string.Empty : iconUrl; - } - - private static string GetRequiredObject(IGraph graph, INode subject, Uri predicate) - { - var predicateNode = graph.CreateUriNode(predicate); - var triple = graph.GetTriplesWithSubjectPredicate(subject, predicateNode).First(); - - return triple.Object.ToString(); - } - - private static string GetOptionalObject(IGraph graph, INode subject, Uri predicate) - { - var predicateNode = graph.CreateUriNode(predicate); - var triple = graph.GetTriplesWithSubjectPredicate(subject, predicateNode).FirstOrDefault(); - - return triple?.Object.ToString(); - } - } -} diff --git a/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs b/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs deleted file mode 100644 index 3068e8d86..000000000 --- a/src/Catalog/Registration/RegistrationMakerCatalogWriter.cs +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Versioning; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationMakerCatalogWriter : CatalogWriterBase - { - private readonly IList _cleanUpList; - - public RegistrationMakerCatalogWriter(IStorage storage, int partitionSize = 100, IList cleanUpList = null, ICatalogGraphPersistence graphPersistence = null, CatalogContext context = null) - : base(storage, graphPersistence, context) - { - _cleanUpList = cleanUpList; - PartitionSize = partitionSize; - } - - public int PartitionSize { get; private set; } - - protected override Uri[] GetAdditionalRootType() - { - return new Uri[] { Schema.DataTypes.PackageRegistration, Schema.DataTypes.Permalink }; - } - - protected override async Task> SavePages(Guid commitId, DateTime commitTimeStamp, IDictionary itemEntries, CancellationToken cancellationToken) - { - SortedDictionary> versions = new SortedDictionary>(); - - // load items from existing pages - - IDictionary pageEntries = await LoadIndexResource(RootUri, cancellationToken); - - foreach (KeyValuePair pageEntry in pageEntries) - { - IDictionary pageItemEntries = await LoadIndexResource(new Uri(pageEntry.Key), cancellationToken); - - foreach (KeyValuePair pageItemEntry in pageItemEntries) - { - NuGetVersion version = GetPackageVersion(new Uri(pageItemEntry.Key), pageItemEntry.Value.Content); - versions.Add(version, pageItemEntry); - } - } - - // add new items - - foreach (KeyValuePair itemEntry in itemEntries) - { - NuGetVersion version = GetPackageVersion(new Uri(itemEntry.Key), itemEntry.Value.Content); - versions[version] = itemEntry; - } - - // (re)create pages - - IDictionary newPageEntries = await PartitionAndSavePages(commitId, commitTimeStamp, versions, cancellationToken); - - // add to list of pages to clean up - - if (_cleanUpList != null) - { - foreach (string existingPage in pageEntries.Keys) - { - if (!newPageEntries.ContainsKey(existingPage)) - { - _cleanUpList.Add(new Uri(existingPage)); - } - } - } - - return newPageEntries; - } - - protected override ResourceSaveOperation CreateSaveOperationForItem(IStorage storage, CatalogContext context, CatalogItem item, CancellationToken cancellationToken) - { - // This method decides what to do with the item. - // If it's a RegistrationMakerCatalogItem and it already exists, then don't write content. - var registrationMakerCatalogItem = item as RegistrationMakerCatalogItem; - if (registrationMakerCatalogItem != null) - { - var content = item.CreateContent(Context); // note: always do this first - var resourceUri = item.GetItemAddress(); - - var saveOperation = new ResourceSaveOperation(); - saveOperation.ResourceUri = resourceUri; - - if (!registrationMakerCatalogItem.IsExistingItem && content != null) - { - saveOperation.SaveTask = storage.SaveAsync(resourceUri, content, cancellationToken); - } - else - { - Trace.WriteLine(string.Format("Resource {0} already exists. Skipping.", resourceUri), "Debug"); - } - - return saveOperation; - } - - return base.CreateSaveOperationForItem(storage, context, item, cancellationToken); - } - - private async Task> PartitionAndSavePages(Guid commitId, DateTime commitTimeStamp, SortedDictionary> versions, CancellationToken cancellationToken) - { - IDictionary newPageEntries = new Dictionary(); - - foreach (IEnumerable>> partition in Utils.Partition(versions, PartitionSize)) - { - string lower = partition.First().Key.ToString(); - string upper = partition.Last().Key.ToString(); - - Uri newPageUri = CreatePageUri(Storage.BaseAddress, ("page/" + lower + "/" + upper).ToLowerInvariant()); - - IDictionary newPageItemEntries = new Dictionary(); - foreach (KeyValuePair> version in partition) - { - newPageItemEntries.Add(version.Value); - } - - IGraph extra = CreateExtraGraph(newPageUri, lower, upper); - - await SaveIndexResource(newPageUri, Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, newPageItemEntries, RootUri, extra, null, cancellationToken); - - newPageEntries[newPageUri.AbsoluteUri] = new CatalogItemSummary(Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, newPageItemEntries.Count, CreatePageSummary(newPageUri, lower, upper)); - } - - return newPageEntries; - } - - private static IGraph CreateExtraGraph(Uri pageUri, string lower, string upper) - { - IGraph graph = new Graph(); - INode resourceNode = graph.CreateUriNode(pageUri); - graph.Assert(resourceNode, graph.CreateUriNode(Schema.Predicates.Lower), graph.CreateLiteralNode(lower)); - graph.Assert(resourceNode, graph.CreateUriNode(Schema.Predicates.Upper), graph.CreateLiteralNode(upper)); - return graph; - } - - private static NuGetVersion GetPackageVersion(Uri packageUri, IGraph pageContent) - { - Triple t1 = pageContent.GetTriplesWithSubjectPredicate( - pageContent.CreateUriNode(packageUri), - pageContent.CreateUriNode(Schema.Predicates.CatalogEntry)).FirstOrDefault(); - - Triple t2 = pageContent.GetTriplesWithSubjectPredicate( - pageContent.CreateUriNode(((IUriNode)t1.Object).Uri), - pageContent.CreateUriNode(Schema.Predicates.Version)).FirstOrDefault(); - - string s = t2.Object.ToString(); - return NuGetVersion.Parse(s); - } - - private static IGraph CreatePageSummary(Uri newPageUri, string lower, string upper) - { - IGraph graph = new Graph(); - - INode resourceUri = graph.CreateUriNode(newPageUri); - - graph.Assert(resourceUri, graph.CreateUriNode(Schema.Predicates.Lower), graph.CreateLiteralNode(lower)); - graph.Assert(resourceUri, graph.CreateUriNode(Schema.Predicates.Upper), graph.CreateLiteralNode(upper)); - - return graph; - } - - protected override StorageContent CreateIndexContent(IGraph graph, Uri type) - { - JObject frame = Context.GetJsonLdContext("context.Registration.json", type); - return new JTokenStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store"); - } - } -} \ No newline at end of file diff --git a/src/Catalog/Registration/RegistrationPersistence.cs b/src/Catalog/Registration/RegistrationPersistence.cs deleted file mode 100644 index a2f145b2a..000000000 --- a/src/Catalog/Registration/RegistrationPersistence.cs +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.Metadata.Catalog.Persistence; -using VDS.RDF; -using VDS.RDF.Query; - -namespace NuGet.Services.Metadata.Catalog.Registration -{ - public class RegistrationPersistence : IRegistrationPersistence - { - private readonly Uri _registrationUri; - private readonly int _packageCountThreshold; - private readonly int _partitionSize; - private readonly RecordingStorage _storage; - private readonly RegistrationMakerCatalogItem.PostProcessGraph _postProcessGraph; - private readonly Uri _registrationBaseAddress; - private readonly Uri _contentBaseAddress; - private readonly Uri _galleryBaseAddress; - private readonly bool _forcePackagePathProviderForIcons; - - public RegistrationPersistence( - StorageFactory storageFactory, - RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph, - RegistrationKey registrationKey, - int partitionSize, - int packageCountThreshold, - Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons) - { - _storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString())); - _postProcessGraph = postProcessGraph; - _registrationUri = _storage.ResolveUri("index.json"); - _packageCountThreshold = packageCountThreshold; - _partitionSize = partitionSize; - _registrationBaseAddress = storageFactory.BaseAddress; - _contentBaseAddress = contentBaseAddress; - _galleryBaseAddress = galleryBaseAddress; - _forcePackagePathProviderForIcons = forcePackagePathProviderForIcons; - } - - public Task> Load(CancellationToken cancellationToken) - { - return Load(_storage, _registrationUri, cancellationToken); - } - - public async Task Save(IDictionary registration, CancellationToken cancellationToken) - { - await Save(_storage, _postProcessGraph, _registrationBaseAddress, registration, _partitionSize, _packageCountThreshold, _contentBaseAddress, _galleryBaseAddress, _forcePackagePathProviderForIcons, cancellationToken); - - await Cleanup(_storage, cancellationToken); - } - - // Load implementation - - private static async Task> Load(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) - { - IGraph graph = await LoadCatalog(storage, resourceUri, cancellationToken); - - IDictionary resources = GetResources(graph); - - Trace.TraceInformation("RegistrationPersistence.Load: resourceUri = {0}, resources = {1}", resourceUri, resources.Count); - - return resources; - } - - private static IDictionary GetResources(IGraph graph) - { - IDictionary resources = new Dictionary(); - - TripleStore store = new TripleStore(); - store.Add(graph); - - IList existingItems = ListExistingItems(store); - - foreach (Uri existingItem in existingItems) - { - AddExistingItem(resources, store, existingItem); - } - - return resources; - } - - private static IList ListExistingItems(TripleStore store) - { - string sparql = Utils.GetResource("sparql.SelectInlinePackage.rq"); - - SparqlResultSet resultSet = SparqlHelpers.Select(store, sparql); - - IList results = new List(); - foreach (SparqlResult result in resultSet) - { - IUriNode item = (IUriNode)result["catalogPackage"]; - results.Add(item.Uri); - } - - Trace.TraceInformation("RegistrationPersistence.ListExistingItems results = {0}", results.Count); - - return results; - } - - private static void AddExistingItem(IDictionary resources, TripleStore store, Uri catalogEntry) - { - Trace.TraceInformation("RegistrationPersistence.AddExistingItem: catalogEntry = {0}", catalogEntry); - - SparqlParameterizedString sparql = new SparqlParameterizedString - { - CommandText = Utils.GetResource("sparql.ConstructCatalogEntryGraph.rq") - }; - - sparql.SetUri("catalogEntry", catalogEntry); - - IGraph graph = SparqlHelpers.Construct(store, sparql.ToString()); - - resources.Add(RegistrationCatalogEntry.Promote( - catalogEntry.AbsoluteUri, - graph, - shouldInclude: (k, u, g) => true, - isExistingItem: true)); - } - - private static async Task LoadCatalog(IStorage storage, Uri resourceUri, CancellationToken cancellationToken) - { - string json = await storage.LoadStringAsync(resourceUri, cancellationToken); - - IGraph graph = Utils.CreateGraph(resourceUri, json); - - if (graph == null) - { - return new Graph(); - } - - IEnumerable pages = graph.GetTriplesWithPredicateObject(graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.CatalogPage)); - - IList> tasks = new List>(); - - foreach (Triple page in pages) - { - Uri pageUri = ((IUriNode)page.Subject).Uri; - - // note that this is explicit Uri comparison and deliberately ignores differences in the fragment - if (pageUri != resourceUri) - { - tasks.Add(LoadCatalogPage(storage, pageUri, cancellationToken)); - } - } - - await Task.WhenAll(tasks.ToArray()); - - foreach (Task task in tasks) - { - graph.Merge(task.Result, false); - } - - return graph; - } - - private static async Task LoadCatalogPage(IStorage storage, Uri pageUri, CancellationToken cancellationToken) - { - string json = await storage.LoadStringAsync(pageUri, cancellationToken); - IGraph graph = Utils.CreateGraph(pageUri, json); - return graph; - } - - // Save implementation - private static async Task Save( - IStorage storage, - RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, - Uri registrationBaseAddress, - IDictionary registration, - int partitionSize, - int packageCountThreshold, - Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons, - CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationPersistence.Save"); - - var items = registration.Values.Where(v => v != null).ToList(); - - if (items.Count == 0) - { - return; - } - - if (items.Count < packageCountThreshold) - { - await SaveSmallRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken); - } - else - { - await SaveLargeRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken); - } - } - - private static async Task SaveSmallRegistration( - IStorage storage, - RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, - Uri registrationBaseAddress, - IList items, - int partitionSize, Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons, - CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationPersistence.SaveSmallRegistration"); - - SingleGraphPersistence graphPersistence = new SingleGraphPersistence(storage); - - //await graphPersistence.Initialize(); - - await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, null, graphPersistence, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken); - - // now the commit has happened the graphPersistence.Graph should contain all the data - - JObject frame = (new CatalogContext()).GetJsonLdContext("context.Registration.json", graphPersistence.TypeUri); - StorageContent content = new JTokenStorageContent(Utils.CreateJson(graphPersistence.Graph, frame), "application/json", "no-store"); - await storage.SaveAsync(graphPersistence.ResourceUri, content, cancellationToken); - } - - private static async Task SaveLargeRegistration( - IStorage storage, - RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, - Uri registrationBaseAddress, - IList items, - int partitionSize, - Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons, - CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationPersistence.SaveLargeRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count); - - IList cleanUpList = new List(); - - await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, cleanUpList, null, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken); - } - - private static async Task SaveRegistration( - IStorage storage, - RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph, - Uri registrationBaseAddress, - IList items, - IList cleanUpList, - SingleGraphPersistence graphPersistence, - int partitionSize, - Uri contentBaseAddress, - Uri galleryBaseAddress, - bool forcePackagePathProviderForIcons, - CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationPersistence.SaveRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count); - - using (RegistrationMakerCatalogWriter writer = new RegistrationMakerCatalogWriter(storage, partitionSize, cleanUpList, graphPersistence)) - { - foreach (var item in items) - { - writer.Add(new RegistrationMakerCatalogItem(new Uri(item.ResourceUri), item.Graph, registrationBaseAddress, item.IsExistingItem, postProcessGraph, forcePackagePathProviderForIcons, contentBaseAddress, galleryBaseAddress)); - } - await writer.Commit(DateTime.UtcNow, null, cancellationToken); - } - } - - private static async Task Cleanup(RecordingStorage storage, CancellationToken cancellationToken) - { - Trace.TraceInformation("RegistrationPersistence.Cleanup"); - - IList tasks = new List(); - foreach (Uri loaded in storage.Loaded) - { - if (!storage.Saved.Contains(loaded)) - { - tasks.Add(storage.DeleteAsync(loaded, cancellationToken)); - } - } - if (tasks.Count > 0) - { - await Task.WhenAll(tasks.ToArray()); - } - } - } -} \ No newline at end of file diff --git a/src/Catalog/ReindexCatalogItem.cs b/src/Catalog/ReindexCatalogItem.cs deleted file mode 100644 index 2ff699cbc..000000000 --- a/src/Catalog/ReindexCatalogItem.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; - -namespace NuGet.Services.Metadata.Catalog -{ - public class ReindexCatalogItem : CatalogItem - { - Uri _itemAddress; - Uri _itemType; - - public ReindexCatalogItem(Uri itemAddress, Uri itemType) - { - _itemAddress = itemAddress; - _itemType = itemType; - } - - public override Uri GetItemType() - { - return _itemType; - } - - public override Uri GetItemAddress() - { - return _itemAddress; - } - } -} diff --git a/src/Catalog/Schema.cs b/src/Catalog/Schema.cs index 0a090f1f2..45b42702f 100644 --- a/src/Catalog/Schema.cs +++ b/src/Catalog/Schema.cs @@ -11,21 +11,14 @@ public static class Prefixes { public static readonly string NuGet = "http://schema.nuget.org/schema#"; public static readonly string Catalog = "http://schema.nuget.org/catalog#"; - public static readonly string Record = "http://schema.nuget.org/record#"; public static readonly string Xsd = "http://www.w3.org/2001/XMLSchema#"; public static readonly string Rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; } public static class DataTypes { - public static readonly Uri CatalogInfastructure = new Uri(Prefixes.Catalog + "CatalogInfastructure"); - public static readonly Uri Package = new Uri(Prefixes.NuGet + "Package"); - public static readonly Uri PackageRegistration = new Uri(Prefixes.NuGet + "PackageRegistration"); public static readonly Uri PackageDetails = new Uri(Prefixes.NuGet + "PackageDetails"); public static readonly Uri PackageDelete = new Uri(Prefixes.NuGet + "PackageDelete"); - public static readonly Uri PackageDependencyGroup = new Uri(Prefixes.NuGet + "PackageDependencyGroup"); - public static readonly Uri PackageDependency = new Uri(Prefixes.NuGet + "PackageDependency"); - public static readonly Uri NuGetClassicPackage = new Uri(Prefixes.NuGet + "NuGetClassicPackage"); public static readonly Uri PackageEntry = new Uri(Prefixes.NuGet + "PackageEntry"); @@ -33,25 +26,10 @@ public static class DataTypes public static readonly Uri CatalogPage = new Uri(Prefixes.Catalog + "CatalogPage"); public static readonly Uri Permalink = new Uri(Prefixes.Catalog + "Permalink"); public static readonly Uri AppendOnlyCatalog = new Uri(Prefixes.Catalog + "AppendOnlyCatalog"); - public static readonly Uri CatalogDelete = new Uri(Prefixes.Catalog + "CatalogDelete"); public static readonly Uri Integer = new Uri(Prefixes.Xsd + "integer"); public static readonly Uri DateTime = new Uri(Prefixes.Xsd + "dateTime"); public static readonly Uri Boolean = new Uri(Prefixes.Xsd + "boolean"); - - public static readonly Uri Icon = new Uri(Prefixes.NuGet + "Icon"); - public static readonly Uri Screenshot = new Uri(Prefixes.NuGet + "Screenshot"); - public static readonly Uri ZipArchive = new Uri(Prefixes.NuGet + "ZipArchive"); - public static readonly Uri HeroIcon = new Uri(Prefixes.NuGet + "HeroIcon"); - public static readonly Uri LargeIcon = new Uri(Prefixes.NuGet + "LargeIcon"); - public static readonly Uri MediumIcon = new Uri(Prefixes.NuGet + "MediumIcon"); - public static readonly Uri SmallIcon = new Uri(Prefixes.NuGet + "SmallIcon"); - public static readonly Uri WideIcon = new Uri(Prefixes.NuGet + "WideIcon"); - public static readonly Uri CsmTemplate = new Uri(Prefixes.NuGet + "CsmTemplate"); - - public static readonly Uri Record = new Uri(Prefixes.Record + "Record"); - public static readonly Uri RecordRegistration = new Uri(Prefixes.Record + "Registration"); - public static readonly Uri RecordOwner = new Uri(Prefixes.Record + "Owner"); } public static class Predicates @@ -64,95 +42,30 @@ public static class Predicates public static readonly Uri CatalogCount = new Uri(Prefixes.Catalog + "count"); public static readonly Uri CatalogParent = new Uri(Prefixes.Catalog + "parent"); - public static readonly Uri GalleryKey = new Uri(Prefixes.Catalog + "galleryKey"); - public static readonly Uri GalleryChecksum = new Uri(Prefixes.Catalog + "galleryChecksum"); - - public static readonly Uri Prefix = new Uri(Prefixes.NuGet + "prefix"); public static readonly Uri Id = new Uri(Prefixes.NuGet + "id"); public static readonly Uri Version = new Uri(Prefixes.NuGet + "version"); - public static readonly Uri VerbatimVersion = new Uri(Prefixes.NuGet + "verbatimVersion"); public static readonly Uri OriginalId = new Uri(Prefixes.NuGet + "originalId"); - public static readonly Uri Upper = new Uri(Prefixes.NuGet + "upper"); - public static readonly Uri Lower = new Uri(Prefixes.NuGet + "lower"); - - public static readonly Uri CatalogEntry = new Uri(Prefixes.NuGet + "catalogEntry"); - public static readonly Uri PackageContent = new Uri(Prefixes.NuGet + "packageContent"); - public static readonly Uri PackageEntry = new Uri(Prefixes.NuGet + "packageEntry"); public static readonly Uri FullName = new Uri(Prefixes.NuGet + "fullName"); public static readonly Uri Name = new Uri(Prefixes.NuGet + "name"); public static readonly Uri Length = new Uri(Prefixes.NuGet + "length"); public static readonly Uri CompressedLength = new Uri(Prefixes.NuGet + "compressedLength"); - public static readonly Uri FileName = new Uri(Prefixes.Catalog + "fileName"); - public static readonly Uri Details = new Uri(Prefixes.Catalog + "details"); - - public static readonly Uri Domain = new Uri(Prefixes.Record + "domain"); - public static readonly Uri RecordDomain = new Uri(Prefixes.Record + "recordDomain"); - public static readonly Uri RecordRegistration = new Uri(Prefixes.Record + "recordRegistration"); - public static readonly Uri ObjectId = new Uri(Prefixes.Record + "objectId"); - - public static readonly Uri Visibility = new Uri(Prefixes.NuGet + "visibility"); - public static readonly Uri Organization = new Uri(Prefixes.NuGet + "organization"); - public static readonly Uri Subscription = new Uri(Prefixes.NuGet + "subscription"); - - // General-purpose fields - - public static readonly Uri Author = new Uri(Prefixes.NuGet + "author"); - public static readonly Uri Copyright = new Uri(Prefixes.NuGet + "copyright"); + // General-purpose fields used in C# explicitly (not just in the .nuspec to RDF XSLT) public static readonly Uri Created = new Uri(Prefixes.NuGet + "created"); - public static readonly Uri Description = new Uri(Prefixes.NuGet + "description"); - public static readonly Uri IconUrl = new Uri(Prefixes.NuGet + "iconUrl"); - - public static readonly Uri Package = new Uri(Prefixes.NuGet + "package"); - public static readonly Uri Registration = new Uri(Prefixes.NuGet + "registration"); public static readonly Uri LastCreated = new Uri(Prefixes.NuGet + "lastCreated"); public static readonly Uri LastEdited = new Uri(Prefixes.NuGet + "lastEdited"); public static readonly Uri LastDeleted = new Uri(Prefixes.NuGet + "lastDeleted"); public static readonly Uri Listed = new Uri(Prefixes.NuGet + "listed"); - public static readonly Uri Language = new Uri(Prefixes.NuGet + "language"); public static readonly Uri Published = new Uri(Prefixes.NuGet + "published"); - public static readonly Uri Publisher = new Uri(Prefixes.NuGet + "publisher"); - public static readonly Uri UserName = new Uri(Prefixes.NuGet + "userName"); - public static readonly Uri Tenant = new Uri(Prefixes.NuGet + "tenant"); - public static readonly Uri UserId = new Uri(Prefixes.NuGet + "userId"); - public static readonly Uri TenantId = new Uri(Prefixes.NuGet + "tenantId"); public static readonly Uri PackageHash = new Uri(Prefixes.NuGet + "packageHash"); public static readonly Uri PackageHashAlgorithm = new Uri(Prefixes.NuGet + "packageHashAlgorithm"); public static readonly Uri PackageSize = new Uri(Prefixes.NuGet + "packageSize"); - public static readonly Uri ProjectUrl = new Uri(Prefixes.NuGet + "projectUrl"); - public static readonly Uri GalleryDetailsUrl = new Uri(Prefixes.NuGet + "galleryDetailsUrl"); - public static readonly Uri ReleaseNotes = new Uri(Prefixes.NuGet + "releaseNotes"); - public static readonly Uri RequireLicenseAcceptance = new Uri(Prefixes.NuGet + "requireLicenseAcceptance"); - public static readonly Uri Summary = new Uri(Prefixes.NuGet + "summary"); - public static readonly Uri Title = new Uri(Prefixes.NuGet + "title"); - public static readonly Uri LicenseUrl = new Uri(Prefixes.NuGet + "licenseUrl"); - public static readonly Uri LicenseNames = new Uri(Prefixes.NuGet + "licenseNames"); - public static readonly Uri LicenseReportUrl = new Uri(Prefixes.NuGet + "licenseReportUrl"); - public static readonly Uri MinimumClientVersion = new Uri(Prefixes.NuGet + "minimumClientVersion"); - public static readonly Uri Tag = new Uri(Prefixes.NuGet + "tag"); - public static readonly Uri LicenseName = new Uri(Prefixes.NuGet + "licenseName"); - public static readonly Uri SupportedFramework = new Uri(Prefixes.NuGet + "supportedFramework"); - public static readonly Uri Owner = new Uri(Prefixes.NuGet + "owner"); - public static readonly Uri Namespace = new Uri(Prefixes.NuGet + "namespace"); - - public static readonly Uri DependencyGroup = new Uri(Prefixes.NuGet + "dependencyGroup"); - public static readonly Uri Dependency = new Uri(Prefixes.NuGet + "dependency"); - public static readonly Uri Range = new Uri(Prefixes.NuGet + "range"); - public static readonly Uri NameIdentifier = new Uri(Prefixes.NuGet + "nameIdentifier"); - public static readonly Uri GivenName = new Uri(Prefixes.NuGet + "givenName"); - public static readonly Uri Surname = new Uri(Prefixes.NuGet + "surname"); - public static readonly Uri Email = new Uri(Prefixes.NuGet + "email"); - public static readonly Uri Iss = new Uri(Prefixes.NuGet + "iss"); - - public static readonly Uri LicenseExpression = new Uri(Prefixes.NuGet + "licenseExpression"); - public static readonly Uri LicenseFile = new Uri(Prefixes.NuGet + "licenseFile"); - - public static readonly Uri IconFile = new Uri(Prefixes.NuGet + "iconFile"); + public static readonly Uri Range = new Uri(Prefixes.NuGet + "range"); public static readonly Uri Deprecation = new Uri(Prefixes.NuGet + "deprecation"); diff --git a/src/Catalog/SingleGraphPersistence.cs b/src/Catalog/SingleGraphPersistence.cs deleted file mode 100644 index 4cb48ae83..000000000 --- a/src/Catalog/SingleGraphPersistence.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Threading; -using System.Threading.Tasks; -using NuGet.Services.Metadata.Catalog.Persistence; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog -{ - public class SingleGraphPersistence : ICatalogGraphPersistence - { - Uri[] _propertiesToUpdate = - { - Schema.Predicates.CatalogCommitId, - Schema.Predicates.CatalogTimeStamp, - Schema.Predicates.CatalogCount - }; - - IStorage _storage; - IGraph _initialGraph; - - public SingleGraphPersistence(IStorage storage) - { - _storage = storage; - Graph = new Graph(); - } - - public IGraph Graph { get; private set; } - public Uri ResourceUri { get; private set; } - public Uri TypeUri { get; private set; } - - public async Task Initialize(CancellationToken cancellationToken) - { - Uri rootUri = _storage.ResolveUri("index.json"); - - string json = await _storage.LoadStringAsync(rootUri, cancellationToken); - - if (json != null) - { - _initialGraph = Utils.CreateGraph(rootUri, json); - } - else - { - _initialGraph = null; - } - } - - public async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken) - { - await Task.Run(() => - { - Utils.RemoveExistingProperties(Graph, graph, _propertiesToUpdate); - - Graph.Merge(graph, true); - - ResourceUri = resourceUri; - TypeUri = typeUri; - }, cancellationToken); - } - - public Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken) - { - return Task.FromResult(_initialGraph); - } - - public Uri CreatePageUri(Uri baseAddress, string relativeAddress) - { - return new Uri(_storage.BaseAddress, "index.json#" + relativeAddress); - } - } -} \ No newline at end of file diff --git a/src/Catalog/SortingGraphCollector.cs b/src/Catalog/SortingGraphCollector.cs deleted file mode 100644 index 75e70ae65..000000000 --- a/src/Catalog/SortingGraphCollector.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using VDS.RDF; - -namespace NuGet.Services.Metadata.Catalog -{ - public abstract class SortingGraphCollector : SortingIdCollector - { - private readonly Uri[] _types; - - public SortingGraphCollector( - Uri index, - Uri[] types, - ITelemetryService telemetryService, - Func handlerFunc = null, - IHttpRetryStrategy httpRetryStrategy = null) - : base(index, telemetryService, handlerFunc, httpRetryStrategy) - { - _types = types; - } - - protected override async Task ProcessSortedBatchAsync( - CollectorHttpClient client, - KeyValuePair> sortedBatch, - JToken context, - CancellationToken cancellationToken) - { - var graphs = new Dictionary(); - var graphTasks = new Dictionary>(); - - foreach (var item in sortedBatch.Value) - { - var isMatch = false; - - foreach (Uri type in _types) - { - if (item.TypeUris.Any(typeUri => typeUri.AbsoluteUri == type.AbsoluteUri)) - { - isMatch = true; - break; - } - } - - if (isMatch) - { - // Load package details from catalog. - // Download the graph to a read-only container. This allows operations on each graph to be safely - // parallelized. - var task = client.GetGraphAsync(item.Uri, readOnly: true, token: cancellationToken); - - graphTasks.Add(item.Uri.AbsoluteUri, task); - } - } - - await Task.WhenAll(graphTasks.Values.ToArray()); - - foreach (var task in graphTasks) - { - graphs.Add(task.Key, task.Value.Result); - } - - if (graphs.Count > 0) - { - var sortedGraphs = new KeyValuePair>(sortedBatch.Key, graphs); - - await ProcessGraphsAsync(sortedGraphs, cancellationToken); - } - } - - protected abstract Task ProcessGraphsAsync( - KeyValuePair> sortedGraphs, - CancellationToken cancellationToken); - } -} \ No newline at end of file diff --git a/src/Catalog/SortingIdCollector.cs b/src/Catalog/SortingIdCollector.cs deleted file mode 100644 index ea45ed787..000000000 --- a/src/Catalog/SortingIdCollector.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Net.Http; - -namespace NuGet.Services.Metadata.Catalog -{ - public abstract class SortingIdCollector : SortingCollector - { - public SortingIdCollector( - Uri index, - ITelemetryService telemetryService, - Func handlerFunc = null, - IHttpRetryStrategy retryStrategy = null) - : base(index, telemetryService, handlerFunc, retryStrategy) - { - } - - protected override string GetKey(CatalogCommitItem item) - { - return item.PackageIdentity.Id; - } - } -} \ No newline at end of file diff --git a/src/Catalog/StorageWriteLock.cs b/src/Catalog/StorageWriteLock.cs deleted file mode 100644 index 81588ff15..000000000 --- a/src/Catalog/StorageWriteLock.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using System; -using System.Net; -using System.Threading; -using System.Threading.Tasks; - -namespace NuGet.Services.Metadata.Catalog -{ - public class StorageWriteLock - { - string _connectionString; - string _container; - string _leaseId; - TimeSpan _duration; - CloudBlockBlob _writeLockBlob; - - public StorageWriteLock(string connectionString, string container, int seconds = 30) - { - _connectionString = connectionString; - _container = container; - _duration = TimeSpan.FromSeconds(seconds < 15 ? 15 : seconds); - } - - public async Task AquireAsync() - { - CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_connectionString); - CloudBlobClient client = storageAccount.CreateCloudBlobClient(); - CloudBlobContainer container = client.GetContainerReference(_container); - _writeLockBlob = container.GetBlockBlobReference("write.lock"); - - _leaseId = await AquireAsync(_writeLockBlob, _duration); - } - public async Task ReleaseAsync() - { - await _writeLockBlob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId)); - } - - static async Task AquireAsync(CloudBlockBlob blob, TimeSpan timeStamp) - { - bool retry = false; - do - { - try - { - return await blob.AcquireLeaseAsync(timeStamp, null); - } - catch (StorageException e) - { - WebException webException = e.InnerException as WebException; - if (webException != null) - { - HttpStatusCode statusCode = ((HttpWebResponse)webException.Response).StatusCode; - if (statusCode == HttpStatusCode.Conflict) - { - Thread.Sleep(500); - retry = true; - } - else if (statusCode == HttpStatusCode.NotFound) - { - blob.UploadText(string.Empty); - retry = true; - } - } - else - { - throw; - } - } - } - while (retry); - - return null; - } - } -} diff --git a/src/Catalog/Telemetry/TelemetryConstants.cs b/src/Catalog/Telemetry/TelemetryConstants.cs index df2bc63ee..38055f511 100644 --- a/src/Catalog/Telemetry/TelemetryConstants.cs +++ b/src/Catalog/Telemetry/TelemetryConstants.cs @@ -39,7 +39,6 @@ public static class TelemetryConstants public const string ProcessPackageDeleteSeconds = "ProcessPackageDeleteSeconds"; public const string ProcessPackageDetailsSeconds = "ProcessPackageDetailsSeconds"; public const string ProcessPackageVersionIndexSeconds = "ProcessPackageVersionIndexSeconds"; - public const string RegistrationDeltaCount = "RegistrationDeltaCount"; public const string SizeInBytes = "SizeInBytes"; public const string StatusCode = "StatusCode"; public const string Success = "Success"; diff --git a/src/Catalog/VerboseFileSystemEmulatorHandler.cs b/src/Catalog/VerboseFileSystemEmulatorHandler.cs deleted file mode 100644 index 6cc2d72be..000000000 --- a/src/Catalog/VerboseFileSystemEmulatorHandler.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Diagnostics; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace NuGet.Services.Metadata.Catalog -{ - public class VerboseFileSystemEmulatorHandler : FileSystemEmulatorHandler - { - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - Trace.TraceInformation("HTTP {0} {1}", request.Method, request.RequestUri); - return base.SendAsync(request, cancellationToken); - } - } -} diff --git a/src/Catalog/context/Ownership.json b/src/Catalog/context/Ownership.json deleted file mode 100644 index 7a453e753..000000000 --- a/src/Catalog/context/Ownership.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "@context" : { - "@vocab" : "http://schema.nuget.org/schema#", - "catalog": "http://schema.nuget.org/catalog#", - "record" : "http://schema.nuget.org/record#", - "objectId" : { "@id": "record:objectId" }, - "registrations" : { "@id": "registration", "@container" : "@set" }, - "owners" : { "@id": "owner", "@container" : "@set" }, - "versions" : { "@id": "version", "@container" : "@set" }, - "catalog:commitTimeStamp" : { "@type" : "http://www.w3.org/2001/XMLSchema#dateTime" }, - } -} diff --git a/src/Catalog/context/Package.json b/src/Catalog/context/Package.json deleted file mode 100644 index df9bbf616..000000000 --- a/src/Catalog/context/Package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "@context": { - "@vocab": "http://schema.nuget.org/schema#", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "catalogEntry" : { "@type" : "@id" }, - "registration" : { "@type" : "@id" }, - "packageContent": { "@type": "@id" }, - "published": { "@type": "xsd:dateTime" } - } -} diff --git a/src/Catalog/context/Registration.json b/src/Catalog/context/Registration.json deleted file mode 100644 index ba7144b8b..000000000 --- a/src/Catalog/context/Registration.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "@context" : { - "@vocab": "http://schema.nuget.org/schema#", - "catalog" : "http://schema.nuget.org/catalog#", - "xsd": "http://www.w3.org/2001/XMLSchema#", - - "items" : { "@id": "catalog:item", "@container" : "@set" }, - - "commitTimeStamp" : { "@id" : "catalog:commitTimeStamp", "@type" : "xsd:dateTime" }, - "commitId" : { "@id" : "catalog:commitId" }, - "count" : { "@id" : "catalog:count" }, - "parent" : { "@id" : "catalog:parent", "@type" : "@id" }, - - "tags": { "@container" : "@set", "@id": "tag" }, - "reasons" : { "@container" : "@set" }, - - "packageTargetFrameworks": { "@container": "@set", "@id": "packageTargetFramework" }, - - "dependencyGroups": { "@container" : "@set", "@id": "dependencyGroup" }, - "dependencies": { "@container" : "@set", "@id": "dependency" }, - - "packageContent": { "@type": "@id" }, - "published" : { "@type" : "xsd:dateTime" }, - "registration" : { "@type": "@id" } - } -} diff --git a/src/Catalog/context/Segment.json b/src/Catalog/context/Segment.json deleted file mode 100644 index ecab5ef52..000000000 --- a/src/Catalog/context/Segment.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "@context" : { - "@vocab" : "http://schema.nuget.org/catalog#", - "nuget" : "http://schema.nuget.org/schema#", - "id" : "nuget:id", - "version" : "nuget:version", - "description" : "nuget:description", - "registration" : { "@id" : "nuget:registration", "@type" : "@id" }, - "entry" : { "@container" : "@set" }, - "url" : "@id" - } -} diff --git a/src/Catalog/schema/schema.ttl b/src/Catalog/schema/schema.ttl deleted file mode 100644 index 1da9514ca..000000000 --- a/src/Catalog/schema/schema.ttl +++ /dev/null @@ -1,26 +0,0 @@ - -@prefix rdf: . -@prefix rdfs: . -@prefix owl: . -@PREFIX nuget: . - -nuget:Package rdfs:subClassOf nuget:Resource . - -nuget:Container rdfs:subClassOf nuget:Resource . - -nuget:item rdfs:domain nuget:Container . - -nuget:container owl:inverseOf nuget:item . - -nuget:PackageRegistration nuget:transform "PackageRegistration.rq" ; - nuget:frame "PackageRegistrationFrame.json" . - -nuget:Package nuget:transform "Package.rq" ; - nuget:frame "PackageFrame.json" . - -nuget:Owner nuget:transform "Owner.rq" ; - nuget:frame "OwnerFrame.json" . - -nuget:Container nuget:transform "Container.rq" ; - nuget:frame "ContainerFrame.json" . - diff --git a/src/Catalog/sparql/All.rq b/src/Catalog/sparql/All.rq deleted file mode 100644 index 946aa3cd1..000000000 --- a/src/Catalog/sparql/All.rq +++ /dev/null @@ -1,8 +0,0 @@ -CONSTRUCT -{ - ?s ?p ?o . -} -WHERE -{ - ?s ?p ?o . -} \ No newline at end of file diff --git a/src/Catalog/sparql/ConstructCatalogEntryGraph.rq b/src/Catalog/sparql/ConstructCatalogEntryGraph.rq deleted file mode 100644 index c508d45ca..000000000 --- a/src/Catalog/sparql/ConstructCatalogEntryGraph.rq +++ /dev/null @@ -1,93 +0,0 @@ -PREFIX nuget: -PREFIX catalog: - -CONSTRUCT -{ - ?catalogEntry a nuget:PackageDetails ; - nuget:id ?id ; - nuget:version ?version ; - nuget:published ?published ; - nuget:packageContent ?packageContent ; - nuget:dependencyGroup ?dependency_group ; - nuget:listed ?listed ; - nuget:description ?description ; - nuget:title ?title ; - nuget:summary ?summary ; - nuget:iconUrl ?iconUrl ; - nuget:licenseUrl ?licenseUrl ; - nuget:licenseExpression ?licenseExpression ; - nuget:projectUrl ?projectUrl ; - nuget:requireLicenseAcceptance ?requireLicenseAcceptance ; - nuget:language ?language ; - nuget:authors ?authors ; - nuget:tag ?tag ; - nuget:minClientVersion ?minClientVersion ; - nuget:deprecation ?deprecation . - - ?dependency_group a nuget:PackageDependencyGroup ; - nuget:dependency ?dependency ; - nuget:targetFramework ?dependency_group_targetFramework . - - ?dependency a nuget:PackageDependency ; - nuget:id ?dependency_id ; - nuget:range ?dependency_range ; - nuget:version ?dependency_version . - - ?deprecation a nuget:deprecation ; - nuget:reasons ?deprecation_reasons ; - nuget:message ?deprecation_message ; - nuget:alternatePackage ?deprecation_alternatePackage . - - ?deprecation_alternatePackage a nuget:alternatePackage ; - nuget:id ?deprecation_alternatePackage_id ; - nuget:range ?deprecation_alternatePackage_range . -} -WHERE -{ - BIND (@catalogEntry AS ?catalogEntry) - - ?catalogEntry nuget:version ?version ; - nuget:id ?id ; - nuget:published ?published ; - - OPTIONAL { ?catalogEntry nuget:packageContent ?packageContent . } - OPTIONAL { ?catalogEntry nuget:listed ?listed . } - OPTIONAL { ?catalogEntry nuget:description ?description . } - OPTIONAL { ?catalogEntry nuget:title ?title . } - OPTIONAL { ?catalogEntry nuget:summary ?summary . } - OPTIONAL { ?catalogEntry nuget:iconUrl ?iconUrl . } - OPTIONAL { ?catalogEntry nuget:licenseUrl ?licenseUrl . } - OPTIONAL { ?catalogEntry nuget:licenseExpression ?licenseExpression . } - OPTIONAL { ?catalogEntry nuget:projectUrl ?projectUrl . } - OPTIONAL { ?catalogEntry nuget:requireLicenseAcceptance ?requireLicenseAcceptance . } - OPTIONAL { ?catalogEntry nuget:language ?language . } - OPTIONAL { ?catalogEntry nuget:authors ?authors . } - OPTIONAL { ?catalogEntry nuget:tag ?tag . } - OPTIONAL { ?catalogEntry nuget:minClientVersion ?minClientVersion . } - - OPTIONAL - { - ?catalogEntry nuget:dependencyGroup ?dependency_group . - OPTIONAL { ?dependency_group nuget:targetFramework ?dependency_group_targetFramework . } - OPTIONAL - { - ?dependency_group nuget:dependency ?dependency . - ?dependency nuget:id ?dependency_id . - OPTIONAL { ?dependency nuget:range ?dependency_range . } - OPTIONAL { ?dependency nuget:version ?dependency_version . } - } - } - - OPTIONAL - { - ?catalogEntry nuget:deprecation ?deprecation . - ?deprecation nuget:reasons ?deprecation_reasons . - OPTIONAL { ?deprecation nuget:message ?deprecation_message . } - OPTIONAL - { - ?deprecation nuget:alternatePackage ?deprecation_alternatePackage . - ?deprecation_alternatePackage nuget:id ?deprecation_alternatePackage_id . - ?deprecation_alternatePackage nuget:range ?deprecation_alternatePackage_range . - } - } -} diff --git a/src/Catalog/sparql/ConstructPackagePageContentGraph.rq b/src/Catalog/sparql/ConstructPackagePageContentGraph.rq deleted file mode 100644 index c7447bcd7..000000000 --- a/src/Catalog/sparql/ConstructPackagePageContentGraph.rq +++ /dev/null @@ -1,91 +0,0 @@ -PREFIX nuget: -PREFIX catalog: - -CONSTRUCT -{ - ?package a nuget:Package ; - nuget:registration ?registration ; - nuget:packageContent ?packageContent ; - nuget:catalogEntry ?catalogPackage . - - ?catalogPackage a nuget:PackageDetails ; - nuget:id ?id ; - nuget:version ?version ; - nuget:dependencyGroup ?dependency_group ; - nuget:published ?published ; - nuget:listed ?listed ; - nuget:description ?description ; - nuget:title ?title ; - nuget:summary ?summary ; - nuget:iconUrl ?iconUrl ; - nuget:licenseUrl ?licenseUrl ; - nuget:projectUrl ?projectUrl ; - nuget:requireLicenseAcceptance ?requireLicenseAcceptance ; - nuget:language ?language ; - nuget:authors ?authors ; - nuget:tag ?tag ; - nuget:minClientVersion ?minClientVersion . - - ?dependency_group a nuget:PackageDependencyGroup ; - nuget:dependency ?dependency ; - nuget:targetFramework ?dependency_group_targetFramework . - - ?dependency a nuget:PackageDependency ; - nuget:id ?dependency_id ; - nuget:registration ?dependency_registration ; - nuget:range ?dependency_version . -} -WHERE -{ - BIND (@catalogPackage AS ?catalogPackage) - BIND (@package AS ?package) - BIND (@packageContent AS ?packageContent) - - BIND (IRI(CONCAT(STR(@baseAddress), "index.json")) AS ?registration) - - ?catalogPackage nuget:version ?version ; - nuget:id ?id ; - nuget:published ?published . - - OPTIONAL { ?catalogPackage nuget:description ?optionalDescription . } - OPTIONAL { ?catalogPackage nuget:title ?optionalTitle . } - OPTIONAL { ?catalogPackage nuget:summary ?optionalSummary . } - OPTIONAL { ?catalogPackage nuget:iconUrl ?optionalIconUrl . } - OPTIONAL { ?catalogPackage nuget:licenseUrl ?optionalLicenseUrl . } - OPTIONAL { ?catalogPackage nuget:projectUrl ?optionalProjectUrl . } - OPTIONAL { ?catalogPackage nuget:requireLicenseAcceptance ?optionalRequireLicenseAcceptance . } - OPTIONAL { ?catalogPackage nuget:language ?optionalLanguage . } - OPTIONAL { ?catalogPackage nuget:authors ?optionalAuthors . } - OPTIONAL { ?catalogPackage nuget:tag ?optionalTag . } - OPTIONAL { ?catalogPackage nuget:minClientVersion ?optionalMinClientVersion . } - - BIND(COALESCE(?optionalDescription, "") AS ?description) - BIND(COALESCE(?optionalTitle, "") AS ?title) - BIND(COALESCE(?optionalSummary, "") AS ?summary) - BIND(COALESCE(?optionalIconUrl, "") AS ?iconUrl) - BIND(COALESCE(?optionalLicenseUrl, "") AS ?licenseUrl) - BIND(COALESCE(?optionalProjectUrl, "") AS ?projectUrl) - BIND(COALESCE(?optionalRequireLicenseAcceptance, false) AS ?requireLicenseAcceptance) - BIND(COALESCE(?optionalLanguage, "") AS ?language) - BIND(COALESCE(?optionalAuthors, "") AS ?authors) - BIND(COALESCE(?optionalTag, "") AS ?tag) - BIND(COALESCE(?optionalMinClientVersion, "") AS ?minClientVersion) - BIND (IF (year(?published) = 1900, false, true) as ?listed) - - OPTIONAL - { - ?catalogPackage nuget:dependencyGroup ?dependency_group . - - OPTIONAL { ?dependency_group nuget:targetFramework ?dependency_group_targetFramework . } - - OPTIONAL - { - ?dependency_group nuget:dependency ?dependency . - ?dependency nuget:id ?dependency_id . - - BIND (IRI(CONCAT(STR(@registrationBaseAddress), LCASE(?dependency_id), "/index.json")) AS ?dependency_registration) - - OPTIONAL { ?dependency nuget:range ?dependency_version . } - } - } -} diff --git a/src/Catalog/sparql/ConstructRegistrationPageContentGraph.rq b/src/Catalog/sparql/ConstructRegistrationPageContentGraph.rq deleted file mode 100644 index 57082293f..000000000 --- a/src/Catalog/sparql/ConstructRegistrationPageContentGraph.rq +++ /dev/null @@ -1,123 +0,0 @@ -PREFIX nuget: -PREFIX catalog: - -CONSTRUCT -{ - ?package a nuget:Package ; - nuget:registration ?registration ; - nuget:packageContent ?packageContent ; - nuget:catalogEntry ?catalogEntry . - - ?catalogEntry a nuget:PackageDetails ; - nuget:id ?id ; - nuget:version ?version ; - nuget:published ?published ; - nuget:packageContent ?packageContent ; - nuget:dependencyGroup ?dependency_group ; - nuget:listed ?listed ; - nuget:description ?description ; - nuget:title ?title ; - nuget:summary ?summary ; - nuget:iconUrl ?iconUrl ; - nuget:licenseUrl ?licenseUrl ; - nuget:licenseExpression ?licenseExpression ; - nuget:projectUrl ?projectUrl ; - nuget:requireLicenseAcceptance ?requireLicenseAcceptance ; - nuget:language ?language ; - nuget:authors ?authors ; - nuget:tag ?tag ; - nuget:minClientVersion ?minClientVersion ; - nuget:deprecation ?deprecation . - - ?dependency_group a nuget:PackageDependencyGroup ; - nuget:dependency ?dependency ; - nuget:targetFramework ?dependency_group_targetFramework . - - ?dependency a nuget:PackageDependency ; - nuget:id ?dependency_id ; - nuget:registration ?dependency_registration ; - nuget:range ?dependency_range ; - nuget:version ?dependency_version . - - ?deprecation a nuget:deprecation ; - nuget:reasons ?deprecation_reasons ; - nuget:message ?deprecation_message ; - nuget:alternatePackage ?deprecation_alternatePackage . - - ?deprecation_alternatePackage a nuget:alternatePackage ; - nuget:id ?deprecation_alternatePackage_id ; - nuget:range ?deprecation_alternatePackage_range . -} -WHERE -{ - BIND (@catalogEntry AS ?catalogEntry) - BIND (@package AS ?package) - - BIND (IRI(CONCAT(STR(@baseAddress), "index.json")) AS ?registration) - - BIND(@licenseUrl AS ?licenseUrl) - - BIND(@iconUrl AS ?iconUrl) - - ?catalogEntry nuget:version ?version ; - nuget:id ?id ; - nuget:published ?published . - - OPTIONAL { ?catalogEntry nuget:packageContent ?optionalPackageContent . } - BIND(COALESCE(?optionalPackageContent, @packageContent) AS ?packageContent) - - OPTIONAL { ?catalogEntry nuget:description ?optionalDescription . } - OPTIONAL { ?catalogEntry nuget:title ?optionalTitle . } - OPTIONAL { ?catalogEntry nuget:summary ?optionalSummary . } - OPTIONAL { ?catalogEntry nuget:licenseExpression ?optionalLicenseExpression . } - OPTIONAL { ?catalogEntry nuget:projectUrl ?optionalProjectUrl . } - OPTIONAL { ?catalogEntry nuget:requireLicenseAcceptance ?optionalRequireLicenseAcceptance . } - OPTIONAL { ?catalogEntry nuget:language ?optionalLanguage . } - OPTIONAL { ?catalogEntry nuget:authors ?optionalAuthors . } - OPTIONAL { ?catalogEntry nuget:tag ?optionalTag . } - OPTIONAL { ?catalogEntry nuget:minClientVersion ?optionalMinClientVersion . } - OPTIONAL { ?catalogEntry nuget:listed ?optionalListed . } - - BIND(COALESCE(?optionalDescription, "") AS ?description) - BIND(COALESCE(?optionalTitle, "") AS ?title) - BIND(COALESCE(?optionalSummary, "") AS ?summary) - BIND(COALESCE(?optionalLicenseExpression, "") AS ?licenseExpression) - BIND(COALESCE(?optionalProjectUrl, "") AS ?projectUrl) - BIND(COALESCE(?optionalRequireLicenseAcceptance, false) AS ?requireLicenseAcceptance) - BIND(COALESCE(?optionalLanguage, "") AS ?language) - BIND(COALESCE(?optionalAuthors, "") AS ?authors) - BIND(COALESCE(?optionalTag, "") AS ?tag) - BIND(COALESCE(?optionalMinClientVersion, "") AS ?minClientVersion) - - BIND (IF (year(?published) = 1900, false, true) as ?listedFromPublished) - BIND(COALESCE(?optionalListed, ?listedFromPublished) AS ?listed) - - OPTIONAL - { - ?catalogEntry nuget:dependencyGroup ?dependency_group . - - OPTIONAL { ?dependency_group nuget:targetFramework ?dependency_group_targetFramework . } - - OPTIONAL - { - ?dependency_group nuget:dependency ?dependency . - ?dependency nuget:id ?dependency_id . - BIND (IRI(CONCAT(STR(@registrationBaseAddress), LCASE(?dependency_id), "/index.json")) AS ?dependency_registration) - OPTIONAL { ?dependency nuget:range ?dependency_range . } - OPTIONAL { ?dependency nuget:version ?dependency_version . } - } - } - - OPTIONAL - { - ?catalogEntry nuget:deprecation ?deprecation . - ?deprecation nuget:reasons ?deprecation_reasons . - OPTIONAL { ?deprecation nuget:message ?deprecation_message . } - OPTIONAL - { - ?deprecation nuget:alternatePackage ?deprecation_alternatePackage . - ?deprecation_alternatePackage nuget:id ?deprecation_alternatePackage_id . - ?deprecation_alternatePackage nuget:range ?deprecation_alternatePackage_range . - } - } -} diff --git a/src/Catalog/sparql/SelectDistinctDependency.rq b/src/Catalog/sparql/SelectDistinctDependency.rq deleted file mode 100644 index 6ff5579f5..000000000 --- a/src/Catalog/sparql/SelectDistinctDependency.rq +++ /dev/null @@ -1,10 +0,0 @@ -PREFIX nuget: - -SELECT DISTINCT ?id -WHERE -{ - ?dependency a nuget:PackageDependency ; - nuget:id ?s . - - BIND (LCASE(?s) AS ?id) -} diff --git a/src/Catalog/sparql/SelectDistinctDependencyVersionRanges.rq b/src/Catalog/sparql/SelectDistinctDependencyVersionRanges.rq deleted file mode 100644 index df8719a9f..000000000 --- a/src/Catalog/sparql/SelectDistinctDependencyVersionRanges.rq +++ /dev/null @@ -1,13 +0,0 @@ -PREFIX nuget: - -SELECT DISTINCT ?versionRange -WHERE -{ - BIND (@resourceUri AS ?resourceUri) - - ?resourceUri nuget:dependencyGroup ?dependencyGroup . - ?dependencyGroup a nuget:PackageDependencyGroup . - ?dependencyGroup nuget:dependency ?dependency . - ?dependency a nuget:PackageDependency . - ?dependency nuget:range ?versionRange . -} diff --git a/src/Catalog/sparql/SelectInlinePackage.rq b/src/Catalog/sparql/SelectInlinePackage.rq deleted file mode 100644 index e6f8a5401..000000000 --- a/src/Catalog/sparql/SelectInlinePackage.rq +++ /dev/null @@ -1,8 +0,0 @@ -PREFIX catalog: -PREFIX nuget: - -SELECT ?catalogPackage -WHERE -{ - ?resource catalog:item/catalog:item/nuget:catalogEntry ?catalogPackage . -} diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index 570682594..76546f7d4 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -11,9 +11,7 @@ public static class Arguments public const char Prefix = '-'; public const char Quote = '"'; - public const string DirectoryType = "directoryType"; public const string Gallery = "gallery"; - public const string Id = "id"; public const string InstrumentationKey = "instrumentationkey"; public const string HeartbeatIntervalSeconds = "HeartbeatIntervalSeconds"; public const string Path = "path"; @@ -27,7 +25,6 @@ public static class Arguments public const int DefaultReinitializeIntervalSec = 60 * 60; // 1 hour public const string ReinitializeIntervalSec = "ReinitializeIntervalSec"; - public const string AzureStorageType = "azure"; public const string FileStorageType = "file"; @@ -41,7 +38,6 @@ public static class Arguments public const string StoragePath = "storagePath"; public const string StorageQueueName = "storageQueueName"; public const string StorageType = "storageType"; - public const string Version = "version"; public const string StorageSuffix = "storageSuffix"; public const string StorageOperationMaxExecutionTimeInSeconds = "storageOperationMaxExecutionTimeInSeconds"; @@ -55,30 +51,6 @@ public static class Arguments public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; public const string StorageUseServerSideCopy = "storageUseServerSideCopy"; - public const string MaxConcurrentBatches = "maxConcurrentBatches"; - - #endregion - - #region Catalog2Registration - public const string CompressedStorageAccountName = "compressedStorageAccountName"; - public const string CompressedStorageBaseAddress = "compressedStorageBaseAddress"; - public const string CompressedStorageContainer = "compressedStorageContainer"; - public const string CompressedStorageKeyValue = "compressedStorageKeyValue"; - public const string CompressedStoragePath = "compressedStoragePath"; - - public const string SemVer2StorageAccountName = "semVer2StorageAccountName"; - public const string SemVer2StorageBaseAddress = "semVer2StorageBaseAddress"; - public const string SemVer2StorageContainer = "semVer2StorageContainer"; - public const string SemVer2StorageKeyValue = "semVer2StorageKeyValue"; - public const string SemVer2StoragePath = "semVer2StoragePath"; - - public const string UseCompressedStorage = "useCompressedStorage"; - public const string UseSemVer2Storage = "useSemVer2Storage"; - - public const string ContentIsFlatContainer = "contentIsFlatContainer"; - public const string CursorUri = "cursorUri"; - public const string FlatContainerName = "flatContainerName"; - public const string AllIconsInFlatContainer = "allIconsInFlatContainer"; #endregion @@ -185,13 +157,26 @@ public static class Arguments #endregion #region Lightning + public const string CompressedStorageAccountName = "compressedStorageAccountName"; + public const string CompressedStorageBaseAddress = "compressedStorageBaseAddress"; + public const string CompressedStorageContainer = "compressedStorageContainer"; + public const string CompressedStorageKeyValue = "compressedStorageKeyValue"; + public const string CompressedStoragePath = "compressedStoragePath"; + + public const string SemVer2StorageAccountName = "semVer2StorageAccountName"; + public const string SemVer2StorageBaseAddress = "semVer2StorageBaseAddress"; + public const string SemVer2StorageContainer = "semVer2StorageContainer"; + public const string SemVer2StorageKeyValue = "semVer2StorageKeyValue"; + public const string SemVer2StoragePath = "semVer2StoragePath"; + + public const string FlatContainerName = "flatContainerName"; + public const string Command = "command"; public const string OutputFolder = "outputFolder"; public const string TemplateFile = "templateFile"; public const string BatchSize = "batchSize"; public const string IndexFile = "indexFile"; public const string CursorFile = "cursorFile"; - public const string Driver = "driver"; #endregion } } \ No newline at end of file diff --git a/src/Ng/Catalog2Registration.nuspec b/src/Ng/Catalog2Registration.nuspec deleted file mode 100644 index ada0f01d0..000000000 --- a/src/Ng/Catalog2Registration.nuspec +++ /dev/null @@ -1,16 +0,0 @@ - - - - Catalog2Registration - $version$ - .NET Foundation - .NET Foundation - The Catalog2Registration job. - Copyright .NET Foundation - - - - - - - \ No newline at end of file diff --git a/src/Ng/CommandHelpers.cs b/src/Ng/CommandHelpers.cs index 2ebaf1772..fdc0dd29e 100644 --- a/src/Ng/CommandHelpers.cs +++ b/src/Ng/CommandHelpers.cs @@ -18,7 +18,6 @@ using NuGet.Services.Metadata.Catalog.Monitoring; using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Services.Storage; -using CatalogAggregateStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.AggregateStorageFactory; using CatalogAzureStorage = NuGet.Services.Metadata.Catalog.Persistence.AzureStorage; using CatalogAzureStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.AzureStorageFactory; using CatalogFileStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.FileStorageFactory; @@ -110,37 +109,6 @@ public static void AssertAzureStorage(IDictionary arguments) } } - public static RegistrationStorageFactories CreateRegistrationStorageFactories(IDictionary arguments, bool verbose) - { - CatalogStorageFactory legacyStorageFactory; - var semVer2StorageFactory = CreateSemVer2StorageFactory(arguments, verbose); - - var storageFactory = CreateStorageFactory(arguments, verbose); - var compressedStorageFactory = CreateCompressedStorageFactory(arguments, verbose); - if (compressedStorageFactory != null) - { - var secondaryStorageBaseUrlRewriter = new SecondaryStorageBaseUrlRewriter(new List> - { - // always rewrite storage root url in seconary - new KeyValuePair(storageFactory.BaseAddress.ToString(), compressedStorageFactory.BaseAddress.ToString()) - }); - - var aggregateStorageFactory = new CatalogAggregateStorageFactory( - storageFactory, - new[] { compressedStorageFactory }, - secondaryStorageBaseUrlRewriter.Rewrite, - verbose); - - legacyStorageFactory = aggregateStorageFactory; - } - else - { - legacyStorageFactory = storageFactory; - } - - return new RegistrationStorageFactories(legacyStorageFactory, semVer2StorageFactory); - } - public static CatalogStorageFactory CreateStorageFactory( IDictionary arguments, bool verbose, @@ -167,54 +135,6 @@ public static CatalogStorageFactory CreateStorageFactory( throttle: throttle); } - public static CatalogStorageFactory CreateCompressedStorageFactory(IDictionary arguments, bool verbose) - { - if (!arguments.GetOrDefault(Arguments.UseCompressedStorage, false)) - { - return null; - } - - IDictionary names = new Dictionary - { - { Arguments.StorageBaseAddress, Arguments.CompressedStorageBaseAddress }, - { Arguments.StorageAccountName, Arguments.CompressedStorageAccountName }, - { Arguments.StorageKeyValue, Arguments.CompressedStorageKeyValue }, - { Arguments.StorageContainer, Arguments.CompressedStorageContainer }, - { Arguments.StoragePath, Arguments.CompressedStoragePath }, - { Arguments.StorageSuffix, Arguments.StorageSuffix }, - { Arguments.StorageUseServerSideCopy, Arguments.StorageUseServerSideCopy }, - { Arguments.StorageOperationMaxExecutionTimeInSeconds, Arguments.StorageOperationMaxExecutionTimeInSeconds }, - { Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds } - }; - - return CreateStorageFactoryImpl(arguments, names, verbose, compressed: true); - } - - public static CatalogStorageFactory CreateSemVer2StorageFactory( - IDictionary arguments, - bool verbose) - { - if (!arguments.GetOrDefault(Arguments.UseSemVer2Storage, false)) - { - return null; - } - - IDictionary names = new Dictionary - { - { Arguments.StorageBaseAddress, Arguments.SemVer2StorageBaseAddress }, - { Arguments.StorageAccountName, Arguments.SemVer2StorageAccountName }, - { Arguments.StorageKeyValue, Arguments.SemVer2StorageKeyValue }, - { Arguments.StorageContainer, Arguments.SemVer2StorageContainer }, - { Arguments.StoragePath, Arguments.SemVer2StoragePath }, - { Arguments.StorageSuffix, Arguments.StorageSuffix }, - { Arguments.StorageUseServerSideCopy, Arguments.StorageUseServerSideCopy }, - { Arguments.StorageOperationMaxExecutionTimeInSeconds, Arguments.StorageOperationMaxExecutionTimeInSeconds }, - { Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds } - }; - - return CreateStorageFactoryImpl(arguments, names, verbose, compressed: true); - } - public static CatalogStorageFactory CreateSuffixedStorageFactory( string suffix, IDictionary arguments, diff --git a/src/Ng/Helpers/EmptySecretInjector.cs b/src/Ng/Helpers/EmptySecretInjector.cs deleted file mode 100644 index 3c275c315..000000000 --- a/src/Ng/Helpers/EmptySecretInjector.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using NuGet.Services.KeyVault; - -namespace Ng.Helpers -{ - /// - /// This type doesn't need to do anything as secret injection happens at a higher level. - /// - public class EmptySecretInjector : ISecretInjector - { - public Task InjectAsync(string input) - { - return Task.FromResult(input); - } - - public Task InjectAsync(string input, ILogger logger) - { - return Task.FromResult(input); - } - } -} diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs index a6edebfb9..0b2c3a3f1 100644 --- a/src/Ng/Jobs/Catalog2MonitoringJob.cs +++ b/src/Ng/Jobs/Catalog2MonitoringJob.cs @@ -7,7 +7,6 @@ using Microsoft.Extensions.Logging; using NuGet.Services.Configuration; using NuGet.Services.Logging; -using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Monitoring; namespace Ng.Jobs diff --git a/src/Ng/Jobs/Catalog2RegistrationJob.cs b/src/Ng/Jobs/Catalog2RegistrationJob.cs deleted file mode 100644 index fd675d4c0..000000000 --- a/src/Ng/Jobs/Catalog2RegistrationJob.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using NuGet.Services.Configuration; -using NuGet.Services.Logging; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Registration; - -namespace Ng.Jobs -{ - public class Catalog2RegistrationJob : LoopingNgJob - { - private CommitCollector _collector; - private ReadWriteCursor _front; - private ReadCursor _back; - private Uri _destination; - - public Catalog2RegistrationJob( - ILoggerFactory loggerFactory, - ITelemetryClient telemetryClient, - IDictionary telemetryGlobalDimensions) - : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) - { - } - - public override string GetUsage() - { - return "Usage: ng catalog2registration " - + $"-{Arguments.Source} " - + $"-{Arguments.ContentBaseAddress} " - + $"-{Arguments.GalleryBaseAddress} " - + $"-{Arguments.StorageBaseAddress} " - + $"-{Arguments.StorageType} file|azure " - + $"[-{Arguments.StoragePath} ]" - + "|" - + $"[-{Arguments.StorageAccountName} " - + $"-{Arguments.StorageKeyValue} " - + $"-{Arguments.StorageContainer} " - + $"-{Arguments.StoragePath} " - + $"[-{Arguments.VaultName} " - + $"-{Arguments.UseManagedIdentity} true|false " - + $"-{Arguments.ClientId} Should not be set if {Arguments.UseManagedIdentity} is true" - + $"-{Arguments.CertificateThumbprint} Should not be set if {Arguments.UseManagedIdentity} is true" - + $"[-{Arguments.ValidateCertificate} true|false]]] " - + $"[-{Arguments.Verbose} true|false] " - + $"[-{Arguments.Interval} ]" - + Environment.NewLine - + "To compress data in a separate container, add: " - + $"-{Arguments.UseCompressedStorage} [true|false] " - + $"-{Arguments.CompressedStorageBaseAddress} " - + $"-{Arguments.CompressedStorageAccountName} " - + $"-{Arguments.CompressedStorageKeyValue} " - + $"-{Arguments.CompressedStorageContainer} " - + $"-{Arguments.CompressedStoragePath} " - + Environment.NewLine - + "To generate registration blobs that contain SemVer 2.0.0 packages, add: " - + $"-{Arguments.UseSemVer2Storage} [true|false] " - + $"-{Arguments.SemVer2StorageBaseAddress} " - + $"-{Arguments.SemVer2StorageAccountName} " - + $"-{Arguments.SemVer2StorageKeyValue} " - + $"-{Arguments.SemVer2StorageContainer} " - + $"-{Arguments.SemVer2StoragePath} " - + Environment.NewLine - + "Optional Arguments" - + $"-{Arguments.StorageOperationMaxExecutionTimeInSeconds} " - + $"-{Arguments.StorageServerTimeoutInSeconds} " - + $"-{Arguments.AllIconsInFlatContainer} [true|false]"; - } - - protected override void Init(IDictionary arguments, CancellationToken cancellationToken) - { - var source = arguments.GetOrThrow(Arguments.Source); - var verbose = arguments.GetOrDefault(Arguments.Verbose, false); - - var contentBaseAddress = arguments.GetOrDefault(Arguments.ContentBaseAddress); - var galleryBaseAddress = arguments.GetOrDefault(Arguments.GalleryBaseAddress); - var isContentFlatContainer = arguments.GetOrDefault(Arguments.ContentIsFlatContainer); - var allIconsInFlatContainer = arguments.GetOrDefault(Arguments.AllIconsInFlatContainer); - var maxConcurrentBatches = MaxConcurrentBatches(arguments.GetOrDefault(Arguments.MaxConcurrentBatches)); - - // The term "legacy" here refers to the registration hives that do not contain any SemVer 2.0.0 packages. - // In production, this is two registration hives: - // 1) the first hive released, which is not gzipped and does not have SemVer 2.0.0 packages - // 2) the secondary hive released, which is gzipped but does not have SemVer 2.0.0 packages - var storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose); - - Logger.LogInformation( - "CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", - source, - storageFactories.LegacyStorageFactory); - - if (isContentFlatContainer) - { - var flatContainerCursorUriString = arguments.GetOrThrow(Arguments.CursorUri); - var flatContainerName = arguments.GetOrThrow(Arguments.FlatContainerName); - RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName); - // In case that the flat container is used as the packages' source the registration needs to wait for the flatcontainer cursor - _back = new HttpReadCursor(new Uri(flatContainerCursorUriString)); - } - else - { - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - _back = MemoryCursor.CreateMax(); - } - - _collector = new RegistrationCollector( - new Uri(source), - storageFactories.LegacyStorageFactory, - storageFactories.SemVer2StorageFactory, - contentBaseAddress == null ? null : new Uri(contentBaseAddress), - galleryBaseAddress == null ? null : new Uri(galleryBaseAddress), - allIconsInFlatContainer, - TelemetryService, - Logger, - CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), - maxConcurrentBatches: maxConcurrentBatches); - - var cursorStorage = storageFactories.LegacyStorageFactory.Create(); - _front = new DurableCursor(cursorStorage.ResolveUri("cursor.json"), cursorStorage, MemoryCursor.MinValue); - storageFactories.SemVer2StorageFactory?.Create(); - - _destination = storageFactories.LegacyStorageFactory.DestinationAddress; - TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination?.AbsoluteUri; - } - - private int MaxConcurrentBatches(int argBatches) - { - if (argBatches < 0) - { - throw new ArgumentException($"{nameof(argBatches)} cannot be negative."); - } - if (argBatches == 0) - { - return RegistrationCollector.DefaultMaxConcurrentBatches; - } - - return argBatches; - } - - protected override async Task RunInternalAsync(CancellationToken cancellationToken) - { - using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination?.AbsoluteUri)) - using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) - { - bool run; - do - { - run = await _collector.RunAsync(_front, _back, cancellationToken); - } - while (run); - } - } - } -} \ No newline at end of file diff --git a/src/Ng/Jobs/Db2CatalogJob.cs b/src/Ng/Jobs/Db2CatalogJob.cs index 7c3f2e564..048f72a74 100644 --- a/src/Ng/Jobs/Db2CatalogJob.cs +++ b/src/Ng/Jobs/Db2CatalogJob.cs @@ -9,7 +9,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Ng.Helpers; using NuGet.Protocol; using NuGet.Services.Configuration; using NuGet.Services.Logging; diff --git a/src/Ng/Jobs/Db2MonitoringJob.cs b/src/Ng/Jobs/Db2MonitoringJob.cs index 820a7e8c6..3e20e6aca 100644 --- a/src/Ng/Jobs/Db2MonitoringJob.cs +++ b/src/Ng/Jobs/Db2MonitoringJob.cs @@ -9,20 +9,18 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Ng.Helpers; using NuGet.Protocol; using NuGet.Services.Configuration; +using NuGet.Services.Logging; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Helpers; using NuGet.Services.Metadata.Catalog.Monitoring; using NuGet.Services.Metadata.Catalog.Monitoring.Monitoring; using NuGet.Services.Sql; using NuGet.Services.Storage; - -using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; +using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; using Constants = NuGet.Services.Metadata.Catalog.Constants; -using NuGet.Services.Logging; namespace Ng.Jobs { diff --git a/src/Ng/Jobs/LightningJob.cs b/src/Ng/Jobs/LightningJob.cs index 3c901812d..5c8f570a2 100644 --- a/src/Ng/Jobs/LightningJob.cs +++ b/src/Ng/Jobs/LightningJob.cs @@ -15,6 +15,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs.Catalog2Registration; using NuGet.Packaging.Core; using NuGet.Protocol.Catalog; @@ -23,14 +24,13 @@ using NuGet.Services.Logging; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.Metadata.Catalog.Registration; -using VDS.RDF; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGetGallery; namespace Ng.Jobs { public class LightningJob : NgJob { - private const string GraphDriver = "graph"; private const string JsonDriver = "json"; public LightningJob( @@ -71,21 +71,11 @@ public override string GetUsage() sw.WriteLine($" The base address for package contents."); sw.WriteLine($" -{Arguments.GalleryBaseAddress} "); sw.WriteLine($" The base address for gallery."); - sw.WriteLine($" -{Arguments.ContentIsFlatContainer} true|false"); - sw.WriteLine($" Boolean to indicate if the registration blobs will have the content from the packages or flat container."); - sw.WriteLine($" The default value is false."); sw.WriteLine($" -{Arguments.FlatContainerName} "); - sw.WriteLine($" It is required when the ContentIsFlatContainer flag is true."); + sw.WriteLine($" The storage container name for the flat container resource."); sw.WriteLine($" -{Arguments.StorageSuffix} "); sw.WriteLine($" String to indicate the target storage suffix. If china for example core.chinacloudapi.cn needs to be used."); sw.WriteLine($" The default value is 'core.windows.net'."); - sw.WriteLine($" -{Arguments.AllIconsInFlatContainer} true|false"); - sw.WriteLine($" Assume all icons (including external) are in flat container."); - sw.WriteLine($" The default value is false."); - sw.WriteLine($" -{Arguments.Driver} {GraphDriver}|{JsonDriver}"); - sw.WriteLine($" Specifies which implementation to use when building the hives."); - sw.WriteLine($" '{GraphDriver}' uses RDF and JSON-LD. '{JsonDriver}' uses simple JSON serialization."); - sw.WriteLine($" The default value is '{GraphDriver}'."); sw.WriteLine($" -{Arguments.Verbose} true|false"); sw.WriteLine($" Switch output verbosity on/off."); sw.WriteLine($" The default value is false."); @@ -99,8 +89,6 @@ public override string GetUsage() sw.WriteLine($" -{Arguments.StorageContainer} "); sw.WriteLine($" Container to generate registrations in."); sw.WriteLine(); - sw.WriteLine($" -{Arguments.UseCompressedStorage} true|false"); - sw.WriteLine($" Enable or disable compressed registration blobs."); sw.WriteLine($" -{Arguments.CompressedStorageBaseAddress} "); sw.WriteLine($" Compressed only: Base address to write into registration blobs."); sw.WriteLine($" -{Arguments.CompressedStorageAccountName} "); @@ -110,8 +98,6 @@ public override string GetUsage() sw.WriteLine($" -{Arguments.CompressedStorageContainer} "); sw.WriteLine($" Compressed only: Container to generate registrations in."); sw.WriteLine(); - sw.WriteLine($" -{Arguments.UseSemVer2Storage} true|false"); - sw.WriteLine($" Enable or disable SemVer 2.0.0 registration blobs."); sw.WriteLine($" -{Arguments.SemVer2StorageBaseAddress} "); sw.WriteLine($" SemVer 2.0.0 only: Base address to write into registration blobs."); sw.WriteLine($" -{Arguments.SemVer2StorageAccountName} "); @@ -158,13 +144,6 @@ public override string GetUsage() private string _command; private bool _verbose; private TextWriter _log; - private string _contentBaseAddress; - private string _galleryBaseAddress; - private RegistrationStorageFactories _storageFactories; - private ShouldIncludeRegistrationPackage _shouldIncludeSemVer2ForLegacyStorageFactory; - private RegistrationMakerCatalogItem.PostProcessGraph _postProcessGraphForLegacyStorageFactory; - private bool _forceIconsFromFlatContainer; - private string _driver; private IDictionary _arguments; #endregion @@ -187,37 +166,13 @@ protected override void Init(IDictionary arguments, Cancellation // Hard code against Azure storage. arguments[Arguments.StorageType] = Arguments.AzureStorageType; - // Configure the package path provider. - var useFlatContainerAsPackageContent = arguments.GetOrDefault(Arguments.ContentIsFlatContainer, false); - if (!useFlatContainerAsPackageContent) - { - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - } - else - { - var flatContainerName = arguments.GetOrThrow(Arguments.FlatContainerName); - RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName); - } - _command = arguments.GetOrThrow(Arguments.Command); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _log = _verbose ? Console.Out : new StringWriter(); - _contentBaseAddress = arguments.GetOrThrow(Arguments.ContentBaseAddress); - _galleryBaseAddress = arguments.GetOrThrow(Arguments.GalleryBaseAddress); - _storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, _verbose); - _shouldIncludeSemVer2ForLegacyStorageFactory = RegistrationCollector.GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory); - _postProcessGraphForLegacyStorageFactory = RegistrationCollector.GetPostProcessGraphForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory); - _forceIconsFromFlatContainer = arguments.GetOrDefault(Arguments.AllIconsInFlatContainer); - _driver = arguments.GetOrDefault(Arguments.Driver, GraphDriver).ToLowerInvariant(); // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments // used by "prepare" should be used when executing "strike". _arguments = arguments; - if (_driver != GraphDriver && _driver != JsonDriver) - { - throw new NotSupportedException($"The lightning driver '{_driver}' is not supported."); - } - switch (_command.ToLowerInvariant()) { case "charge": @@ -323,9 +278,22 @@ private async Task PrepareAsync() _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); + // Create the containers + _log.WriteLine("Creating the containers..."); + var container = GetAutofacContainer(); + var blobClient = container.Resolve(); + var config = container.Resolve>().Value; + foreach (var name in new[] { config.LegacyStorageContainer, config.GzippedStorageContainer, config.SemVer2StorageContainer }) + { + var reference = blobClient.GetContainerReference(name); + await reference.CreateIfNotExistAsync(); + await reference.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); + } + // Write cursor to storage _log.WriteLine("Start writing new cursor..."); - var storage = _storageFactories.LegacyStorageFactory.Create(); + var storageFactory = container.ResolveKeyed(DependencyInjectionExtensions.CursorBindingKey); + var storage = storageFactory.Create(); var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit) { Value = latestCommit @@ -334,10 +302,6 @@ private async Task PrepareAsync() await cursor.SaveAsync(CancellationToken.None); _log.WriteLine("Finished writing new cursor."); - // Ensure the SemVer 2.0.0 storage containers is created, if applicable. The gzipped storage account is - // created above when we write the cursor. - _storageFactories.SemVer2StorageFactory?.Create(); - // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); @@ -381,11 +345,8 @@ private async Task PrepareAsync() //the not required arguments need to be added only if they were passed in //they cannot be hardcoded in the template var optionalArguments = new StringBuilder(); - AppendOptionalArgument(optionalArguments, Arguments.ContentIsFlatContainer); AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName); AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix); - AppendOptionalArgument(optionalArguments, Arguments.AllIconsInFlatContainer); - AppendOptionalArgument(optionalArguments, Arguments.Driver); AppendOptionalArgument(optionalArguments, Arguments.Verbose); commandStreamContents = commandStreamContents @@ -440,11 +401,9 @@ private async Task StrikeAsync() } // Time to strike - var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose); - var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory()); - var serviceProvider = GetServiceProvider(); - var catalogClient = serviceProvider.GetRequiredService(); - var registrationUpdater = serviceProvider.GetRequiredService(); + var container = GetAutofacContainer(); + var catalogClient = container.Resolve(); + var registrationUpdater = container.Resolve(); var startElement = string.Format("Element@{0}.", batchStart); var endElement = string.Format("Element@{0}.", batchEnd + 1); @@ -468,16 +427,7 @@ private async Task StrikeAsync() { var packageId = line.Split(new[] { ". " }, StringSplitOptions.None).Last().Trim(); - IStrike strike; - switch (_driver) - { - case JsonDriver: - strike = new JsonStrike(catalogClient, registrationUpdater, packageId); - break; - default: - strike = new GraphStrike(this, collectorHttpClient, packageId); - break; - } + IStrike strike = new JsonStrike(catalogClient, registrationUpdater, packageId); line = await indexStreamReader.ReadLineAsync(); while (!string.IsNullOrEmpty(line) && !line.Contains("Element@")) @@ -519,7 +469,7 @@ private static Task UpdateCursorFileAsync(string cursorFileName, int startIndex, return Task.FromResult(true); } - private IServiceProvider GetServiceProvider() + private IContainer GetAutofacContainer() { var services = new ServiceCollection(); services.AddSingleton(LoggerFactory); @@ -537,35 +487,26 @@ private IServiceProvider GetServiceProvider() Arguments.StorageKeyValue, Arguments.StorageSuffix); - if (_arguments.GetOrDefault(Arguments.UseCompressedStorage)) - { - config.GzippedBaseUrl = _arguments.GetOrDefault(Arguments.CompressedStorageBaseAddress); - config.GzippedStorageContainer = _arguments.GetOrDefault(Arguments.CompressedStorageContainer); - config.StorageConnectionString = GetConnectionString( - config.StorageConnectionString, - Arguments.CompressedStorageAccountName, - Arguments.CompressedStorageKeyValue, - Arguments.StorageSuffix); - } + config.GzippedBaseUrl = _arguments.GetOrDefault(Arguments.CompressedStorageBaseAddress); + config.GzippedStorageContainer = _arguments.GetOrDefault(Arguments.CompressedStorageContainer); + config.StorageConnectionString = GetConnectionString( + config.StorageConnectionString, + Arguments.CompressedStorageAccountName, + Arguments.CompressedStorageKeyValue, + Arguments.StorageSuffix); - if (_arguments.GetOrDefault(Arguments.UseSemVer2Storage)) - { - config.SemVer2BaseUrl = _arguments.GetOrDefault(Arguments.SemVer2StorageBaseAddress); - config.SemVer2StorageContainer = _arguments.GetOrDefault(Arguments.SemVer2StorageContainer); - config.StorageConnectionString = GetConnectionString( - config.StorageConnectionString, - Arguments.SemVer2StorageAccountName, - Arguments.SemVer2StorageKeyValue, - Arguments.StorageSuffix); - } + config.SemVer2BaseUrl = _arguments.GetOrDefault(Arguments.SemVer2StorageBaseAddress); + config.SemVer2StorageContainer = _arguments.GetOrDefault(Arguments.SemVer2StorageContainer); + config.StorageConnectionString = GetConnectionString( + config.StorageConnectionString, + Arguments.SemVer2StorageAccountName, + Arguments.SemVer2StorageKeyValue, + Arguments.StorageSuffix); config.GalleryBaseUrl = _arguments.GetOrThrow(Arguments.GalleryBaseAddress); - if (_arguments.GetOrThrow(Arguments.ContentIsFlatContainer)) - { - var contentBaseAddress = _arguments.GetOrThrow(Arguments.ContentBaseAddress); - var flatContainerName = _arguments.GetOrThrow(Arguments.FlatContainerName); - config.FlatContainerBaseUrl = contentBaseAddress.TrimEnd('/') + '/' + flatContainerName; - } + var contentBaseAddress = _arguments.GetOrThrow(Arguments.ContentBaseAddress); + var flatContainerName = _arguments.GetOrThrow(Arguments.FlatContainerName); + config.FlatContainerBaseUrl = contentBaseAddress.TrimEnd('/') + '/' + flatContainerName; config.EnsureSingleSnapshot = true; }); @@ -576,7 +517,7 @@ private IServiceProvider GetServiceProvider() containerBuilder.AddCatalog2Registration(); containerBuilder.Populate(services); - return new AutofacServiceProvider(containerBuilder.Build()); + return containerBuilder.Build(); } private string GetConnectionString( @@ -600,88 +541,6 @@ private string GetConnectionString( return connectionString; } - private class GraphStrike : IStrike - { - private readonly CollectorHttpClient _collectorHttpClient; - private readonly string _packageId; - private readonly LightningJob _job; - private Dictionary _sortedGraphs; - - public GraphStrike(LightningJob job, CollectorHttpClient collectorHttpClient, string packageId) - { - _collectorHttpClient = collectorHttpClient; - _packageId = packageId; - _job = job; - _sortedGraphs = new Dictionary(); - } - - public async Task ProcessCatalogLeafUrlAsync(string url) - { - // Fetch graph for package version - var graph = await _collectorHttpClient.GetGraphAsync(new Uri(url)); - if (_sortedGraphs.ContainsKey(url)) - { - _sortedGraphs[url] = graph; - } - else - { - _sortedGraphs.Add(url, graph); - } - - // To reduce memory footprint, we're flushing out large registrations - // in very small batches. - if (graph.Nodes.Count() > 3000 && _sortedGraphs.Count >= 20) - { - // Process graphs - await ProcessGraphsAsync(); - - // Destroy! - _sortedGraphs = new Dictionary(); - } - } - - public async Task FinishAsync() - { - // Process graphs - if (_sortedGraphs.Any()) - { - await ProcessGraphsAsync(); - } - } - - private async Task ProcessGraphsAsync() - { - await RegistrationMaker.ProcessAsync( - new RegistrationKey(_packageId), - _sortedGraphs, - _job._shouldIncludeSemVer2ForLegacyStorageFactory, - _job._storageFactories.LegacyStorageFactory, - _job._postProcessGraphForLegacyStorageFactory, - new Uri(_job._contentBaseAddress), - new Uri(_job._galleryBaseAddress), - RegistrationCollector.PartitionSize, - RegistrationCollector.PackageCountThreshold, - _job._forceIconsFromFlatContainer, - _job.TelemetryService, - CancellationToken.None); - - if (_job._storageFactories.SemVer2StorageFactory != null) - { - await RegistrationMaker.ProcessAsync( - new RegistrationKey(_packageId), - _sortedGraphs, - _job._storageFactories.SemVer2StorageFactory, - new Uri(_job._contentBaseAddress), - new Uri(_job._galleryBaseAddress), - RegistrationCollector.PartitionSize, - RegistrationCollector.PackageCountThreshold, - _job._forceIconsFromFlatContainer, - _job.TelemetryService, - CancellationToken.None); - } - } - } - private class JsonStrike : IStrike { private readonly ICatalogClient _catalogClient; diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs index 221dfb96a..f9420e082 100644 --- a/src/Ng/Jobs/MonitoringProcessorJob.cs +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Queue.Protocol; -using Ng.Helpers; using NuGet.Protocol.Core.Types; using NuGet.Services.Configuration; using NuGet.Services.Logging; diff --git a/src/Ng/Json/InterceptingJsonReader.cs b/src/Ng/Json/InterceptingJsonReader.cs deleted file mode 100644 index 1cd422a40..000000000 --- a/src/Ng/Json/InterceptingJsonReader.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using Newtonsoft.Json; - -namespace Ng.Json -{ - public abstract class InterceptingJsonReader - : JsonReader - { - private static readonly ConcurrentDictionary CachedRegexes = new ConcurrentDictionary(); - - protected JsonReader InnerReader { get; } - protected List PropertiesToIntercept { get; } - - protected InterceptingJsonReader(JsonReader innerReader, IEnumerable interceptPaths) - { - InnerReader = innerReader; - InnerReader.DateParseHandling = DateParseHandling.None; - - PropertiesToIntercept = interceptPaths - .Distinct(StringComparer.Ordinal) - .Select(path => MakeRegex(path)) - .ToList(); - } - - private static Regex MakeRegex(string jsonPath) - { - Regex expression = null; - if (!CachedRegexes.TryGetValue(jsonPath, out expression)) - { - var pattern = jsonPath - .Replace("[*]", @"\[\d+\]") - .Replace(".", @"\."); - - expression = new Regex("^" + pattern + "$"); - - CachedRegexes.TryAdd(jsonPath, expression); - } - - return expression; - } - - private bool TestPath(string jsonPath) - { - foreach (var property in PropertiesToIntercept) - { - if (property.IsMatch(jsonPath)) - { - return true; - } - } - - return false; - } - - public override bool Read() - { - if (!InnerReader.Read()) - { - return false; - } - - if (InnerReader.TokenType == JsonToken.PropertyName && TestPath(InnerReader.Path)) - { - return OnReadInterceptedPropertyName(); - } - - return true; - } - - protected abstract bool OnReadInterceptedPropertyName(); - - public override object Value - { - get - { - return InnerReader.Value; - } - } - - public override JsonToken TokenType - { - get - { - return InnerReader.TokenType; - } - } - - public override string Path - { - get - { - return InnerReader.Path; - } - } - - public override int Depth - { - get - { - return InnerReader.Depth; - } - } - - public override Type ValueType - { - get - { - return InnerReader.ValueType; - } - } - } -} \ No newline at end of file diff --git a/src/Ng/Json/PropertySkippingJsonReader.cs b/src/Ng/Json/PropertySkippingJsonReader.cs deleted file mode 100644 index 012b129c6..000000000 --- a/src/Ng/Json/PropertySkippingJsonReader.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Collections.Generic; -using Newtonsoft.Json; - -namespace Ng.Json -{ - public class PropertySkippingJsonReader - : InterceptingJsonReader - { - public PropertySkippingJsonReader(JsonReader innerReader, IEnumerable interceptPaths) - : base(innerReader, interceptPaths) - { - } - - protected override bool OnReadInterceptedPropertyName() - { - InnerReader.Skip(); - - return Read(); - } - } -} \ No newline at end of file diff --git a/src/Ng/Json/RegistrationBaseUrlRewritingJsonReader.cs b/src/Ng/Json/RegistrationBaseUrlRewritingJsonReader.cs deleted file mode 100644 index e05288bed..000000000 --- a/src/Ng/Json/RegistrationBaseUrlRewritingJsonReader.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Collections.Generic; -using Newtonsoft.Json.Linq; - -namespace Ng.Json -{ - public class RegistrationBaseUrlRewritingJsonReader - : InterceptingJsonReader - { - private readonly List> _replacements; - - private static readonly string[] RegistrationPathsToIntercept = new[] - { - // Replace in index.json + {version}.json - "@id", - - // Replace in index.json: - "registration", - - // Replace in {version}.json: - "items[*].@id", - - // Replace in {version}.json: - "items[*].parent", - - // Replace in {version}.json: - "items[*].items[*].@id", - - // Replace in {version}.json: - "items[*].items[*].registration", - - // Replace in {version}.json: - "items[*].items[*].catalogEntry.dependencyGroups[*].dependencies[*].registration", - - // Replace in page/*/{version}.json: - "parent", - - // Replace in page/*/{version}.json: - "items[*].@id", - - // Replace in page/*/{version}.json: - "items[*].registration", - - // Replace in page/*/{version}.json: - "items[*].catalogEntry.dependencyGroups[*].dependencies[*].registration" - }; - - public RegistrationBaseUrlRewritingJsonReader(JTokenReader innerReader, List> replacements) - : base(innerReader, RegistrationPathsToIntercept) - { - _replacements = replacements; - } - - protected override bool OnReadInterceptedPropertyName() - { - var tokenReader = InnerReader as JTokenReader; - var currentProperty = tokenReader?.CurrentToken as JProperty; - if (currentProperty != null) - { - var propertyValue = (string)currentProperty.Value; - - // Run replacements - foreach (var replacement in _replacements) - { - propertyValue = propertyValue.Replace(replacement.Key, replacement.Value); - } - - // Set new property value - currentProperty.Value.Replace(propertyValue); - } - - return true; - } - } -} \ No newline at end of file diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index ca454022b..77913c3cc 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -61,29 +61,22 @@ - - - - - - - @@ -93,7 +86,6 @@ - diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs index e45aac81d..8daef6ab2 100644 --- a/src/Ng/NgJobFactory.cs +++ b/src/Ng/NgJobFactory.cs @@ -15,7 +15,6 @@ public static class NgJobFactory { { "db2catalog", typeof(Db2CatalogJob) }, { "db2monitoring", typeof(Db2MonitoringJob) }, - { "catalog2registration", typeof(Catalog2RegistrationJob) }, { "catalog2dnx", typeof(Catalog2DnxJob) }, { "lightning", typeof(LightningJob) }, { "catalog2monitoring", typeof(Catalog2MonitoringJob) }, diff --git a/src/Ng/RegistrationStorageFactories.cs b/src/Ng/RegistrationStorageFactories.cs deleted file mode 100644 index 017e9dbac..000000000 --- a/src/Ng/RegistrationStorageFactories.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace Ng -{ - /// - /// Contains the storage factories used to create registration blobs. - /// - public class RegistrationStorageFactories - { - public RegistrationStorageFactories(StorageFactory legacyStorageFactory, StorageFactory semVer2StorageFactory) - { - if (legacyStorageFactory == null) - { - throw new ArgumentNullException(nameof(legacyStorageFactory)); - } - - LegacyStorageFactory = legacyStorageFactory; - SemVer2StorageFactory = semVer2StorageFactory; - } - - /// - /// The factory used for registration hives that do not have SemVer 2.0.0 packages. This factory is typically - /// an that writes both gzipped and non-gzipped blobs. - /// - public StorageFactory LegacyStorageFactory { get; } - - /// - /// The factory used for registration hives that have all packages (SemVer 2.0.0 and otherwise). - /// - public StorageFactory SemVer2StorageFactory { get; } - } -} diff --git a/src/Ng/SecondaryStorageBaseUrlRewriter.cs b/src/Ng/SecondaryStorageBaseUrlRewriter.cs deleted file mode 100644 index aea9514b1..000000000 --- a/src/Ng/SecondaryStorageBaseUrlRewriter.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Ng.Json; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace Ng -{ - public class SecondaryStorageBaseUrlRewriter - { - private readonly List> _replacements; - - public SecondaryStorageBaseUrlRewriter(List> replacements) - { - _replacements = replacements; - } - - public StorageContent Rewrite( - Uri primaryStorageBaseUri, - Uri primaryResourceUri, - Uri secondaryStorageBaseUri, - Uri secondaryResourceUri, - StorageContent content) - { - JTokenReader tokenReader = null; - - var storageContent = content as JTokenStorageContent; - if (storageContent != null) - { - // Production code should always have JTokenStorageContent - tokenReader = storageContent.Content.CreateReader() as JTokenReader; - } - else - { - // Test code may end up here - we need to make sure we have a JTokenReader at our disposal - using (var streamReader = new StreamReader(content.GetContentStream())) - { - using (var jsonTextReader = new JsonTextReader(streamReader)) - { - tokenReader = JToken.Load(jsonTextReader).CreateReader() as JTokenReader; - } - } - } - - if (tokenReader != null) - { - // Create a rewriting reader - var rewritingReader = new RegistrationBaseUrlRewritingJsonReader(tokenReader, - _replacements.Union(new List> - { - new KeyValuePair(primaryStorageBaseUri.ToString(), secondaryStorageBaseUri.ToString()), - new KeyValuePair(GetParentUriString(primaryResourceUri), GetParentUriString(secondaryResourceUri)) - }).ToList()); - - // Clone the original token (passing through our intercepting reader) - var updatedJson = JToken.Load(rewritingReader); - - // Create new content - return new JTokenStorageContent(updatedJson, content.ContentType, content.CacheControl); - } - - return content; - } - - private static string GetParentUriString(Uri uri) - { - return uri.AbsoluteUri.Remove(uri.AbsoluteUri.Length - uri.Segments.Last().Length); - } - } -} \ No newline at end of file diff --git a/src/Ng/lightning-template.txt b/src/Ng/lightning-template.txt index 2cd5f6f48..ce9c7b990 100644 --- a/src/Ng/lightning-template.txt +++ b/src/Ng/lightning-template.txt @@ -16,12 +16,10 @@ start /w ng.exe lightning ^ -storageAccountName "[storageAccountName]" ^ -storageKeyValue "[storageKeyValue]" ^ -storageContainer "[storageContainer]" ^ - -useCompressedStorage "[useCompressedStorage]" ^ -compressedStorageBaseAddress "[compressedStorageBaseAddress]" ^ -compressedStorageAccountName "[compressedStorageAccountName]" ^ -compressedStorageKeyValue "[compressedStorageKeyValue]" ^ -compressedStorageContainer "[compressedStorageContainer]" ^ - -useSemVer2Storage "[useSemVer2Storage]" ^ -semVer2StorageBaseAddress "[semVer2StorageBaseAddress]" ^ -semVer2StorageAccountName "[semVer2StorageAccountName]" ^ -semVer2StorageKeyValue "[semVer2StorageKeyValue]" ^ diff --git a/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs index 9c7795426..de3ffa949 100644 --- a/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs +++ b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs @@ -18,7 +18,7 @@ namespace NuGet.Jobs.Catalog2Registration { public static class DependencyInjectionExtensions { - private const string CursorBindingKey = "Cursor"; + public const string CursorBindingKey = "Cursor"; public static ContainerBuilder AddCatalog2Registration(this ContainerBuilder containerBuilder) { diff --git a/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs b/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs index a4ddb1ce4..fa94e5116 100644 --- a/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs +++ b/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs @@ -8,7 +8,6 @@ using NuGet.Protocol.Registration; using NuGet.Services; using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Registration; using NuGet.Versioning; namespace NuGet.Jobs.Catalog2Registration diff --git a/src/NuGet.Protocol.Catalog/Models/PackageType.cs b/src/NuGet.Protocol.Catalog/Models/PackageType.cs index 854788638..897d74980 100644 --- a/src/NuGet.Protocol.Catalog/Models/PackageType.cs +++ b/src/NuGet.Protocol.Catalog/Models/PackageType.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Collections.Generic; using Newtonsoft.Json; namespace NuGet.Protocol.Catalog diff --git a/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs b/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs index c6c393b89..9b8ac54ac 100644 --- a/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs +++ b/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs @@ -1,8 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; - namespace NuGet.Services.AzureSearch { public class AzureSearchJobDevelopmentConfiguration diff --git a/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs index 5840e0599..0c23dc572 100644 --- a/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs +++ b/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs @@ -10,7 +10,6 @@ using NuGet.Protocol.Catalog; using NuGet.Services.Entities; using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Registration; using NuGet.Versioning; using NuGetGallery; using PackageDependency = NuGet.Protocol.Catalog.PackageDependency; diff --git a/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs index 7f3313445..a8b0d01bf 100644 --- a/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs +++ b/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Search.Models; using Microsoft.Extensions.Logging; diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs index 13084d4fc..77fc5d484 100644 --- a/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs @@ -7,7 +7,7 @@ using Microsoft.Azure.Search.Models; using Microsoft.Extensions.Options; using NuGet.Protocol.Registration; -using NuGet.Services.Metadata.Catalog.Registration; +using NuGet.Services.Metadata.Catalog; using NuGet.Versioning; namespace NuGet.Services.AzureSearch.SearchService diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs index dcb17b40b..f77b5c35b 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Newtonsoft.Json; diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs index 3c9fd8e7c..f2552497c 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs @@ -1,13 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Protocol; -using NuGet.Protocol.Core.Types; namespace NuGet.Services.Metadata.Catalog.Monitoring { diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj index e6944c82d..010285e2d 100644 --- a/tests/CatalogTests/CatalogTests.csproj +++ b/tests/CatalogTests/CatalogTests.csproj @@ -118,23 +118,12 @@ - - - - - - - - - - - diff --git a/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs b/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs index b7da54bff..ba4dcd5be 100644 --- a/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs +++ b/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs @@ -1,10 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Helpers; -using VDS.RDF; using Xunit; namespace CatalogTests.Helpers @@ -71,172 +68,5 @@ public void GetFullVersionString(string input, string expected) // Assert Assert.Equal(expected, actual); } - - [Theory] - [InlineData("1.0.0-alpha.1", true)] - [InlineData("1.0.0-alpha.1+githash", true)] - [InlineData("1.0.0-alpha+githash", true)] - [InlineData("1.0.0+githash", true)] - [InlineData("1.0.0-alpha", false)] - [InlineData("1.0.0", false)] - [InlineData("invalid", false)] - public void IsVersionSemVer2(string input, bool expected) - { - // Arrange & Act - var actual = NuGetVersionUtility.IsVersionSemVer2(input); - - // Assert - Assert.Equal(expected, actual); - } - - [Theory] - [InlineData("[1.0.0-alpha.1, 2.0.0)", true)] - [InlineData("[1.0.0+githash, 2.0.0)", true)] - [InlineData("[1.0.0-alpha, 2.0.0-alpha.1)", true)] - [InlineData("[1.0.0, 2.0.0+githash)", true)] - [InlineData("[1.0.0-alpha, 2.0.0)", false)] - [InlineData("invalid", false)] - public void IsVersionRangeSemVer2(string input, bool expected) - { - // Arrange & Act - var actual = NuGetVersionUtility.IsVersionRangeSemVer2(input); - - // Assert - Assert.Equal(expected, actual); - } - - [Fact] - public void IsGraphSemVer2_WithSemVer2PackageVersion_IsSemVer2() - { - // Arrange - var graph = new Graph(); - - // Act - var actual = NuGetVersionUtility.IsGraphSemVer2(TD.SemVer2Version, TD.ResourceUri, graph); - - // Assert - Assert.True(actual); - } - - [Fact] - public void IsGraphSemVer2_WithSemVer1PackageVersionAndNoDependencies_IsSemVer1() - { - // Arrange - var graph = new Graph(); - - // Act - var actual = NuGetVersionUtility.IsGraphSemVer2(TD.SemVer1Version, TD.ResourceUri, graph); - - // Assert - Assert.False(actual); - } - - [Fact] - public void IsGraphSemVer2_WithSemVer1PackageVersionAndSemVer2VerbatimVersion_IsSemVer2() - { - // Arrange - var graph = new Graph(); - graph.Assert( - graph.CreateUriNode(new Uri(TD.ResourceUri)), - graph.CreateUriNode(Schema.Predicates.VerbatimVersion), - graph.CreateLiteralNode(TD.SemVer2Version)); - - // Act - var actual = NuGetVersionUtility.IsGraphSemVer2(TD.SemVer1Version, TD.ResourceUri, graph); - - // Assert - Assert.True(actual); - } - - [Fact] - public void IsGraphSemVer2_WithSemVer1PackageVersionAndSemVer1Dependencies_IsSemVer1() - { - // Arrange - var graph = new Graph(); - graph.Assert( - graph.CreateUriNode(new Uri(TD.ResourceUri)), - graph.CreateUriNode(Schema.Predicates.DependencyGroup), - graph.CreateUriNode(new Uri(TD.DependencyGroupUri))); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyGroupUri)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDependencyGroup)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyGroupUri)), - graph.CreateUriNode(Schema.Predicates.Dependency), - graph.CreateUriNode(new Uri(TD.DependencyUriA))); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriA)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDependency)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriA)), - graph.CreateUriNode(Schema.Predicates.Range), - graph.CreateLiteralNode(TD.SemVer1Version)); - - // Act - var actual = NuGetVersionUtility.IsGraphSemVer2(TD.SemVer1Version, TD.ResourceUri, graph); - - // Assert - Assert.False(actual); - } - - [Fact] - public void IsGraphSemVer2_WithSemVer1PackageVersionAndSemVer2Dependencies_IsSemVer2() - { - // Arrange - var graph = new Graph(); - graph.Assert( - graph.CreateUriNode(new Uri(TD.ResourceUri)), - graph.CreateUriNode(Schema.Predicates.DependencyGroup), - graph.CreateUriNode(new Uri(TD.DependencyGroupUri))); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyGroupUri)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDependencyGroup)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyGroupUri)), - graph.CreateUriNode(Schema.Predicates.Dependency), - graph.CreateUriNode(new Uri(TD.DependencyUriA))); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriA)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDependency)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriA)), - graph.CreateUriNode(Schema.Predicates.Range), - graph.CreateLiteralNode(TD.SemVer1Version)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyGroupUri)), - graph.CreateUriNode(Schema.Predicates.Dependency), - graph.CreateUriNode(new Uri(TD.DependencyUriB))); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriB)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDependency)); - graph.Assert( - graph.CreateUriNode(new Uri(TD.DependencyUriB)), - graph.CreateUriNode(Schema.Predicates.Range), - graph.CreateLiteralNode(TD.SemVer2Version)); - - // Act - var actual = NuGetVersionUtility.IsGraphSemVer2(TD.SemVer1Version, TD.ResourceUri, graph); - - // Assert - Assert.True(actual); - } - - /// - /// Test data. - /// - private static class TD - { - public const string SemVer1Version = "1.0.0-alpha"; - public const string SemVer2Version = "1.0.0-alpha.1"; - public const string ResourceUri = "https://example.com/packageid.1.0.0-a.1.json"; - public const string DependencyGroupUri = ResourceUri + "#dependencyGroup/net45"; - public const string DependencyUriA = ResourceUri + "#dependencyGroup/net45/newtonsoft.json"; - public const string DependencyUriB = ResourceUri + "#dependencyGroup/net45/nuget.versioning"; - } } } diff --git a/tests/CatalogTests/Helpers/UtilsTests.cs b/tests/CatalogTests/Helpers/UtilsTests.cs index 64c09d2e9..92d71487a 100644 --- a/tests/CatalogTests/Helpers/UtilsTests.cs +++ b/tests/CatalogTests/Helpers/UtilsTests.cs @@ -69,24 +69,6 @@ public void GetNupkgMetadata_WhenNuspecNotFound_Throws() } } - [Theory] - [InlineData(null)] - [InlineData("")] - public void GetResource_WhenResourceNameIsNullOrEmpty_Throws(string resourceName) - { - var exception = Assert.Throws(() => Utils.GetResource(resourceName)); - - Assert.Equal("resourceName", exception.ParamName); - } - - [Fact] - public void GetResource_WhenResourceNameIsValid_ReturnsString() - { - var resource = Utils.GetResource("sparql.SelectInlinePackage.rq"); - - Assert.NotEmpty(resource); - } - [Theory] [InlineData(null)] [InlineData("")] @@ -100,7 +82,7 @@ public void GetResourceStream_WhenResourceNameIsNullOrEmpty_Throws(string resour [Fact] public void GetResourceStream_WhenResourceNameIsValid_ReturnsStream() { - using (var stream = Utils.GetResourceStream("sparql.SelectInlinePackage.rq")) + using (var stream = Utils.GetResourceStream("context.Catalog.json")) { Assert.NotNull(stream); } @@ -124,16 +106,13 @@ public void GetNupkgMetadataWithLicenseUrl_ReturnsLicenseUrl() var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); var licenseFileTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseFile"))) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseFile")))); var licenseExpressionTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseExpression"))) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseExpression")))); var licenseUrlTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(Schema.Predicates.LicenseUrl) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseUrl")))); // Assert Assert.Empty(licenseFileTriples); @@ -158,12 +137,10 @@ public void GetNupkgMetadataWithLicenseType_ReturnsLicense(string packageName, s var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); var licenseTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet, licenseType))) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet, licenseType)))); var licenseUrlTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(Schema.Predicates.LicenseUrl) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseUrl")))); var result = (LiteralNode)licenseTriples.First().Object; // Assert @@ -191,12 +168,10 @@ public void GetNupkgMetadataWithIcon_ProcessesCorrectly(string packageFilename, var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); var iconTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(Schema.Predicates.IconFile) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "iconFile")))); var iconUrlTriples = graph.GetTriplesWithSubjectPredicate( graph.CreateUriNode(uriNodeName), - graph.CreateUriNode(Schema.Predicates.IconUrl) - ); + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "iconUrl")))); var result = (LiteralNode)iconTriples.FirstOrDefault()?.Object; // Assert diff --git a/tests/CatalogTests/Persistence/AggregateStorageTests.cs b/tests/CatalogTests/Persistence/AggregateStorageTests.cs deleted file mode 100644 index 65beeda5f..000000000 --- a/tests/CatalogTests/Persistence/AggregateStorageTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Threading; -using System.Threading.Tasks; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog.Persistence; -using Xunit; - -namespace CatalogTests.Persistence -{ - public class AggregateStorageTests - { - private readonly AggregateStorage _storage; - - public AggregateStorageTests() - { - _storage = CreateNewAggregateStorage(); - } - - [Fact] - public async Task CopyAsync_Always_Throws() - { - var destinationStorage = CreateNewAggregateStorage(); - var sourceFileUri = _storage.ResolveUri("a"); - var destinationFileUri = destinationStorage.ResolveUri("a"); - - await Assert.ThrowsAsync( - () => _storage.CopyAsync( - sourceFileUri, - destinationStorage, - destinationFileUri, - destinationProperties: null, - cancellationToken: CancellationToken.None)); - } - - private static AggregateStorage CreateNewAggregateStorage() - { - return new AggregateStorage( - new Uri("https://nuget.test"), - new MemoryStorage(), - secondaryStorage: null, - writeSecondaryStorageContentInterceptor: null); - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs b/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs index bda9c0d7d..4b8766a58 100644 --- a/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs +++ b/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using NuGet.Services.Metadata.Catalog.Registration; +using NuGet.Services.Metadata.Catalog; using Xunit; namespace CatalogTests.Registration diff --git a/tests/CatalogTests/Registration/PackagesFolderPackagePathProviderTests.cs b/tests/CatalogTests/Registration/PackagesFolderPackagePathProviderTests.cs deleted file mode 100644 index a46aa0b1b..000000000 --- a/tests/CatalogTests/Registration/PackagesFolderPackagePathProviderTests.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using NuGet.Services.Metadata.Catalog.Registration; -using Xunit; - -namespace CatalogTests.Registration -{ - public class PackagesFolderPackagePathProviderTests - { - [Theory] - [InlineData("Newtonsoft.Json", "9.0.1", "newtonsoft.json.9.0.1")] - [InlineData("Newtonsoft.Json", "9.0.1+githash", "newtonsoft.json.9.0.1")] - [InlineData("Newtonsoft.Json", "9.0.1-a", "newtonsoft.json.9.0.1-a")] - [InlineData("Newtonsoft.Json", "9.0.1-a+githash", "newtonsoft.json.9.0.1-a")] - [InlineData("Newtonsoft.Json", "9.0.1-a.1", "newtonsoft.json.9.0.1-a.1")] - [InlineData("Newtonsoft.Json", "9.0.1-a.1+githash", "newtonsoft.json.9.0.1-a.1")] - public void GetPackagePath_UsesNormalizedVersion(string id, string version, string expected) - { - // Arrange - var provider = new PackagesFolderPackagePathProvider(); - - // Act - var path = provider.GetPackagePath(id, version); - - // Assert - Assert.Equal($"packages/{expected}.nupkg", path); - } - - [Theory] - [InlineData("Newtonsoft.Json", "9.0.1", "newtonsoft.json/9.0.1")] - [InlineData("Newtonsoft.Json", "9.0.1+githash", "newtonsoft.json/9.0.1")] - [InlineData("Newtonsoft.Json", "9.0.1-a", "newtonsoft.json/9.0.1-a")] - [InlineData("Newtonsoft.Json", "9.0.1-a+githash", "newtonsoft.json/9.0.1-a")] - [InlineData("Newtonsoft.Json", "9.0.1-a.1", "newtonsoft.json/9.0.1-a.1")] - [InlineData("Newtonsoft.Json", "9.0.1-a.1+githash", "newtonsoft.json/9.0.1-a.1")] - public void GetIconPath_UsesNormalizedVersion(string id, string version, string expected) - { - // Arrange - var provider = new PackagesFolderPackagePathProvider(); - - // Act - var path = provider.GetIconPath(id, version); - - // Assert - Assert.Equal($"packages/{expected}/icon", path); - } - } -} diff --git a/tests/CatalogTests/Registration/RecordingStorageTests.cs b/tests/CatalogTests/Registration/RecordingStorageTests.cs deleted file mode 100644 index fe92eff81..000000000 --- a/tests/CatalogTests/Registration/RecordingStorageTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Threading; -using System.Threading.Tasks; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog.Registration; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RecordingStorageTests - { - private readonly RecordingStorage _storage; - - public RecordingStorageTests() - { - _storage = CreateNewRecordingStorage(); - } - - [Fact] - public async Task CopyAsync_Always_Throws() - { - var destinationStorage = CreateNewRecordingStorage(); - var sourceFileUri = _storage.ResolveUri("a"); - var destinationFileUri = destinationStorage.ResolveUri("a"); - - await Assert.ThrowsAsync( - () => _storage.CopyAsync( - sourceFileUri, - destinationStorage, - destinationFileUri, - destinationProperties: null, - cancellationToken: CancellationToken.None)); - } - - private static RecordingStorage CreateNewRecordingStorage() - { - return new RecordingStorage(new MemoryStorage()); - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationCatalogEntryTests.cs b/tests/CatalogTests/Registration/RegistrationCatalogEntryTests.cs deleted file mode 100644 index 983021b1e..000000000 --- a/tests/CatalogTests/Registration/RegistrationCatalogEntryTests.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Registration; -using VDS.RDF; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationCatalogEntryTests - { - private const string ResourceUri = "http://example.com/packageid.1.0.0-a.1.json"; - private const string Id = "PackageId"; - private const string Version = "1.0.0-a.1"; - private const bool IsExistingItem = true; - - [Fact] - public void Promote_WhenShouldBeIncluded_ReturnsCompleteKeyValuePair() - { - // Arrange - var graph = InitializeGraph(ResourceUri, Id, Version); - var isDelegateInvoked = new DelegateInvoked(shouldInclude: true); - - // Act - var pair = RegistrationCatalogEntry.Promote( - ResourceUri, - graph, - isDelegateInvoked.Delegate, - IsExistingItem); - - // Assert - Assert.Equal(pair.Key.RegistrationKey.Id, Id); - Assert.Equal(pair.Key.Version, Version); - Assert.NotNull(pair.Value); - Assert.Equal(pair.Value.IsExistingItem, IsExistingItem); - Assert.Same(pair.Value.Graph, graph); - Assert.Equal(pair.Value.ResourceUri, ResourceUri); - Assert.True(isDelegateInvoked.Invoked, "The delegate to determine whether an entry should have been invoked."); - } - - [Fact] - public void Promote_WhenShouldNotBeIncluded_ReturnsCompleteKeyValuePairWithNullValue() - { - // Arrange - var graph = InitializeGraph(ResourceUri, Id, Version); - var isDelegateInvoked = new DelegateInvoked(shouldInclude: false); - - // Act - var pair = RegistrationCatalogEntry.Promote( - ResourceUri, - graph, - isDelegateInvoked.Delegate, - IsExistingItem); - - // Assert - Assert.Equal(pair.Key.RegistrationKey.Id, Id); - Assert.Equal(pair.Key.Version, Version); - Assert.Null(pair.Value); - Assert.True(isDelegateInvoked.Invoked, "The delegate to determine whether an entry should have been invoked."); - } - - [Fact] - public void Promote_WithDelete_ReturnsCompleteKeyValuePairWithNullValue() - { - // Arrange - var graph = InitializeGraph(ResourceUri, Id, Version); - graph.Assert( - graph.CreateUriNode(new Uri(ResourceUri)), - graph.CreateUriNode(Schema.Predicates.Type), - graph.CreateUriNode(Schema.DataTypes.PackageDelete)); - var delegateInvoked = new DelegateInvoked(shouldInclude: true); - - // Act - var pair = RegistrationCatalogEntry.Promote( - ResourceUri, - graph, - delegateInvoked.Delegate, - IsExistingItem); - - // Assert - Assert.Equal(pair.Key.RegistrationKey.Id, Id); - Assert.Equal(pair.Key.Version, Version); - Assert.Null(pair.Value); - Assert.False(delegateInvoked.Invoked, "The delegate to determine whether an entry should not have been invoked."); - } - - private static Graph InitializeGraph(string resourceUri, string id, string version) - { - var graph = new Graph(); - graph.Assert( - graph.CreateUriNode(new Uri(resourceUri)), - graph.CreateUriNode(Schema.Predicates.Id), - graph.CreateLiteralNode(id)); - graph.Assert( - graph.CreateUriNode(new Uri(resourceUri)), - graph.CreateUriNode(Schema.Predicates.Version), - graph.CreateLiteralNode(version)); - return graph; - } - - private class DelegateInvoked - { - public DelegateInvoked(bool shouldInclude) - { - Invoked = false; - Delegate = (k, u, g) => - { - Invoked = true; - return shouldInclude; - }; - } - - public bool Invoked { get; private set; } - public ShouldIncludeRegistrationPackage Delegate { get; } - } - } -} diff --git a/tests/CatalogTests/Registration/RegistrationCollectorTests.cs b/tests/CatalogTests/Registration/RegistrationCollectorTests.cs deleted file mode 100644 index b22d39cf4..000000000 --- a/tests/CatalogTests/Registration/RegistrationCollectorTests.cs +++ /dev/null @@ -1,598 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using CatalogTests.Helpers; -using Microsoft.Extensions.Logging; -using Moq; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NgTests; -using NgTests.Data; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Services.Metadata.Catalog.Registration; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationCollectorTests : RegistrationTestBase - { - private readonly static Uri _baseUri = new Uri("https://nuget.test"); - private readonly static JObject _contextKeyword = new JObject( - new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), - new JProperty(CatalogConstants.NuGet, CatalogConstants.NuGetSchemaUri), - new JProperty(CatalogConstants.Items, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Item), - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), - new JProperty(CatalogConstants.Parent, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.CommitTimeStamp, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.NuGetLastCreated, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.NuGetLastEdited, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.NuGetLastDeleted, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); - - private MemoryStorage _legacyStorage; - private TestStorageFactory _legacyStorageFactory; - private MockServerHttpClientHandler _mockServer; - private RegistrationCollector _target; - private MemoryStorage _semVer2Storage; - private TestStorageFactory _semVer2StorageFactory; - - private void SharedInit(bool useLegacy, bool useSemVer2, Uri baseUri = null, Uri indexUri = null, Uri contentBaseUri = null, Uri galleryBaseUri = null, bool forceFlatContainerIcons = false) - { - if (useLegacy) - { - _legacyStorage = new MemoryStorage(baseUri ?? new Uri("http://tempuri.org")); - _legacyStorageFactory = new TestStorageFactory(name => _legacyStorage.WithName(name)); - } - - if (useSemVer2) - { - _semVer2Storage = new MemoryStorage(baseUri ?? new Uri("http://tempuri.org")); - _semVer2StorageFactory = new TestStorageFactory(name => _semVer2Storage.WithName(name)); - } - - _mockServer = new MockServerHttpClientHandler(); - _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); - - _target = new RegistrationCollector( - indexUri ?? new Uri("http://tempuri.org/index.json"), - _legacyStorageFactory, - _semVer2StorageFactory, - contentBaseUri ?? new Uri("http://tempuri.org/packages"), - galleryBaseUri ?? new Uri("http://tempuri.org/gallery"), - forceFlatContainerIcons, - Mock.Of(), - Mock.Of(), - handlerFunc: () => _mockServer, - httpRetryStrategy: new NoRetryStrategy()); - - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - } - - [Theory] - [InlineData(0)] - [InlineData(-1)] - public void Constructor_WhenMaxConcurrentBatchesIsNotPositive_Throws(int maxConcurrentBatches) - { - var storageFactory = new TestStorageFactory(); - - var exception = Assert.Throws( - () => new RegistrationCollector( - _baseUri, - storageFactory, - storageFactory, - _baseUri, - _baseUri, - false, - Mock.Of(), - Mock.Of(), - maxConcurrentBatches: maxConcurrentBatches)); - - Assert.Equal("maxConcurrentBatches", exception.ParamName); - Assert.Equal(maxConcurrentBatches, exception.ActualValue); - } - - [Theory] - [InlineData("/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", null)] - [InlineData("/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", "2015-10-12T10:08:54.1506742Z")] - [InlineData("/data/2015.10.12.10.08.55/anotherpackage.1.0.0.json", "2015-10-12T10:08:54.1506742Z")] - public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry) - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: false); - - var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorageAsync(catalogStorage); - - // Make the first request for a catalog leaf node fail. This will cause the registration collector - // to fail the first time but pass the second time. - FailFirstRequest(catalogUri); - - expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O"); - - ReadWriteCursor front = new DurableCursor( - _legacyStorage.ResolveUri("cursor.json"), - _legacyStorage, - MemoryCursor.MinValue); - ReadCursor back = MemoryCursor.CreateMax(); - - // Act - await Assert.ThrowsAsync( - () => _target.RunAsync(front, back, CancellationToken.None)); - var cursorBeforeRetry = front.Value; - await _target.RunAsync(front, back, CancellationToken.None); - var cursorAfterRetry = front.Value; - - // Assert - var unlistedPackage100 = _legacyStorage - .Content - .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); - Assert.NotNull(unlistedPackage100.Key); - - var listedPackage101 = _legacyStorage - .Content - .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); - Assert.NotNull(listedPackage101.Key); - - var anotherPackage100 = _legacyStorage - .Content - .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/anotherpackage/1.0.0.json")); - Assert.NotNull(anotherPackage100.Key); - - Assert.Equal(MemoryCursor.MinValue, cursorBeforeRetry); - Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry); - } - - private void FailFirstRequest(string relativeUri) - { - var originalAction = _mockServer.Actions[relativeUri]; - var hasFailed = false; - Func> failFirst = request => - { - if (!hasFailed) - { - hasFailed = true; - throw new HttpRequestException("Simulated HTTP failure."); - } - - return originalAction(request); - }; - _mockServer.SetAction(relativeUri, failFirst); - } - - [Fact] - public async Task CreatesRegistrationsAndRespectsDeletes() - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: false); - - var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); - await _mockServer.AddStorageAsync(catalogStorage); - - ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - ReadCursor back = MemoryCursor.CreateMax(); - - // Act - await _target.RunAsync(front, back, CancellationToken.None); - - // Assert - Assert.Equal(6, _legacyStorage.Content.Count); - - // Ensure storage has cursor.json - var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.NotNull(cursorJson.Key); - - // Check package entries - ListedPackage - var package1Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json")); - Assert.NotNull(package1Index.Key); - Assert.Contains("\"listed\":true,", package1Index.Value.GetContentString()); - Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString()); - Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString()); - Assert.Contains("\"lower\":\"1.0.0\",", package1Index.Value.GetContentString()); - Assert.Contains("\"upper\":\"1.0.1\"", package1Index.Value.GetContentString()); - - var package1 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json")); - Assert.Contains("\"listed\":true,", package1.Value.GetContentString()); - Assert.NotNull(package1.Key); - - var package2 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); - Assert.Contains("\"listed\":true,", package2.Value.GetContentString()); - Assert.NotNull(package2.Key); - - // Check package entries - UnlistedPackage - var package2Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json")); - Assert.NotNull(package2Index.Key); - Assert.Contains("\"listed\":false,", package2Index.Value.GetContentString()); - Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString()); - Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString()); - Assert.Contains("\"lower\":\"1.0.0\",", package2Index.Value.GetContentString()); - Assert.Contains("\"upper\":\"1.0.0\"", package2Index.Value.GetContentString()); - - var package3 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); - Assert.Contains("\"listed\":false,", package3.Value.GetContentString()); - Assert.NotNull(package3.Key); - - // Ensure storage does not have the deleted "OtherPackage" - var otherPackageIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json")); - Assert.Null(otherPackageIndex.Key); - } - - [Theory] - [MemberData(nameof(PageContent))] - public async Task WhenPackageHasMultipleCommitsRespectsOrder(string pageContent) - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: false); - - var catalogStorage = Catalogs.CreateTestCatalogWithThreeItemsForSamePackage(pageContent); - await _mockServer.AddStorageAsync(catalogStorage); - - ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - ReadCursor back = MemoryCursor.CreateMax(); - - // Act - await _target.RunAsync(front, back, CancellationToken.None); - - // Assert - Assert.Equal(3, _legacyStorage.Content.Count); - - // Ensure storage has cursor.json - var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.NotNull(cursorJson.Key); - - // Check package entries - ListedPackage - var myPackageIndexFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/index.json")); - Assert.NotNull(myPackageIndexFile.Key); - Assert.Contains("\"catalog:CatalogRoot\"", myPackageIndexFile.Value.GetContentString()); - Assert.Contains("\"PackageRegistration\"", myPackageIndexFile.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageIndexFile.Value.GetContentString()); - Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString()); - Assert.Contains("\"lower\":\"3.0.0\",", myPackageIndexFile.Value.GetContentString()); - Assert.Contains("\"upper\":\"3.0.0\"", myPackageIndexFile.Value.GetContentString()); - - var myPackageVersionFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/3.0.0.json")); - Assert.NotNull(myPackageVersionFile.Key); - Assert.Contains("\"catalogEntry\":\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageVersionFile.Value.GetContentString()); - Assert.Contains("\"listed\":true", myPackageVersionFile.Value.GetContentString()); - Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString()); - } - - public static IEnumerable PageContent - { - get - { - // Reverse chronological order of commits in page - yield return new object[] - { - TestCatalogEntries.TestCatalogPageForMyPackage_Option1, - }; - - // Random chronoligical order of commits in page - yield return new object[] - { - TestCatalogEntries.TestCatalogPageForMyPackage_Option2 - }; - } - } - - [Fact] - public async Task CreatesRegistrationsWithSemVer2() - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: true); - - var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorageAsync(catalogStorage); - - var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - var back = MemoryCursor.CreateMax(); - - // Act - await _target.RunAsync(front, back, CancellationToken.None); - - // Assert - // Verify the contents of the legacy (non-SemVer 2.0.0) storage - Assert.Single(_legacyStorage.Content); - - var legacyCursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.NotNull(legacyCursorJson.Key); - - // Verify the contents of the SemVer 2.0.0 storage - Assert.Equal(2, _semVer2Storage.Content.Count); - - var semVer2CursorJson = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.Null(semVer2CursorJson.Key); - - var index = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); - Assert.NotNull(index.Key); - Assert.Contains("\"catalog:CatalogRoot\"", index.Value.GetContentString()); - Assert.Contains("\"PackageRegistration\"", index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", index.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", index.Value.GetContentString()); - Assert.Contains("\"version\":\"1.0.0-alpha.1+githash\"", index.Value.GetContentString()); - Assert.Contains("1.0.0-alpha.1/1.0.0-alpha.1", index.Value.GetContentString()); - Assert.Contains("\"lower\":\"1.0.0-alpha.1\",", index.Value.GetContentString()); - Assert.Contains("\"upper\":\"1.0.0-alpha.1\"", index.Value.GetContentString()); - - var package = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); - Assert.NotNull(package.Key); - Assert.Contains("\"Package\"", package.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", package.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", package.Value.GetContentString()); - Assert.Contains("\"http://tempuri.org/testpackage.semver2/index.json\"", package.Value.GetContentString()); - } - - [Fact] - public async Task IgnoresSemVer2PackagesInLegacyStorageWhenSemVer2IsEnabled() - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: true); - - var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorageAsync(catalogStorage); - - var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - var back = MemoryCursor.CreateMax(); - - // Act - await _target.RunAsync(front, back, CancellationToken.None); - - // Assert - var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.NotNull(legacyCursor.Key); - var legacyIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); - Assert.Null(legacyIndex.Key); - var legacyLeaf = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); - Assert.Null(legacyLeaf.Key); - Assert.Single(_legacyStorage.Content); - - var semVer2Cursor = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.Null(semVer2Cursor.Key); - var semVer2Index = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); - Assert.NotNull(semVer2Index.Key); - var semVer2Leaf = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); - Assert.NotNull(semVer2Leaf.Key); - Assert.Equal(2, _semVer2Storage.Content.Count); - } - - [Fact] - public async Task PutsSemVer2PackagesInLegacyStorageWhenSemVer2IsDisabled() - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: false); - - var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); - await _mockServer.AddStorageAsync(catalogStorage); - - var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - var back = MemoryCursor.CreateMax(); - - // Act - await _target.RunAsync(front, back, CancellationToken.None); - - // Assert - var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); - Assert.NotNull(legacyCursor.Key); - var legacyIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); - Assert.NotNull(legacyIndex.Key); - var legacyLeaf = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); - Assert.NotNull(legacyLeaf.Key); - Assert.Equal(3, _legacyStorage.Content.Count); - } - - [Fact] - public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion() - { - // Arrange - SharedInit(useLegacy: true, useSemVer2: false); - - var catalogStorage = Catalogs.CreateTestCatalogWithNonNormalizedDelete(); - await _mockServer.AddStorageAsync(catalogStorage); - - ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); - - // Act - await _target.RunAsync( - front, - new MemoryCursor(DateTime.Parse("2015-10-12T10:08:54.1506742")), - CancellationToken.None); - var intermediateUris = _legacyStorage - .Content - .Select(pair => pair.Key.ToString()) - .OrderBy(uri => uri) - .ToList(); - await _target.RunAsync( - front, - MemoryCursor.CreateMax(), - CancellationToken.None); - - // Assert - Assert.Equal(3, intermediateUris.Count); - Assert.Equal("http://tempuri.org/cursor.json", intermediateUris[0]); - Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", intermediateUris[1]); - Assert.Equal("http://tempuri.org/otherpackage/index.json", intermediateUris[2]); - - // This should really be 1, but see: - // https://github.com/NuGet/Engineering/issues/404 - var finalUris = _legacyStorage - .Content - .Select(pair => pair.Key.ToString()) - .OrderBy(uri => uri) - .ToList(); - Assert.Equal(2, finalUris.Count); - Assert.Equal("http://tempuri.org/cursor.json", finalUris[0]); - Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", finalUris[1]); - } - - [Fact] - public async Task RunAsync_WhenPackageIdSpansCommitsAndVariesInCasing_BatchesAcrossCommitsByLowerCasePackageId() - { - var pageUri = new Uri(_baseUri, "v3-catalog0/page0.json"); - var indexUri = new Uri(_baseUri, "v3-catalog0/index.json"); - var contentBaseUri = new Uri(_baseUri, "packages"); - - SharedInit(useLegacy: true, useSemVer2: true, baseUri: _baseUri, indexUri: indexUri, contentBaseUri: contentBaseUri); - - var commitId1 = Guid.NewGuid().ToString(); - var commitId2 = Guid.NewGuid().ToString(); - var commitTimeStamp1 = DateTimeOffset.UtcNow.AddMinutes(-1); - var commitTimeStamp2 = commitTimeStamp1.AddMinutes(1); - var independentPackageDetails1 = new CatalogIndependentPackageDetails( - id: "ABC", - version: "1.0.0", - baseUri: _baseUri.AbsoluteUri, - commitId: commitId1, - commitTimeStamp: commitTimeStamp1); - var independentPackageDetails2 = new CatalogIndependentPackageDetails( - id: "AbC", - version: "1.0.1", - baseUri: _baseUri.AbsoluteUri, - commitId: commitId1, - commitTimeStamp: commitTimeStamp1); - var independentPackageDetails3 = new CatalogIndependentPackageDetails( - id: "abc", - version: "1.0.2", - baseUri: _baseUri.AbsoluteUri, - commitId: commitId2, - commitTimeStamp: commitTimeStamp2); - var packageDetails = new[] - { - CatalogPackageDetails.Create(independentPackageDetails1), - CatalogPackageDetails.Create(independentPackageDetails2), - CatalogPackageDetails.Create(independentPackageDetails3) - }; - - var independentPage = new CatalogIndependentPage( - pageUri.AbsoluteUri, - CatalogConstants.CatalogPage, - commitId2, - commitTimeStamp2.ToString(CatalogConstants.CommitTimeStampFormat), - packageDetails.Length, - indexUri.AbsoluteUri, - packageDetails, - _contextKeyword); - - var index = CatalogIndex.Create(independentPage, _contextKeyword); - - var catalogStorage = new MemoryStorage(_baseUri); - - catalogStorage.Content.TryAdd(indexUri, CreateStringStorageContent(index)); - catalogStorage.Content.TryAdd(pageUri, CreateStringStorageContent(independentPage)); - catalogStorage.Content.TryAdd( - new Uri(independentPackageDetails1.IdKeyword), - CreateStringStorageContent(independentPackageDetails1)); - catalogStorage.Content.TryAdd( - new Uri(independentPackageDetails2.IdKeyword), - CreateStringStorageContent(independentPackageDetails2)); - catalogStorage.Content.TryAdd( - new Uri(independentPackageDetails3.IdKeyword), - CreateStringStorageContent(independentPackageDetails3)); - - await _mockServer.AddStorageAsync(catalogStorage); - - await _target.RunAsync(CancellationToken.None); - - var expectedPage = new ExpectedPage(independentPackageDetails1, independentPackageDetails2, independentPackageDetails3); - - Verify(_legacyStorage, expectedPage); - Verify(_semVer2Storage, expectedPage); - } - - private void Verify(MemoryStorage storage, ExpectedPage expectedPage) - { - var firstPackageDetails = expectedPage.Details.First(); - var registrationIndexUri = GetRegistrationPackageIndexUri(storage.BaseAddress, firstPackageDetails.Id.ToLowerInvariant()); - var registrationIndex = GetStorageContent(storage, registrationIndexUri); - - var commitId = registrationIndex.CommitId; - var commitTimeStamp = registrationIndex.CommitTimeStamp; - - Assert.Equal(1, registrationIndex.Count); - - var registrationPage = registrationIndex.Items.Single(); - - Assert.Equal(CatalogConstants.CatalogCatalogPage, registrationPage.TypeKeyword); - Assert.Equal(commitId, registrationPage.CommitId); - Assert.Equal(commitTimeStamp, registrationPage.CommitTimeStamp); - Assert.Equal(expectedPage.Details.Count, registrationPage.Count); - Assert.Equal(registrationIndexUri.AbsoluteUri, registrationPage.Parent); - Assert.Equal(expectedPage.LowerVersion, registrationPage.Lower); - Assert.Equal(expectedPage.UpperVersion, registrationPage.Upper); - - Assert.Equal(expectedPage.Details.Count, registrationPage.Count); - Assert.Equal(registrationPage.Count, registrationPage.Items.Length); - - for (var i = 0; i < registrationPage.Count; ++i) - { - var catalogPackageDetails = expectedPage.Details[i]; - var packageId = catalogPackageDetails.Id.ToLowerInvariant(); - var packageVersion = catalogPackageDetails.Version.ToLowerInvariant(); - var registrationPackageVersionUri = GetRegistrationPackageVersionUri(storage.BaseAddress, packageId, packageVersion); - var packageContentUri = GetPackageContentUri(storage.BaseAddress, packageId, packageVersion); - - var registrationPackage = registrationPage.Items[i]; - - Assert.Equal(registrationPackageVersionUri.AbsoluteUri, registrationPackage.IdKeyword); - Assert.Equal(CatalogConstants.Package, registrationPackage.TypeKeyword); - Assert.Equal(commitId, registrationPackage.CommitId); - Assert.Equal(commitTimeStamp, registrationPackage.CommitTimeStamp); - Assert.Equal(packageContentUri.AbsoluteUri, registrationPackage.PackageContent); - Assert.Equal(registrationIndexUri.AbsoluteUri, registrationPackage.Registration); - - var registrationCatalogEntry = registrationPackage.CatalogEntry; - - Assert.Equal(catalogPackageDetails.IdKeyword, registrationCatalogEntry.IdKeyword); - Assert.Equal(CatalogConstants.PackageDetails, registrationCatalogEntry.TypeKeyword); - Assert.Equal(catalogPackageDetails.Id, registrationCatalogEntry.Id); - Assert.Equal(catalogPackageDetails.Version, registrationCatalogEntry.Version); - Assert.Equal(catalogPackageDetails.Authors, registrationCatalogEntry.Authors); - Assert.Equal(catalogPackageDetails.Description, registrationCatalogEntry.Description); - Assert.Equal(catalogPackageDetails.Listed, registrationCatalogEntry.Listed); - - Assert.Equal(packageContentUri.AbsoluteUri, registrationCatalogEntry.PackageContent); - Assert.Equal(GetRegistrationDateTime(catalogPackageDetails.Published), registrationCatalogEntry.Published); - Assert.Equal(catalogPackageDetails.RequireLicenseAcceptance, registrationCatalogEntry.RequireLicenseAcceptance); - - var registrationPackageDetails = GetStorageContent(storage, new Uri(registrationPackage.IdKeyword)); - - Assert.Equal(registrationPackageVersionUri.AbsoluteUri, registrationPackageDetails.IdKeyword); - Assert.Equal(new[] { CatalogConstants.Package, CatalogConstants.NuGetCatalogSchemaPermalinkUri }, registrationPackageDetails.TypeKeyword); - Assert.Equal(catalogPackageDetails.IdKeyword, registrationPackageDetails.CatalogEntry); - Assert.Equal(catalogPackageDetails.Listed, registrationPackageDetails.Listed); - Assert.Equal(packageContentUri.AbsoluteUri, registrationPackageDetails.PackageContent); - Assert.Equal(GetRegistrationDateTime(catalogPackageDetails.Published), registrationPackageDetails.Published); - Assert.Equal(registrationIndexUri.AbsoluteUri, registrationPackageDetails.Registration); - } - } - - private static StringStorageContent CreateStringStorageContent(T value) - { - return new StringStorageContent(JsonConvert.SerializeObject(value, _jsonSettings)); - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationEntryKeyTests.cs b/tests/CatalogTests/Registration/RegistrationEntryKeyTests.cs deleted file mode 100644 index f7caa9a3e..000000000 --- a/tests/CatalogTests/Registration/RegistrationEntryKeyTests.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Linq; -using NuGet.Services.Metadata.Catalog.Registration; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationEntryKeyTests - { - [Theory] - [MemberData(nameof(RegistrationEntryKeys))] - public void EqualsIgnoresCaseAndComparesNormalized(string idA, string versionA, string idB, string versionB, bool expectedEquals) - { - // Arrange - var a = new RegistrationEntryKey(new RegistrationKey(idA), versionA); - var b = new RegistrationEntryKey(new RegistrationKey(idB), versionB); - - // Act - var actualEqualsA = a.Equals(b); - var actualEqualsB = b.Equals(a); - - // Assert - Assert.Equal(expectedEquals, actualEqualsA); - Assert.Equal(expectedEquals, actualEqualsB); - } - - [Theory] - [MemberData(nameof(EqualRegistrationEntryKeys))] - public void GetHashCodeIgnoresCaseAndUsesNormalized(string idA, string versionA, string idB, string versionB) - { - // Arrange - var a = new RegistrationEntryKey(new RegistrationKey(idA), versionA); - var b = new RegistrationEntryKey(new RegistrationKey(idB), versionB); - - // Act - var hashCodeA = a.GetHashCode(); - var hashCodeB = b.GetHashCode(); - - // Assert - Assert.Equal(hashCodeA, hashCodeB); - } - - public static IEnumerable RegistrationEntryKeys - { - get - { - yield return new object[] { "NuGet.Core", "2.12.0", "NuGet.Core", "2.12.0", true }; - yield return new object[] { "nuget.core", "2.12.0", "NuGet.Core", "2.12.0", true }; - yield return new object[] { "NuGet.Core", "2.12.0-alpha", "NuGet.Core", "2.12.0-ALPHA", true }; - yield return new object[] { "NuGet.Core", "2.12.0+a", "NuGet.Core", "2.12.0+b", true }; - yield return new object[] { "NuGet.Core", "2.12.0+a", "NuGet.Core", "2.12.0", true }; - yield return new object[] { "NuGet.Core", "2.12.0.0", "NuGet.Core", "2.12.0", true }; - yield return new object[] { "NuGet.Core", "2.012.0", "NuGet.Core", "2.12.0", true }; - yield return new object[] { "NuGet.Versioning", "2.12.0", "NuGet.Core", "2.12.0", false }; - yield return new object[] { "NuGet.Core", "2.13.0", "NuGet.Core", "2.12.0", false }; - yield return new object[] { "NuGet.Core", "2.12.0-alpha", "NuGet.Core", "2.12.0", false }; - } - } - - public static IEnumerable EqualRegistrationEntryKeys => - RegistrationEntryKeys - .Where(a => (bool)a[4]) - .Select(a => new object[] { a[0], a[1], a[2], a[3] }); - } -} diff --git a/tests/CatalogTests/Registration/RegistrationKeyTests.cs b/tests/CatalogTests/Registration/RegistrationKeyTests.cs deleted file mode 100644 index 2cab56746..000000000 --- a/tests/CatalogTests/Registration/RegistrationKeyTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using NuGet.Services.Metadata.Catalog.Registration; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationKeyTests - { - [Fact] - public void Constructor_WhenIdIsNull_Throws() - { - var exception = Assert.Throws(() => new RegistrationKey(id: null)); - - Assert.Equal("id", exception.ParamName); - } - - [Theory] - [MemberData(nameof(RegistrationKeys))] - public void EqualsIgnoresCase(string idA, string idB, bool expectedEquals) - { - // Arrange - var a = new RegistrationKey(idA); - var b = new RegistrationKey(idB); - - // Act - var actualEqualsA = a.Equals(b); - var actualEqualsB = b.Equals(a); - - // Assert - Assert.Equal(expectedEquals, actualEqualsA); - Assert.Equal(expectedEquals, actualEqualsB); - } - - [Theory] - [MemberData(nameof(EqualRegistrationKeys))] - public void GetHashCodeIgnoresCaseAndUsesNormalized(string idA, string idB) - { - // Arrange - var a = new RegistrationKey(idA); - var b = new RegistrationKey(idB); - - // Act - var hashCodeA = a.GetHashCode(); - var hashCodeB = b.GetHashCode(); - - // Assert - Assert.Equal(hashCodeA, hashCodeB); - } - - [Theory] - [InlineData("a")] - [InlineData("A")] - public void ToString_Always_ReturnsLowerCase(string id) - { - var key = new RegistrationKey(id); - - Assert.Equal(id.ToLowerInvariant(), key.ToString()); - } - - public static IEnumerable RegistrationKeys - { - get - { - yield return new object[] { "NuGet.Core", "NuGet.Core", true }; - yield return new object[] { "nuget.core", "NuGet.Core", true }; - yield return new object[] { "NuGet.Versioning", "NuGet.Core", false }; - } - } - - public static IEnumerable EqualRegistrationKeys => - RegistrationKeys - .Where(a => (bool)a[2]) - .Select(a => new object[] { a[0], a[1] }); - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationMakerCatalogItemTests.cs b/tests/CatalogTests/Registration/RegistrationMakerCatalogItemTests.cs deleted file mode 100644 index 7cba665cf..000000000 --- a/tests/CatalogTests/Registration/RegistrationMakerCatalogItemTests.cs +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Services.Metadata.Catalog.Registration; -using VDS.RDF; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationMakerCatalogItemTests - { - [Theory] - [InlineData("0001-01-01T00:00:00-01:00", true)] - [InlineData("0001-01-01T00:00:00+00:00", true)] - [InlineData("0001-01-01T00:00:00+01:00", true)] - [InlineData("1900-01-01T00:00:00-01:00", false)] - [InlineData("1900-01-01T00:00:00+00:00", false)] - [InlineData("1900-01-01T00:00:00+01:00", true)] // Right at the end of 1899, UTC. - [InlineData("1901-01-01T00:00:00-01:00", true)] - [InlineData("1901-01-01T00:00:00+00:00", true)] - [InlineData("1901-01-01T00:00:00+01:00", false)] // Right at the end of 1900, UTC. - [InlineData("2015-01-01T00:00:00-01:00", true)] - [InlineData("2015-01-01T00:00:00+00:00", true)] - [InlineData("2015-01-01T00:00:00+01:00", true)] - public void CreateContent_SetsListedProperly(string published, bool expectedListed) - { - // Arrange - var catalogUri = new Uri("http://example/catalog/mypackage.1.0.0.json"); - var graph = new Graph(); - graph.Assert( - graph.CreateUriNode(catalogUri), - graph.CreateUriNode(Schema.Predicates.Version), - graph.CreateLiteralNode("1.0.0")); - graph.Assert( - graph.CreateUriNode(catalogUri), - graph.CreateUriNode(Schema.Predicates.Id), - graph.CreateLiteralNode("MyPackage")); - graph.Assert( - graph.CreateUriNode(catalogUri), - graph.CreateUriNode(Schema.Predicates.Published), - graph.CreateLiteralNode(published)); - - var registrationBaseAddress = new Uri("http://example/registration/"); - var packageContentBaseAddress = new Uri("http://example/content/"); - var item = new RegistrationMakerCatalogItem( - catalogUri, - graph, - registrationBaseAddress, - isExistingItem: false, - postProcessGraph: g => g, - packageContentBaseAddress: packageContentBaseAddress, - forcePathProviderForIcons: false) - { - BaseAddress = new Uri("http://example/registration/mypackage/"), - }; - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - var context = new CatalogContext(); - - // Act - var content = item.CreateContent(context); - - // Assert - var contentJson = JObject.Parse(GetContentString(content)); - var actualListed = contentJson["listed"].Value(); - Assert.Equal(expectedListed, actualListed); - } - - private static string GetContentString(StorageContent content) - { - if (content is StringStorageContent stringStorageContent) - { - return stringStorageContent.Content; - } - - using (var reader = new StreamReader(content.GetContentStream())) - { - return reader.ReadToEnd(); - } - } - - public class TheCreatePageContentMethod - { - private const string PackageId = "MyPackage"; - private const string PackageVersion = "01.02.03+ABC"; - private readonly Uri _catalogUri = new Uri("http://example/catalog/mypackage.1.0.0.json"); - private readonly Uri _registrationBaseAddress = new Uri("http://example/registration/"); - private readonly Uri _packageContentBaseAddress = new Uri("http://example/content/"); - private readonly Uri _baseAddress = new Uri("http://example/registration/mypackage/"); - private readonly Graph _graph; - - public TheCreatePageContentMethod() - { - _graph = new Graph(); - AddTriple(_catalogUri, Schema.Predicates.Id, PackageId); - AddTriple(_catalogUri, Schema.Predicates.Version, PackageVersion); - AddTriple(_catalogUri, Schema.Predicates.Published, "2015-01-01T00:00:00+00:00"); - } - - [Theory] - [InlineData(null, null, null, "", "http://gallery.test")] - [InlineData("https://test", null, null, "https://test", "http://gallery.test")] - [InlineData("https://test", "TestExpression", null, "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData("https://test", null, "TestLicense", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, "TestExpression", null, "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestLicense", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestFile", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestLicense.txt", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestLicense.exe", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestLicense.any", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "/Folder/TestLicense", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test")] - [InlineData(null, null, "TestLicense", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test/")] - [InlineData(null, null, "TestLicense", "http://gallery.test/packages/MyPackage/1.2.3/license", "http://gallery.test//")] - public void CreatePageContent_SetsLicenseUrlAndExpressionProperly( - string licenseUrl, - string licenseExpression, - string licenseFile, - string expectedLicenseUrlValue, - string galleryBaseAddress) - { - // Arrange - AddTriple(_catalogUri, Schema.Predicates.LicenseUrl, licenseUrl); - AddTriple(_catalogUri, Schema.Predicates.LicenseExpression, licenseExpression); - AddTriple(_catalogUri, Schema.Predicates.LicenseFile, licenseFile); - - var item = new RegistrationMakerCatalogItem( - _catalogUri, - _graph, - _registrationBaseAddress, - isExistingItem: false, - postProcessGraph: g => g, - packageContentBaseAddress: _packageContentBaseAddress, - galleryBaseAddress: new Uri(galleryBaseAddress), - forcePathProviderForIcons: false) - { - BaseAddress = _baseAddress, - }; - - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - var context = new CatalogContext(); - - // Act - var content = item.CreatePageContent(context); - var licenseUrlTriples = GetTriples(content, _catalogUri, Schema.Predicates.LicenseUrl); - var licenseExpressionTriples = GetTriples(content, _catalogUri, Schema.Predicates.LicenseExpression); - var licenseFileTriples = GetTriples(content, _catalogUri, Schema.Predicates.LicenseFile); - - // Assert - Assert.Equal(expectedLicenseUrlValue, Assert.Single(licenseUrlTriples).Object.ToString()); - Assert.Equal(licenseExpression ?? "", Assert.Single(licenseExpressionTriples).Object.ToString()); - Assert.Empty(licenseFileTriples); - } - - [Theory] - [InlineData(null, null, false, "")] - [InlineData("http://icon.test/", null, false, "http://icon.test/")] - [InlineData("http://icon.test/", null, true, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.png", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.png", true, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData("http://icon.test/", "icon.png", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData("http://icon.test/", "icon.png", true, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.jpg", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.jpeg", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.gif", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.svg", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - [InlineData(null, "icon.exe", false, "http://example/content/test-container/mypackage/1.2.3/icon")] - public void CreatePageContent_SetsIconUrlProperly(string iconUrl, string iconFile, bool forceFlatContainer, string expectedIconUrl) - { - // Arrange - AddTriple(_catalogUri, Schema.Predicates.IconUrl, iconUrl); - AddTriple(_catalogUri, Schema.Predicates.IconFile, iconFile); - - var item = new RegistrationMakerCatalogItem( - _catalogUri, - _graph, - _registrationBaseAddress, - isExistingItem: false, - postProcessGraph: g => g, - packageContentBaseAddress: _packageContentBaseAddress, - galleryBaseAddress: new Uri("http://gallery.test"), - forcePathProviderForIcons: forceFlatContainer) - { - BaseAddress = _baseAddress, - }; - RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider("test-container"); - var context = new CatalogContext(); - - // Act - var graph = item.CreatePageContent(context); - - // Assert - var iconUrlTriples = GetTriples(graph, _catalogUri, Schema.Predicates.IconUrl); - var iconFileTriples = GetTriples(graph, _catalogUri, Schema.Predicates.IconFile); - - Assert.Equal(expectedIconUrl, Assert.Single(iconUrlTriples).Object.ToString()); - Assert.Empty(iconFileTriples); - } - - private void AddTriple(Uri subject, Uri predicate, string @object) - { - if (@object != null) - { - _graph.Assert( - _graph.CreateUriNode(subject), - _graph.CreateUriNode(predicate), - _graph.CreateLiteralNode(@object)); - } - } - - private IEnumerable GetTriples(IGraph graph, Uri subject, Uri predicate) - { - return graph.GetTriplesWithSubjectPredicate(graph.CreateUriNode(subject), graph.CreateUriNode(predicate)); - } - - public static IEnumerable CreatePageContent_SetsDeprecationInformationProperly_Data - { - get - { - foreach (var shouldPostProcess in new[] { false, true }) - { - foreach (var reason in - new[] - { - new[] { "first" }, - new[] { "first", "second" } - }) - { - foreach (var message in new[] { null, "this is the message" }) - { - yield return new object[] { shouldPostProcess, reason, message, null, null }; - yield return new object[] { shouldPostProcess, reason, message, "theId", "homeOnTheRange" }; - } - } - } - } - } - - [Theory] - [MemberData(nameof(CreatePageContent_SetsDeprecationInformationProperly_Data))] - public void CreatePageContent_SetsDeprecationInformationProperly( - bool shouldPostProcess, - IEnumerable reasons, - string message, - string alternatePackageId, - string alternatePackageRange) - { - if (alternatePackageId == null && alternatePackageRange != null) - { - throw new ArgumentException("Must specify alternate package range if alternate package ID is specified."); - } - - if (alternatePackageId != null && alternatePackageRange == null) - { - throw new ArgumentException("Must specify alternate package ID if alternate package range is specified."); - } - - // Arrange - var rootNode = _graph.CreateUriNode(_catalogUri); - var deprecationPredicate = _graph.CreateUriNode(Schema.Predicates.Deprecation); - var deprecationRootNode = _graph.CreateUriNode(new Uri(_catalogUri.ToString() + "#deprecation")); - _graph.Assert(rootNode, deprecationPredicate, deprecationRootNode); - - var deprecationReasonRootNode = _graph.CreateUriNode(Schema.Predicates.Reasons); - foreach (var reason in reasons) - { - var reasonNode = _graph.CreateLiteralNode(reason); - _graph.Assert(deprecationRootNode, deprecationReasonRootNode, reasonNode); - } - - if (message != null) - { - _graph.Assert( - deprecationRootNode, - _graph.CreateUriNode(Schema.Predicates.Message), - _graph.CreateLiteralNode(message)); - } - - if (alternatePackageId != null) - { - var deprecationAlternatePackagePredicate = _graph.CreateUriNode(Schema.Predicates.AlternatePackage); - var deprecationAlternatePackageRootNode = _graph.CreateUriNode(new Uri(_catalogUri.ToString() + "#deprecation/alternatePackage")); - _graph.Assert(deprecationRootNode, deprecationAlternatePackagePredicate, deprecationAlternatePackageRootNode); - - _graph.Assert( - deprecationAlternatePackageRootNode, - _graph.CreateUriNode(Schema.Predicates.Id), - _graph.CreateLiteralNode(alternatePackageId)); - - _graph.Assert( - deprecationAlternatePackageRootNode, - _graph.CreateUriNode(Schema.Predicates.Range), - _graph.CreateLiteralNode(alternatePackageRange)); - } - - var item = new RegistrationMakerCatalogItem( - _catalogUri, - _graph, - _registrationBaseAddress, - isExistingItem: false, - postProcessGraph: shouldPostProcess ? RegistrationCollector.FilterOutDeprecationInformation : g => g, - packageContentBaseAddress: _packageContentBaseAddress, - forcePathProviderForIcons: false) - { - BaseAddress = _baseAddress, - }; - - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - var context = new CatalogContext(); - - // Act - var content = item.CreatePageContent(context); - - // Assert - var deprecationPredicateTriples = content - .GetTriplesWithSubjectPredicate( - content.CreateUriNode(_catalogUri), - content.CreateUriNode(Schema.Predicates.Deprecation)); - - if (shouldPostProcess) - { - Assert.Empty(deprecationPredicateTriples); - return; - } - - var deprecationObjectNode = deprecationPredicateTriples - .Single() - .Object; - - var deprecationTriples = content.GetTriplesWithSubject(deprecationObjectNode); - var reasonTriples = deprecationTriples - .Where(t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.Reasons))); - - foreach (var reason in reasons) - { - Assert.Contains(reasonTriples, t => t.HasObject(content.CreateLiteralNode(reason))); - } - - if (message == null) - { - Assert.DoesNotContain( - deprecationTriples, - t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.Message))); - } - else - { - Assert.Contains( - deprecationTriples, - t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.Message)) && t.HasObject(content.CreateLiteralNode(message))); - } - - if (alternatePackageId == null) - { - Assert.DoesNotContain( - deprecationTriples, - t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.AlternatePackage))); - } - else - { - var alternatePackageObjectNode = content - .GetTriplesWithSubjectPredicate( - deprecationObjectNode, - content.CreateUriNode(Schema.Predicates.AlternatePackage)) - .Single() - .Object; - - var alternatePackageTriples = content.GetTriplesWithSubject(alternatePackageObjectNode); - Assert.Contains(alternatePackageTriples, - t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.Id)) && t.HasObject(content.CreateLiteralNode(alternatePackageId))); - - Assert.Contains(alternatePackageTriples, - t => t.HasPredicate(content.CreateUriNode(Schema.Predicates.Range)) && t.HasObject(content.CreateLiteralNode(alternatePackageRange))); - } - } - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationMakerTests.cs b/tests/CatalogTests/Registration/RegistrationMakerTests.cs deleted file mode 100644 index db0f0ab06..000000000 --- a/tests/CatalogTests/Registration/RegistrationMakerTests.cs +++ /dev/null @@ -1,706 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using CatalogTests.Helpers; -using Moq; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NgTests; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Registration; -using VDS.RDF; -using Xunit; - -namespace CatalogTests.Registration -{ - public class RegistrationMakerTests : RegistrationTestBase - { - private const int _defaultPackageCountThreshold = 128; - - private static readonly Uri _contentBaseAddress = new Uri("https://nuget.test/"); - private static readonly Uri _galleryBaseAddress = new Uri("https://nuget.org/"); - - private readonly MemoryStorageFactory _storageFactory = new MemoryStorageFactory(new Uri("https://nuget.test/v3-registration3/")); - private readonly Mock _telemetryService = new Mock(); - - public RegistrationMakerTests() - { - RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); - } - - [Fact] - public async Task ProcessAsync_WithEmptyStorageAndEmptyNewItems_DoesNotCreateAnything() - { - var emptyItems = new Dictionary(); - var storage = await ProcessAsync(emptyItems, packageId: null, partitionSize: 1); - - Assert.Empty(storage.Content); - Assert.Empty(storage.ContentBytes); - } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(3)] - public async Task ProcessAsync_WithCustomPartitionSize_FillsPages(int partitionSize) - { - // The idea here is that we should see pages filled before a new pages is created. - // Ultimately, we should have 2 filled pages plus one unfilled page. - var packageVersionCount = partitionSize * 2 + 1; - var pages = new List(); - - for (var i = 0; i < packageVersionCount; ++i) - { - var packageDetails = new CatalogIndependentPackageDetails(id: "A", version: $"1.0.{i}"); - var newItem = CreateNewItem(packageDetails); - var storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); - - pages.Add(new ExpectedPage(packageDetails)); - - var expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize); - } - } - - [Fact] - public async Task ProcessAsync_WithVariablePartitionSize_RepaginatesExistingPages() - { - var partitionSize = 2; - var packageVersionCount = partitionSize * 2; - var pages = new List(); - CatalogIndependentPackageDetails packageDetails; - IReadOnlyDictionary newItem; - MemoryStorage storage; - IReadOnlyList expectedPages; - - for (var i = 0; i < packageVersionCount; ++i) - { - packageDetails = new CatalogIndependentPackageDetails(id: "a", version: $"1.0.{i}"); - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); - - pages.Add(new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize); - } - - partitionSize = 3; - ++packageVersionCount; - - packageDetails = new CatalogIndependentPackageDetails(id: "a", version: $"1.0.{packageVersionCount}"); - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); - - pages.Add(new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize); - } - - [Fact] - public async Task ProcessAsync_WhenNewPackageVersionWouldChangeExistingPage_RepaginatesExistingPage() - { - const int partitionSize = 2; - var pages = new List(); - CatalogIndependentPackageDetails packageDetails; - IReadOnlyDictionary newItem; - MemoryStorage storage; - IReadOnlyList expectedPages; - - for (var i = 0; i < 2; ++i) - { - packageDetails = new CatalogIndependentPackageDetails(id: "a", version: $"1.0.{i * 2}"); - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); - - pages.Add(new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize); - } - - packageDetails = new CatalogIndependentPackageDetails(id: "a", version: "1.0.1"); - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize); - - pages.Insert(index: 1, item: new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize); - } - - [Fact] - public async Task ProcessAsync_WithCustomPackageCountThreshold_TransitionsToPageWhenThresholdHit() - { - const int partitionSize = 2; - const int packageCountThreshold = 2; - var pages = new List(); - CatalogIndependentPackageDetails packageDetails; - IReadOnlyDictionary newItem; - MemoryStorage storage; - IReadOnlyList expectedPages; - - for (var i = 0; i < 2; ++i) - { - packageDetails = new CatalogIndependentPackageDetails(id: "a", version: $"1.0.{i}"); - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync(newItem, packageDetails.Id, partitionSize, packageCountThreshold); - - pages.Add(new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify(storage, expectedPages, partitionSize, packageCountThreshold); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ProcessAsync_WithDeprecationInformation_FormatsPageCorrectly(bool filterOutDeprecation) - { - const int partitionSize = 1; - var pages = new List(); - IReadOnlyDictionary newItem; - MemoryStorage storage; - IReadOnlyList expectedPages; - - var packageDetailsList = new[] - { - // No deprecation - new CatalogIndependentPackageDetails( - id: "deprecationTests", - version: "2.4.3"), - - // Single reason - new CatalogIndependentPackageDetails( - id: "deprecationTests", - version: "3.6.5", - deprecation: new RegistrationPackageDeprecation( - new[] {"r1" })), - - // Message - new CatalogIndependentPackageDetails( - id: "deprecationTests", - version: "4.7.6", - deprecation: new RegistrationPackageDeprecation( - new[] {"r1", "r2" }, - "the cow goes moo")), - - // Alternate package - new CatalogIndependentPackageDetails( - id: "deprecationTests", - version: "5.9.8", - deprecation: new RegistrationPackageDeprecation( - new[] {"r1", "r2" }, - null, - new RegistrationPackageDeprecationAlternatePackage("altPkg", "breezepackages"))), - - // Message and alternate package - new CatalogIndependentPackageDetails( - id: "deprecationTests", - version: "6.0.2", - deprecation: new RegistrationPackageDeprecation( - new[] {"r1", "r2" }, - "the package goes nuu", - new RegistrationPackageDeprecationAlternatePackage("altPkg", "breezepackages"))), - }; - - foreach (var packageDetails in packageDetailsList) - { - newItem = CreateNewItem(packageDetails); - storage = await ProcessAsync( - newItem, - packageDetails.Id, - partitionSize, - filterOutDeprecation: filterOutDeprecation); - - pages.Add(new ExpectedPage(packageDetails)); - - expectedPages = Repaginate(pages, partitionSize); - - Verify( - storage, - expectedPages, - partitionSize, - filterOutDeprecation: filterOutDeprecation); - } - } - - private static IReadOnlyDictionary CreateNewItem(CatalogIndependentPackageDetails packageDetails) - { - var json = JsonConvert.SerializeObject(packageDetails, _jsonSettings); - - using (var reader = new JsonTextReader(new StringReader(json))) - { - reader.DateParseHandling = DateParseHandling.DateTimeOffset; // make sure we always preserve timezone info - - var jObject = JObject.Load(reader); - var graph = Utils.CreateGraph(jObject, readOnly: true); - - return new Dictionary() - { - { packageDetails.IdKeyword, graph } - }; - } - } - - private async Task ProcessAsync( - IReadOnlyDictionary newItems, - string packageId, - int partitionSize, - int packageCountThreshold = _defaultPackageCountThreshold, - bool filterOutDeprecation = false) - { - var registrationKey = new RegistrationKey(packageId?.ToLowerInvariant() ?? string.Empty); - - await RegistrationMaker.ProcessAsync( - registrationKey, - newItems, - (g, u, k) => true, - _storageFactory, - filterOutDeprecation ? RegistrationCollector.FilterOutDeprecationInformation : g => g, - _contentBaseAddress, - _galleryBaseAddress, - partitionSize, - packageCountThreshold, - forcePackagePathProviderForIcons: false, - telemetryService: _telemetryService.Object, - cancellationToken: CancellationToken.None); - - return (MemoryStorage)_storageFactory.Create(registrationKey.ToString()); - } - - private void Verify( - MemoryStorage registrationStorage, - IReadOnlyList expectedPages, - int partitionSize, - int packageCountThreshold = _defaultPackageCountThreshold, - bool filterOutDeprecation = false) - { - var expectedStorageContentCount = expectedPages.SelectMany(page => page.Details).Count(); - - expectedStorageContentCount += expectedStorageContentCount / packageCountThreshold; - - ++expectedStorageContentCount; // index - - Assert.Equal(expectedStorageContentCount, registrationStorage.Content.Count); - - var firstPage = expectedPages.First(); - var packageId = firstPage.Details.First().Id.ToLowerInvariant(); - var indexUri = GetRegistrationPackageIndexUri(_storageFactory.BaseAddress, packageId); - var index = GetStorageContent(registrationStorage, indexUri); - - VerifyRegistrationIndex( - registrationStorage, - index, - indexUri, - packageId, - expectedPages, - partitionSize, - packageCountThreshold, - filterOutDeprecation); - } - - private void VerifyRegistrationIndex( - MemoryStorage registrationStorage, - RegistrationIndex index, - Uri indexUri, - string packageId, - IReadOnlyList expectedPages, - int partitionSize, - int packageCountThreshold, - bool filterOutDeprecation) - { - Assert.Equal(indexUri.AbsoluteUri, index.IdKeyword); - Assert.Equal( - new[] { "catalog:CatalogRoot", "PackageRegistration", CatalogConstants.CatalogPermalink }, - index.TypeKeyword); - - Assert.True(Guid.TryParse(index.CommitId, out var guid)); - Assert.True(DateTime.TryParse(index.CommitTimeStamp, out var datetime)); - Assert.Equal(expectedPages.Count, index.Count); - Assert.Equal(index.Count, index.Items.Length); - - for (var i = 0; i < index.Count; ++i) - { - var page = index.Items[i]; - var expectedPage = expectedPages[i]; - var expectFullPage = i < index.Count - 1; - - if (expectFullPage) - { - Assert.Equal(partitionSize, page.Count); - } - else - { - Assert.InRange(page.Count, low: 1, high: partitionSize); - } - - if (page.Count == packageCountThreshold) - { - VerifyRegistrationPageReference( - registrationStorage, - page, - packageId, - expectedPage, - index.CommitId, - index.CommitTimeStamp, - filterOutDeprecation); - } - else - { - var pageUri = GetRegistrationPageUri(packageId, expectedPage); - - VerifyRegistrationPage( - registrationStorage, - page, - pageUri, - packageId, - expectedPage, - index.CommitId, - index.CommitTimeStamp, - filterOutDeprecation); - } - } - - var expectedContext = new JObject( - new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), - new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), - new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), - new JProperty(CatalogConstants.Items, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogItem), - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), - new JProperty(CatalogConstants.CommitTimeStamp, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitTimeStamp), - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.CommitId, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitId))), - new JProperty(CatalogConstants.Count, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCount))), - new JProperty(CatalogConstants.Parent, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogParent), - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Tags, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Tag))), - new JProperty(CatalogConstants.Reasons, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), - new JProperty(CatalogConstants.PackageTargetFrameworks, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageTargetFramework))), - new JProperty(CatalogConstants.DependencyGroups, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.DependencyGroup))), - new JProperty(CatalogConstants.Dependencies, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Dependency))), - new JProperty(CatalogConstants.PackageContent, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Published, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.Registration, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword)))); - - Assert.Equal(expectedContext.ToString(), index.ContextKeyword.ToString()); - } - - private void VerifyRegistrationPage( - MemoryStorage registrationStorage, - RegistrationPage page, - Uri pageUri, - string packageId, - ExpectedPage expectedPage, - string commitId, - string commitTimeStamp, - bool filterOutDeprecation) - { - var packageIndexUri = GetRegistrationPackageIndexUri(registrationStorage.BaseAddress); - - Assert.Equal(pageUri.AbsoluteUri, page.IdKeyword); - Assert.Equal(CatalogConstants.CatalogCatalogPage, page.TypeKeyword); - Assert.Equal(commitId, page.CommitId); - Assert.Equal(commitTimeStamp, page.CommitTimeStamp); - Assert.Equal(expectedPage.LowerVersion, page.Lower); - Assert.Equal(expectedPage.UpperVersion, page.Upper); - Assert.Equal(page.Count, page.Items.Length); - Assert.Equal(packageIndexUri.AbsoluteUri, page.Parent); - - for (var i = 0; i < page.Count; ++i) - { - var item = page.Items[i]; - var packageDetails = expectedPage.Details[i]; - - VerifyRegistrationPackage( - registrationStorage, - item, - packageDetails, - commitId, - commitTimeStamp, - filterOutDeprecation); - } - } - - private void VerifyRegistrationPageReference( - MemoryStorage registrationStorage, - RegistrationPage page, - string packageId, - ExpectedPage expectedPage, - string commitId, - string commitTimeStamp, - bool filterOutDeprecation) - { - var pageUri = GetRegistrationPageReferenceUri(packageId, expectedPage); - - Assert.Equal(pageUri.AbsoluteUri, page.IdKeyword); - Assert.Equal(CatalogConstants.CatalogCatalogPage, page.TypeKeyword); - Assert.Equal(commitId, page.CommitId); - Assert.Equal(commitTimeStamp, page.CommitTimeStamp); - Assert.Equal(expectedPage.LowerVersion, page.Lower); - Assert.Equal(expectedPage.UpperVersion, page.Upper); - - Assert.Null(page.Items); - Assert.Null(page.Parent); - - var independentPage = GetStorageContent(registrationStorage, pageUri); - - VerifyRegistrationPage( - registrationStorage, - independentPage, - pageUri, - packageId, - expectedPage, - commitId, - commitTimeStamp, - filterOutDeprecation); - - JObject expectedContext = GetExpectedIndexOrPageContext(); - - Assert.Equal(expectedContext.ToString(), independentPage.ContextKeyword.ToString()); - } - - private void VerifyRegistrationPackage( - MemoryStorage registrationStorage, - RegistrationPackage package, - CatalogIndependentPackageDetails packageDetails, - string commitId, - string commitTimeStamp, - bool filterOutDeprecation) - { - var packageId = packageDetails.Id.ToLowerInvariant(); - var packageVersion = packageDetails.Version.ToLowerInvariant(); - var packageVersionUri = GetRegistrationPackageVersionUri(packageId, packageVersion); - var packageContentUri = GetPackageContentUri(_contentBaseAddress, packageId, packageVersion); - - Assert.Equal(packageVersionUri.AbsoluteUri, package.IdKeyword); - Assert.Equal(CatalogConstants.Package, package.TypeKeyword); - Assert.Equal(commitId, package.CommitId); - Assert.Equal(commitTimeStamp, package.CommitTimeStamp); - Assert.Equal(packageDetails.IdKeyword, package.CatalogEntry.IdKeyword); - Assert.Equal(CatalogConstants.PackageDetails, package.CatalogEntry.TypeKeyword); - Assert.Equal(packageDetails.Authors, package.CatalogEntry.Authors); - Assert.Equal(packageDetails.Description, package.CatalogEntry.Description); - Assert.Empty(package.CatalogEntry.IconUrl); - Assert.Equal(packageDetails.Id, package.CatalogEntry.Id); - Assert.Empty(package.CatalogEntry.Language); - Assert.Empty(package.CatalogEntry.LicenseUrl); - Assert.Equal(packageDetails.Listed, package.CatalogEntry.Listed); - Assert.Empty(package.CatalogEntry.MinClientVersion); - Assert.Equal(packageContentUri.AbsoluteUri, package.CatalogEntry.PackageContent); - Assert.Empty(package.CatalogEntry.ProjectUrl); - Assert.Equal(GetRegistrationDateTime(packageDetails.Published), package.CatalogEntry.Published); - Assert.Equal(packageDetails.RequireLicenseAcceptance, package.CatalogEntry.RequireLicenseAcceptance); - Assert.Empty(package.CatalogEntry.Summary); - Assert.Equal(new[] { string.Empty }, package.CatalogEntry.Tags); - Assert.Empty(package.CatalogEntry.Title); - Assert.Equal(packageDetails.Version, package.CatalogEntry.Version); - - var actualDeprecation = package.CatalogEntry.Deprecation; - var expectedDeprecation = packageDetails.Deprecation; - if (filterOutDeprecation || expectedDeprecation == null) - { - Assert.Null(actualDeprecation); - } - else - { - Assert.NotNull(actualDeprecation); - - Assert.Equal(expectedDeprecation.Reasons.OrderBy(r => r), actualDeprecation.Reasons.OrderBy(r => r)); - Assert.Equal(expectedDeprecation.Message, actualDeprecation.Message); - - var actualDeprecationAltPackage = actualDeprecation.AlternatePackage; - var expectedDeprecationAltPackage = expectedDeprecation.AlternatePackage; - if (expectedDeprecationAltPackage == null) - { - Assert.Null(actualDeprecationAltPackage); - } - else - { - Assert.NotNull(actualDeprecationAltPackage); - - Assert.Equal(expectedDeprecationAltPackage.Id, actualDeprecationAltPackage.Id); - Assert.Equal(expectedDeprecationAltPackage.Range, actualDeprecationAltPackage.Range); - } - } - - var independentPackageUri = GetRegistrationPackageVersionUri(packageId, packageVersion); - var independentPackage = GetStorageContent( - registrationStorage, - independentPackageUri); - - VerifyRegistrationIndependentPackage( - registrationStorage, - independentPackage, - independentPackageUri, - packageDetails, - packageId, - packageVersion); - } - - private void VerifyRegistrationIndependentPackage( - MemoryStorage registrationStorage, - RegistrationIndependentPackage package, - Uri packageUri, - CatalogIndependentPackageDetails packageDetails, - string packageId, - string packageVersion) - { - var packageContentUri = GetPackageContentUri(_contentBaseAddress, packageId, packageVersion); - var packageIndexUri = GetRegistrationPackageIndexUri(registrationStorage.BaseAddress); - - Assert.Equal(packageUri.AbsoluteUri, package.IdKeyword); - Assert.Equal( - new[] { CatalogConstants.Package, CatalogConstants.NuGetCatalogSchemaPermalinkUri }, - package.TypeKeyword); - Assert.Equal(packageDetails.Listed, package.Listed); - Assert.Equal(packageContentUri.AbsoluteUri, package.PackageContent); - Assert.Equal(GetRegistrationDateTime(packageDetails.Published), package.Published); - Assert.Equal(packageIndexUri.AbsoluteUri, package.Registration); - - var expectedContext = new JObject( - new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), - new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), - new JProperty(CatalogConstants.CatalogEntry, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Registration, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.PackageContent, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Published, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); - - Assert.Equal(expectedContext.ToString(), package.ContextKeyword.ToString()); - } - - private static JObject GetExpectedIndexOrPageContext() - { - return new JObject( - new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), - new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), - new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), - new JProperty(CatalogConstants.Items, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogItem), - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), - new JProperty(CatalogConstants.CommitTimeStamp, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitTimeStamp), - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.CommitId, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCommitId))), - new JProperty(CatalogConstants.Count, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogCount))), - new JProperty(CatalogConstants.Parent, - new JObject( - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.CatalogParent), - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Tags, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Tag))), - new JProperty(CatalogConstants.Reasons, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), - new JProperty(CatalogConstants.PackageTargetFrameworks, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageTargetFramework))), - new JProperty(CatalogConstants.DependencyGroups, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.DependencyGroup))), - new JProperty(CatalogConstants.Dependencies, - new JObject( - new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword), - new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Dependency))), - new JProperty(CatalogConstants.PackageContent, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), - new JProperty(CatalogConstants.Published, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), - new JProperty(CatalogConstants.Registration, - new JObject( - new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword)))); - } - - private Uri GetRegistrationPageUri(string packageId, ExpectedPage expectedPage) - { - return new Uri(GetRegistrationPackageIndexUri(_storageFactory.BaseAddress, packageId), - $"#page/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}"); - } - - private Uri GetRegistrationPageReferenceUri(string packageId, ExpectedPage expectedPage) - { - return new Uri($"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/page" - + $"/{expectedPage.LowerVersion}/{expectedPage.UpperVersion}.json"); - } - - private Uri GetRegistrationPackageVersionUri(string packageId, string packageVersion) - { - return new Uri($"{_storageFactory.BaseAddress.AbsoluteUri}{packageId}/{packageVersion}.json"); - } - - private static IReadOnlyList Repaginate( - IReadOnlyList pages, - int partitionSize) - { - return pages.Select((page, index) => new { Page = page, Index = index }) - .GroupBy(x => x.Index / partitionSize) - .Select(group => new ExpectedPage(group.SelectMany(x => x.Page.Details).ToArray())) - .ToList(); - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/Registration/RegistrationTestBase.cs b/tests/CatalogTests/Registration/RegistrationTestBase.cs deleted file mode 100644 index eac7f9cca..000000000 --- a/tests/CatalogTests/Registration/RegistrationTestBase.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using CatalogTests.Helpers; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Versioning; -using Xunit; - -namespace CatalogTests.Registration -{ - public abstract class RegistrationTestBase - { - protected const string _cacheControl = "no-store"; - protected const string _contentType = "application/json"; - - protected static readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings() - { - DateParseHandling = DateParseHandling.None, - NullValueHandling = NullValueHandling.Ignore - }; - - protected static string GetRegistrationDateTime(string catalogDateTime) - { - return DateTimeOffset.Parse(catalogDateTime) - .ToLocalTime() - .ToString("yyyy-MM-ddTHH:mm:ss.FFFzzz"); - } - - protected static T GetStorageContent(MemoryStorage registrationStorage, Uri contentUri) - { - Assert.True(registrationStorage.Content.TryGetValue(contentUri, out var content)); - - var jTokenStorageContent = content as JTokenStorageContent; - - Assert.NotNull(jTokenStorageContent); - Assert.Equal(_cacheControl, jTokenStorageContent.CacheControl); - Assert.Equal(_contentType, jTokenStorageContent.ContentType); - - var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); - - // Verify that no new properties were added unexpectedly. - Assert.Equal(properties.Length, ((JObject)jTokenStorageContent.Content).Count); - - var json = jTokenStorageContent.Content.ToString(); - - return JsonConvert.DeserializeObject(json, _jsonSettings); - } - - protected static Uri GetPackageContentUri(Uri contentBaseUri, string packageId, string packageVersion) - { - return new Uri($"{contentBaseUri.AbsoluteUri}packages/{packageId}.{packageVersion}.nupkg"); - } - - protected static Uri GetRegistrationPackageIndexUri(Uri baseUri) - { - return new Uri($"{baseUri.AbsoluteUri}index.json"); - } - - protected static Uri GetRegistrationPackageIndexUri(Uri baseUri, string packageId) - { - return new Uri($"{baseUri.AbsoluteUri}{packageId}/index.json"); - } - - protected static Uri GetRegistrationPackageVersionUri(Uri baseUri, string packageId, string packageVersion) - { - return new Uri($"{baseUri.AbsoluteUri}{packageId}/{packageVersion}.json"); - } - - protected sealed class ExpectedPage - { - internal IReadOnlyList Details { get; } - internal string LowerVersion { get; } - internal string UpperVersion { get; } - - internal ExpectedPage(params CatalogIndependentPackageDetails[] packageDetails) - { - Details = packageDetails; - - var versions = packageDetails.Select(details => new NuGetVersion(details.Version)).ToArray(); - - LowerVersion = versions.Min().ToNormalizedString().ToLowerInvariant(); - UpperVersion = versions.Max().ToNormalizedString().ToLowerInvariant(); - } - } - } -} \ No newline at end of file diff --git a/tests/CatalogTests/SortingGraphCollectorTests.cs b/tests/CatalogTests/SortingGraphCollectorTests.cs deleted file mode 100644 index 8c5452848..000000000 --- a/tests/CatalogTests/SortingGraphCollectorTests.cs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Moq; -using NgTests.Data; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog; -using VDS.RDF; -using Xunit; - -namespace CatalogTests -{ - public class SortingGraphCollectorTests - { - private MockServerHttpClientHandler _mockServer; - private TestSortingGraphCollector _collector; - - private void Initialize() - { - _mockServer = new MockServerHttpClientHandler(); - _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); - - _collector = new TestSortingGraphCollector( - new Uri("http://tempuri.org/index.json"), - new Uri[] { Schema.DataTypes.PackageDetails, Schema.DataTypes.PackageDelete }, - handlerFunc: () => _mockServer); - } - - [Fact] - public async Task AllGraphsAreReadOnly() - { - // Arrange - Initialize(); - - var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorageAsync(catalogStorage); - - // Act - var result = await _collector.RunAsync(CancellationToken.None); - - // Assert - foreach (var graphs in _collector.AllSortedGraphs) - { - foreach (var pair in graphs.Value) - { - Assert.IsType(pair.Value); - } - } - - Assert.True(result, "The result of running the test collector should be true."); - } - - [Fact] - public async Task GraphsAreBatchedById() - { - // Arrange - Initialize(); - - var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); - await _mockServer.AddStorageAsync(catalogStorage); - - // Act - var result = await _collector.RunAsync(CancellationToken.None); - - // Assert - Assert.Equal( - new[] { "AnotherPackage", "ListedPackage", "UnlistedPackage" }, - _collector.AllSortedGraphs.Select(x => x.Key).OrderBy(x => x).ToArray()); - } - - private class TestSortingGraphCollector : SortingGraphCollector - { - private readonly ConcurrentBag>> _allSortedGraphs; - - public TestSortingGraphCollector(Uri index, Uri[] types, Func handlerFunc) : base( - index, - types, - new Mock().Object, - handlerFunc) - { - _allSortedGraphs = new ConcurrentBag>>(); - } - - public IEnumerable>> AllSortedGraphs => _allSortedGraphs; - - protected override Task ProcessGraphsAsync(KeyValuePair> sortedGraphs, CancellationToken cancellationToken) - { - _allSortedGraphs.Add(sortedGraphs); - return Task.FromResult(0); - } - } - } -} \ No newline at end of file diff --git a/tests/NgTests/AggregateStorageTests.cs b/tests/NgTests/AggregateStorageTests.cs deleted file mode 100644 index 91e3d344e..000000000 --- a/tests/NgTests/AggregateStorageTests.cs +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Ng; -using NgTests.Data; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog.Persistence; -using Xunit; - -namespace NgTests -{ - public class AggregateStorageTests - { - [Fact] - public async Task PropagatesSaveToAllStorage() - { - // Arrange - var storage1 = new MemoryStorage(new Uri("http://tempuri.org/firstone")); - var storage2 = new MemoryStorage(new Uri("http://tempuri.org/secondone")); - var storage3 = new MemoryStorage(new Uri("http://tempuri.org/thirdone")); - - var aggregateStorageFactory = Create(storage1, storage2, storage3); - var aggregateStorageRoot = aggregateStorageFactory.Create(); - var aggregateStorageSub = aggregateStorageFactory.Create("sub"); - - // Act - await aggregateStorageRoot.SaveAsync(new Uri("http://tempuri.org/firstone/test.txt"), new StringStorageContent("test1"), CancellationToken.None); - await aggregateStorageSub.SaveAsync(new Uri("http://tempuri.org/firstone/sub/test.txt"), new StringStorageContent("test2"), CancellationToken.None); - - // Assert - Assert.Equal(2, storage1.Content.Count); - Assert.Equal(2, storage2.Content.Count); - Assert.Equal(2, storage3.Content.Count); - - AssertUriAndContentExists(storage1, new Uri("http://tempuri.org/firstone/test.txt"), "test1"); - AssertUriAndContentExists(storage2, new Uri("http://tempuri.org/secondone/test.txt"), "test1"); - AssertUriAndContentExists(storage3, new Uri("http://tempuri.org/thirdone/test.txt"), "test1"); - - AssertUriAndContentExists(storage1, new Uri("http://tempuri.org/firstone/sub/test.txt"), "test2"); - AssertUriAndContentExists(storage2, new Uri("http://tempuri.org/secondone/sub/test.txt"), "test2"); - AssertUriAndContentExists(storage3, new Uri("http://tempuri.org/thirdone/sub/test.txt"), "test2"); - } - - [Fact] - public async Task PropagatesDeleteToAllStorage() - { - // Arrange - var storage1 = new MemoryStorage(new Uri("http://tempuri.org/firstone")); - var storage2 = new MemoryStorage(new Uri("http://tempuri.org/secondone")); - var storage3 = new MemoryStorage(new Uri("http://tempuri.org/thirdone")); - - var aggregateStorageFactory = Create(storage1, storage2, storage3); - var aggregateStorageRoot = aggregateStorageFactory.Create(); - var aggregateStorageSub = aggregateStorageFactory.Create("sub"); - - await aggregateStorageRoot.SaveAsync(new Uri("http://tempuri.org/firstone/test.txt"), new StringStorageContent("test1"), CancellationToken.None); - await aggregateStorageSub.SaveAsync(new Uri("http://tempuri.org/firstone/sub/test.txt"), new StringStorageContent("test2"), CancellationToken.None); - - // Act - await aggregateStorageRoot.DeleteAsync(new Uri("http://tempuri.org/firstone/test.txt"), CancellationToken.None); - - // Assert - Assert.Single(storage1.Content); - Assert.Single(storage2.Content); - Assert.Single(storage3.Content); - - AssertUriAndContentDoesNotExist(storage1, new Uri("http://tempuri.org/firstone/test.txt"), "test1"); - AssertUriAndContentDoesNotExist(storage2, new Uri("http://tempuri.org/secondone/test.txt"), "test1"); - AssertUriAndContentDoesNotExist(storage3, new Uri("http://tempuri.org/thirdone/test.txt"), "test1"); - - AssertUriAndContentExists(storage1, new Uri("http://tempuri.org/firstone/sub/test.txt"), "test2"); - AssertUriAndContentExists(storage2, new Uri("http://tempuri.org/secondone/sub/test.txt"), "test2"); - AssertUriAndContentExists(storage3, new Uri("http://tempuri.org/thirdone/sub/test.txt"), "test2"); - } - - [Fact] - public async Task AlwaysReadsFromPrimary() - { - // Arrange - var storage1 = new MemoryStorage(new Uri("http://tempuri.org/firstone")); - var storage2 = new MemoryStorage(new Uri("http://tempuri.org/secondone")); - var storage3 = new MemoryStorage(new Uri("http://tempuri.org/thirdone")); - - var aggregateStorageFactory = Create(storage1, storage2, storage3); - var aggregateStorage = aggregateStorageFactory.Create(); - - await aggregateStorage.SaveAsync(new Uri("http://tempuri.org/firstone/test.txt"), new StringStorageContent("test1"), CancellationToken.None); - - // Act - var content1 = await aggregateStorage.LoadAsync(new Uri("http://tempuri.org/firstone/test.txt"), CancellationToken.None); - var content2 = await aggregateStorage.LoadAsync(new Uri("http://tempuri.org/secondone/test.txt"), CancellationToken.None); - var content3 = await aggregateStorage.LoadAsync(new Uri("http://tempuri.org/thirdone/test.txt"), CancellationToken.None); - - // Assert - Assert.NotNull(content1); - Assert.Null(content2); - Assert.Null(content3); - } - - [Fact] - public async Task CorrectlyReplacesRegistrationBaseUrlsInSecondaryStorage() - { - // Arrange - var storageToReplay = Registrations.CreateTestRegistrations(); - var storage1 = new MemoryStorage(storageToReplay.BaseAddress); - var storage2 = new MemoryStorage(new Uri("http://tempuri.org/secondone")); - var storage3 = new MemoryStorage(new Uri("http://tempuri.org/thirdone")); - - var secondaryStorageBaseUrlRewriter = new SecondaryStorageBaseUrlRewriter(new List> - { - new KeyValuePair(storage1.BaseAddress.ToString(), storage2.BaseAddress.ToString() ), - new KeyValuePair(storage1.BaseAddress.ToString(), storage3.BaseAddress.ToString() ) - }); - - var aggregateStorageFactory = CreateWithInterceptor(secondaryStorageBaseUrlRewriter.Rewrite, storage1, storage2, storage3); - var aggregateStorage = aggregateStorageFactory.Create(); - - var storage1BaseAddress = storage1.BaseAddress.ToString(); - var storage2BaseAddress = storage2.BaseAddress.ToString(); - var storage3BaseAddress = storage3.BaseAddress.ToString(); - - // Act - foreach (var content in storageToReplay.Content) - { - await aggregateStorage.SaveAsync(content.Key, content.Value, CancellationToken.None); - } - - // Assert - Assert.Equal(storageToReplay.Content.Count, storage1.Content.Count); - Assert.Equal(storageToReplay.Content.Count, storage2.Content.Count); - Assert.Equal(storageToReplay.Content.Count, storage3.Content.Count); - - foreach (var content in storage1.Content) - { - AssertContentDoesNotContain(content.Value, storage1BaseAddress); - AssertContentDoesNotContain(content.Value, storage2BaseAddress); - AssertContentDoesNotContain(content.Value, storage3BaseAddress); - } - - foreach (var content in storage2.Content) - { - AssertContentDoesNotContain(content.Value, storage1BaseAddress); - AssertContentDoesNotContain(content.Value, storage2BaseAddress); - AssertContentDoesNotContain(content.Value, storage3BaseAddress); - } - - foreach (var content in storage3.Content) - { - AssertContentDoesNotContain(content.Value, storage1BaseAddress); - AssertContentDoesNotContain(content.Value, storage2BaseAddress); - AssertContentDoesNotContain(content.Value, storage3BaseAddress); - } - } - - protected AggregateStorageFactory Create(MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) - { - const AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor = null; - - return CreateWithInterceptor(interceptor, primaryStorage, secondaryStorage, storages); - } - - protected AggregateStorageFactory CreateWithInterceptor(AggregateStorage.WriteSecondaryStorageContentInterceptor interceptor, MemoryStorage primaryStorage, MemoryStorage secondaryStorage, params MemoryStorage[] storages) - { - var storageFactories = new List(); - storageFactories.Add(new TestStorageFactory(name => primaryStorage.WithName(name))); - storageFactories.Add(new TestStorageFactory(name => secondaryStorage.WithName(name))); - - foreach (var storage in storages) - { - storageFactories.Add(new TestStorageFactory(name => storage.WithName(name))); - } - - return new AggregateStorageFactory( - storageFactories.First(), - storageFactories.Skip(1).ToArray(), - interceptor, - verbose: false); - } - - protected void AssertUriAndContentExists(MemoryStorage storage, Uri uri, string expectedContent) - { - var value = storage.Content.FirstOrDefault(pair => pair.Key == uri); - - Assert.NotNull(value.Key); - Assert.Equal(expectedContent, value.Value.GetContentString()); - } - - protected void AssertUriAndContentDoesNotExist(MemoryStorage storage, Uri uri, string expectedContent) - { - var value = storage.Content.FirstOrDefault(pair => pair.Key == uri); - - Assert.Null(value.Key); - } - - protected void AssertContentDoesNotContain(StorageContent content, string value) - { - var contentValue = content.GetContentString(); - - Assert.DoesNotContain(contentValue, value, StringComparison.OrdinalIgnoreCase); - } - } -} \ No newline at end of file diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index 86dfafc0b..ae3e8440d 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -54,7 +54,6 @@ - diff --git a/tests/NgTests/StorageFactoryTests.cs b/tests/NgTests/StorageFactoryTests.cs index 8f9a41b68..180bbc811 100644 --- a/tests/NgTests/StorageFactoryTests.cs +++ b/tests/NgTests/StorageFactoryTests.cs @@ -1,10 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.ComponentModel; -using System.Linq; using Ng; using NuGet.Services.Metadata.Catalog.Persistence; using Xunit; @@ -13,60 +11,6 @@ namespace NgTests { public class StorageFactoryTests { - [Theory] - [Description("The azure compressed factory should compress the content.")] - [InlineData("http://localhost/reg", "testAccount", "DummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9AU4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==", "testContainer", "testStoragePath", "azure")] - public void AzureCompressedFactory(string storageBaseAddress, - string storageAccountName, - string storageKeyValue, - string storageContainer, - string storagePath, - string storageType) - { - Dictionary arguments = new Dictionary() - { - { Arguments.CompressedStorageBaseAddress, storageBaseAddress }, - { Arguments.CompressedStorageAccountName, storageAccountName }, - { Arguments.CompressedStorageKeyValue, storageKeyValue }, - { Arguments.CompressedStorageContainer, storageContainer }, - { Arguments.CompressedStoragePath, storagePath }, - { Arguments.StorageType, storageType}, - { Arguments.UseCompressedStorage, "true"} - }; - - StorageFactory factory = CommandHelpers.CreateCompressedStorageFactory(arguments, true); - AzureStorageFactory azureFactory = factory as AzureStorageFactory; - // Assert - Assert.True(azureFactory!=null, "The CreateCompressedStorageFactory should return an AzureStorageFactory."); - Assert.True(azureFactory.CompressContent, "The compressed storage factory should compress the content."); - } - - [Theory] - [Description("The azure compressed factory will be null if the UseCompressedStorage is false.")] - [InlineData("http://localhost/reg", "testAccount", "DummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9AU4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==", "testContainer", "testStoragePath", "azure")] - public void AzureCompressedFactoryNull(string storageBaseAddress, - string storageAccountName, - string storageKeyValue, - string storageContainer, - string storagePath, - string storageType) - { - Dictionary arguments = new Dictionary() - { - { Arguments.CompressedStorageBaseAddress, storageBaseAddress }, - { Arguments.CompressedStorageAccountName, storageAccountName }, - { Arguments.CompressedStorageKeyValue, storageKeyValue }, - { Arguments.CompressedStorageContainer, storageContainer }, - { Arguments.CompressedStoragePath, storagePath }, - { Arguments.StorageType, storageType }, - { Arguments.UseCompressedStorage, "false" } - }; - - StorageFactory factory = CommandHelpers.CreateCompressedStorageFactory(arguments, true); - // Assert - Assert.True(factory == null, "The CompressedStorageFactory should be null when the UseCompressedStorage is false."); - } - [Theory] [Description("The regular azure factory should not compress the content if.")] [InlineData("http://localhost/reg", "testAccount", "DummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9AU4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==", "testContainer", "testStoragePath", "azure")] @@ -93,171 +37,5 @@ public void AzureFactory(string storageBaseAddress, Assert.True(azureFactory != null, "The CreateCompressedStorageFactory should return an AzureStorageFactory type."); Assert.False(azureFactory.CompressContent, "The azure storage factory should not compress the content."); } - - [Theory] - [Description("The SemVer 2.0.0 Azure factory should compress the content.")] - [InlineData("http://localhost/reg", "testAccount", "DummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9AU4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==", "testContainer", "testStoragePath")] - public void AzureSemVer2Factory(string storageBaseAddress, - string storageAccountName, - string storageKeyValue, - string storageContainer, - string storagePath) - { - Dictionary arguments = new Dictionary() - { - { Arguments.SemVer2StorageBaseAddress, storageBaseAddress }, - { Arguments.SemVer2StorageAccountName, storageAccountName }, - { Arguments.SemVer2StorageKeyValue, storageKeyValue }, - { Arguments.SemVer2StorageContainer, storageContainer }, - { Arguments.SemVer2StoragePath, storagePath }, - { Arguments.StorageType, "azure" }, - { Arguments.UseSemVer2Storage, "true" } - }; - - var factory = CommandHelpers.CreateSemVer2StorageFactory(arguments, verbose: true); - - // Assert - var azureFactory = Assert.IsType(factory); - Assert.True(azureFactory.CompressContent, "The Azure storage factory should compress the content."); - } - - [Fact] - public void CreateRegistrationStorageFactories_WithAllThreeFactories() - { - // Arrange - var arguments = new Dictionary() - { - { Arguments.StorageType, "azure" }, - - { Arguments.StorageBaseAddress, "http://localhost/reg" }, - { Arguments.StorageAccountName, "testAccount" }, - { Arguments.StorageKeyValue, "ADummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.StorageContainer, "testContainer" }, - { Arguments.StoragePath, "testStoragePath" }, - - { Arguments.UseCompressedStorage, "true" }, - { Arguments.CompressedStorageBaseAddress, "http://localhost/reg-gz" }, - { Arguments.CompressedStorageAccountName, "testAccount-gz" }, - { Arguments.CompressedStorageKeyValue, "BDummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.CompressedStorageContainer, "testContainer-gz" }, - { Arguments.CompressedStoragePath, "testStoragePath-gz" }, - - { Arguments.UseSemVer2Storage, "true" }, - { Arguments.SemVer2StorageBaseAddress, "http://localhost/reg-gz-semver2" }, - { Arguments.SemVer2StorageAccountName, "testAccount-semver2" }, - { Arguments.SemVer2StorageKeyValue, "CDummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.SemVer2StorageContainer, "testContainer-semver2" }, - { Arguments.SemVer2StoragePath, "testStoragePath-semver2" }, - }; - - // Act - var factories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose: true); - - // Assert - var legacy = Assert.IsType(factories.LegacyStorageFactory); - Assert.True(legacy.Verbose, "verbose should be true on the aggregate storage factory"); - Assert.Equal(legacy.BaseAddress, new Uri("http://localhost/reg/testStoragePath/")); - - var originalFactory = Assert.IsType(legacy.PrimaryStorageFactory); - Assert.True(originalFactory.Verbose, "verbose should be true on the original storage factory"); - Assert.False(originalFactory.CompressContent, "compress should be false on the original storage factory"); - Assert.Equal(originalFactory.BaseAddress, new Uri("http://localhost/reg/testStoragePath/")); - - Assert.Single(legacy.SecondaryStorageFactories); - var compressFactory = Assert.IsType(legacy.SecondaryStorageFactories.First()); - Assert.True(compressFactory.Verbose, "verbose should be true on the compress storage factory"); - Assert.True(compressFactory.CompressContent, "compress should be true on the compress storage factory"); - Assert.Equal(compressFactory.BaseAddress, new Uri("http://localhost/reg-gz/testStoragePath-gz/")); - - var semVer2 = Assert.IsType(factories.SemVer2StorageFactory); - Assert.True(semVer2.Verbose, "verbose should be true on the SemVer 2.0.0 storage factory"); - Assert.True(semVer2.CompressContent, "compress should be true on the SemVer 2.0.0 storage factory"); - Assert.Equal(semVer2.BaseAddress, new Uri("http://localhost/reg-gz-semver2/testStoragePath-semver2/")); - } - - [Fact] - public void CreateRegistrationStorageFactories_WithNoCompressFactory() - { - // Arrange - var arguments = new Dictionary() - { - { Arguments.StorageType, "azure" }, - - { Arguments.StorageBaseAddress, "http://localhost/reg" }, - { Arguments.StorageAccountName, "testAccount" }, - { Arguments.StorageKeyValue, "ADummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.StorageContainer, "testContainer" }, - { Arguments.StoragePath, "testStoragePath" }, - - { Arguments.UseCompressedStorage, "false" }, - - { Arguments.UseSemVer2Storage, "true" }, - { Arguments.SemVer2StorageBaseAddress, "http://localhost/reg-gz-semver2" }, - { Arguments.SemVer2StorageAccountName, "testAccount-semver2" }, - { Arguments.SemVer2StorageKeyValue, "CDummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.SemVer2StorageContainer, "testContainer-semver2" }, - { Arguments.SemVer2StoragePath, "testStoragePath-semver2" }, - }; - - // Act - var factories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose: true); - - // Assert - var originalFactory = Assert.IsType(factories.LegacyStorageFactory); - Assert.True(originalFactory.Verbose, "verbose should be true on the original storage factory"); - Assert.False(originalFactory.CompressContent, "compress should be false on the original storage factory"); - Assert.Equal(originalFactory.BaseAddress, new Uri("http://localhost/reg/testStoragePath/")); - - var semVer2 = Assert.IsType(factories.SemVer2StorageFactory); - Assert.True(semVer2.Verbose, "verbose should be true on the SemVer 2.0.0 storage factory"); - Assert.True(semVer2.CompressContent, "compress should be true on the SemVer 2.0.0 storage factory"); - Assert.Equal(semVer2.BaseAddress, new Uri("http://localhost/reg-gz-semver2/testStoragePath-semver2/")); - } - - [Fact] - public void CreateRegistrationStorageFactories_WithNoSemVer2Factory() - { - // Arrange - var arguments = new Dictionary() - { - { Arguments.StorageType, "azure" }, - - { Arguments.StorageBaseAddress, "http://localhost/reg" }, - { Arguments.StorageAccountName, "testAccount" }, - { Arguments.StorageKeyValue, "ADummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.StorageContainer, "testContainer" }, - { Arguments.StoragePath, "testStoragePath" }, - - { Arguments.UseCompressedStorage, "true" }, - { Arguments.CompressedStorageBaseAddress, "http://localhost/reg-gz" }, - { Arguments.CompressedStorageAccountName, "testAccount-gz" }, - { Arguments.CompressedStorageKeyValue, "BDummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9A4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==" }, - { Arguments.CompressedStorageContainer, "testContainer-gz" }, - { Arguments.CompressedStoragePath, "testStoragePath-gz" }, - - { Arguments.UseSemVer2Storage, "false" }, - }; - - // Act - var factories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose: true); - - // Assert - var legacy = Assert.IsType(factories.LegacyStorageFactory); - Assert.True(legacy.Verbose, "verbose should be true on the aggregate storage factory"); - Assert.Equal(legacy.BaseAddress, new Uri("http://localhost/reg/testStoragePath/")); - - var originalFactory = Assert.IsType(legacy.PrimaryStorageFactory); - Assert.True(originalFactory.Verbose, "verbose should be true on the original storage factory"); - Assert.False(originalFactory.CompressContent, "compress should be false on the original storage factory"); - Assert.Equal(originalFactory.BaseAddress, new Uri("http://localhost/reg/testStoragePath/")); - - Assert.Single(legacy.SecondaryStorageFactories); - var compressFactory = Assert.IsType(legacy.SecondaryStorageFactories.First()); - Assert.True(compressFactory.Verbose, "verbose should be true on the compress storage factory"); - Assert.True(compressFactory.CompressContent, "compress should be true on the compress storage factory"); - Assert.Equal(compressFactory.BaseAddress, new Uri("http://localhost/reg-gz/testStoragePath-gz/")); - - Assert.Null(factories.SemVer2StorageFactory); - } } } diff --git a/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs b/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs index fe2b15cd5..c620c99dc 100644 --- a/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs +++ b/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs @@ -2,8 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; -using System.IO; using System.Net; using System.Net.Http; using System.Text; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs index 967a542eb..a34d3aede 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.Extensions.Options; using Moq; using NuGet.Services.AzureSearch.AuxiliaryFiles; using NuGet.Services.AzureSearch.Support; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs index 35182cb29..850c36e72 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs @@ -5,19 +5,15 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Castle.Core.Resource; using Microsoft.Azure.Search; using Microsoft.Azure.Search.Models; -using Microsoft.Data.OData; using Moq; using NuGet.Packaging.Core; using NuGet.Protocol.Catalog; using NuGet.Services.Metadata.Catalog; using NuGet.Versioning; using NuGetGallery; -using Serilog.Core; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs index 3702487ae..f521909b2 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq;