From 7077dcd21376f78bcd0b36ad66f1a642eff56397 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 19:35:01 +0200 Subject: [PATCH 1/7] try to fix https://github.com/fsprojects/Paket/issues/2640 by always calling getVersions. --- .../Dependencies/PackageResolver.fs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index 63c5add36a..48a97028ca 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -499,22 +499,25 @@ let private getCompatibleVersions allRequirementsOfCurrentPackage |> Set.forall (fun r -> (mapF r).VersionRequirement.IsInRange ver) - let getSingleVersion v = - match currentRequirement.Parent with - | PackageRequirementSource.Package(_,_,parentSource) -> - let sources = parentSource :: currentRequirement.Sources |> List.distinct - Seq.singleton (v,sources) - | _ -> - let sources : PackageSource list = currentRequirement.Sources |> List.sortBy (fun x -> not x.IsLocalFeed, String.containsIgnoreCase "nuget.org" x.Url |> not) - Seq.singleton (v,sources) + //let getSingleVersion v = + // match currentRequirement.Parent with + // | PackageRequirementSource.Package(_,_,parentSource) -> + // let sources = parentSource :: currentRequirement.Sources |> List.distinct + // Seq.singleton (v,sources) + // | _ -> + // let sources : PackageSource list = currentRequirement.Sources |> List.sortBy (fun x -> not x.IsLocalFeed, String.containsIgnoreCase "nuget.org" x.Url |> not) + // Seq.singleton (v,sources) let availableVersions = + let resolverStrategy = getResolverStrategy globalStrategyForDirectDependencies globalStrategyForTransitives allRequirementsOfCurrentPackage currentRequirement + let allVersions = getVersionsF currentRequirement.Sources resolverStrategy groupName currentRequirement.Name match currentRequirement.VersionRequirement.Range with - | OverrideAll v -> getSingleVersion v - | Specific v -> getSingleVersion v - | _ -> - let resolverStrategy = getResolverStrategy globalStrategyForDirectDependencies globalStrategyForTransitives allRequirementsOfCurrentPackage currentRequirement - getVersionsF currentRequirement.Sources resolverStrategy groupName currentRequirement.Name + | Specific v + | OverrideAll v -> + allVersions + |> Seq.filter (fun (v1, _) -> v1 = v) + //getSingleVersion v + | _ -> allVersions let compatibleVersions = Seq.filter (isInRange id) availableVersions |> Seq.cache From 3172ef327ba80ea33160fce23130c1ab7411f95c Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 19:46:32 +0200 Subject: [PATCH 2/7] fix test --- tests/Paket.Tests/Resolver/PrereleaseSpecs.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Paket.Tests/Resolver/PrereleaseSpecs.fs b/tests/Paket.Tests/Resolver/PrereleaseSpecs.fs index 4929281b79..23bde2821a 100644 --- a/tests/Paket.Tests/Resolver/PrereleaseSpecs.fs +++ b/tests/Paket.Tests/Resolver/PrereleaseSpecs.fs @@ -80,7 +80,7 @@ let ``should resolve overwritten config4``() = let cfg = DependenciesFile.FromSource(config4) let resolved = ResolveWithGraph(cfg,noSha1, VersionsFromGraphAsSeq graph, PackageDetailsFromGraph graph).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() getVersion resolved.[PackageName "packageA"] |> shouldEqual "1.0.11250" - getVersion resolved.[PackageName "packageB"] |> shouldEqual "2.0" + getVersion resolved.[PackageName "packageB"] |> shouldEqual "2.0.0" [] From 22e4c2ab2ada85d551d9b9eb8224b5822add96f5 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 21:11:07 +0200 Subject: [PATCH 3/7] refactor getVersion and getDetails and add types to collect parameters --- src/Paket.Core/Dependencies/NuGet.fs | 18 ++-- .../Dependencies/PackageResolver.fs | 86 ++++++++++++------- src/Paket.Core/Installation/InstallProcess.fs | 2 +- src/Paket.Core/Installation/UpdateProcess.fs | 18 ++-- .../PackageAnalysis/FindOutdated.fs | 6 +- src/Paket.Core/PublicAPI.fs | 3 +- .../DependencyGroupsAndRestrictions.fs | 2 +- .../Resolver/GlobalOptimisticStrategySpecs.fs | 2 +- .../GlobalPessimisticStrategySpecs.fs | 2 +- .../Resolver/ResolverErrorSituationTests.fs | 10 +-- tests/Paket.Tests/Resolver/StrategySpecs.fs | 2 +- tests/Paket.Tests/TestHelpers.fs | 20 ++--- 12 files changed, 103 insertions(+), 68 deletions(-) diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index c8d2a9d96e..adbc7f2d05 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -15,6 +15,7 @@ open System.Runtime.ExceptionServices open System.Text open FSharp.Polyfill open Paket.NuGetCache +open Paket.PackageResolver type NuGetContent = | NuGetDirectory of name:string * contents:NuGetContent list @@ -233,7 +234,10 @@ let tryNuGetV3 (auth, nugetV3Url, package:PackageName) = NuGetV3.findVersionsForPackage(nugetV3Url, auth, package) -let rec private getPackageDetails alternativeProjectRoot root force (sources:PackageSource list) packageName (version:SemVerInfo) : Async = +let rec private getPackageDetails alternativeProjectRoot root force (parameters:GetPackageDetailsParameters) : Async = + let sources = parameters.Package.Sources + let packageName = parameters.Package.PackageName + let version = parameters.Version async { let inCache = sources @@ -357,17 +361,17 @@ let rec private getPackageDetails alternativeProjectRoot root force (sources:Pac LicenseUrl = nugetObject.LicenseUrl DirectDependencies = NuGetPackageCache.getDependencies nugetObject |> Set.ofList } } -let rec GetPackageDetails alternativeProjectRoot root force (sources:PackageSource list) groupName packageName (version:SemVerInfo) : Async = +let rec GetPackageDetails alternativeProjectRoot root force (parameters:GetPackageDetailsParameters): Async = async { try - return! getPackageDetails alternativeProjectRoot root force sources packageName version + return! getPackageDetails alternativeProjectRoot root force parameters with | exn -> if verbose then traceWarnfn "GetPackageDetails failed: %O" exn else traceWarnfn "Something failed in GetPackageDetails, trying again with force: %s" exn.Message - return! getPackageDetails alternativeProjectRoot root true sources packageName version + return! getPackageDetails alternativeProjectRoot root true parameters } let protocolCache = System.Collections.Concurrent.ConcurrentDictionary<_,_>() @@ -413,7 +417,9 @@ let getVersionsCached key f (source, auth, nugetURL, package) = /// Allows to retrieve all version no. for a package from the given sources. -let GetVersions force alternativeProjectRoot root (sources, packageName:PackageName) = async { +let GetVersions force alternativeProjectRoot root (parameters:GetPackageVersionsParameters) = async { + let packageName = parameters.Package.PackageName + let sources = parameters.Package.Sources let trial force = async { let getVersionsFailedCacheFileName (source:PackageSource) = let h = source.Url |> NuGetCache.normalizeUrl |> hash |> abs @@ -671,7 +677,7 @@ let DownloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverride:bool let group = if groupName = Constants.MainDependencyGroup then "" else sprintf " (%O)" groupName tracefn "Downloading %O %O%s" packageName version group - let! nugetPackage = GetPackageDetails alternativeProjectRoot root force [source] groupName packageName version + let! nugetPackage = GetPackageDetails alternativeProjectRoot root force (GetPackageDetailsParameters.ofParams [source] groupName packageName version) let encodeURL (url:string) = url.Replace("+","%2B") let downloadUri = diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index 48a97028ca..cbff0e20c5 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -12,6 +12,8 @@ open Paket.PackageSources open System.Threading.Tasks open System.Threading open FSharp.Polyfill +open System.Text.RegularExpressions + type DependencySet = Set @@ -26,6 +28,31 @@ type PackageDetails = { DirectDependencies : DependencySet } +type SourcePackageInfo = + { Sources : PackageSource list + GroupName : GroupName + PackageName : PackageName } + static member ofParams sources groupName packageName = + { Sources = sources; GroupName = groupName; PackageName = packageName } + +type GetPackageDetailsParameters = + { Package : SourcePackageInfo + Version : SemVerInfo } + static member ofParams sources groupName packageName version = + SourcePackageInfo.ofParams sources groupName packageName + |> fun p -> { Package = p; Version = version } + +type GetPackageVersionsParameters = + { Package : SourcePackageInfo } + static member ofParams sources groupName packageName = + SourcePackageInfo.ofParams sources groupName packageName + |> fun p -> { Package = p } + +type PackageDetailsFunc = GetPackageDetailsParameters -> Async +type PackageDetailsSyncFunc = GetPackageDetailsParameters -> PackageDetails +type PackageVersionsFunc = GetPackageVersionsParameters -> Async> +type PackageVersionsSyncFunc = GetPackageVersionsParameters -> seq + /// Represents data about resolved packages [] type ResolvedPackage = { @@ -404,7 +431,7 @@ let private updateRestrictions (pkgConfig:PackageConfig) (package:ResolvedPackag } -let private explorePackageConfig getPackageDetailsBlock (pkgConfig:PackageConfig) = +let private explorePackageConfig (getPackageDetailsBlock:PackageDetailsSyncFunc) (pkgConfig:PackageConfig) = let dependency, version = pkgConfig.Dependency, pkgConfig.Version let packageSources = pkgConfig.Sources @@ -421,7 +448,7 @@ let private explorePackageConfig getPackageDetailsBlock (pkgConfig:PackageConfig filterRestrictions dependency.Settings.FrameworkRestrictions pkgConfig.GlobalRestrictions try let packageDetails : PackageDetails = - getPackageDetailsBlock packageSources pkgConfig.GroupName dependency.Name version + getPackageDetailsBlock (GetPackageDetailsParameters.ofParams packageSources pkgConfig.GroupName dependency.Name version) let filteredDependencies = DependencySetFilter.filterByRestrictions newRestrictions packageDetails.DirectDependencies let settings = @@ -455,7 +482,7 @@ type StackPack = { } -let private getExploredPackage (pkgConfig:PackageConfig) getPackageDetailsBlock (stackpack:StackPack) = +let private getExploredPackage (pkgConfig:PackageConfig) (getPackageDetailsBlock:PackageDetailsSyncFunc) (stackpack:StackPack) = let key = (pkgConfig.Dependency.Name, pkgConfig.Version) match stackpack.ExploredPackages.TryGetValue key with @@ -481,7 +508,7 @@ let private getCompatibleVersions (currentStep:ResolverStep) groupName (currentRequirement:PackageRequirement) - (getVersionsF: PackageSource list -> ResolverStrategy -> GroupName -> PackageName -> seq) + (getVersionsF: ResolverStrategy -> PackageVersionsSyncFunc) globalOverride globalStrategyForDirectDependencies globalStrategyForTransitives = @@ -510,7 +537,7 @@ let private getCompatibleVersions let availableVersions = let resolverStrategy = getResolverStrategy globalStrategyForDirectDependencies globalStrategyForTransitives allRequirementsOfCurrentPackage currentRequirement - let allVersions = getVersionsF currentRequirement.Sources resolverStrategy groupName currentRequirement.Name + let allVersions = getVersionsF resolverStrategy (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName currentRequirement.Name) match currentRequirement.VersionRequirement.Range with | Specific v | OverrideAll v -> @@ -837,8 +864,9 @@ let selectVersionsToPreload (verReq:VersionRequirement) versions = let RequestTimeout = 180000 let WorkerCount = 6 +type PreferredVersionsFunc = ResolverStrategy -> GetPackageVersionsParameters -> list /// Resolves all direct and transitive dependencies -let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, groupName:GroupName, globalStrategyForDirectDependencies, globalStrategyForTransitives, globalFrameworkRestrictions, (rootDependencies:PackageRequirement Set), updateMode : UpdateMode) = +let Resolve (getVersionsRaw : PackageVersionsFunc, getPreferredVersionsRaw : PreferredVersionsFunc, getPackageDetailsRaw : PackageDetailsFunc, groupName:GroupName, globalStrategyForDirectDependencies, globalStrategyForTransitives, globalFrameworkRestrictions, (rootDependencies:PackageRequirement Set), updateMode : UpdateMode) = tracefn "Resolving packages for group %O:" groupName let cliToolSettings = @@ -911,43 +939,43 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou reraise() let startedGetPackageDetailsRequests = System.Collections.Concurrent.ConcurrentDictionary<_,ResolverTaskMemory<_>>() - let startRequestGetPackageDetails sources groupName packageName (semVer:SemVerInfo) = - let key = (sources, packageName, semVer) + let startRequestGetPackageDetails (details:GetPackageDetailsParameters) = + let key = (details.Package.Sources, details.Package.PackageName, details.Version) startedGetPackageDetailsRequests.GetOrAdd (key, fun _ -> workerQueue |> ResolverRequestQueue.addWork WorkPriority.BackgroundWork (fun ct -> - (getPackageDetailsRaw sources groupName packageName semVer : Async) + (getPackageDetailsRaw details : Async) |> fun a -> Async.StartAsTaskProperCancel(a, cancellationToken = ct)) |> ResolverTaskMemory.ofWork) - let getPackageDetailsBlock sources groupName packageName semVer = - let workHandle = startRequestGetPackageDetails sources groupName packageName semVer + let getPackageDetailsBlock (details:GetPackageDetailsParameters) = + let workHandle = startRequestGetPackageDetails details try - getAndReport sources Profile.BlockReason.PackageDetails workHandle + getAndReport details.Package.Sources Profile.BlockReason.PackageDetails workHandle with e -> - raise <| Exception (sprintf "Unable to retrieve package details for '%O'-%s" packageName semVer.AsString, e) + raise <| Exception (sprintf "Unable to retrieve package details for '%O'-%s" details.Package.PackageName details.Version.AsString, e) let startedGetVersionsRequests = System.Collections.Concurrent.ConcurrentDictionary<_,ResolverTaskMemory<_>>() - let startRequestGetVersions sources groupName packageName = - let key = (sources, packageName) + let startRequestGetVersions (versions:GetPackageVersionsParameters) = + let key = (versions.Package.Sources, versions.Package.PackageName) startedGetVersionsRequests.GetOrAdd (key, fun _ -> workerQueue |> ResolverRequestQueue.addWork WorkPriority.BackgroundWork (fun ct -> - getVersionsRaw sources groupName packageName + getVersionsRaw versions |> fun a -> Async.StartAsTaskProperCancel(a, cancellationToken = ct)) |> ResolverTaskMemory.ofWork) - let getVersionsBlock sources resolverStrategy groupName packageName = - let workHandle = startRequestGetVersions sources groupName packageName + let getVersionsBlock resolverStrategy versionParams = + let workHandle = startRequestGetVersions versionParams let versions = - try getAndReport sources Profile.BlockReason.GetVersion workHandle |> Seq.toList + try getAndReport versionParams.Package.Sources Profile.BlockReason.GetVersion workHandle |> Seq.toList with e -> - raise <| Exception (sprintf "Unable to retrieve package versions for '%O'" packageName, e) + raise <| Exception (sprintf "Unable to retrieve package versions for '%O'" versionParams.Package.PackageName, e) let sorted = match resolverStrategy with | ResolverStrategy.Max -> List.sortDescending versions | ResolverStrategy.Min -> List.sort versions - let pref = getPreferredVersionsRaw sources resolverStrategy groupName packageName + let pref = getPreferredVersionsRaw resolverStrategy versionParams pref @ sorted |> List.toSeq let packageFilter = @@ -1056,8 +1084,8 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou getConflicts currentStep currentRequirement stackpack.KnownConflicts let currentConflict = - let getVersionsF = - getVersionsBlock currentRequirement.Sources ResolverStrategy.Max groupName + let getVersionsF packName = + getVersionsBlock ResolverStrategy.Max (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName packName) if Seq.isEmpty conflicts then { currentConflict with Status = ResolutionRaw.ConflictRaw { @@ -1165,12 +1193,12 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou // Start pre-loading infos about dependencies. for (pack,verReq,restr) in exploredPackage.Dependencies do async { - let requestVersions = startRequestGetVersions currentRequirement.Sources groupName pack + let requestVersions = startRequestGetVersions (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName pack) requestVersions.Work.TryReprioritize true WorkPriority.LikelyRequired let! versions = (requestVersions).Work.Task |> Async.AwaitTask // Preload the first version in range of this requirement for (verToPreload, prio) in selectVersionsToPreload verReq (versions |> Seq.map fst) do - let w = startRequestGetPackageDetails currentRequirement.Sources groupName pack verToPreload + let w = startRequestGetPackageDetails (GetPackageDetailsParameters.ofParams currentRequirement.Sources groupName pack verToPreload) w.Work.TryReprioritize true prio return () } |> Async.Start @@ -1208,8 +1236,8 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou Environment.NewLine currentRequirement Environment.NewLine nextStep.OpenRequirements step (Step((currentConflict,nextStep,currentRequirement), (currentConflict,currentStep,currentRequirement,compatibleVersions,flags)::priorConflictSteps)) stackpack currentConflict.VersionsToExplore flags else - let getVersionsF = - getVersionsBlock currentRequirement.Sources ResolverStrategy.Max groupName + let getVersionsF packName = + getVersionsBlock ResolverStrategy.Max (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName packName) let conflictingPackage,(_,vr,_) = conflictingResolvedPackages |> Seq.head let currentConflict = { currentConflict with @@ -1234,13 +1262,13 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou } for openReq in startingStep.OpenRequirements do - startRequestGetVersions openReq.Sources groupName openReq.Name + startRequestGetVersions (GetPackageVersionsParameters.ofParams openReq.Sources groupName openReq.Name) |> ignore let currentRequirement = getCurrentRequirement packageFilter startingStep.OpenRequirements (Dictionary()) let status = - let getVersionsF = getVersionsBlock currentRequirement.Sources ResolverStrategy.Max groupName + let getVersionsF packName = getVersionsBlock ResolverStrategy.Max (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName packName) ResolutionRaw.ConflictRaw { ResolveStep = startingStep; RequirementSet = Set.empty; Requirement = currentRequirement; GetPackageVersions = getVersionsF } diff --git a/src/Paket.Core/Installation/InstallProcess.fs b/src/Paket.Core/Installation/InstallProcess.fs index 76c216f1ac..6f8aa57919 100644 --- a/src/Paket.Core/Installation/InstallProcess.fs +++ b/src/Paket.Core/Installation/InstallProcess.fs @@ -174,7 +174,7 @@ let brokenDeps = HashSet<_>() let private applyBindingRedirects isFirstGroup createNewBindingFiles redirects cleanBindingRedirects root groupName findDependencies allKnownLibNames (projectCache: Dictionary) - extractedPackages = + (extractedPackages:seq) = let dependencyGraph = Dictionary<_,Set<_>>() let referenceFiles = Dictionary<_,ReferencesFile option>() diff --git a/src/Paket.Core/Installation/UpdateProcess.fs b/src/Paket.Core/Installation/UpdateProcess.fs index 84ae42b739..f70072a759 100644 --- a/src/Paket.Core/Installation/UpdateProcess.fs +++ b/src/Paket.Core/Installation/UpdateProcess.fs @@ -105,19 +105,19 @@ let selectiveUpdate force getSha1 getVersionsF getPackageDetailsF getRuntimeGrap v,s :: (List.map PackageSources.PackageSource.FromCache caches)) - let getPreferredVersionsF sources resolverStrategy groupName packageName = - match preferredVersions |> Map.tryFind (groupName, packageName), resolverStrategy with + let getPreferredVersionsF resolverStrategy (parameters:GetPackageVersionsParameters) = + match preferredVersions |> Map.tryFind (parameters.Package.GroupName, parameters.Package.PackageName), resolverStrategy with | Some x, ResolverStrategy.Min -> [x] | Some x, _ -> - if not (changes |> Set.contains (groupName, packageName)) then + if not (changes |> Set.contains (parameters.Package.GroupName, parameters.Package.PackageName)) then [x] else [] | _ -> [] - let getPackageDetailsF sources groupName packageName version = async { - let! (exploredPackage:PackageDetails) = getPackageDetailsF sources groupName packageName version - match preferredVersions |> Map.tryFind (groupName,packageName) with - | Some (preferedVersion,_) when version = preferedVersion -> return { exploredPackage with Unlisted = false } + let getPackageDetailsF (parameters:GetPackageDetailsParameters) = async { + let! (exploredPackage:PackageDetails) = getPackageDetailsF parameters + match preferredVersions |> Map.tryFind (parameters.Package.GroupName, parameters.Package.PackageName) with + | Some (preferedVersion,_) when parameters.Version = preferedVersion -> return { exploredPackage with Unlisted = false } | _ -> return exploredPackage } getPreferredVersionsF,getPackageDetailsF,groups @@ -179,8 +179,8 @@ let SelectiveUpdate(dependenciesFile : DependenciesFile, alternativeProjectRoot, let getSha1 origin owner repo branch auth = RemoteDownload.getSHA1OfBranch origin owner repo branch auth |> Async.RunSynchronously let root = Path.GetDirectoryName dependenciesFile.FileName - let inline getVersionsF sources groupName packageName = async { - let! result = NuGet.GetVersions force alternativeProjectRoot root (sources, packageName) + let inline getVersionsF (parameters:GetPackageVersionsParameters) = async { + let! result = NuGet.GetVersions force alternativeProjectRoot root parameters return result |> List.toSeq } let dependenciesFile = detectProjectFrameworksForDependenciesFile dependenciesFile diff --git a/src/Paket.Core/PackageAnalysis/FindOutdated.fs b/src/Paket.Core/PackageAnalysis/FindOutdated.fs index 87aa12b821..61ac16f8f3 100644 --- a/src/Paket.Core/PackageAnalysis/FindOutdated.fs +++ b/src/Paket.Core/PackageAnalysis/FindOutdated.fs @@ -36,10 +36,10 @@ let FindOutdated strict force includingPrereleases groupNameFilter environment = let root = Path.GetDirectoryName dependenciesFile.FileName let alternativeProjectRoot = None - let getVersionsF sources groupName packageName = async { - let! versions = NuGet.GetVersions force alternativeProjectRoot root (sources, packageName) + let getVersionsF parameters = async { + let! versions = NuGet.GetVersions force alternativeProjectRoot root parameters return versions |> List.toSeq } - let getPreferredVersionsF sources resolverStrategy groupName packageName = [] + let getPreferredVersionsF resolverStrategy parameters = [] let dependenciesFile = UpdateProcess.detectProjectFrameworksForDependenciesFile dependenciesFile let checkedDepsGroups = match groupNameFilter with diff --git a/src/Paket.Core/PublicAPI.fs b/src/Paket.Core/PublicAPI.fs index 4906138d7d..02f2785dba 100644 --- a/src/Paket.Core/PublicAPI.fs +++ b/src/Paket.Core/PublicAPI.fs @@ -8,6 +8,7 @@ open System open System.Xml open System.IO open Chessie.ErrorHandling +open PackageResolver /// Paket API which is optimized for F# Interactive use. type Dependencies(dependenciesFileName: string) = @@ -613,7 +614,7 @@ type Dependencies(dependenciesFileName: string) = |> List.distinct let versions = - NuGet.GetVersions true alternativeProjectRoot root (sources, PackageName name) + NuGet.GetVersions true alternativeProjectRoot root (GetPackageVersionsParameters.ofParams sources (GroupName "") (PackageName name)) |> Async.RunSynchronously |> List.map (fun (v,_) -> v.ToString()) |> List.toArray diff --git a/tests/Paket.Tests/Resolver/DependencyGroupsAndRestrictions.fs b/tests/Paket.Tests/Resolver/DependencyGroupsAndRestrictions.fs index 767a167087..ed92ad9702 100644 --- a/tests/Paket.Tests/Resolver/DependencyGroupsAndRestrictions.fs +++ b/tests/Paket.Tests/Resolver/DependencyGroupsAndRestrictions.fs @@ -10,7 +10,7 @@ open Paket.Requirements let resolve graph updateMode (cfg : DependenciesFile) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() + cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() let graph1 = GraphOfNuspecs [ diff --git a/tests/Paket.Tests/Resolver/GlobalOptimisticStrategySpecs.fs b/tests/Paket.Tests/Resolver/GlobalOptimisticStrategySpecs.fs index dd83cc0375..ae011e204f 100644 --- a/tests/Paket.Tests/Resolver/GlobalOptimisticStrategySpecs.fs +++ b/tests/Paket.Tests/Resolver/GlobalOptimisticStrategySpecs.fs @@ -9,7 +9,7 @@ open Paket.PackageResolver let resolve graph updateMode (cfg : DependenciesFile) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() + cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() let graph = OfSimpleGraph [ diff --git a/tests/Paket.Tests/Resolver/GlobalPessimisticStrategySpecs.fs b/tests/Paket.Tests/Resolver/GlobalPessimisticStrategySpecs.fs index bf0b047c23..5c9fce7182 100644 --- a/tests/Paket.Tests/Resolver/GlobalPessimisticStrategySpecs.fs +++ b/tests/Paket.Tests/Resolver/GlobalPessimisticStrategySpecs.fs @@ -9,7 +9,7 @@ open Paket.PackageResolver let resolve graph updateMode (cfg : DependenciesFile) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() + cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() let graph = OfSimpleGraph [ diff --git a/tests/Paket.Tests/Resolver/ResolverErrorSituationTests.fs b/tests/Paket.Tests/Resolver/ResolverErrorSituationTests.fs index 4f9d4c9ad7..324a7006bb 100644 --- a/tests/Paket.Tests/Resolver/ResolverErrorSituationTests.fs +++ b/tests/Paket.Tests/Resolver/ResolverErrorSituationTests.fs @@ -24,7 +24,7 @@ let rec findExnWhichContains msg (exn:exn) = let resolve graph updateMode (cfg : DependenciesFile) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() + cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() let graph1 = GraphOfNuspecs [ @@ -84,9 +84,9 @@ nuget Chessie""" try let groupResults = cfg.Resolve( - true,noSha1,VersionsFromGraphAsSeq graph1,(fun _ _ _ _ -> []), + true,noSha1,VersionsFromGraphAsSeq graph1,(fun _ _ -> []), // Will never finish... - (fun _ _ _ _ -> (new TaskCompletionSource<_>()).Task |> Async.AwaitTask), + (fun _ -> (new TaskCompletionSource<_>()).Task |> Async.AwaitTask), (fun _ _ -> None),groups, UpdateMode.UpdateAll) let resolved = groupResults.[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() Assert.Fail "Expected exception" @@ -215,9 +215,9 @@ nuget Chessie""" try let groupResults = cfg.Resolve( - true,noSha1,VersionsFromGraphAsSeq graph1,(fun _ _ _ _ -> []), + true,noSha1,VersionsFromGraphAsSeq graph1,(fun _ _ -> []), // Will throw a proper exception when canceled - (fun _ _ _ _ -> + (fun _ -> async { let tcs = new TaskCompletionSource<_>() //let! tok = Async.CancellationToken diff --git a/tests/Paket.Tests/Resolver/StrategySpecs.fs b/tests/Paket.Tests/Resolver/StrategySpecs.fs index 8be2e207d8..01e51a7cf2 100644 --- a/tests/Paket.Tests/Resolver/StrategySpecs.fs +++ b/tests/Paket.Tests/Resolver/StrategySpecs.fs @@ -9,7 +9,7 @@ open Paket.PackageResolver let resolve graph updateMode (cfg : DependenciesFile) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() + cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail() let graph = OfSimpleGraph [ diff --git a/tests/Paket.Tests/TestHelpers.fs b/tests/Paket.Tests/TestHelpers.fs index 581d3c1f34..cde5c68c2f 100644 --- a/tests/Paket.Tests/TestHelpers.fs +++ b/tests/Paket.Tests/TestHelpers.fs @@ -48,27 +48,27 @@ let OfGraphWithRuntimeDeps (g:seq Seq.toList -let PackageDetailsFromGraph (graph : DependencyGraph) sources groupName (package:PackageName) (version:SemVerInfo) = +let PackageDetailsFromGraph (graph : DependencyGraph) (parameters:GetPackageDetailsParameters) = let name,dependencies = graph - |> Seq.filter (fun (p, v, _, _) -> (PackageName p) = package && SemVer.Parse v = version) + |> Seq.filter (fun (p, v, _, _) -> (PackageName p) = parameters.Package.PackageName && SemVer.Parse v = parameters.Version) |> Seq.map (fun (n, _, d, _) -> PackageName n,d |> List.map (fun (x,y,z) -> PackageName x,y,z)) |> Seq.head { Name = name - Source = Seq.head sources + Source = Seq.head parameters.Package.Sources DownloadLink = "" LicenseUrl = "" Unlisted = false DirectDependencies = Set.ofList dependencies } |> async.Return -let VersionsFromGraph (graph : DependencyGraph) sources groupName packageName = +let VersionsFromGraph (graph : DependencyGraph) (parameters:GetPackageVersionsParameters) = let versions = graph - |> Seq.filter (fun (p, _, _, _) -> (PackageName p) = packageName) + |> Seq.filter (fun (p, _, _, _) -> (PackageName p) = parameters.Package.PackageName) |> Seq.map (fun (_, v, _, _) -> SemVer.Parse v) - |> Seq.map (fun v -> v,sources) + |> Seq.map (fun v -> v,parameters.Package.Sources) versions |> async.Return @@ -83,8 +83,8 @@ let GetRuntimeGraphFromGraph (graph : DependencyGraph) groupName (package:Resolv |> Some -let VersionsFromGraphAsSeq (graph : DependencyGraph) sources groupName packageName = - VersionsFromGraph graph sources groupName packageName +let VersionsFromGraphAsSeq (graph : DependencyGraph) parameters = + VersionsFromGraph graph parameters let safeResolve graph (dependencies : (string * VersionRange) list) = let sources = [ PackageSource.NuGetV2Source "" ] @@ -103,13 +103,13 @@ let safeResolve graph (dependencies : (string * VersionRange) list) = ResolverStrategyForTransitives = Some ResolverStrategy.Max }) |> Set.ofList - PackageResolver.Resolve(VersionsFromGraphAsSeq graph, (fun _ _ _ _ -> []), PackageDetailsFromGraph graph, Constants.MainDependencyGroup, None, None, ExplicitRestriction FrameworkRestriction.NoRestriction, packages, UpdateMode.UpdateAll) + PackageResolver.Resolve(VersionsFromGraphAsSeq graph, (fun _ _ -> []), PackageDetailsFromGraph graph, Constants.MainDependencyGroup, None, None, ExplicitRestriction FrameworkRestriction.NoRestriction, packages, UpdateMode.UpdateAll) let resolve graph dependencies = (safeResolve graph dependencies).GetModelOrFail() let ResolveWithGraphR(dependenciesFile:DependenciesFile,getSha1,getVersionsF, getPackageDetailsF, getRuntimeGraph) = let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq - dependenciesFile.Resolve(true,getSha1,getVersionsF,(fun _ _ _ _ -> []),getPackageDetailsF,getRuntimeGraph,groups,UpdateMode.UpdateAll) + dependenciesFile.Resolve(true,getSha1,getVersionsF,(fun _ _ -> []),getPackageDetailsF,getRuntimeGraph,groups,UpdateMode.UpdateAll) let ResolveWithGraph(dependenciesFile:DependenciesFile,getSha1,getVersionsF, getPackageDetailsF) = ResolveWithGraphR(dependenciesFile,getSha1,getVersionsF, getPackageDetailsF, (fun _ _ -> None)) From 17616e47812e7971af8ebd44f78395cfde9c2ddc Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 21:40:56 +0200 Subject: [PATCH 4/7] introduce version, cache to attach information flow from GetVersions -> GetDetails --- .../Dependencies/PackageResolver.fs | 76 +++++++++++-------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index cbff0e20c5..74a1fd5665 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -83,10 +83,15 @@ type ResolvedPackage = { type PackageResolution = Map +/// Caches information retrieved by GetVersions until it is required by GetDetails +type VersionCache = + { Version : SemVerInfo; Sources : PackageSource list; AssumedVersion : bool } + static member ofParams version sources isAssumed = + { Version = version; Sources = sources; AssumedVersion = isAssumed } type ResolverStep = { Relax: bool - FilteredVersions : Map + FilteredVersions : Map CurrentResolution : Map; ClosedRequirements : Set OpenRequirements : Set } @@ -286,7 +291,7 @@ type Resolution with member self.IsConflict = not self.IsDone -let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestrictions,(versionToExplore,_),dependency,resolverStep:ResolverStep) = +let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestrictions,(verCache:VersionCache),dependency,resolverStep:ResolverStep) = let dependenciesByName = // there are packages which define multiple dependencies to the same package // we compress these here - see #567 @@ -331,7 +336,7 @@ let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestric { dependency with Name = n VersionRequirement = v - Parent = Package(dependency.Name, versionToExplore, exploredPackage.Source) + Parent = Package(dependency.Name, verCache.Version, exploredPackage.Source) Graph = Set.add dependency dependency.Graph TransitivePrereleases = dependency.TransitivePrereleases && exploredPackage.Version.PreRelease.IsSome Settings = { dependency.Settings with FrameworkRestrictions = newRestrictions } }) @@ -477,7 +482,7 @@ let private explorePackageConfig (getPackageDetailsBlock:PackageDetailsSyncFunc) type StackPack = { ExploredPackages : Dictionary - KnownConflicts : HashSet * ((SemVerInfo * PackageSource list) list * bool) option> + KnownConflicts : HashSet * (VersionCache list * bool) option> ConflictHistory : Dictionary } @@ -522,28 +527,34 @@ let private getCompatibleVersions |> Set.filter (fun r -> currentRequirement.Name = r.Name) // we didn't select a version yet so all versions are possible - let isInRange mapF (ver,_) = + let isInRange mapF (cache:VersionCache) = allRequirementsOfCurrentPackage - |> Set.forall (fun r -> (mapF r).VersionRequirement.IsInRange ver) + |> Set.forall (fun r -> (mapF r).VersionRequirement.IsInRange cache.Version) - //let getSingleVersion v = - // match currentRequirement.Parent with - // | PackageRequirementSource.Package(_,_,parentSource) -> - // let sources = parentSource :: currentRequirement.Sources |> List.distinct - // Seq.singleton (v,sources) - // | _ -> - // let sources : PackageSource list = currentRequirement.Sources |> List.sortBy (fun x -> not x.IsLocalFeed, String.containsIgnoreCase "nuget.org" x.Url |> not) - // Seq.singleton (v,sources) + let getSingleVersion v = + let sources = + match currentRequirement.Parent with + | PackageRequirementSource.Package(_,_,parentSource) -> + parentSource :: currentRequirement.Sources |> List.distinct + | _ -> + currentRequirement.Sources |> List.sortBy (fun x -> not x.IsLocalFeed, String.containsIgnoreCase "nuget.org" x.Url |> not) + Seq.singleton (VersionCache.ofParams v sources true) let availableVersions = let resolverStrategy = getResolverStrategy globalStrategyForDirectDependencies globalStrategyForTransitives allRequirementsOfCurrentPackage currentRequirement - let allVersions = getVersionsF resolverStrategy (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName currentRequirement.Name) + let allVersions = + getVersionsF resolverStrategy (GetPackageVersionsParameters.ofParams currentRequirement.Sources groupName currentRequirement.Name) + |> Seq.map (fun (v, sources) -> VersionCache.ofParams v sources false) match currentRequirement.VersionRequirement.Range with | Specific v - | OverrideAll v -> - allVersions - |> Seq.filter (fun (v1, _) -> v1 = v) - //getSingleVersion v + | OverrideAll v -> + let results = + allVersions + |> Seq.filter (fun cache -> cache.Version = v) + |> Seq.toList + match results with + | [] -> getSingleVersion v + | _ -> List.toSeq results | _ -> allVersions let compatibleVersions = Seq.filter (isInRange id) availableVersions |> Seq.cache @@ -562,7 +573,7 @@ let private getCompatibleVersions PreReleaseStatus.All let available = availableVersions |> Seq.toList - let allPrereleases = available |> List.filter (fun (v,_) -> v.PreRelease <> None) = available + let allPrereleases = available |> List.filter (fun (cache:VersionCache) -> cache.Version.PreRelease <> None) = available let prereleases = List.filter (isInRange (fun r -> r.IncludingPrereleases(prereleaseStatus r))) available if allPrereleases then Seq.ofList prereleases, globalOverride @@ -578,10 +589,13 @@ let private getCompatibleVersions let compatibleVersions, tryRelaxed = if globalOverride then List.toSeq versions, false else let compat = - Seq.filter (fun (v,_) -> currentRequirement.VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions + versions + |> Seq.filter (fun (cache) -> currentRequirement.VersionRequirement.IsInRange(cache.Version,currentRequirement.Parent.IsRootRequirement() |> not)) if Seq.isEmpty compat then - let withPrereleases = Seq.filter (fun (v,_) -> currentRequirement.IncludingPrereleases().VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions + let withPrereleases = + versions + |> Seq.filter (fun (cache) -> currentRequirement.IncludingPrereleases().VersionRequirement.IsInRange(cache.Version,currentRequirement.Parent.IsRootRequirement() |> not)) if currentStep.Relax || Seq.isEmpty withPrereleases then withPrereleases, false else @@ -591,7 +605,7 @@ let private getCompatibleVersions compatibleVersions, false, tryRelaxed -let private getConflicts (currentStep:ResolverStep) (currentRequirement:PackageRequirement) (knownConflicts:HashSet * ((SemVerInfo * PackageSource list) list * bool) option>) = +let private getConflicts (currentStep:ResolverStep) (currentRequirement:PackageRequirement) (knownConflicts:HashSet * (VersionCache list * bool) option>) = let allRequirements = currentStep.OpenRequirements @@ -638,7 +652,7 @@ type ConflictState = { LastConflictReported : DateTime TryRelaxed : bool Conflicts : Set - VersionsToExplore : seq + VersionsToExplore : seq GlobalOverride : bool } with member x.AddError exn = @@ -660,7 +674,7 @@ type ConflictState = { let private boostConflicts - (filteredVersions:Map) + (filteredVersions:Map) (currentRequirement:PackageRequirement) (stackpack:StackPack) (conflictState:ConflictState) = @@ -722,9 +736,9 @@ type private StepFlags = { self.Ready self.UseUnlisted self.HasUnlisted self.ForceBreak self.FirstTrial self.UnlistedSearch type private Stage = - | Step of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list - | Outer of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list - | Inner of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list + | Step of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list + | Outer of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list + | Inner of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq * StepFlags) list type WorkPriority = | BackgroundWork = 10 @@ -1166,7 +1180,7 @@ let Resolve (getVersionsRaw : PackageVersionsFunc, getPreferredVersionsRaw : Pre step (Outer((currentConflict,currentStep,currentRequirement), priorConflictSteps)) stackpack compatibleVersions flags else let flags = { flags with FirstTrial = false } - let (version,sources) & versionToExplore = Seq.head currentConflict.VersionsToExplore + let ({ Version = version; Sources = sources }: VersionCache) & versionToExplore = Seq.head currentConflict.VersionsToExplore let currentConflict = { currentConflict with VersionsToExplore = Seq.tail currentConflict.VersionsToExplore } @@ -1279,12 +1293,12 @@ let Resolve (getVersionsRaw : PackageVersionsFunc, getPreferredVersionsRaw : Pre TryRelaxed = false GlobalOverride = false Conflicts = (Set.empty : Set) - VersionsToExplore = (Seq.empty : seq) + VersionsToExplore = (Seq.empty : seq) } let stackpack = { ExploredPackages = Dictionary() - KnownConflicts = (HashSet() : HashSet * ((SemVerInfo * PackageSource list) list * bool) option>) + KnownConflicts = (HashSet() : HashSet * (VersionCache list * bool) option>) ConflictHistory = (Dictionary() : Dictionary) } From 07e020c5237e51c288c430cd3992c26b18689bfa Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 21:58:05 +0200 Subject: [PATCH 5/7] add and forward isAssumedVersion --- .../Dependencies/PackageResolver.fs | 22 ++++++++++--------- tests/Paket.Tests/Resolver/PropertyTests.fs | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index 74a1fd5665..4fe7326eea 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -37,10 +37,13 @@ type SourcePackageInfo = type GetPackageDetailsParameters = { Package : SourcePackageInfo + VersionIsAssumed : bool Version : SemVerInfo } - static member ofParams sources groupName packageName version = + static member ofParamsEx isAssumed sources groupName packageName version = SourcePackageInfo.ofParams sources groupName packageName - |> fun p -> { Package = p; Version = version } + |> fun p -> { Package = p; Version = version; VersionIsAssumed = isAssumed } + static member ofParams sources groupName packageName version = + GetPackageDetailsParameters.ofParamsEx false sources groupName packageName version type GetPackageVersionsParameters = { Package : SourcePackageInfo } @@ -389,8 +392,7 @@ type private PackageConfig = { GlobalRestrictions : FrameworkRestrictions RootSettings : IDictionary CliTools : Set - Version : SemVerInfo - Sources : PackageSource list + VersionCache : VersionCache UpdateMode : UpdateMode } with member self.HasGlobalRestrictions = @@ -437,8 +439,9 @@ let private updateRestrictions (pkgConfig:PackageConfig) (package:ResolvedPackag let private explorePackageConfig (getPackageDetailsBlock:PackageDetailsSyncFunc) (pkgConfig:PackageConfig) = - let dependency, version = pkgConfig.Dependency, pkgConfig.Version - let packageSources = pkgConfig.Sources + let dependency, version = pkgConfig.Dependency, pkgConfig.VersionCache.Version + let packageSources = pkgConfig.VersionCache.Sources + let isAssumedVersion = pkgConfig.VersionCache.AssumedVersion match pkgConfig.UpdateMode with | Install -> tracefn " - %O %A" dependency.Name version @@ -453,7 +456,7 @@ let private explorePackageConfig (getPackageDetailsBlock:PackageDetailsSyncFunc) filterRestrictions dependency.Settings.FrameworkRestrictions pkgConfig.GlobalRestrictions try let packageDetails : PackageDetails = - getPackageDetailsBlock (GetPackageDetailsParameters.ofParams packageSources pkgConfig.GroupName dependency.Name version) + getPackageDetailsBlock (GetPackageDetailsParameters.ofParamsEx isAssumedVersion packageSources pkgConfig.GroupName dependency.Name version) let filteredDependencies = DependencySetFilter.filterByRestrictions newRestrictions packageDetails.DirectDependencies let settings = @@ -488,7 +491,7 @@ type StackPack = { let private getExploredPackage (pkgConfig:PackageConfig) (getPackageDetailsBlock:PackageDetailsSyncFunc) (stackpack:StackPack) = - let key = (pkgConfig.Dependency.Name, pkgConfig.Version) + let key = (pkgConfig.Dependency.Name, pkgConfig.VersionCache.Version) match stackpack.ExploredPackages.TryGetValue key with | true, package -> @@ -1190,8 +1193,7 @@ let Resolve (getVersionsRaw : PackageVersionsFunc, getPreferredVersionsRaw : Pre Dependency = currentRequirement GlobalRestrictions = globalFrameworkRestrictions RootSettings = rootSettings - Version = version - Sources = sources + VersionCache = versionToExplore UpdateMode = updateMode CliTools = cliToolSettings } diff --git a/tests/Paket.Tests/Resolver/PropertyTests.fs b/tests/Paket.Tests/Resolver/PropertyTests.fs index 7ec43ca301..3a2e41418a 100644 --- a/tests/Paket.Tests/Resolver/PropertyTests.fs +++ b/tests/Paket.Tests/Resolver/PropertyTests.fs @@ -31,7 +31,7 @@ let isValid ((g,deps):ResolverPuzzle) resolution = match s with | Open deps -> match resolution |> Map.tryFind d with - | Some r -> if not <| vr.IsInRange(r.Version) then Error else s + | Some (r:ResolvedPackage) -> if not <| vr.IsInRange(r.Version) then Error else s | None -> Open ((d,vr)::deps) | _ -> s) (Open []) |> fun s -> From e49da8d1288c82c9856f0ae8be1b3ed36ca43339 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 22:08:43 +0200 Subject: [PATCH 6/7] forward and use isAssumed to ignore blacklisting --- src/Paket.Core/Dependencies/NuGet.fs | 2 ++ src/Paket.Core/Dependencies/NuGetCache.fs | 11 +++++++++-- src/Paket.Core/Dependencies/NuGetV2.fs | 13 +++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index adbc7f2d05..83359b7a01 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -249,10 +249,12 @@ let rec private getPackageDetails alternativeProjectRoot root force (parameters: let tryV2 (nugetSource:NugetSource) force = NuGetV2.getDetailsFromNuGet force + parameters.VersionIsAssumed nugetSource packageName version + let tryV3 (nugetSource:NugetV3Source) force = NuGetV3.GetPackageDetails force nugetSource packageName version diff --git a/src/Paket.Core/Dependencies/NuGetCache.fs b/src/Paket.Core/Dependencies/NuGetCache.fs index 33dc6299e0..019df5ce3b 100644 --- a/src/Paket.Core/Dependencies/NuGetCache.fs +++ b/src/Paket.Core/Dependencies/NuGetCache.fs @@ -465,12 +465,19 @@ let private tryUrlOrBlacklist (f: _ -> Async<'a>) (isOk : 'a -> bool) (source:Nu | FirstCall t -> FirstCall (t |> Task.Map (fun (l, r) -> l, (r :?> 'a))) -let tryAndBlacklistUrl doWarn (source:NugetSource) (tryAgain : 'a -> bool) (f : string -> Async<'a>) (urls: UrlToTry list) : Async<'a>= +let tryAndBlacklistUrl doBlackList doWarn (source:NugetSource) (tryAgain : 'a -> bool) (f : string -> Async<'a>) (urls: UrlToTry list) : Async<'a>= async { let! tasks, resultIndex = urls |> Seq.map (fun url -> async { - let cached = tryUrlOrBlacklist (fun () -> async { return! f url.InstanceUrl }) (tryAgain >> not) (source, url.UrlId) + let cached = + if doBlackList then + tryUrlOrBlacklist (fun () -> async { return! f url.InstanceUrl }) (tryAgain >> not) (source, url.UrlId) + else + async { + let! result = f url.InstanceUrl + return (tryAgain result |> not, result) + } |> Async.StartAsTask |> FirstCall match cached with | SubsequentCall task -> let! result = task |> Async.AwaitTask diff --git a/src/Paket.Core/Dependencies/NuGetV2.fs b/src/Paket.Core/Dependencies/NuGetV2.fs index e2c02a05b2..03cff4a2df 100644 --- a/src/Paket.Core/Dependencies/NuGetV2.fs +++ b/src/Paket.Core/Dependencies/NuGetV2.fs @@ -255,7 +255,8 @@ let parseODataEntryDetails (url,nugetURL,packageName:PackageName,version:SemVerI | ODataSearchResult.Match entry -> entry -let getDetailsFromNuGetViaODataFast nugetSource (packageName:PackageName) (version:SemVerInfo) = +let getDetailsFromNuGetViaODataFast isVersionAssumed nugetSource (packageName:PackageName) (version:SemVerInfo) = + let doBlacklist = not isVersionAssumed async { let normalizedVersion = version.Normalize() let urls = @@ -369,7 +370,7 @@ let getDetailsFromNuGetViaODataFast nugetSource (packageName:PackageName) (versi | Choice1Of2 _ -> false | _ -> true - let! result = NuGetCache.tryAndBlacklistUrl true nugetSource tryAgain handleUrl urls + let! result = NuGetCache.tryAndBlacklistUrl doBlacklist true nugetSource tryAgain handleUrl urls match result with | Choice1Of2 res -> return res | Choice2Of2 ex -> return raise (exn("error", ex)) @@ -377,16 +378,16 @@ let getDetailsFromNuGetViaODataFast nugetSource (packageName:PackageName) (versi /// Gets package details from NuGet via OData -let getDetailsFromNuGetViaOData nugetSource (packageName:PackageName) (version:SemVerInfo) = - getDetailsFromNuGetViaODataFast nugetSource packageName version +let getDetailsFromNuGetViaOData isVersionAssumed nugetSource (packageName:PackageName) (version:SemVerInfo) = + getDetailsFromNuGetViaODataFast isVersionAssumed nugetSource packageName version -let getDetailsFromNuGet force nugetSource packageName version = +let getDetailsFromNuGet force isVersionAssumed nugetSource packageName version = getDetailsFromCacheOr force nugetSource.Url packageName version - (fun () -> getDetailsFromNuGetViaOData nugetSource packageName version) + (fun () -> getDetailsFromNuGetViaOData isVersionAssumed nugetSource packageName version) /// Uses the NuGet v2 API to retrieve all packages with the given prefix. let FindPackages(auth, nugetURL, packageNamePrefix, maxResults) = From 8911613f9f342ed4373c61956ff52ca3d37de166 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 4 Sep 2017 22:29:45 +0200 Subject: [PATCH 7/7] add failing test --- global.json | 5 ----- .../Paket.IntegrationTests/BasicResolverSpecs.fs | 6 +++++- ...640-never-exists-on-nuget-gallery.1.0.0.nupkg | Bin 0 -> 3607 bytes .../scenarios/i002640/before/paket.dependencies | 4 ++++ 4 files changed, 9 insertions(+), 6 deletions(-) delete mode 100644 global.json create mode 100644 integrationtests/scenarios/i002640/before/my-package-source/paket-issue-i002640-never-exists-on-nuget-gallery.1.0.0.nupkg create mode 100644 integrationtests/scenarios/i002640/before/paket.dependencies diff --git a/global.json b/global.json deleted file mode 100644 index 0aa382680b..0000000000 --- a/global.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk": { - "version": "2.1.0-preview1-007002" - } -} \ No newline at end of file diff --git a/integrationtests/Paket.IntegrationTests/BasicResolverSpecs.fs b/integrationtests/Paket.IntegrationTests/BasicResolverSpecs.fs index 361e443790..dc1be1f2aa 100644 --- a/integrationtests/Paket.IntegrationTests/BasicResolverSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/BasicResolverSpecs.fs @@ -113,6 +113,7 @@ let ``#1189 should allow # in path``() = lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "FAKE"].Version |> shouldBeGreaterThan (SemVer.Parse "4.7.2") + [] let ``#1247 shouldn't load lockfile in full update``() = updateEx true "i001247-lockfile-error" |> ignore @@ -137,7 +138,10 @@ let ``#1450 should resolve with twiddle wakka``() = let lockFile = update "i001450-twiddle-wakka" lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "EnterpriseLibrary.SemanticLogging"].Version |> shouldBeSmallerThan (SemVer.Parse "3") - + +[] +let ``#2640 shouldn't try GetDetails if package only exists locally``() = + updateEx true "i002640" |> ignore #if INTERACTIVE ;; diff --git a/integrationtests/scenarios/i002640/before/my-package-source/paket-issue-i002640-never-exists-on-nuget-gallery.1.0.0.nupkg b/integrationtests/scenarios/i002640/before/my-package-source/paket-issue-i002640-never-exists-on-nuget-gallery.1.0.0.nupkg new file mode 100644 index 0000000000000000000000000000000000000000..ec18a25cb446e67081e4cb3aa83e36f2ae22e217 GIT binary patch literal 3607 zcmcInc{CeJ-wqmX?R71+lqw=rVu`J-rIgmL)uNUlB9uf#5^I$bTxuz*cBN=7t)&&N zYGZHhOEoB=P3>a6s%k55?)lF7?tA;a-yiQe?>lqO%$b?r%yZ`X&CKtatLALS1OWiR zFMzd|vgYSu+j9^W06>Zr0N`Twy5dn7f->~aGCj55r(K0VdOfCzGz8CB0oU71&rYJd zxWJneCtidP^+<>8a}FC`QXqGty!1Q+f8Wo|?G_T4px<`1j}q7bagSQ-!Z*7Fr`BCu z>p8Llpkg|-xRNZL?X5gQ*o;KKXx=+TqYV@+c9sZ5<^)sci{$A^{1EHUqU5W`L#ESa zbldnUYC%A5WR(T3pe3#9G`iKeGk-hqSYNtCK+fs!W3zRu&_UX4rlI$8jBXO2Aa@`6wN8oU*=IA_2P|CN$+If(yo2pPy zkMKr$PkT~I)o9DQWRp!13|VURtm=UyCqxWkm4q3yF&Tmjz2^ai2Vl^(YNdf!`!Dn2 z!-$icrTXi7OwNvu*Z^1m#wyk}!&Q^XswPpJ zQFsU{*qcBkK>V-}Y=9Tj#tVVLpztA3Yybg=LguBlnz5j(B^rLc>B}xx0)$^zDTm)}6x_R1>Q#_Aavz zv{lGB<&%c6l>^1u^n#n8Ig3w)yoxm8acA|>_RA2-bL@I{?W1U-N&XW}9tT*LfAj|E zW}r~StHNFms&maNrE0aebzH$capR6ASo9b5lndHi@-60SENe-u3AjqZ9dKn-Ri;do z+fkE9Gy9F!od*5Q7e<7yv|m?lZp{4Q7GO2j^5)7KZ>yZEvFK)fkbzZA0OCX1-k0rN zh5J;sMbD$N9}~M5^(9)9rv>H07n}a2FBHVM9F~F1Aprougjr#{-IcK@B7um&dLZx~ zYETvBf8RYG7|g1@B}Uv>FnmV4Ja?G<#CwFZ4wIXaAyu6x#R`cqT`A$2?M#}oX=#I3 z%MXB-;pm|JcZL;;EV};SGA*dF;q)&L?@`a#{?dTE#oHnztJncaFSu?Hx6!{c;Xlou z$~HID^sOm74X!M>%oe#329FMNTt=I|jSfL}5P~set@`?%UVfRW+%* zx=RY}wr0t29J#0noq@V-mVDM~+=koSG_f?jD1U+RE~#>#RjS*d=)|azR(Rt#hmrN~ zWc%?JccT;P9bAn^fGiufdWX?VdRT)mp;uG5e2SX1Vk&kZ59wb%SN4hKGW5rk$ ze18WFU6&8tbe%1^Y{t`A^2@}P<%x>U1)cN@qd?6L?4 z3p9A%re;l?T3*wUmF@S4${d>Vk6m<12dM;==ycX_R&_*Y8Gk(0d0ky#yM0L;AEq6t zM%peY(si47<%Y?sm02}fHlE=dwWR&f)?{CDV@c*Qn7X3bfAA8xyoMe%gj;ODw-jhb z&5xe}Uzr-dd~gSFsCWD@W;SQmOICiIrm(C;yw1Lf?gF`l+f}6w zA^qLgqZWE%qB3A1&A%=6J=C@OQPrDpt~O2cO{l*kfpb#4Exgs5HegYFn1a~+b<&@Q z=CdME5ArQ73t6Q0+15?hYj_LxBva8VRyg~h#No|>2Dp;wQ*svlsir&{TkJnpLQwh ztL95=`s%BpnQffNR8Y)Kn4(jCoTt<59@G@z#DqvYl9MW$U^8 z-gqnPbKBmT27dX@e^3$wE0Cn68#U2VTp)V(I8=k3&v&&>E=OfDt=#tAkf3KidJs^j z=9971aX5GjZS%p4%xcdNKBK94ak(QrzEdkDX(Dr{e?dzmE(!E@Y**+LFwdH6TB*EP zh^B8@B2Yf2WdU!jTOBDD`Aj1ow0Ctn8%gQswPx$)o8B`~i=Fe&`^*~C8L2ia9 z4}=*6XG?T?b;8ieI>?2mFtrVfslZ6?>Zym0JqwGS^bG+rn1M(7mN29ju4`g19RDCO z!kuuhD%qsrz@s90yS=}MIwdDNpzz%ENVv`dI%>M2xyhLNA`2B+SYESUXxF97;~G6$ zIlNix5P6r8xA;hWyT3ylg%5YD_dYZ@>WPrslE%&mEa;Q$lO%RjarfyP-tT3{Uq%#= z&5G?wRO9B$l3N&Z*dF+R3rgY6(zzv?B7&V+YNW=GJd*5*m!M0I@P0E=UVlc*82Sql57cLgLoQ{9& zYj8ec%pm_m9#zPs2ru#8HNzyjfHf>fO^>7a1Vv8u2GWIRDZf>q-IEeM7oGk(;N*FK z{rC}=NQ%gw`y-ulv}uI7=)t<`v-C_z5$R@r;^oudH9DSW6iTZ(6e@h&dp!({579|U z5RxZ9Q*}Dx3XC`flCG0Ji#()UXyQdSxK?-6;DJj0%{zRcQL|>F_DFROS0Rt!0P)Xj zl9g#JZ;!00haD8|_i&R4w!a7WPhpmCMwx-+Ju|@n%B-C(`C*ADEYbBw2o6PnLxX)W zd8w+{b`@^p@2nJ3VO`A}5qz)BDpppj3wSK2KBh-Nde{KJuVPxt>Yb3>v47MYxjL_Y zh0xI^T{=+3cqe9&vY+y90IGIlJ+L-B+=tK0P*coma@;gpFAtf8+jjP9EAi+;Z5Mfo zo(kF7l4%@!8bU~U;Fe}qC2lvC=pCfJBX@7SKyYdfDC1O1`K`KXAhhUYO9DQ~;)mAt zZ>aLRklqc!I9GMIq}yfEk8T$p#h(^=UIvB;Y}ngM3F&Wn@TV6ryb*>wG{a@Al%p$i zlmA!aoNIMskeC$IF*VLLrUedxL?gUV$^;ZX&>M*&DEp#_2oD4ip^WsyqaZlE9}a~l zdNT)L+A3N|Pc3bCn3k40LR|%+qNaj$S5w#2P*+h$A+(@4g0BY>muF*npLUv`V$X=U zsK-$=W=b~mttMAeFTJ)csj6^s$O=wXy~$1OZdQIAUt1B%;0c!M%gvtuqAj+`rQ_p8 zD{@B}w3wBWS7~Qr*(l1Ajo;VBmULDqxh)(#CTTs>R^9c%JctQ@J!8MDue$bWq{tJP zi!LwjiEwG%mHML~Dp9FVh7qGN2+LjWdwsU|-ci`z9=xdyTvmTF`XTNbgtJfO<&6r< zuGZaa$NG35q>vWBtdQqqjy*QwKgdh8GAEyTB3E(^FRvR%57UZgVI7G(Q3maOz1mnJ zY%>U!&|ohauo2(ySc*}lTk}K|H#I6;T>2(xQ}FsnfnirE&1v+_oO6KV6FUYe_6TD1 znfv7e>LWZyqVf{;P_pgb+eFWs~8c4O9 z?L?)HKDS!S71}*%87Jw;r5cQ|^N?~)%y6Y|t+3bl!YLAr{eHW$(ATk{Umt&oxbrYj z`Tk;k*p%P@dYpw-5b*#17MN!Ll>X`OaMk?p6Mq)Of6cOE9-)6