From 6e5ec1a84312a044eb37df30601c249df673e86e Mon Sep 17 00:00:00 2001 From: Steffen Forkmann Date: Sat, 19 Dec 2015 10:59:18 +0100 Subject: [PATCH] Only check PackageDetails for Sources that responded with versions for a package - references #1317 --- RELEASE_NOTES.md | 15 ++------- src/Paket.Core/DependencyChangeDetection.fs | 2 +- src/Paket.Core/NuGetV2.fs | 32 +++++++++++-------- src/Paket.Core/PackageResolver.fs | 30 ++++++++--------- src/Paket.Core/UpdateProcess.fs | 10 +++--- .../DependencyChangesSpecs.fs | 2 ++ tests/Paket.Tests/TestHelpers.fs | 1 + 7 files changed, 45 insertions(+), 47 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e3e31eda63..af9bbf6999 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,20 +1,11 @@ -#### 2.36.7 - 19.12.2015 +#### 2.37.0-alpha001 - 19.12.2015 +* Paket checks PackageDetails only for sources that responded with versions for a package - https://github.com/fsprojects/Paket/issues/1317 * Implemented support for specifying per-template versions in paket pack - https://github.com/fsprojects/Paket/pull/1314 - -#### 2.36.6 - 15.12.2015 * Added support for relative src link to package content - https://github.com/fsprojects/Paket/pull/1311 - -#### 2.36.5 - 14.12.2015 * BUGFIX: Fix NullReferenceException - https://github.com/fsprojects/Paket/issues/1307 - -#### 2.36.3 - 14.12.2015 -* COSMETICS: No need to show cache warnings - -#### 2.36.2 - 13.12.2015 * BUGFIX: NuGet packages with FrameworkAssembly nodes did not work - https://github.com/fsprojects/Paket/issues/1306 - -#### 2.36.1 - 12.12.2015 * Paket install did an unnecessary update when framework restriction were present - https://github.com/fsprojects/Paket/issues/1305 +* COSMETICS: No need to show cache warnings #### 2.36.0 - 10.12.2015 * Getting assembly metadata without loading the assembly - https://github.com/fsprojects/Paket/pull/1293 diff --git a/src/Paket.Core/DependencyChangeDetection.fs b/src/Paket.Core/DependencyChangeDetection.fs index d0f4b3f1b1..86108fff63 100644 --- a/src/Paket.Core/DependencyChangeDetection.fs +++ b/src/Paket.Core/DependencyChangeDetection.fs @@ -162,5 +162,5 @@ let findRemoteFileChangesInDependenciesFile(dependenciesFile:DependenciesFile,lo let GetPreferredNuGetVersions (oldLockFile:LockFile) = oldLockFile.GetGroupedResolution() - |> Seq.map (fun kv -> kv.Key, kv.Value.Version) + |> Seq.map (fun kv -> kv.Key, (kv.Value.Version, kv.Value.Source)) |> Map.ofSeq \ No newline at end of file diff --git a/src/Paket.Core/NuGetV2.fs b/src/Paket.Core/NuGetV2.fs index c83a2ba41a..be8c2c0c2e 100644 --- a/src/Paket.Core/NuGetV2.fs +++ b/src/Paket.Core/NuGetV2.fs @@ -85,19 +85,19 @@ let tryGetPackageVersionsViaJson (auth, nugetURL, package:PackageName) = with _ -> return None } -let tryNuGetV3 (auth, nugetV3Url, package:PackageName) = +let tryNuGetV3 (source, auth, nugetV3Url, package:PackageName) = async { try let! data = NuGetV3.findVersionsForPackage(nugetV3Url, auth, package) match data with - | Some data -> return Some data + | Some data -> return Some (source,data) | _ -> return None with exn -> return None } /// Gets versions of the given package from local Nuget feed. -let getAllVersionsFromLocalPath (localNugetPath, package:PackageName, root) = +let getAllVersionsFromLocalPath (source, localNugetPath, package:PackageName, root) = async { let localNugetPath = Utils.normalizeLocalPath localNugetPath let di = getDirectoryInfo localNugetPath root @@ -111,7 +111,7 @@ let getAllVersionsFromLocalPath (localNugetPath, package:PackageName, root) = let _match = Regex(sprintf @"^%O\.(\d.*)\.nupkg" package, RegexOptions.IgnoreCase).Match(fi.Name) if _match.Groups.Count > 1 then Some _match.Groups.[1].Value else None) |> Seq.toArray - return Some versions + return Some(source,versions) } @@ -607,7 +607,7 @@ let GetPackageDetails root force sources packageName (version:SemVerInfo) : Pack let protocolCache = System.Collections.Concurrent.ConcurrentDictionary<_,_>() -let getVersionsCached key f (auth, nugetURL, package) = +let getVersionsCached key f (source, auth, nugetURL, package) = async { match protocolCache.TryGetValue((auth, nugetURL)) with | true, v when v <> key -> return None @@ -616,7 +616,7 @@ let getVersionsCached key f (auth, nugetURL, package) = match result with | Some x -> protocolCache.TryAdd((auth, nugetURL), key) |> ignore - return Some x + return Some (source, x) | _ -> return None } @@ -624,19 +624,20 @@ let getVersionsCached key f (auth, nugetURL, package) = let GetVersions root (sources, packageName:PackageName) = let getVersions() = sources - |> Seq.map (function + |> Seq.map (fun nugetSource -> + match nugetSource with | Nuget source -> let auth = source.Authentication |> Option.map toBasicAuth let v3Feeds = match NuGetV3.getAllVersionsAPI(source.Authentication,source.Url) with | None -> [] - | Some v3Url -> [ tryNuGetV3 (auth, v3Url, packageName) ] + | Some v3Url -> [ tryNuGetV3 (nugetSource, auth, v3Url, packageName) ] let v2Feeds = - [ yield getVersionsCached "OData" tryGetPackageVersionsViaOData (auth, source.Url, packageName) - yield getVersionsCached "ODataWithFilter" tryGetAllVersionsFromNugetODataWithFilter (auth, source.Url, packageName) + [ yield getVersionsCached "OData" tryGetPackageVersionsViaOData (nugetSource, auth, source.Url, packageName) + yield getVersionsCached "ODataWithFilter" tryGetAllVersionsFromNugetODataWithFilter (nugetSource, auth, source.Url, packageName) if not (source.Url.ToLower().Contains "teamcity" || source.Url.ToLower().Contains "feedservice.svc") then - yield getVersionsCached "Json" tryGetPackageVersionsViaJson (auth, source.Url, packageName) + yield getVersionsCached "Json" tryGetPackageVersionsViaJson (nugetSource, auth, source.Url, packageName) ] v2Feeds @ v3Feeds @@ -645,18 +646,20 @@ let GetVersions root (sources, packageName:PackageName) = async { let! autoCompleteUrl = PackageSources.getNugetV3Resource source AllVersionsAPI return! - tryNuGetV3 (source.Authentication |> Option.map toBasicAuth, + tryNuGetV3 (nugetSource, + source.Authentication |> Option.map toBasicAuth, autoCompleteUrl, packageName) } [ resp ] - | LocalNuget path -> [ getAllVersionsFromLocalPath (path, packageName, root) ]) + | LocalNuget path -> [ getAllVersionsFromLocalPath (nugetSource, path, packageName, root) ]) |> Seq.toArray |> Array.map Async.Choice |> Async.Parallel |> Async.RunSynchronously |> Array.choose id + |> Array.map (fun (s,versions) -> versions |> Array.map (fun v -> v,s)) |> Array.concat let versions = @@ -666,4 +669,5 @@ let GetVersions root (sources, packageName:PackageName) = versions |> Seq.toList - |> List.map SemVer.Parse + |> List.groupBy fst + |> List.map (fun (v,s) -> SemVer.Parse v,s |> List.map snd) diff --git a/src/Paket.Core/PackageResolver.fs b/src/Paket.Core/PackageResolver.fs index 51c1ccd6f6..fe78ef54d5 100644 --- a/src/Paket.Core/PackageResolver.fs +++ b/src/Paket.Core/PackageResolver.fs @@ -86,7 +86,7 @@ let cleanupNames (model : PackageResolution) : PackageResolution = [] type Resolution = | Ok of PackageResolution -| Conflict of Map * Set * Set * PackageRequirement * (PackageName -> SemVerInfo seq) +| Conflict of Map * Set * Set * PackageRequirement * (PackageName -> (SemVerInfo * PackageSource list) seq) with member this.GetConflicts() = @@ -153,7 +153,7 @@ type Resolution = " Please try to relax some conditions." |> failwithf "%s" -let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestrictions,versionToExplore,dependency,closed:Set,stillOpen:Set) = +let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestrictions,(versionToExplore,packageSource),dependency,closed:Set,stillOpen:Set) = let dependenciesByName = // there are packages which define multiple dependencies to the same package // we just take the latest one - see #567 @@ -217,16 +217,16 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra let conflictHistory = Dictionary() let knownConflicts = HashSet<_>() - let getExploredPackage(dependency:PackageRequirement,version) = - - match exploredPackages.TryGetValue <| (dependency.Name,version) with + let getExploredPackage(dependency:PackageRequirement,(version,packageSources)) = + let key = dependency.Name,version + match exploredPackages.TryGetValue key with | true,package -> let newRestrictions = if List.isEmpty globalFrameworkRestrictions && (List.isEmpty package.Settings.FrameworkRestrictions || List.isEmpty dependency.Settings.FrameworkRestrictions) then [] else optimizeRestrictions (package.Settings.FrameworkRestrictions @ dependency.Settings.FrameworkRestrictions @ globalFrameworkRestrictions) let package = { package with Settings = { package.Settings with FrameworkRestrictions = newRestrictions } } - exploredPackages.[(dependency.Name,version)] <- package + exploredPackages.[key] <- package package | false,_ -> match updateMode with @@ -238,7 +238,7 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra let newRestrictions = filterRestrictions dependency.Settings.FrameworkRestrictions globalFrameworkRestrictions - let packageDetails : PackageDetails = getPackageDetailsF sources dependency.Name version + let packageDetails : PackageDetails = getPackageDetailsF packageSources dependency.Name version let filteredDependencies = DependencySetFilter.filterByRestrictions newRestrictions packageDetails.DirectDependencies @@ -258,10 +258,10 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra Unlisted = packageDetails.Unlisted Settings = settings Source = packageDetails.Source } - exploredPackages.Add((dependency.Name,version),explored) + exploredPackages.Add(key,explored) explored - let rec step (filteredVersions:Map,currentResolution:Map,closedRequirements:Set,openRequirements:Set) = + let rec step (filteredVersions:Map,currentResolution:Map,closedRequirements:Set,openRequirements:Set) = if Set.isEmpty openRequirements then Resolution.Ok(cleanupNames currentResolution) else verbosefn " %d packages in resolution. %d requirements left" currentResolution.Count openRequirements.Count @@ -336,14 +336,14 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra | false -> combined // we didn't select a version yet so all versions are possible - let isInRange mapF ver = + let isInRange mapF (ver,ps) = currentRequirements |> Seq.forall (fun r -> (mapF r).VersionRequirement.IsInRange ver) availableVersions := match currentRequirement.VersionRequirement.Range with - | OverrideAll v -> Seq.singleton v - | Specific v -> Seq.singleton v + | OverrideAll v -> Seq.singleton (v,sources) + | Specific v -> Seq.singleton (v,sources) | _ -> getVersionsF sources resolverStrategy groupName currentRequirement.Name |> Seq.cache @@ -361,7 +361,7 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra else if Seq.isEmpty !compatibleVersions then let prereleases = Seq.filter (isInRange (fun r -> r.IncludingPrereleases())) (!availableVersions) |> Seq.toList - let allPrereleases = prereleases |> List.filter (fun v -> v.PreRelease <> None) = prereleases + let allPrereleases = prereleases |> List.filter (fun (v,ps) -> v.PreRelease <> None) = prereleases if allPrereleases then availableVersions := Seq.ofList prereleases compatibleVersions := Seq.ofList prereleases @@ -373,11 +373,11 @@ let Resolve(groupName:GroupName, sources, getVersionsF, getPackageDetailsF, stra compatibleVersions := List.toSeq versions else compatibleVersions := - Seq.filter (fun v -> currentRequirement.VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions + Seq.filter (fun (v,ps) -> currentRequirement.VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions if Seq.isEmpty !compatibleVersions then compatibleVersions := - Seq.filter (fun v -> currentRequirement.IncludingPrereleases().VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions + Seq.filter (fun (v,ps) -> currentRequirement.IncludingPrereleases().VersionRequirement.IsInRange(v,currentRequirement.Parent.IsRootRequirement() |> not)) versions if Seq.isEmpty !compatibleVersions then // boost the conflicting package, in order to solve conflicts faster diff --git a/src/Paket.Core/UpdateProcess.fs b/src/Paket.Core/UpdateProcess.fs index 0448b914be..ff8b690cf9 100644 --- a/src/Paket.Core/UpdateProcess.fs +++ b/src/Paket.Core/UpdateProcess.fs @@ -10,8 +10,8 @@ open Chessie.ErrorHandling open Paket.Logging let selectiveUpdate force getSha1 getSortedVersionsF getPackageDetailsF (lockFile:LockFile) (dependenciesFile:DependenciesFile) updateMode semVerUpdateMode = - let allVersions = Dictionary() - let getSortedAndCachedVersionsF sources resolverStrategy groupName packageName = + let allVersions = Dictionary() + let getSortedAndCachedVersionsF sources resolverStrategy groupName packageName : seq = match allVersions.TryGetValue(packageName) with | false,_ -> let versions = @@ -22,11 +22,10 @@ let selectiveUpdate force getSha1 getSortedVersionsF getPackageDetailsF (lockFil failwithf "Couldn't retrieve versions for %O." packageName allVersions.Add(packageName,versions) versions - | true,versions -> versions + | true,versions -> versions |> List.toSeq - - let getPreferredVersionsF preferredVersions changedDependencies sources resolverStrategy groupName packageName = + let getPreferredVersionsF (preferredVersions:Map<(GroupName * PackageName),SemVerInfo * (PackageSources.PackageSource list)>) changedDependencies sources resolverStrategy groupName packageName = seq { match preferredVersions |> Map.tryFind (groupName, packageName), resolverStrategy, changedDependencies |> Set.exists ((=) (groupName, packageName)) with | Some v, ResolverStrategy.Min, _ @@ -132,6 +131,7 @@ let selectiveUpdate force getSha1 getSortedVersionsF getPackageDetailsF (lockFil let preferredVersions = DependencyChangeDetection.GetPreferredNuGetVersions lockFile + |> Map.map (fun k (v,s) -> v,[s]) |> getPreferredVersionsF preferredVersions changes,groups diff --git a/tests/Paket.Tests/DependenciesFile/DependencyChangesSpecs.fs b/tests/Paket.Tests/DependenciesFile/DependencyChangesSpecs.fs index 81f5c3acf6..09d951536c 100644 --- a/tests/Paket.Tests/DependenciesFile/DependencyChangesSpecs.fs +++ b/tests/Paket.Tests/DependenciesFile/DependencyChangesSpecs.fs @@ -113,6 +113,7 @@ nuget NUnit""" newDependencies |> Map.filter (fun k v -> not <| changedDependencies.Contains(k)) + |> Map.map (fun k (v,_) -> v) |> shouldEqual expected [] @@ -159,6 +160,7 @@ nuget Castle.Windsor-log4net >= 3.3.0""" newDependencies |> Map.filter (fun k v -> not <| changedDependencies.Contains(k)) + |> Map.map (fun k (v,_) -> v) |> shouldEqual expected [] diff --git a/tests/Paket.Tests/TestHelpers.fs b/tests/Paket.Tests/TestHelpers.fs index 83c0127df9..a27f556ad2 100644 --- a/tests/Paket.Tests/TestHelpers.fs +++ b/tests/Paket.Tests/TestHelpers.fs @@ -29,6 +29,7 @@ let VersionsFromGraph (graph : seq Seq.filter (fun (p, _, _) -> (PackageName p) = packageName) |> Seq.map (fun (_, v, _) -> SemVer.Parse v) |> Seq.toList + |> List.map (fun v -> v,sources) match resolverStrategy with | ResolverStrategy.Max -> List.sortDescending versions