Skip to content

Commit

Permalink
paket: performance by cloudRoutine and 0x53A
Browse files Browse the repository at this point in the history 
fixes #2289
  • Loading branch information
@0x53A
0x53A committed Apr 28, 2017
1 parent d99c8f6 commit cd95e3b
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 107 deletions.
11 changes: 10 additions & 1 deletion integrationtests/Paket.IntegrationTests/ResolverSkipsConflictsFastSpecs.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,16 @@ let ``#1166 Should resolve Nancy without timeout``() =
|> shouldBeGreaterThan (SemVer.Parse "1.1")

[<Test>]
[<Ignore("fails with SO, skipping until works")>]
let ``#2289 Paket 4.x install command takes hours to complete``() =
let lockFile = install "i002289-resolve-nunit-timeout"
let nunitVersion =
lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "NUnit"].Version
nunitVersion
|> shouldBeGreaterThan (SemVer.Parse "2.0")
nunitVersion
|> shouldBeSmallerThan (SemVer.Parse "3.0")

[<Test>]
let ``#1174 Should find Ninject error``() =
updateShouldFindPackageConflict "Ninject" "i001174-resolve-fast-conflict"

Expand Down
52 changes: 52 additions & 0 deletions integrationtests/scenarios/i002289-resolve-nunit-timeout/before/paket.dependencies
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
redirects: on
framework: net46

source https://api.nuget.org/v3/index.json

nuget Akka.Persistence.SqlServer
nuget SQLProvider
nuget Microsoft.FSharpLu.Json
nuget Microsoft.Net.Http
nuget Microsoft.AspNet.WebApi.Client
nuget FSharp.Core redirects: force
nuget FSharp.Configuration
nuget FSharp.Formatting
nuget FSharp.Data
nuget FSharp.Data.TypeProviders
nuget Newtonsoft.Json
nuget FAKE
nuget SourceLink.Fake
nuget Akka
nuget Akka.FSharp
nuget Akka.Persistence
nuget Akka.Persistence.FSharp
nuget Akka.Persistence.Sql.Common
nuget Akka.Persistence.Query
nuget Akka.Persistence.Query.Sql
nuget Akka.Logger.Serilog
nuget Akka.Monitoring
nuget Akka.Monitoring.StatsD
nuget Akka.TestKit.NUnit
nuget Akka.TestKit.Xunit2
nuget Akka.Serialization.Wire
nuget FsPickler
nuget Serilog
nuget Serilog.Sinks.ElasticSearch
nuget Nest ~> 5
nuget RabbitMQ.Client
nuget SSH.NET
nuget Topshelf
nuget Topshelf.Serilog
nuget NUnit ~> 2
nuget NUnit.Runners
nuget NUnitTestAdapter
nuget TickSpec.NUnit content: none
nuget FsCheck
nuget FsCheck.Nunit
nuget System.Collections.Immutable
nuget Suave
nuget OctopusTools
nuget Suave.Swagger
nuget Quartz
nuget Akka.Quartz.Actor
nuget Oracle.ManagedDataAccess
175 changes: 101 additions & 74 deletions src/Paket.Core/Dependencies/PackageResolver.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestric
Settings = { dependency.Settings with FrameworkRestrictions = newRestrictions } })
|> Set.filter (fun d ->
resolverStep.ClosedRequirements
|> Seq.exists (fun x ->
|> Set.exists (fun x ->
x.Name = d.Name &&
x.Settings.FrameworkRestrictions = d.Settings.FrameworkRestrictions &&
(x = d ||
Expand All @@ -281,7 +281,7 @@ let calcOpenRequirements (exploredPackage:ResolvedPackage,globalFrameworkRestric
|> not)
|> Set.filter (fun d ->
resolverStep.OpenRequirements
|> Seq.exists (fun x -> x.Name = d.Name && (x = d || x.VersionRequirement.Range.IsGlobalOverride) && x.Settings.FrameworkRestrictions = d.Settings.FrameworkRestrictions)
|> Set.exists (fun x -> x.Name = d.Name && (x = d || x.VersionRequirement.Range.IsGlobalOverride) && x.Settings.FrameworkRestrictions = d.Settings.FrameworkRestrictions)
|> not)
|> Set.union rest

Expand Down Expand Up @@ -390,7 +390,7 @@ let private explorePackageConfig getPackageDetailsF (pkgConfig:PackageConfig) =

type StackPack = {
ExploredPackages : Dictionary<PackageName*SemVerInfo,ResolvedPackage>
KnownConflicts : HashSet<Set<PackageRequirement> * ((SemVerInfo * PackageSource list) list * bool) option>
KnownConflicts : HashSet<HashSet<PackageRequirement> * ((SemVerInfo * PackageSource list) list * bool) option>
ConflictHistory : Dictionary<PackageName, int>
}

Expand Down Expand Up @@ -494,23 +494,26 @@ let private getCompatibleVersions
compatibleVersions, false, tryRelaxed


let private getConflicts (currentStep:ResolverStep) (currentRequirement:PackageRequirement) (knownConflicts:HashSet<_>) =
let private getConflicts (currentStep:ResolverStep) (currentRequirement:PackageRequirement) (knownConflicts:HashSet<HashSet<PackageRequirement> * ((SemVerInfo * PackageSource list) list * bool) option>) =

let allRequirements =
currentStep.OpenRequirements
|> Set.filter (fun r -> r.Graph |> List.contains currentRequirement |> not)
|> Set.union currentStep.ClosedRequirements
Set.toSeq currentStep.OpenRequirements
|> Seq.filter (fun r -> r.Graph |> List.contains currentRequirement |> not)
|> Seq.append currentStep.ClosedRequirements
|> HashSet

knownConflicts
|> Seq.map (fun (conflicts,selectedVersion) ->
match selectedVersion with
| None when Set.isSubset conflicts allRequirements -> conflicts
| None when conflicts.IsSubsetOf allRequirements -> conflicts
| Some(selectedVersion,_) ->
let n = (Seq.head conflicts).Name
match currentStep.FilteredVersions |> Map.tryFind n with
| Some(v,_) when v = selectedVersion && Set.isSubset conflicts allRequirements -> conflicts
| _ -> Set.empty
| _ -> Set.empty)
|> Set.unionMany
| Some(v,_) when v = selectedVersion && conflicts.IsSubsetOf allRequirements -> conflicts
| _ -> HashSet()
| _ -> HashSet())
|> Seq.collect id
|> HashSet


let private getCurrentRequirement packageFilter (openRequirements:Set<PackageRequirement>) (conflictHistory:Dictionary<_,_>) =
Expand Down Expand Up @@ -575,7 +578,7 @@ let private boostConflicts
match conflicts with
| c::_ ->
let selectedVersion = Map.tryFind c.Name filteredVersions
let key = conflicts |> Set.ofList,selectedVersion
let key = conflicts |> HashSet,selectedVersion
stackpack.KnownConflicts.Add key |> ignore

let reportThatResolverIsTakingLongerThanExpected =
Expand All @@ -597,25 +600,24 @@ let private boostConflicts


[<Struct>]
type private StepFlags (ready:bool,useUnlisted:bool,hasUnlisted:bool,forceBreak:bool,firstTrial:bool,unlistedSearch:bool) =
member __.Ready = ready
member __.UseUnlisted = useUnlisted
member __.HasUnlisted = hasUnlisted
member __.ForceBreak = forceBreak
member __.FirstTrial = firstTrial
member __.UnlistedSearch = unlistedSearch
member private self.Display
with get () =
sprintf
"[< FLAGS >]\n\
| Ready - %b\n\
| UseUnlisted - %b\n\
| HasUnlisted - %b\n\
| ForceBreak - %b\n\
| FirstTrial - %b\n\
| UnlistedSearch - %b\n"
ready useUnlisted hasUnlisted forceBreak firstTrial unlistedSearch
override self.ToString() = self.Display
type private StepFlags = {
Ready : bool
UseUnlisted : bool
HasUnlisted : bool
ForceBreak : bool
FirstTrial : bool
UnlistedSearch : bool
} with
override self.ToString () =
sprintf
"[< FLAGS >]\n\
| Ready - %b\n\
| UseUnlisted - %b\n\
| HasUnlisted - %b\n\
| ForceBreak - %b\n\
| FirstTrial - %b\n\
| UnlistedSearch - %b\n"
self.Ready self.UseUnlisted self.HasUnlisted self.ForceBreak self.FirstTrial self.UnlistedSearch

type private Stage =
| Step of currentConflict : (ConflictState * ResolverStep * PackageRequirement) * priorConflictSteps : (ConflictState * ResolverStep * PackageRequirement * seq<SemVerInfo * PackageSource list> * StepFlags) list
Expand Down Expand Up @@ -649,14 +651,34 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate

let rec step (stage:Stage) (stackpack:StackPack) compatibleVersions (flags:StepFlags) =

let inline fuseConflicts currentConflict priorConflictSteps =
match currentConflict, priorConflictSteps with
| currentConflict, (lastConflict,lastStep,lastRequirement,lastCompatibleVersions,lastFlags)::priorConflictSteps ->
let inline fuseConflicts currentConflict priorConflictSteps conflicts =
let findMatchingStep priorConflictSteps =
priorConflictSteps // row
|> Seq.tryFind (fun (_,_,lastRequirement:PackageRequirement,_,_) ->
let currentNames =
conflicts |> Seq.collect (fun c ->
let graphNameList =
c.Graph |> List.map (fun (pr:PackageRequirement) -> pr.Name)
c.Name :: graphNameList)
|> Seq.toArray
currentNames |> Array.contains lastRequirement.Name
)
|> Option.map (fun r -> r, priorConflictSteps |> List.filter(fun r2 -> not (obj.ReferenceEquals(r, r2))))

match priorConflictSteps, findMatchingStep priorConflictSteps with
| [], _ -> currentConflict
| _, Some (head, priorConflictSteps) ->
let (lastConflict, lastStep, lastRequirement, lastCompatibleVersions, lastFlags) = head
let continueConflict =
{ currentConflict with VersionsToExplore = lastConflict.VersionsToExplore }
step (Inner((continueConflict,lastStep,lastRequirement),priorConflictSteps)) stackpack lastCompatibleVersions lastFlags
| currentConflict, [] -> currentConflict

step (Inner((continueConflict,lastStep,lastRequirement), priorConflictSteps)) stackpack lastCompatibleVersions lastFlags
// could not find a specific package - go back one step
| head :: priorConflictSteps, None ->
let (lastConflict, lastStep, lastRequirement, lastCompatibleVersions, lastFlags) = head
let continueConflict =
{ currentConflict with VersionsToExplore = lastConflict.VersionsToExplore }
step (Inner((continueConflict,lastStep,lastRequirement), priorConflictSteps)) stackpack lastCompatibleVersions lastFlags

match stage with
| Step((currentConflict,currentStep,_currentRequirement), priorConflictSteps) ->
if Set.isEmpty currentStep.OpenRequirements then
Expand All @@ -678,10 +700,8 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate
&& not (conflicts |> Set.exists (fun r ->
r = lastRequirement
|| r.Graph |> List.contains lastRequirement)) ->
let flags =
StepFlags(flags.Ready,flags.UseUnlisted,flags.HasUnlisted,true,flags.FirstTrial,flags.UnlistedSearch)

step (Inner((continueConflict,lastStep,lastRequirement),priorConflictSteps)) stackpack lastCompatibleVersions lastFlags
step (Inner((continueConflict,lastStep,lastRequirement),priorConflictSteps)) stackpack lastCompatibleVersions { flags with ForceBreak = true }
| _ ->
step (Inner((continueConflict,lastStep,lastRequirement),priorConflictSteps)) stackpack lastCompatibleVersions lastFlags

Expand All @@ -703,15 +723,15 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate

let currentConflict =
let getVersionsF = getVersionsF currentRequirement.Sources ResolverStrategy.Max groupName
if Set.isEmpty conflicts then
if Seq.isEmpty conflicts then
{ currentConflict with
Status = Resolution.Conflict(currentStep,Set.empty,currentRequirement,getVersionsF)}
Status = Resolution.Conflict (currentStep,Set.empty,currentRequirement,getVersionsF)}
else
{ currentConflict with
Status = Resolution.Conflict(currentStep,conflicts,Seq.head conflicts,getVersionsF)}
Status = Resolution.Conflict (currentStep,set conflicts,Seq.head conflicts,getVersionsF)}

if not (Set.isEmpty conflicts) then
fuseConflicts currentConflict priorConflictSteps
if not (Seq.isEmpty conflicts) then
fuseConflicts currentConflict priorConflictSteps conflicts
else
let compatibleVersions,globalOverride,tryRelaxed =
getCompatibleVersions currentStep groupName currentRequirement getVersionsF
Expand All @@ -721,7 +741,7 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate

let currentConflict = {
currentConflict with
Conflicts = conflicts
Conflicts = set conflicts
TryRelaxed = tryRelaxed
GlobalOverride = globalOverride
}
Expand All @@ -730,21 +750,23 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate
boostConflicts currentStep.FilteredVersions currentRequirement stackpack currentConflict
else
currentConflict, stackpack
let flags =
StepFlags
( ready = false
, useUnlisted = false
, hasUnlisted = false
, forceBreak = flags.ForceBreak
, firstTrial = flags.FirstTrial
, unlistedSearch = false
)
let flags = {
flags with
Ready = false
UseUnlisted = false
HasUnlisted = false
UnlistedSearch = false
}
step (Outer ((conflictState,currentStep,currentRequirement),priorConflictSteps)) stackpack compatibleVersions flags
| Outer ((currentConflict,currentStep,currentRequirement), priorConflictSteps) ->
if flags.Ready then
fuseConflicts currentConflict priorConflictSteps
if flags.Ready then
fuseConflicts currentConflict priorConflictSteps (HashSet [ currentRequirement ])
else
let flags = StepFlags(flags.Ready,flags.UseUnlisted,flags.HasUnlisted,false,true,flags.UnlistedSearch)
let flags = {
flags with
ForceBreak = false
FirstTrial = true
}
let currentConflict = { currentConflict with VersionsToExplore = compatibleVersions }
step (Inner ((currentConflict,currentStep,currentRequirement), priorConflictSteps)) stackpack compatibleVersions flags

Expand All @@ -758,14 +780,21 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate
then
// if it's been determined that an unlisted package must be used, ready must be set to false
verbosefn "\nSearching for compatible unlisted package\n"
StepFlags(false,true,flags.HasUnlisted,flags.ForceBreak,flags.FirstTrial,true)
{ flags with
Ready = false
UseUnlisted = true
UnlistedSearch = true
}
else
StepFlags(true,flags.UseUnlisted,flags.HasUnlisted,flags.ForceBreak,flags.FirstTrial,false)
{ flags with
Ready = true
UnlistedSearch = true
}
step (Outer((currentConflict,currentStep,currentRequirement), priorConflictSteps)) stackpack compatibleVersions flags
else


let flags = StepFlags(flags.Ready,flags.UseUnlisted,flags.HasUnlisted,flags.ForceBreak,false,flags.UnlistedSearch)
let flags = { flags with FirstTrial = false }
let (version,sources) & versionToExplore = Seq.head currentConflict.VersionsToExplore

let currentConflict =
Expand All @@ -787,8 +816,7 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate

| stackpack, Some(alreadyExplored,exploredPackage) ->
let hasUnlisted = exploredPackage.Unlisted || flags.HasUnlisted
let flags =
StepFlags(flags.Ready,flags.UseUnlisted,hasUnlisted,flags.ForceBreak,flags.FirstTrial,flags.UnlistedSearch)
let flags = { flags with HasUnlisted = hasUnlisted }

if exploredPackage.Unlisted && not flags.UseUnlisted then
if not alreadyExplored then
Expand Down Expand Up @@ -835,19 +863,18 @@ let Resolve (getVersionsF, getPackageDetailsF, groupName:GroupName, globalStrate

let stackpack = {
ExploredPackages = Dictionary<PackageName*SemVerInfo,ResolvedPackage>()
KnownConflicts = (HashSet() : HashSet<Set<PackageRequirement> * ((SemVerInfo * PackageSource list) list * bool) option>)
KnownConflicts = (HashSet() : HashSet<HashSet<PackageRequirement> * ((SemVerInfo * PackageSource list) list * bool) option>)
ConflictHistory = (Dictionary() : Dictionary<PackageName, int>)
}

let flags =
StepFlags
( ready = false
, useUnlisted = false
, hasUnlisted = false
, forceBreak = false
, firstTrial = true
, unlistedSearch = false
)
let flags = {
Ready = false
UseUnlisted = false
HasUnlisted = false
ForceBreak = false
FirstTrial = true
UnlistedSearch = false
}

match step (Step((currentConflict,startingStep,currentRequirement),[])) stackpack Seq.empty flags with
| { Status = Resolution.Conflict _ } as conflict ->
Expand Down
Loading

0 comments on commit cd95e3b

Please sign in to comment.