diff --git a/integrationtests/Paket.IntegrationTests/NuGetV3Specs.fs b/integrationtests/Paket.IntegrationTests/NuGetV3Specs.fs index 0a982f023f..0f20cd1160 100644 --- a/integrationtests/Paket.IntegrationTests/NuGetV3Specs.fs +++ b/integrationtests/Paket.IntegrationTests/NuGetV3Specs.fs @@ -15,4 +15,20 @@ let ``#1387 update package in v3``() = update "i001387-nugetv3" |> ignore let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i001387-nugetv3","paket.lock")) lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "Bender"].Version - |> shouldEqual (SemVer.Parse "3.0.29.0") \ No newline at end of file + |> shouldEqual (SemVer.Parse "3.0.29.0") + +[] +let ``#2700-1 v3 works properly``() = + paketEx true "update" "i002700-1" |> ignore + let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i002700-1","paket.lock")) + let mainGroup = lockFile.Groups.[Constants.MainDependencyGroup] + mainGroup.Resolution.[PackageName "Microsoft.CSharp"].Source.Url + |> shouldEqual "https://www.myget.org/F/dotnet-core-svc/api/v3/index.json" + +[] +let ``#2700-2 v2 is not upgraded to v3``() = + updateEx true "i002700-2" |> ignore + let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i002700-2","paket.lock")) + let mainGroup = lockFile.Groups.[Constants.MainDependencyGroup] + mainGroup.Resolution.[PackageName "Microsoft.CSharp"].Source.Url + |> shouldEqual "https://www.myget.org/F/dotnet-core-svc" \ No newline at end of file diff --git a/integrationtests/Paket.IntegrationTests/PackSpecs.fs b/integrationtests/Paket.IntegrationTests/PackSpecs.fs index 7da4e3a460..e4037c4492 100644 --- a/integrationtests/Paket.IntegrationTests/PackSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/PackSpecs.fs @@ -391,3 +391,29 @@ let ``#1816 pack localized when satellite dll is missing`` () = Path.Combine(outPath, "lib", "net45", "sv-FI", "LocalizedLib.resources.dll") |> checkFileExists CleanDir rootPath + +[] +let ``#2694 paket fixnuspec should not remove project references``() = + let project = "console" + let scenario = "i002694" + prepareSdk scenario + + let wd = (scenarioTempPath scenario) @@ project + + directDotnet true (sprintf "pack %s.csproj" project) wd + |> ignore + + let nupkgPath = wd @@ "bin" @@ "Debug" @@ project + ".1.0.0.nupkg" + if File.Exists nupkgPath |> not then Assert.Fail(sprintf "Expected '%s' to exist" nupkgPath) + let nuspec = NuGetLocal.getNuSpecFromNupgk nupkgPath + match nuspec.Dependencies |> Seq.tryFind (fun (name,_,_) -> name = PackageName "library") with + | None -> Assert.Fail("Expected package to still contain the project reference!") + | Some s -> ignore s + match nuspec.Dependencies |> Seq.tryFind (fun (name,_,_) -> name = PackageName "FSharp.Core") with + | None -> Assert.Fail("Expected package to still contain the FSharp.Core reference!") + | Some s -> ignore s + + // Should we remove Microsoft.NETCore.App? + // Problably not as "packaged" console applications have this dependency by default, see https://www.nuget.org/packages/dotnet-mergenupkg + nuspec.Dependencies.Length + |> shouldEqual 3 \ No newline at end of file diff --git a/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs b/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs index b4a5a22043..7cf09fd5e6 100644 --- a/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs @@ -10,36 +10,8 @@ open FsUnit let ``#2496 Paket fails on projects that target multiple frameworks``() = let project = "EmptyTarget" let scenario = "i002496" - let tmpPaketFolder = (scenarioTempPath scenario) @@ ".paket" - let targetsFile = FullName(__SOURCE_DIRECTORY__ + "../../../src/Paket/embedded/Paket.Restore.targets") - let paketExe = FullName(__SOURCE_DIRECTORY__ + "../../../bin/paket.exe") + prepareSdk scenario - setEnvironVar "PaketExePath" paketExe - prepare scenario - if (not (Directory.Exists tmpPaketFolder)) then - Directory.CreateDirectory tmpPaketFolder |> ignore - - FileHelper.CopyFile tmpPaketFolder targetsFile - let wd = (scenarioTempPath scenario) @@ project - let restore = - let dotnetExePath = - match Environment.GetEnvironmentVariable "DOTNET_EXE_PATH" with - | null | "" -> "dotnet" - | s -> s - ProcessHelper.ExecProcessAndReturnMessages (fun info -> - info.FileName <- dotnetExePath - info.WorkingDirectory <- wd - info.Arguments <- (sprintf "restore %s.csproj" project) - ) - let result = restore <| TimeSpan.FromMinutes 3. - let exitCode = result.ExitCode - - result.Errors.ForEach (fun m-> tracefn "%s" m ) - result.Messages.ForEach (fun m-> tracefn "%s" m ) - - let hasErrorOut = result.Errors.Count > 0; - - match (result.OK, hasErrorOut) with - | (true, false) -> printfn "all good!" - | _ -> failwithf "dotnet restore exitCode %d > 0? or hasErrorOut %b" exitCode hasErrorOut \ No newline at end of file + directDotnet true (sprintf "restore %s.csproj" project) wd + |> ignore \ No newline at end of file diff --git a/integrationtests/Paket.IntegrationTests/TestHelper.fs b/integrationtests/Paket.IntegrationTests/TestHelper.fs index 77ba4eee5d..0580d338ce 100644 --- a/integrationtests/Paket.IntegrationTests/TestHelper.fs +++ b/integrationtests/Paket.IntegrationTests/TestHelper.fs @@ -35,6 +35,10 @@ let partitionForTravis scenario = let paketToolPath = FullName(__SOURCE_DIRECTORY__ + "../../../bin/paket.exe") +let dotnetToolPath = + match Environment.GetEnvironmentVariable "DOTNET_EXE_PATH" with + | null | "" -> "dotnet" + | s -> s let integrationTestPath = FullName(__SOURCE_DIRECTORY__ + "../../../integrationtests/scenarios") let scenarioTempPath scenario = Path.Combine(integrationTestPath,scenario,"temp") let originalScenarioPath scenario = Path.Combine(integrationTestPath,scenario,"before") @@ -70,38 +74,58 @@ let prepare scenario = for file in Directory.GetFiles(scenarioPath, (sprintf "*.%stemplate" ext), SearchOption.AllDirectories) do File.Move(file, Path.ChangeExtension(file, ext)) -let directPaketInPath command scenarioPath = +let prepareSdk scenario = + let tmpPaketFolder = (scenarioTempPath scenario) @@ ".paket" + let targetsFile = FullName(__SOURCE_DIRECTORY__ + "../../../src/Paket/embedded/Paket.Restore.targets") + let paketExe = FullName(__SOURCE_DIRECTORY__ + "../../../bin/paket.exe") + + setEnvironVar "PaketExePath" paketExe + prepare scenario + if (not (Directory.Exists tmpPaketFolder)) then + Directory.CreateDirectory tmpPaketFolder |> ignore + + FileHelper.CopyFile tmpPaketFolder targetsFile + + +type PaketMsg = + { IsError : bool; Message : string } + static member isError ({ IsError = e}:PaketMsg) = e + static member getMessage ({ Message = msg }:PaketMsg) = msg + +let directToolEx isPaket toolPath command workingDir = #if INTERACTIVE let result = ExecProcessWithLambdas (fun info -> - info.FileName <- paketToolPath - info.WorkingDirectory <- scenarioPath + info.FileName <- toolPath + info.WorkingDirectory <- workingDir info.Arguments <- command) (System.TimeSpan.FromMinutes 7.) false (printfn "%s") (printfn "%s") - string result + let res = new ResizeArray() + res.Add (string result) + res #else Environment.SetEnvironmentVariable("PAKET_DETAILED_ERRORS", "true") - printfn "%s> paket %s" scenarioPath command + printfn "%s> %s %s" workingDir (if isPaket then "paket" else toolPath) command let perfMessages = ResizeArray() - let msgs = ResizeArray() + let msgs = ResizeArray() let mutable perfMessagesStarted = false let addAndPrint isError msg = if not isError then - if msg = "Performance:" then + if isPaket && msg = "Performance:" then perfMessagesStarted <- true - elif perfMessagesStarted then + elif isPaket && perfMessagesStarted then perfMessages.Add(msg) - - msgs.Add((isError, msg)) + + msgs.Add({ IsError = isError; Message = msg}) let result = try ExecProcessWithLambdas (fun info -> - info.FileName <- paketToolPath - info.WorkingDirectory <- scenarioPath + info.FileName <- toolPath + info.WorkingDirectory <- workingDir info.CreateNoWindow <- true info.Arguments <- command) (System.TimeSpan.FromMinutes 7.) @@ -114,61 +138,92 @@ let directPaketInPath command scenarioPath = else printfn "ExecProcessWithLambdas failed. Output was: " - for isError, msg in msgs do + for { IsError = isError; Message = msg } in msgs do printfn "%s%s" (if isError then "ERR: " else "") msg reraise() - // Only throw after the result <> 0 check because the current test might check the argument parsing - // this is the only case where no performance is printed - let isUsageError = result <> 0 && msgs |> Seq.filter fst |> Seq.map snd |> Seq.exists (fun msg -> msg.Contains "USAGE:") - if not isUsageError then - if perfMessages.Count = 0 then - failwith "No Performance messages recieved in test!" - printfn "Performance:" - for msg in perfMessages do - printfn "%s" msg + if isPaket then + // Only throw after the result <> 0 check because the current test might check the argument parsing + // this is the only case where no performance is printed + let isUsageError = result <> 0 && msgs |> Seq.filter PaketMsg.isError |> Seq.map PaketMsg.getMessage |> Seq.exists (fun msg -> msg.Contains "USAGE:") + if not isUsageError then + if perfMessages.Count = 0 then + failwith "No Performance messages recieved in test!" + printfn "Performance:" + for msg in perfMessages do + printfn "%s" msg // always print stderr - for isError, msg in msgs do - if isError then - printfn "ERR: %s" msg + for msg in msgs do + if msg.IsError then + printfn "ERR: %s" msg.Message if result <> 0 then - let errors = String.Join(Environment.NewLine,msgs |> Seq.filter fst |> Seq.map snd) + let errors = String.Join(Environment.NewLine,msgs |> Seq.filter PaketMsg.isError |> Seq.map PaketMsg.getMessage) if String.IsNullOrWhiteSpace errors then failwithf "The process exited with code %i" result else failwith errors - - String.Join(Environment.NewLine,msgs |> Seq.map snd) + msgs #endif -let directPaket command scenario = +let directPaketInPathEx command scenarioPath = + directToolEx true paketToolPath command scenarioPath + +let checkResults msgs = + msgs + |> Seq.filter PaketMsg.isError + |> Seq.toList + |> shouldEqual [] + +let directDotnet checkZeroWarn command workingDir = + let msgs = directToolEx false dotnetToolPath command workingDir + if checkZeroWarn then checkResults msgs + msgs + +let private fromMessages msgs = + String.Join(Environment.NewLine,msgs |> Seq.map PaketMsg.getMessage) + +let directPaketInPath command scenarioPath = directPaketInPathEx command scenarioPath |> fromMessages + +let directPaketEx command scenario = partitionForTravis scenario - directPaketInPath command (scenarioTempPath scenario) + directPaketInPathEx command (scenarioTempPath scenario) -let paket command scenario = +let directPaket command scenario = directPaketEx command scenario |> fromMessages + +let paketEx checkZeroWarn command scenario = prepare scenario - directPaket command scenario + let msgs = directPaketEx command scenario + if checkZeroWarn then checkResults msgs + msgs -let update scenario = +let paket command scenario = + paketEx false command scenario |> fromMessages + +let updateEx checkZeroWarn scenario = #if INTERACTIVE paket "update --verbose" scenario |> printfn "%s" #else - paket "update" scenario |> ignore + paketEx checkZeroWarn "update" scenario |> ignore #endif LockFile.LoadFrom(Path.Combine(scenarioTempPath scenario,"paket.lock")) -let install scenario = +let update scenario = + updateEx false scenario + +let installEx checkZeroWarn scenario = #if INTERACTIVE paket "install --verbose" scenario |> printfn "%s" #else - paket "install" scenario |> ignore + paketEx checkZeroWarn "install" scenario |> ignore #endif LockFile.LoadFrom(Path.Combine(scenarioTempPath scenario,"paket.lock")) -let restore scenario = paket "restore" scenario |> ignore +let install scenario = installEx false scenario + +let restore scenario = paketEx false "restore" scenario |> ignore let updateShouldFindPackageConflict packageName scenario = try diff --git a/integrationtests/scenarios/i002694/before/console/Program.cs b/integrationtests/scenarios/i002694/before/console/Program.cs new file mode 100644 index 0000000000..5ea6f160ba --- /dev/null +++ b/integrationtests/scenarios/i002694/before/console/Program.cs @@ -0,0 +1,12 @@ +using System; + +namespace console +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello World!"); + } + } +} diff --git a/integrationtests/scenarios/i002694/before/console/console.csproj b/integrationtests/scenarios/i002694/before/console/console.csproj new file mode 100644 index 0000000000..997bd99e7e --- /dev/null +++ b/integrationtests/scenarios/i002694/before/console/console.csproj @@ -0,0 +1,11 @@ + + + + + + + Exe + netcoreapp2.0 + + + \ No newline at end of file diff --git a/integrationtests/scenarios/i002694/before/console/paket.references b/integrationtests/scenarios/i002694/before/console/paket.references new file mode 100644 index 0000000000..640cf9145d --- /dev/null +++ b/integrationtests/scenarios/i002694/before/console/paket.references @@ -0,0 +1 @@ +FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i002694/before/library/Class1.cs b/integrationtests/scenarios/i002694/before/library/Class1.cs new file mode 100644 index 0000000000..57298088a8 --- /dev/null +++ b/integrationtests/scenarios/i002694/before/library/Class1.cs @@ -0,0 +1,8 @@ +using System; + +namespace library +{ + public class Class1 + { + } +} diff --git a/integrationtests/scenarios/i002694/before/library/library.csproj b/integrationtests/scenarios/i002694/before/library/library.csproj new file mode 100644 index 0000000000..14a5f11835 --- /dev/null +++ b/integrationtests/scenarios/i002694/before/library/library.csproj @@ -0,0 +1,7 @@ + + + + netstandard2.0 + + + \ No newline at end of file diff --git a/integrationtests/scenarios/i002694/before/library/paket.references b/integrationtests/scenarios/i002694/before/library/paket.references new file mode 100644 index 0000000000..640cf9145d --- /dev/null +++ b/integrationtests/scenarios/i002694/before/library/paket.references @@ -0,0 +1 @@ +FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i002694/before/paket.dependencies b/integrationtests/scenarios/i002694/before/paket.dependencies new file mode 100644 index 0000000000..4ee8ca4f06 --- /dev/null +++ b/integrationtests/scenarios/i002694/before/paket.dependencies @@ -0,0 +1,4 @@ +source https://api.nuget.org/v3/index.json +restriction: || (==netstandard20) (== netcoreapp20) +storage: none +nuget FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i002694/before/paket.lock b/integrationtests/scenarios/i002694/before/paket.lock new file mode 100644 index 0000000000..db697aea77 --- /dev/null +++ b/integrationtests/scenarios/i002694/before/paket.lock @@ -0,0 +1,400 @@ +STORAGE: NONE +RESTRICTION: || (== netstandard2.0) (== netcoreapp2.0) +NUGET + remote: https://api.nuget.org/v3/index.json + FSharp.Core (4.2.3) + System.Collections (>= 4.0.11) + System.Console (>= 4.0) + System.Diagnostics.Debug (>= 4.0.11) + System.Diagnostics.Tools (>= 4.0.1) + System.Globalization (>= 4.0.11) + System.IO (>= 4.1) + System.Linq (>= 4.1) + System.Linq.Expressions (>= 4.1) + System.Linq.Queryable (>= 4.0.1) + System.Net.Requests (>= 4.0.11) + System.Reflection (>= 4.1) + System.Reflection.Extensions (>= 4.0.1) + System.Resources.ResourceManager (>= 4.0.1) + System.Runtime (>= 4.1) + System.Runtime.Extensions (>= 4.1) + System.Runtime.Numerics (>= 4.0.1) + System.Text.RegularExpressions (>= 4.1) + System.Threading (>= 4.0.11) + System.Threading.Tasks (>= 4.0.11) + System.Threading.Tasks.Parallel (>= 4.0.1) + System.Threading.Thread (>= 4.0) + System.Threading.ThreadPool (>= 4.0.10) + System.Threading.Timer (>= 4.0.1) + Microsoft.NETCore.Platforms (2.0) + Microsoft.NETCore.Targets (2.0) + runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.native.System (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + runtime.native.System.Net.Http (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + runtime.native.System.Security.Cryptography.Apple (4.3) + runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple (>= 4.3) + runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.1) + runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple (4.3) + runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.1) + System.Collections (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Collections.Concurrent (4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Diagnostics.Tracing (>= 4.3) + System.Globalization (>= 4.3) + System.Reflection (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Threading (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Console (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.IO (>= 4.3) + System.Runtime (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Diagnostics.Debug (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Diagnostics.DiagnosticSource (4.4.1) + System.Collections (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (< netstandard1.3)) + System.Diagnostics.Debug (>= 4.3) - restriction: == netstandard2.0 + System.Diagnostics.Tracing (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (< netstandard1.3)) + System.Reflection (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (< netstandard1.3)) + System.Runtime (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (< netstandard1.3)) + System.Runtime.Extensions (>= 4.3) - restriction: == netstandard2.0 + System.Threading (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (< netstandard1.3)) + System.Diagnostics.Tools (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Diagnostics.Tracing (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Globalization (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Globalization.Calendars (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Globalization (>= 4.3) + System.Runtime (>= 4.3) + System.Globalization.Extensions (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + System.Globalization (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.IO (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.IO.FileSystem (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.IO (>= 4.3) + System.IO.FileSystem.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.IO.FileSystem.Primitives (4.3) + System.Runtime (>= 4.3) + System.Linq (4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Linq.Expressions (4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Globalization (>= 4.3) + System.IO (>= 4.3) + System.Linq (>= 4.3) + System.ObjectModel (>= 4.3) + System.Reflection (>= 4.3) + System.Reflection.Emit (>= 4.3) + System.Reflection.Emit.ILGeneration (>= 4.3) + System.Reflection.Emit.Lightweight (>= 4.3) + System.Reflection.Extensions (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Reflection.TypeExtensions (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Threading (>= 4.3) + System.Linq.Queryable (4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Linq (>= 4.3) + System.Linq.Expressions (>= 4.3) + System.Reflection (>= 4.3) + System.Reflection.Extensions (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Net.Http (4.3.2) + Microsoft.NETCore.Platforms (>= 1.1) + runtime.native.System (>= 4.3) + runtime.native.System.Net.Http (>= 4.3) + runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Diagnostics.DiagnosticSource (>= 4.3) + System.Diagnostics.Tracing (>= 4.3) + System.Globalization (>= 4.3) + System.Globalization.Extensions (>= 4.3) + System.IO (>= 4.3) + System.IO.FileSystem (>= 4.3) + System.Net.Primitives (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.Security.Cryptography.Algorithms (>= 4.3) + System.Security.Cryptography.Encoding (>= 4.3) + System.Security.Cryptography.OpenSsl (>= 4.3) + System.Security.Cryptography.Primitives (>= 4.3) + System.Security.Cryptography.X509Certificates (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Threading (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Net.Primitives (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Net.Requests (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Diagnostics.Tracing (>= 4.3) + System.Globalization (>= 4.3) + System.IO (>= 4.3) + System.Net.Http (>= 4.3) + System.Net.Primitives (>= 4.3) + System.Net.WebHeaderCollection (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Threading (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Net.WebHeaderCollection (4.3) + System.Collections (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.ObjectModel (4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Threading (>= 4.3) + System.Reflection (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.IO (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Reflection.Emit (4.3) + System.IO (>= 4.3) + System.Reflection (>= 4.3) + System.Reflection.Emit.ILGeneration (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Reflection.Emit.ILGeneration (4.3) + System.Reflection (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Reflection.Emit.Lightweight (4.3) + System.Reflection (>= 4.3) + System.Reflection.Emit.ILGeneration (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Reflection.Extensions (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Reflection (>= 4.3) + System.Runtime (>= 4.3) + System.Reflection.Primitives (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Reflection.TypeExtensions (4.4) + System.Resources.ResourceManager (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Globalization (>= 4.3) + System.Reflection (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime.Extensions (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Runtime.Handles (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Runtime.InteropServices (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Reflection (>= 4.3) + System.Reflection.Primitives (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.Numerics (4.3) + System.Globalization (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Security.Cryptography.Algorithms (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + runtime.native.System.Security.Cryptography.Apple (>= 4.3) + runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3) + System.Collections (>= 4.3) + System.IO (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.Runtime.Numerics (>= 4.3) + System.Security.Cryptography.Encoding (>= 4.3) + System.Security.Cryptography.Primitives (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Security.Cryptography.Cng (4.4) + Microsoft.NETCore.Platforms (>= 2.0) - restriction: || (&& (== netstandard2.0) (>= netcoreapp2.0)) (== netcoreapp2.0) + System.Security.Cryptography.Csp (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + System.IO (>= 4.3) + System.Reflection (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.Security.Cryptography.Algorithms (>= 4.3) + System.Security.Cryptography.Encoding (>= 4.3) + System.Security.Cryptography.Primitives (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Threading (>= 4.3) + System.Security.Cryptography.Encoding (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3) + System.Collections (>= 4.3) + System.Collections.Concurrent (>= 4.3) + System.Linq (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.Security.Cryptography.Primitives (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Security.Cryptography.OpenSsl (4.4) + Microsoft.NETCore.Platforms (>= 2.0) - restriction: || (&& (== netstandard2.0) (>= netcoreapp2.0)) (== netcoreapp2.0) + System.Security.Cryptography.Primitives (4.3) + System.Diagnostics.Debug (>= 4.3) + System.Globalization (>= 4.3) + System.IO (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Threading (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Security.Cryptography.X509Certificates (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + runtime.native.System (>= 4.3) + runtime.native.System.Net.Http (>= 4.3) + runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3) + System.Collections (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Globalization (>= 4.3) + System.Globalization.Calendars (>= 4.3) + System.IO (>= 4.3) + System.IO.FileSystem (>= 4.3) + System.IO.FileSystem.Primitives (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Runtime.InteropServices (>= 4.3) + System.Runtime.Numerics (>= 4.3) + System.Security.Cryptography.Algorithms (>= 4.3) + System.Security.Cryptography.Cng (>= 4.3) + System.Security.Cryptography.Csp (>= 4.3) + System.Security.Cryptography.Encoding (>= 4.3) + System.Security.Cryptography.OpenSsl (>= 4.3) + System.Security.Cryptography.Primitives (>= 4.3) + System.Text.Encoding (>= 4.3) + System.Threading (>= 4.3) + System.Text.Encoding (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Text.RegularExpressions (4.3) + System.Collections (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (>= dnxcore50)) (&& (== netcoreapp2.0) (< netcoreapp1.1)) + System.Globalization (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (>= dnxcore50)) (&& (== netcoreapp2.0) (< netcoreapp1.1)) + System.Resources.ResourceManager (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (>= dnxcore50)) (&& (== netcoreapp2.0) (< netcoreapp1.1)) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (>= dnxcore50)) (&& (== netcoreapp2.0) (< netcoreapp1.1)) + System.Threading (>= 4.3) - restriction: || (== netstandard2.0) (&& (== netcoreapp2.0) (>= dnxcore50)) (&& (== netcoreapp2.0) (< netcoreapp1.1)) + System.Threading (4.3) + System.Runtime (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Threading.Tasks (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) + System.Threading.Tasks.Parallel (4.3) + System.Collections.Concurrent (>= 4.3) + System.Diagnostics.Debug (>= 4.3) + System.Diagnostics.Tracing (>= 4.3) + System.Resources.ResourceManager (>= 4.3) + System.Runtime (>= 4.3) + System.Runtime.Extensions (>= 4.3) + System.Threading (>= 4.3) + System.Threading.Tasks (>= 4.3) + System.Threading.Thread (4.3) + System.Runtime (>= 4.3) + System.Threading.ThreadPool (4.3) + System.Runtime (>= 4.3) + System.Runtime.Handles (>= 4.3) + System.Threading.Timer (4.3) + Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.NETCore.Targets (>= 1.1) + System.Runtime (>= 4.3) diff --git a/integrationtests/scenarios/i002700-1/before/paket.dependencies b/integrationtests/scenarios/i002700-1/before/paket.dependencies new file mode 100644 index 0000000000..04c578eba1 --- /dev/null +++ b/integrationtests/scenarios/i002700-1/before/paket.dependencies @@ -0,0 +1,3 @@ +source https://www.myget.org/F/dotnet-core-svc/api/v3/index.json + +nuget Microsoft.CSharp prerelease \ No newline at end of file diff --git a/integrationtests/scenarios/i002700-2/before/paket.dependencies b/integrationtests/scenarios/i002700-2/before/paket.dependencies new file mode 100644 index 0000000000..00e376709a --- /dev/null +++ b/integrationtests/scenarios/i002700-2/before/paket.dependencies @@ -0,0 +1,3 @@ +source https://www.myget.org/F/dotnet-core-svc + +nuget Microsoft.CSharp prerelease \ No newline at end of file diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index 07130593b4..a2796737a4 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -448,7 +448,7 @@ let GetVersions force alternativeProjectRoot root (sources, packageName:PackageN return v2Feeds | NuGetV3 source -> - let! versionsAPI = PackageSources.getNuGetV3Resource source AllVersionsAPI + let! versionsAPI = NuGetV3.getNuGetV3Resource source NuGetV3.AllVersionsAPI let auth = source.Authentication |> Option.map toCredentials return [ getVersionsCached "V3" tryNuGetV3 (nugetSource, auth, versionsAPI, packageName) ] | LocalNuGet(path,Some _) -> diff --git a/src/Paket.Core/Dependencies/NuGetV3.fs b/src/Paket.Core/Dependencies/NuGetV3.fs index a8eee85c78..913dd55082 100644 --- a/src/Paket.Core/Dependencies/NuGetV3.fs +++ b/src/Paket.Core/Dependencies/NuGetV3.fs @@ -5,6 +5,8 @@ open Newtonsoft.Json open System.IO open System.Collections.Generic +open System +open System.Threading.Tasks open Paket.Domain open Paket.NuGetCache open Paket.Utils @@ -14,18 +16,97 @@ open Paket.Requirements open Paket.Logging open Paket.PlatformMatching + +type NugetV3SourceResourceJSON = + { [] + Type : string + [] + ID : string } + +type NugetV3SourceRootJSON = + { [] + Resources : NugetV3SourceResourceJSON [] } + +//type NugetV3Source = +// { Url : string +// Authentication : NugetSourceAuthentication option } + +type NugetV3ResourceType = + | AutoComplete + | AllVersionsAPI + //| Registration + | PackageIndex + + member this.AsString = + match this with + | AutoComplete -> "SearchAutoCompleteService" + //| Registration -> "RegistrationsBaseUrl" + | AllVersionsAPI -> "PackageBaseAddress/3.0.0" + | PackageIndex -> "PackageDisplayMetadataUriTemplate" + +// Cache for nuget indices of sources +type ResourceIndex = Map +let private nugetV3Resources = System.Collections.Concurrent.ConcurrentDictionary>() + +let getNuGetV3Resource (source : NugetV3Source) (resourceType : NugetV3ResourceType) : Async = + let key = source + let getResourcesRaw () = + async { + let basicAuth = source.Authentication |> Option.map toCredentials + let! rawData = safeGetFromUrl(basicAuth, source.Url, acceptJson) + let rawData = + match rawData with + | NotFound -> + raise <| new Exception(sprintf "Could not load resources (404) from '%s'" source.Url) + | UnknownError e -> + raise <| new Exception(sprintf "Could not load resources from '%s'" source.Url, e.SourceException) + | SuccessResponse x -> x + + let json = JsonConvert.DeserializeObject(rawData) + let resources = + json.Resources + |> Seq.distinctBy(fun x -> x.Type.ToLower()) + |> Seq.map(fun x -> x.Type.ToLower(), x.ID) + let map = + resources + |> Seq.choose (fun (res, value) -> + let resType = + match res.ToLower() with + | "searchautocompleteservice" -> Some AutoComplete + //| "registrationsbaseurl" -> Some Registration + | s when s.StartsWith "packagedisplaymetadatauritemplate" -> Some PackageIndex + | "packagebaseaddress/3.0.0" -> Some AllVersionsAPI + | _ -> None + match resType with + | None -> None + | Some k -> + Some (k, value)) + |> Seq.distinctBy fst + |> Map.ofSeq + return map + } |> Async.StartAsTask + + async { + let t = nugetV3Resources.GetOrAdd(key, (fun _ -> getResourcesRaw())) + let! res = t |> Async.AwaitTask + return + match res.TryFind resourceType with + | Some s -> s + | None -> failwithf "could not find an %s endpoint for %s" (resourceType.ToString()) source.Url + } + /// [omit] -type JSONResource = +type JSONResource = { Type : string; ID: string } /// [omit] -type JSONVersionData = - { Data : string [] +type JSONVersionData = + { Data : string [] Versions : string [] } /// [omit] -type JSONRootData = +type JSONRootData = { Resources : JSONResource [] } /// [omit] @@ -35,7 +116,7 @@ let private searchDict = new System.Collections.Concurrent.ConcurrentDictionary< let private allVersionsDict = new System.Collections.Concurrent.ConcurrentDictionary<_,System.Threading.Tasks.Task<_>>() /// Calculates the NuGet v3 URL from a NuGet v2 URL. -let calculateNuGet3Path(nugetUrl:string) = +let calculateNuGet3Path(nugetUrl:string) = match nugetUrl.TrimEnd([|'/'|]) with | "http://nuget.org/api/v2" -> Some "http://api.nuget.org/v3/index.json" | "https://nuget.org/api/v2" -> Some "https://api.nuget.org/v3/index.json" @@ -49,7 +130,7 @@ let calculateNuGet3Path(nugetUrl:string) = | _ -> None /// Calculates the NuGet v3 URL from a NuGet v2 URL. -let calculateNuGet2Path(nugetUrl:string) = +let calculateNuGet2Path(nugetUrl:string) = match nugetUrl.TrimEnd([|'/'|]) with | "http://api.nuget.org/v3/index.json" -> Some "http://nuget.org/api/v2" | "https://api.nuget.org/v3/index.json" -> Some "https://nuget.org/api/v2" @@ -63,35 +144,35 @@ let calculateNuGet2Path(nugetUrl:string) = /// [omit] -let getSearchAPI(auth,nugetUrl) = +let getSearchAPI(auth,nugetUrl) = searchDict.GetOrAdd(nugetUrl, fun nugetUrl -> async { match calculateNuGet3Path nugetUrl with | None -> return None - | Some v3Path -> + | Some v3Path -> let source = { Url = v3Path; Authentication = auth } - let! v3res = PackageSources.getNuGetV3Resource source AutoComplete |> Async.Catch - return + let! v3res = getNuGetV3Resource source AutoComplete |> Async.Catch + return match v3res with | Choice1Of2 s -> Some s - | Choice2Of2 ex -> + | Choice2Of2 ex -> if verbose then traceWarnfn "getSearchAPI: %s" (ex.ToString()) None } |> Async.StartAsTask) /// [omit] -let getAllVersionsAPI(auth,nugetUrl) = +let getAllVersionsAPI(auth,nugetUrl) = allVersionsDict.GetOrAdd(nugetUrl, fun nugetUrl -> async { match calculateNuGet3Path nugetUrl with | None -> return None | Some v3Path -> let source = { Url = v3Path; Authentication = auth } - let! v3res = PackageSources.getNuGetV3Resource source AllVersionsAPI |> Async.Catch + let! v3res = getNuGetV3Resource source AllVersionsAPI |> Async.Catch return match v3res with | Choice1Of2 s -> Some s - | Choice2Of2 ex -> + | Choice2Of2 ex -> if verbose then traceWarnfn "getAllVersionsAPI: %s" (ex.ToString()) None } |> Async.StartAsTask) @@ -159,13 +240,13 @@ let extractPackages(response:string) = let private getPackages(auth, nugetURL, packageNamePrefix, maxResults) = async { let! apiRes = getSearchAPI(auth,nugetURL) |> Async.AwaitTask match apiRes with - | Some url -> + | Some url -> let query = sprintf "%s?q=%s&take=%d" url packageNamePrefix maxResults let! response = safeGetFromUrl(auth |> Option.map toCredentials,query,acceptJson) match SafeWebResult.asResult response with | Result.Ok text -> return Result.Ok (extractPackages text) | Result.Error err -> return Result.Error err - | None -> + | None -> if verbose then tracefn "Could not calculate search api from %s" nugetURL return Result.Ok [||] } @@ -176,68 +257,132 @@ let FindPackages(auth, nugetURL, packageNamePrefix, maxResults) = return! getPackages(auth, nugetURL, packageNamePrefix, maxResults) } -type Registration = - { [] - CatalogEntry : string - - [] - PackageContent : string } -type CatalogDependency = +type CatalogDependency = { [] - Id : string - + Id : string [] Range : string } -type CatalogDependencyGroup = +type CatalogDependencyGroup = { [] TargetFramework : string - [] Dependencies : CatalogDependency [] } -type Catalog = +type Catalog = { [] LicenseUrl : string - [] Listed : System.Nullable - + [] + Version : string [] DependencyGroups : CatalogDependencyGroup [] } -let getRegistration (source : NugetV3Source) (packageName:PackageName) (version:SemVerInfo) = + +type PackageIndexPackage = + { [] + Type: string + [] + DownloadLink: string + [] + PackageDetails: Catalog } + +type PackageIndexPage = + { [] + Id: string + [] + Type: string + [] + Packages: PackageIndexPackage [] + [] + Count: int + [] + Lower: string + [] + Upper: string } + +type PackageIndex = + { [] + Id: string + [] + Pages: PackageIndexPage [] + [] + Count : int } + +let private getPackageIndexRaw (source : NugetV3Source) (packageName:PackageName) = async { - let! registrationUrl = PackageSources.getNuGetV3Resource source Registration - let url = sprintf "%s%s/%s.json" registrationUrl (packageName.ToString().ToLower()) (version.Normalize()) + let! registrationUrl = getNuGetV3Resource source PackageIndex + let url = registrationUrl.Replace("{id-lower}", packageName.ToString().ToLower()) // sprintf "%s%s/%s.json" registrationUrl (packageName.ToString().ToLower()) (version.Normalize()) let! rawData = safeGetFromUrl (source.Authentication |> Option.map toCredentials, url, acceptJson) return match rawData with | NotFound -> None //raise <| System.Exception(sprintf "could not get registration data (404) from '%s'" url) | UnknownError err -> raise <| System.Exception(sprintf "could not get registration data from %s" url, err.SourceException) - | SuccessResponse x -> Some (JsonConvert.DeserializeObject(x)) + | SuccessResponse x -> Some (JsonConvert.DeserializeObject(x)) } -let getCatalog url auth = +let private getPackageIndexMemoized = + memoizeAsync (fun (source, packageName) -> getPackageIndexRaw source packageName) +let getPackageIndex source packageName = getPackageIndexMemoized (source, packageName) + + +let private getPackageIndexPageRaw (source:NugetV3Source) (url:string) = async { - let! rawData = safeGetFromUrl (auth, url, acceptJson) + let! rawData = safeGetFromUrl (source.Authentication |> Option.map toCredentials, url, acceptJson) return match rawData with - | NotFound -> - raise <| System.Exception(sprintf "could not get catalog data (404) from '%s'" url) + | NotFound -> raise <| System.Exception(sprintf "could not get registration data (404) from '%s'" url) | UnknownError err -> - raise <| System.Exception(sprintf "could not get catalog data from %s" url, err.SourceException) - | SuccessResponse x -> JsonConvert.DeserializeObject(x) + raise <| System.Exception(sprintf "could not get registration data from %s" url, err.SourceException) + | SuccessResponse x -> JsonConvert.DeserializeObject(x) + } + +let private getPackageIndexPageMemoized = + memoizeAsync (fun (source, url) -> getPackageIndexPageRaw source url) +let getPackageIndexPage source (page:PackageIndexPage) = getPackageIndexPageMemoized (source, page.Id) + + +let getRelevantPage (source:NugetV3Source) (index:PackageIndex) (version:SemVerInfo) = + async { + let pages = + index.Pages + |> Seq.filter (fun p -> SemVer.Parse p.Lower <= version && version <= SemVer.Parse p.Upper) + |> Seq.toList + + match pages with + | [ page ] -> + let! resolvedPage = async { + if page.Count > 0 && (isNull page.Packages || page.Packages.Length = 0) then + return! getPackageIndexPage source page + else return page } + if page.Count > 0 && (isNull page.Packages || page.Packages.Length = 0) then + failwithf "Page should contain packages!" + + let packages = + resolvedPage.Packages + |> Seq.filter (fun p -> SemVer.Parse p.PackageDetails.Version = version) + |> Seq.toList + match packages with + | [ package ] -> return Some package + | _ -> return failwithf "Version '%O' should be part of part of page '%s' but wasn't." version page.Id + | [] -> + return None + | _ :: _ -> + return failwithf "Mulitple pages of V3 index '%s' match with version '%O'" index.Id version } let getPackageDetails (source:NugetV3Source) (packageName:PackageName) (version:SemVerInfo) : Async = async { - let! registrationData = getRegistration source packageName version - match registrationData with + let! pageIndex = getPackageIndex source packageName// version + match pageIndex with | None -> return EmptyResult - | Some registrationData -> - let! catalogData = getCatalog registrationData.CatalogEntry (source.Authentication |> Option.map toCredentials) - + | Some pageIndex -> + let! relevantPage = getRelevantPage source pageIndex version + match relevantPage with + | None -> return EmptyResult + | Some relevantPage -> + let catalogData = relevantPage.PackageDetails let dependencyGroups, dependencies = if catalogData.DependencyGroups = null then [], [] @@ -281,7 +426,7 @@ let getPackageDetails (source:NugetV3Source) (packageName:PackageName) (version: PackageName = packageName.ToString() SourceUrl = source.Url Unlisted = unlisted - DownloadUrl = registrationData.PackageContent + DownloadUrl = relevantPage.DownloadLink LicenseUrl = catalogData.LicenseUrl Version = version.Normalize() CacheVersion = NuGetPackageCache.CurrentCacheVersion } diff --git a/src/Paket.Core/PublicAPI.fs b/src/Paket.Core/PublicAPI.fs index c1793d20f2..4906138d7d 100644 --- a/src/Paket.Core/PublicAPI.fs +++ b/src/Paket.Core/PublicAPI.fs @@ -737,6 +737,14 @@ type Dependencies(dependenciesFileName: string) = doc.Save fileStream static member FixNuspecs (referencesFile:ReferencesFile, nuspecFileList:string list) = + let deps = Dependencies.Locate(Path.GetDirectoryName(referencesFile.FileName)) + let locked = deps.GetLockFile() + + // NuGet has thrown away "group" association, so this is best effort. + let known = + locked.GetPackageHull referencesFile + |> Seq.map (fun kv -> kv.Key |> snd) + |> Set.ofSeq for nuspecFile in nuspecFileList do if not (File.Exists nuspecFile) then @@ -761,8 +769,11 @@ type Dependencies(dependenciesFileName: string) = if node.Name = "dependency" then let packageName = match node.Attributes.["id"] with null -> "" | x -> x.InnerText - - if not (directDeps.Contains (PackageName packageName)) then + let packName = PackageName packageName + // Ignore unknown packages, see https://github.com/fsprojects/Paket/issues/2694 + // TODO: Add some version sanity check here. + // Assert that the version we remove it not newer than what we have in our resolution! + if known.Contains packName && not (directDeps.Contains (PackageName packageName)) then nodesToRemove.Add node |> ignore if nodesToRemove.Count = 0 then diff --git a/src/Paket.Core/Versioning/PackageSources.fs b/src/Paket.Core/Versioning/PackageSources.fs index c800a980e4..99d95a4207 100644 --- a/src/Paket.Core/Versioning/PackageSources.fs +++ b/src/Paket.Core/Versioning/PackageSources.fs @@ -81,81 +81,8 @@ type NugetSource = member x.BasicAuth = x.Authentication |> Option.map toCredentials -type NugetV3SourceResourceJSON = - { [] - Type : string - [] - ID : string } - -type NugetV3SourceRootJSON = - { [] - Resources : NugetV3SourceResourceJSON [] } - type NugetV3Source = NugetSource -//type NugetV3Source = -// { Url : string -// Authentication : NugetSourceAuthentication option } -type NugetV3ResourceType = - | AutoComplete - | AllVersionsAPI - | Registration - - member this.AsString = - match this with - | AutoComplete -> "SearchAutoCompleteService" - | Registration -> "RegistrationsBaseUrl" - | AllVersionsAPI -> "PackageBaseAddress/3.0.0" - -// Cache for nuget indices of sources -type ResourceIndex = Map -let private nugetV3Resources = System.Collections.Concurrent.ConcurrentDictionary>() - -let getNuGetV3Resource (source : NugetV3Source) (resourceType : NugetV3ResourceType) : Async = - let key = source - let getResourcesRaw () = - async { - let basicAuth = source.Authentication |> Option.map toCredentials - let! rawData = safeGetFromUrl(basicAuth, source.Url, acceptJson) - let rawData = - match rawData with - | NotFound -> - raise <| new Exception(sprintf "Could not load resources (404) from '%s'" source.Url) - | UnknownError e -> - raise <| new Exception(sprintf "Could not load resources from '%s'" source.Url, e.SourceException) - | SuccessResponse x -> x - - let json = JsonConvert.DeserializeObject(rawData) - let resources = - json.Resources - |> Seq.distinctBy(fun x -> x.Type.ToLower()) - |> Seq.map(fun x -> x.Type.ToLower(), x.ID) - let map = - resources - |> Seq.choose (fun (res, value) -> - let resType = - match res.ToLower() with - | "searchautocompleteservice" -> Some AutoComplete - | "registrationsbaseurl" -> Some Registration - | "packagebaseaddress/3.0.0" -> Some AllVersionsAPI - | _ -> None - match resType with - | None -> None - | Some k -> - Some (k, value)) - |> Seq.distinctBy fst - |> Map.ofSeq - return map - } |> Async.StartAsTask - - async { - let t = nugetV3Resources.GetOrAdd(key, (fun _ -> getResourcesRaw())) - let! res = t |> Async.AwaitTask - return - match res.TryFind resourceType with - | Some s -> s - | None -> failwithf "could not find an %s endpoint for %s" (resourceType.ToString()) source.Url - } let userNameRegex = Regex("username[:][ ]*[\"]([^\"]*)[\"]", RegexOptions.IgnoreCase ||| RegexOptions.Compiled) let passwordRegex = Regex("password[:][ ]*[\"]([^\"]*)[\"]", RegexOptions.IgnoreCase ||| RegexOptions.Compiled) let authTypeRegex = Regex("authtype[:][ ]*[\"]([^\"]*)[\"]", RegexOptions.IgnoreCase ||| RegexOptions.Compiled) diff --git a/src/Paket.Core/Versioning/SemVer.fs b/src/Paket.Core/Versioning/SemVer.fs index 85951fa5aa..231c2d63ae 100644 --- a/src/Paket.Core/Versioning/SemVer.fs +++ b/src/Paket.Core/Versioning/SemVer.fs @@ -156,6 +156,7 @@ type SemVerInfo = | _ -> invalidArg "yobj" "cannot compare values of different types" + /// Parser which allows to deal with [Semantic Versioning](http://semver.org/) (SemVer). module SemVer = /// Parses the given version string into a SemVerInfo which can be printed using ToString() or compared diff --git a/src/Paket/Paket.fsproj b/src/Paket/Paket.fsproj index 6675ccc49d..12806e85d3 100644 --- a/src/Paket/Paket.fsproj +++ b/src/Paket/Paket.fsproj @@ -31,8 +31,8 @@ Project paket.exe Project - install - C:\proj\testing\VS2017PerfIssue\ + update + C:\proj\testing\ true @@ -42,8 +42,8 @@ 3 - install - C:\proj\testing\VS2017PerfIssue\ + update + C:\proj\testing\ 14.0