fsprojects#3127 storage symlink feature

docs/content/dependencies-file.md

@@ -203,6 +203,16 @@ nuget jQuery
 The storage option may be overriden by packages. 
 However, the behavior is undefined and may change (please open an issue if you depend on the current behavior or we break you).
 
+```paket
+// make a symlink instead copy the packages.
+storage: symlink
+source https://nuget.org/api/v2
+
+nuget jQuery
+```
+In this mode, paket will use a directory symbolic link (soft) between nuget cache and packages folder.
+Symlink option can save a disk space on CI server.
+
 ### Controlling whether content files should be copied to the project
 
 The `content` option controls the installation of any content files:

integrationtests/Paket.IntegrationTests/Paket.IntegrationTests.fsproj

@@ -123,6 +123,7 @@
     <Compile Include="LoadingScriptGenerationTests.fs" />
     <Compile Include="LocalOverrideSpecs.fs" />
     <Compile Include="SimplifierSpecs.fs" />
+    <Compile Include="SymbolicLinkSpecs.fs" />
     <None Include="paket.references" />
     <Content Include="App.config" />
     <Compile Include="AddGithubSpecs.fs" />

integrationtests/Paket.IntegrationTests/SymbolicLinkSpecs.fs

@@ -0,0 +1,90 @@
+module Paket.IntegrationTests.SymbolicLinkSpecs
+
+open NUnit.Framework
+open Fake
+open FsUnit
+open Paket
+
+[<Test>]
+let ``#3127 symlink enabled on all dependencies and empty paket.lock``() =
+    clearPackageAtVersion "NUnit" "2.6.3"
+    let scenario = "i003127-storage-symlink"
+    let workingDir = scenarioTempPath scenario
+    paketEx true "update" scenario |> ignore
+
+    workingDir </> "packages" </> "NUnit" 
+    |> SymlinkUtils.isDirectoryLink 
+    |> shouldEqual true
+
+let paketDependencies workingDir content = 
+    let d = DependenciesFile.FromSource(workingDir, content)
+    d.Save()
+    d
+
+let storageConfig (x:DependenciesFile) = x.Groups.[Domain.GroupName Domain.MainGroup].Options.Settings.StorageConfig 
+
+[<Test>]
+let ``#3127 symlink enabled -> disabled on all dependencies on existing paket.lock``() =
+    clearPackageAtVersion "NUnit" "2.6.3"
+
+    let scenario = "i003127-storage-symlink"
+    let workingDir = scenarioTempPath scenario
+
+    let packagesDir = workingDir </> "packages"    
+
+    paketEx true "install" scenario |> ignore
+
+    workingDir </> "paket.dependencies" |> Paket.DependenciesFile.ReadFromFile     
+    |> storageConfig
+    |> shouldEqual (Some PackagesFolderGroupConfig.SymbolicLink)
+
+    packagesDir </> "NUnit" 
+    |> SymlinkUtils.isDirectoryLink 
+    |> shouldEqual true
+
+    let paketDependenciesWithoutConfig = """source https://www.nuget.org/api/v2
+nuget NUnit < 3.0.0"""
+
+    paketDependenciesWithoutConfig
+    |> paketDependencies workingDir
+    |> storageConfig
+    |> shouldEqual None
+
+    directPaketEx "update" scenario |> ignore
+
+    packagesDir </> "NUnit" 
+    |> SymlinkUtils.isDirectoryLink 
+    |> shouldEqual false
+
+[<Test>]
+let ``#3127 symlink disabled -> enabled on all dependencies on existing paket.lock``() =
+    clearPackageAtVersion "NUnit" "2.6.3"
+
+    let scenario = "i003127-storage-symlink"
+    let workingDir = scenarioTempPath scenario
+
+    let packagesDir = workingDir </> "packages"
+
+    let paketDependenciesPath = workingDir </> "paket.dependencies"
+
+    paketDependenciesPath |> Paket.DependenciesFile.ReadFromFile |> storageConfig |> shouldEqual (Some PackagesFolderGroupConfig.SymbolicLink)
+
+    paketEx true "install" scenario |> ignore
+
+    packagesDir </> "NUnit" 
+    |> SymlinkUtils.isDirectoryLink 
+    |> shouldEqual true
+
+    let paketDependenciesWithoutConfig = """source https://www.nuget.org/api/v2
+nuget NUnit < 3.0.0"""
+
+    paketDependenciesWithoutConfig 
+    |> paketDependencies workingDir
+    |> storageConfig
+    |> shouldEqual None
+
+    directPaketEx "update" scenario |> ignore
+
+    packagesDir </> "NUnit" 
+    |> SymlinkUtils.isDirectoryLink 
+    |> shouldEqual false

integrationtests/scenarios/i003127-storage-symlink/before/paket.dependencies

@@ -0,0 +1,3 @@
+source https://www.nuget.org/api/v2
+storage: symlink
+nuget NUnit < 3.0.0

src/Paket.Core.preview3/Paket.Core.fsproj

@@ -28,6 +28,7 @@
     <Compile Include="$(PaketCoreSourcesDir)\Common\Utils.fs" />
     <Compile Include="$(PaketCoreSourcesDir)\Common\Xml.fs" />
     <Compile Include="$(PaketCoreSourcesDir)\Common\ProcessHelper.fs" />
+    <Compile Include="$(PaketCoreSourcesDir)\Common\SymlinkUtils.fs" />
     <Compile Include="$(PaketCoreSourcesDir)\Common\NetUtils.fs" />
     <Compile Include="$(PaketCoreSourcesDir)\Versioning\SemVer.fs" />
     <Compile Include="$(PaketCoreSourcesDir)\Versioning\VersionRange.fs" />

src/Paket.Core/Common/SymlinkUtils.fs

@@ -0,0 +1,35 @@
+module SymlinkUtils
+
+open System
+open Paket
+open System.Diagnostics
+
+let isDirectoryLink directory = 
+    let di = IO.DirectoryInfo(directory)
+    di.Exists && di.Attributes.HasFlag(IO.FileAttributes.ReparsePoint)
+
+/// delete the symlink only (do not remove files before)
+let delete directory = if isDirectoryLink directory then IO.Directory.Delete directory
+
+let makeDirectoryLink target source = 
+    let mklink (p:ProcessStartInfo) = 
+        p.FileName <- "cmd.exe"
+        p.Arguments <- sprintf @"/c ""mklink /D ""%s"" ""%s""""" target source
+
+    let ln (p:ProcessStartInfo) = 
+        p.FileName <- "ln"
+        p.Arguments <- sprintf @"-sT ""%s"" ""%s""" target source
+
+    let xLn = if isUnix then ln else mklink
+
+    let r = ProcessHelper.ExecProcessAndReturnMessages xLn (TimeSpan.FromSeconds(10.))
+
+    match r.OK, Logging.verbose with
+    | true, true -> 
+        let m = ProcessHelper.toLines r.Messages
+        sprintf "symlink used %s -> %s (%s)" source target m |> Logging.traceVerbose
+    | true, false -> ()
+    | false, _ ->
+        let m = ProcessHelper.toLines r.Messages
+        let e = ProcessHelper.toLines r.Errors
+        failwithf "symlink %s -> %s failed with error : [%i] with output : %s%s and error : %s" source target r.ExitCode m Environment.NewLine e

src/Paket.Core/Common/Utils.fs

@@ -401,22 +401,26 @@ type ResolvedPackagesFolder =
     /// No "packages" folder for the current package
     | NoPackagesFolder
     /// the /packages/group/ExtractedPackage.X.Y.Z folder
+    | SymbolicLink of string
     | ResolvedFolder of string
     member x.Path =
         match x with
         | NoPackagesFolder -> None
+        | SymbolicLink f
         | ResolvedFolder f -> Some f
 
 type PackagesFolderGroupConfig =
     | NoPackagesFolder
     | GivenPackagesFolder of string
+    | SymbolicLink
     | DefaultPackagesFolder
     member x.ResolveGroupDir root groupName =
         match x with
         | NoPackagesFolder -> None
         | GivenPackagesFolder p ->
             // relative to root
             Some p
+        | SymbolicLink
         | DefaultPackagesFolder ->
             let groupDir =
                 if groupName = Constants.MainDependencyGroup then
@@ -425,16 +429,20 @@ type PackagesFolderGroupConfig =
                     Path.Combine(root, Constants.DefaultPackagesFolderName, groupName.CompareString)
             Some groupDir
     member x.Resolve root groupName (packageName:PackageName) version includeVersionInPath  =
+        let parentPath () = 
+            let groupDir = x.ResolveGroupDir root groupName |> Option.get
+            let packageFolder = string packageName + (if includeVersionInPath then "." + string version else "")
+            Path.Combine(groupDir, packageFolder)
+
         match x with
         | NoPackagesFolder -> ResolvedPackagesFolder.NoPackagesFolder
         | GivenPackagesFolder p ->
             // relative to root
             ResolvedPackagesFolder.ResolvedFolder p
+        | SymbolicLink ->
+            parentPath () |> ResolvedPackagesFolder.SymbolicLink
         | DefaultPackagesFolder ->
-            let groupDir = x.ResolveGroupDir root groupName |> Option.get
-            let packageFolder = string packageName + (if includeVersionInPath then "." + string version else "")
-            let parent = Path.Combine(groupDir, packageFolder)
-            ResolvedPackagesFolder.ResolvedFolder parent
+            parentPath () |> ResolvedPackagesFolder.ResolvedFolder
     static member Default = DefaultPackagesFolder
 
 

src/Paket.Core/Dependencies/DependenciesFileParser.fs

@@ -295,6 +295,7 @@ module DependenciesFileParser =
             let setting =
                 match trimmed.Replace(":","").Trim() with
                 | String.EqualsIC "none" -> Some PackagesFolderGroupConfig.NoPackagesFolder
+                | String.EqualsIC "symlink" -> Some PackagesFolderGroupConfig.SymbolicLink
                 | String.EqualsIC "packages" -> Some PackagesFolderGroupConfig.DefaultPackagesFolder
                 | _ -> None
 

src/Paket.Core/Dependencies/NuGet.fs

@@ -806,21 +806,36 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr
                 | exn -> raise (Exception(sprintf "Could not download %O %O from %s." packageName version !downloadUrl, exn)) }
 
     async {
+        configResolved.Path |> Option.iter SymlinkUtils.delete
+
         do! download true 0
-        if not isLocalOverride then
+
+        match isLocalOverride, configResolved with
+        | true, ResolvedPackagesFolder.NoPackagesFolder -> return failwithf "paket.local in combination with storage:none is not supported (use storage: symlink instead)"
+        | true, ResolvedPackagesFolder.SymbolicLink directory
+        | true, ResolvedPackagesFolder.ResolvedFolder directory ->
+            let! folder = ExtractPackage(targetFile.FullName, directory, packageName, version, detailed)
+            return targetFileName,folder
+        | false, ResolvedPackagesFolder.SymbolicLink folder -> 
+            folder |> Utils.DirectoryInfo |> Utils.deleteDir 
+            ensureDir folder
+
             let! extractedUserFolder = ExtractPackageToUserFolder(targetFile.FullName, packageName, version, kind)
-            let! files = NuGetCache.CopyFromCache(configResolved, targetFile.FullName, licenseFileName, packageName, version, force, detailed)
+
+            SymlinkUtils.makeDirectoryLink folder extractedUserFolder
+
+            let packageFilePath = Path.Combine(extractedUserFolder, NuGetCache.GetPackageFileName packageName version)
+            return packageFilePath, folder
+        | false, otherConfig ->
+            otherConfig.Path |> Option.iter SymlinkUtils.delete
+
+            let! extractedUserFolder = ExtractPackageToUserFolder(targetFile.FullName, packageName, version, kind)
+            let! files = NuGetCache.CopyFromCache(otherConfig, targetFile.FullName, licenseFileName, packageName, version, force, detailed)
             let finalFolder =
                 match files with
                 | Some f -> f
                 | None -> extractedUserFolder
             return targetFileName,finalFolder
-        else
-            match configResolved.Path with
-            | None -> return failwithf "paket.local in combination with storage:none is not supported"
-            | Some directory ->
-                let! folder = ExtractPackage(targetFile.FullName, directory, packageName, version, detailed)
-                return targetFileName,folder
     }
 
 

src/Paket.Core/Installation/InstallProcess.fs

@@ -39,6 +39,7 @@ let findPackageFolder root (groupName,packageName) (version,settings) =
     let includeVersionInPath = defaultArg settings.IncludeVersionInPath false
     let storageOption = defaultArg settings.StorageConfig PackagesFolderGroupConfig.Default
     match storageOption.Resolve root groupName packageName version includeVersionInPath with
+    | ResolvedPackagesFolder.SymbolicLink targetFolder 
     | ResolvedPackagesFolder.ResolvedFolder targetFolder ->
         let direct = DirectoryInfo targetFolder
         if direct.Exists then

src/Paket.Core/Paket.Core.fsproj

@@ -81,6 +81,7 @@
     <Compile Include="Common\Utils.fs" />
     <Compile Include="Common\Xml.fs" />
     <Compile Include="Common\ProcessHelper.fs" />
+    <Compile Include="Common\SymlinkUtils.fs" />
     <Compile Include="Common\NetUtils.fs" />
     <Compile Include="Versioning\SemVer.fs" />
     <Compile Include="Versioning\VersionRange.fs" />

src/Paket.Core/PaketConfigFiles/DependenciesFile.fs

@@ -217,6 +217,7 @@ type DependenciesFile(fileName,groups:Map<GroupName,DependenciesGroup>, textRepr
     member this.Resolve(force, getSha1, getVersionF, getPreferredVersionF, getPackageDetailsF, getPackageRuntimeGraph, groupsToResolve:Map<GroupName,_>, updateMode) =
         let resolveGroup groupName _ =
             let group = this.GetGroup groupName
+            let storageConfig = group.Options.Settings.StorageConfig
 
             let resolveSourceFile (file:ResolvedSourceFile) : (PackageRequirement list * UnresolvedSource list) =
                 let remoteDependenciesFile =
@@ -272,7 +273,7 @@ type DependenciesFile(fileName,groups:Map<GroupName,DependenciesGroup>, textRepr
                     let runtimeGraph =
                         resolved
                         |> Map.toSeq |> Seq.map snd
-                        |> Seq.choose (getPackageRuntimeGraph groupName)
+                        |> Seq.choose (getPackageRuntimeGraph storageConfig groupName)
                         |> RuntimeGraph.mergeSeq
                     // now we need to get the runtime deps and add them to the resolution
                     let rids = RuntimeGraph.getKnownRids runtimeGraph

src/Paket.Core/PaketConfigFiles/LockFile.fs

@@ -39,6 +39,7 @@ module LockFileSerializer =
         | None -> ()
         match options.Settings.StorageConfig with
         | Some (PackagesFolderGroupConfig.NoPackagesFolder) -> yield "STORAGE: NONE"
+        | Some (PackagesFolderGroupConfig.SymbolicLink) -> yield "STORAGE: SYMLINK"
         | Some (PackagesFolderGroupConfig.DefaultPackagesFolder) -> yield "STORAGE: PACKAGES"
         | Some (PackagesFolderGroupConfig.GivenPackagesFolder f) -> failwithf "Not implemented yet."
         | None -> ()
@@ -297,6 +298,7 @@ module LockFileParser =
             let setting =
                 match trimmed.Trim() with
                 | String.EqualsIC "NONE" -> Some PackagesFolderGroupConfig.NoPackagesFolder
+                | String.EqualsIC "SYMLINK" -> Some PackagesFolderGroupConfig.SymbolicLink
                 | String.EqualsIC "PACKAGES" -> Some PackagesFolderGroupConfig.DefaultPackagesFolder
                 | _ -> None
 

src/Paket.Core/PaketConfigFiles/RuntimeGraph.fs

@@ -203,14 +203,13 @@ module RuntimeGraph =
 
     open System.IO
     /// Downloads the given package into the nuget cache and read its runtime.json.
-    let getRuntimeGraphFromNugetCache root groupName (package:ResolvedPackage) =
-        let config = PackagesFolderGroupConfig.NoPackagesFolder
+    let getRuntimeGraphFromNugetCache root config groupName (package:ResolvedPackage) =
+        let defaultedConfig = config |> Option.defaultValue PackagesFolderGroupConfig.NoPackagesFolder
         // 1. downloading packages into cache
-        let targetFileName, _ =
-            NuGet.DownloadAndExtractPackage (None, root, false, config, package.Source, [], groupName, package.Name, package.Version, package.Kind, false, defaultArg package.Settings.LicenseDownload false, false, false)
+        let _, extractedDir =
+            NuGet.DownloadAndExtractPackage (None, root, false, defaultedConfig, package.Source, [], groupName, package.Name, package.Version, package.Kind, false, defaultArg package.Settings.LicenseDownload false, false, false)
             |> Async.RunSynchronously
 
-        let extractedDir = NuGetCache.ExtractPackageToUserFolder (targetFileName, package.Name, package.Version, package.Kind) |> Async.RunSynchronously
         // 2. Get runtime graph
         try
             let runtime = Path.Combine(extractedDir, "runtime.json")

src/Paket.Core/Versioning/Requirements.fs

@@ -881,6 +881,7 @@ type InstallSettings =
               | None -> ()
               match this.StorageConfig with
               | Some (PackagesFolderGroupConfig.NoPackagesFolder) -> yield "storage: none"
+              | Some (PackagesFolderGroupConfig.SymbolicLink) -> yield "storage: symlink"
               | Some (PackagesFolderGroupConfig.GivenPackagesFolder s) -> failwithf "Not implemented yet."
               | Some (PackagesFolderGroupConfig.DefaultPackagesFolder) -> yield "storage: packages"
               | None -> ()
@@ -961,6 +962,7 @@ type InstallSettings =
               StorageConfig =
                 match getPair "storage" with
                 | Some "packages" -> Some (PackagesFolderGroupConfig.DefaultPackagesFolder)
+                | Some "symlink" -> Some (PackagesFolderGroupConfig.SymbolicLink)
                 | Some "none" -> Some (PackagesFolderGroupConfig.NoPackagesFolder)
                 | _ -> None
               FrameworkRestrictions =

tests/Paket.Tests/DependenciesFile/StorageSpecs.fs

@@ -0,0 +1,17 @@
+module StorageSpecs
+
+open Paket
+open NUnit.Framework
+open FsUnit
+
+[<Test>]
+let ``should configure the symlink option``() = 
+    let dependencies = """framework: >= net40
+storage: symlink
+source https://www.nuget.org/api/v2
+
+nuget NLog framework: net40
+nuget NLog.Contrib"""
+
+    let cfg = DependenciesFile.FromSource(dependencies)
+    cfg.Groups.[Constants.MainDependencyGroup].Options.Settings.StorageConfig |> shouldEqual (Some PackagesFolderGroupConfig.SymbolicLink)

tests/Paket.Tests/Paket.Tests.fsproj

@@ -191,6 +191,7 @@
     <Content Include="NuGetConfig\ConfigWithDisabledFeedFromUpstream.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Compile Include="DependenciesFile\StorageSpecs.fs" />
     <Compile Include="DependenciesFile\VersionRangeSpecs.fs" />
     <Compile Include="DependenciesFile\VersionRequirementSpecs.fs" />
     <Compile Include="DependenciesFile\ParserSpecs.fs" />

tests/Paket.Tests/Resolver/DependencyGroupsAndRestrictions.fs

@@ -10,7 +10,7 @@ open Paket.Requirements
 
 let resolve graph updateMode (cfg : DependenciesFile) =
     let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq
-    cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail()
+    cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail()
 
 let graph1 = 
   GraphOfNuspecs [

tests/Paket.Tests/Resolver/GlobalOptimisticStrategySpecs.fs

@@ -9,7 +9,7 @@ open Paket.PackageResolver
 
 let resolve graph updateMode (cfg : DependenciesFile) =
     let groups = [Constants.MainDependencyGroup, None ] |> Map.ofSeq
-    cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail()
+    cfg.Resolve(true,noSha1,VersionsFromGraphAsSeq graph, (fun _ _ -> []),PackageDetailsFromGraph graph,(fun _ _ _ -> None),groups,updateMode).[Constants.MainDependencyGroup].ResolvedPackages.GetModelOrFail()
 
 let graph =
   OfSimpleGraph [