Merge pull request #571 from sergey-tihon/master

paket.lock beautification for HTTP specs

src/Paket.Core/DependenciesFile.fs

@@ -107,23 +107,24 @@ module DependenciesFileParser =
     let private ``parse http source`` trimmed =
         let parts = parseDependencyLine trimmed
         let getParts (projectSpec:string) fileSpec =
-            let ``project spec`` =
-                match projectSpec.EndsWith("/") with
-                | false -> projectSpec
-                | true ->  projectSpec.Substring(0, projectSpec.Length-1)
-            let splitted = ``project spec``.Split([|':'; '/'|], StringSplitOptions.RemoveEmptyEntries)
+            let projectSpec = projectSpec.TrimEnd('/')
+            let ``project spec``, commit =
+                match projectSpec.IndexOf('/', 8) with // 8 = "https://".Length
+                | -1 -> projectSpec, "/"
+                | pos ->  projectSpec.Substring(0, pos), projectSpec.Substring(pos)
+            let splitted = projectSpec.TrimEnd('/').Split([|':'; '/'|], StringSplitOptions.RemoveEmptyEntries)
             let fileName = match String.IsNullOrEmpty fileSpec with
                             | true ->
                                 let name = Seq.last splitted
                                 if String.IsNullOrEmpty <| Path.GetExtension(name)
                                 then name + ".fs" else name
                             | false -> fileSpec
-            match splitted |> Seq.truncate 4 |> Seq.toArray with
+            let owner =
+                match ``project spec``.IndexOf("://") with
+                | -1 -> ``project spec``
+                | pos ->  ``project spec``.Substring(pos+3)
             //SourceFile(origin(url), (owner,project, commit), path)
-            | [| protocol; domain |] -> HttpLink(``project spec``), (domain, domain, None), fileName
-            | [| protocol; domain; project |] -> HttpLink(``project spec``), (domain,project, None), fileName
-            | [| protocol; owner; project; details |] -> HttpLink(``project spec``), (owner,project+"/"+details, None), fileName
-            | _ -> failwithf "invalid http-reference specification:%s     %s" Environment.NewLine trimmed
+            HttpLink(``project spec``), (owner, "", Some commit), fileName
         match parts with
         | [| _; projectSpec; |] -> getParts projectSpec String.Empty
         | [| _; projectSpec; fileSpec |] -> getParts projectSpec fileSpec

src/Paket.Core/ModuleResolver.fs

@@ -22,7 +22,7 @@ type UnresolvedSourceFile =
     override this.ToString() =
         let name = if this.Name = Constants.FullProjectSourceFileName then "" else " " + this.Name
         match this.Origin with
-        | HttpLink url -> sprintf "http %s%s" url name
+        | HttpLink url -> sprintf "http %s%s %s" url this.Commit.Value this.Name
         | _ ->
             let link = 
                 match this.Origin with
@@ -54,10 +54,11 @@ type ResolvedSourceFile =
 let private getCommit (file : UnresolvedSourceFile) = defaultArg file.Commit "master"
 
 let resolve getDependencies getSha1 (file : UnresolvedSourceFile) : ResolvedSourceFile = 
-    let sha = 
-        file
-        |> getCommit
-        |> getSha1 file.Origin file.Owner file.Project
+    let sha =
+        let commit = getCommit file
+        match file.Origin with
+        | SingleSourceFileOrigin.HttpLink _  ->  commit
+        | _ -> getSha1 file.Origin file.Owner file.Project commit
 
     let resolved = 
         { Commit = sha
@@ -79,10 +80,7 @@ let private detectConflicts (remoteFiles : UnresolvedSourceFile list) : unit =
         remoteFiles
         |> Seq.groupBy (fun file ->
             let directoryName =
-                let path = normalizePath (file.Name.TrimStart('/'))
-                match path.LastIndexOfAny([| '/'; '\\' |]) with
-                | -1 -> ""
-                | x  -> path.Substring(0, x)
+                normalizePath (file.Name.TrimStart('/'))
             file.Owner, file.Project, directoryName)
         |> Seq.map (fun (key, files) -> key, files |> Seq.map getCommit |> Seq.distinct)
         |> Seq.filter (snd >> Seq.length >> (<) 1)

src/Paket.Core/RemoteDownload.fs

@@ -115,7 +115,8 @@ let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destination) = async {
         DirectoryCopy(source,projectPath,true)        
     | SingleSourceFileOrigin.GistLink, _ ->  return! downloadFromUrl(None,rawGistFileUrl remoteFile.Owner remoteFile.Project remoteFile.Name) destination
     | SingleSourceFileOrigin.GitHubLink, _ -> return! downloadFromUrl(None,rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destination
-    | SingleSourceFileOrigin.HttpLink(url), _ ->
+    | SingleSourceFileOrigin.HttpLink(origin), _ ->
+        let url = origin + remoteFile.Commit
         do! downloadFromUrl(None, url) destination
         match Path.GetExtension(destination).ToLowerInvariant() with
         | ".zip" ->

tests/Paket.Tests/DependenciesFile/ParserSpecs.fs

@@ -278,15 +278,15 @@ let ``should read http source file from config without quotes with file specs``(
     dependencies.RemoteFiles
     |> shouldEqual
         [ { Owner = "www.fssnip.net"
-            Project = "raw/1M"
+            Project = ""
             Name = "test1.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M"
-            Commit = None }
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net"
+            Commit = Some "/raw/1M" }
           { Owner = "www.fssnip.net"
-            Project = "raw/1M"
+            Project = ""
             Name = "src/test2.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M/1"
-            Commit = None } ]
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net"
+            Commit = Some "/raw/1M/1" } ]
 
 [<Test>]
 let ``should read gist source file from config without quotes with file specs``() =
@@ -335,30 +335,30 @@ let ``should read http source file from config without quotes, parsing rules``()
     dependencies.RemoteFiles
     |> shouldEqual
         [ { Owner = "example"
-            Project = "example"
+            Project = ""
             Name = "example.fs"
             Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example"
-            Commit = None }
+            Commit = Some "/" }
           { Owner = "example"
-            Project = "item"
+            Project = ""
             Name = "item.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item"
-            Commit = None }
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example"
+            Commit = Some "/item" }
           { Owner = "example"
-            Project = "item"
+            Project = ""
             Name = "item.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item"
-            Commit = None }
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example"
+            Commit = Some "/item" }
           { Owner = "example"
-            Project = "item/3"
+            Project = ""
             Name = "3.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3"
-            Commit = None }
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example"
+            Commit = Some "/item/3" }
           { Owner = "example"
-            Project = "item/3"
+            Project = ""
             Name = "1.fs"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3/1"
-            Commit = None } ]
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example"
+            Commit = Some "/item/3/1" } ]
 
 [<Test>]
 let ``should read http binary references from config``() =
@@ -369,15 +369,15 @@ let ``should read http binary references from config``() =
     dependencies.RemoteFiles
     |> shouldEqual
         [ { Owner = "www.frijters.net"
-            Project = "ikvmbin-8.0.5449.0.zip"
+            Project = ""
             Name = "ikvmbin-8.0.5449.0.zip"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net/ikvmbin-8.0.5449.0.zip"
-            Commit = None }
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net"
+            Commit = Some "/ikvmbin-8.0.5449.0.zip" }
           { Owner = "www.frijters.net"
-            Project = "ikvmbin-8.0.5449.0.zip"
+            Project = ""
             Name = "ikvmbin.zip"
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net/ikvmbin-8.0.5449.0.zip"
-            Commit = None } ]
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net"
+            Commit = Some "/ikvmbin-8.0.5449.0.zip" } ]
 
 
 let configWithoutVersions = """

tests/Paket.Tests/Lockfile/GeneratorSpecs.fs

@@ -130,10 +130,21 @@ let ``should generate other version ranges for packages``() =
     |> LockFileSerializer.serializePackages cfg.Options
     |> shouldEqual (normalizeLineEndings expected3)
 
+let trivialResolve (f:ModuleResolver.UnresolvedSourceFile) =
+    { Commit =
+        match f.Commit with
+        | Some(v) -> v
+        | None -> ""
+      Owner = f.Owner
+      Origin = f.Origin
+      Project = f.Project
+      Dependencies = Set.empty
+      Name = f.Name } : ModuleResolver.ResolvedSourceFile
+
 let expectedWithHttp = """HTTP
-  remote: http://www.fssnip.net/raw/1M
+  remote: http://www.fssnip.net
   specs:
-    test.fs"""
+    test.fs (/raw/1M)"""
 
 [<Test>]
 let ``should generate lock file for http source files``() = 
@@ -142,34 +153,23 @@ let ``should generate lock file for http source files``() =
     let cfg = DependenciesFile.FromCode(config)
 
     cfg.RemoteFiles
-    |> List.map (fun f -> 
-          { Commit = ""
-            Owner = f.Owner
-            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M"
-            Project = f.Project
-            Dependencies = Set.empty
-            Name = f.Name } : ModuleResolver.ResolvedSourceFile)
+    |> List.map trivialResolve
     |> LockFileSerializer.serializeSourceFiles
     |> shouldEqual (normalizeLineEndings expectedWithHttp)
 
 let expectedMultiple = """HTTP
-  remote: http://www.fssnip.net/raw/32
+  remote: http://www.fssnip.net
   specs:
-    myFile2.fs
+    myFile.fs (/raw/1M)
+    myFile2.fs (/raw/32)
+    myFile3.fs (/raw/15)
 GIST
   remote: Thorium/1972308
   specs:
     gistfile1.fs
   remote: Thorium/6088882
   specs:
-    FULLPROJECT
-HTTP
-  remote: http://www.fssnip.net/raw/1M
-  specs:
-    myFile.fs
-  remote: http://www.fssnip.net/raw/15
-  specs:
-    myFile3.fs"""
+    FULLPROJECT"""
 
 [<Test>]
 let ``should generate lock file for http and gist source files``() = 
@@ -186,13 +186,35 @@ http http://www.fssnip.net/raw/15 myFile3.fs """
     let cfg = DependenciesFile.FromCode(config)
 
     cfg.RemoteFiles
-    |> List.map (fun f -> 
-          { Commit = ""
-            Owner = f.Owner
-            Origin = f.Origin
-            Project = f.Project
-            Dependencies = Set.empty
-            Name = f.Name } : ModuleResolver.ResolvedSourceFile)
+    |> List.map trivialResolve
     |> LockFileSerializer.serializeSourceFiles
     |> shouldEqual (normalizeLineEndings expectedMultiple)
 
+
+let expectedForStanfordNLPdotNET = """HTTP
+  remote: http://www.frijters.net
+  specs:
+    ikvmbin-8.0.5449.0.zip (/ikvmbin-8.0.5449.0.zip)
+  remote: http://nlp.stanford.edu
+  specs:
+    stanford-corenlp-full-2014-10-31.zip (/software/stanford-corenlp-full-2014-10-31.zip)
+    stanford-ner-2014-10-26.zip (/software/stanford-ner-2014-10-26.zip)
+    stanford-parser-full-2014-10-31.zip (/software/stanford-parser-full-2014-10-31.zip)
+    stanford-postagger-full-2014-10-26.zip (/software/stanford-postagger-full-2014-10-26.zip)
+    stanford-segmenter-2014-10-26.zip (/software/stanford-segmenter-2014-10-26.zip)"""
+
+[<Test>]
+let ``should generate lock file for http Stanford.NLP.NET project``() =
+    let config = """http http://www.frijters.net/ikvmbin-8.0.5449.0.zip
+http http://nlp.stanford.edu/software/stanford-corenlp-full-2014-10-31.zip
+http http://nlp.stanford.edu/software/stanford-ner-2014-10-26.zip
+http http://nlp.stanford.edu/software/stanford-parser-full-2014-10-31.zip
+http http://nlp.stanford.edu/software/stanford-postagger-full-2014-10-26.zip
+http http://nlp.stanford.edu/software/stanford-segmenter-2014-10-26.zip"""
+
+    let cfg = DependenciesFile.FromCode(config)
+
+    cfg.RemoteFiles
+    |> List.map trivialResolve
+    |> LockFileSerializer.serializeSourceFiles
+    |> shouldEqual (normalizeLineEndings expectedForStanfordNLPdotNET)

tests/Paket.Tests/Resolver/ConflictSourcesSpecs.fs

@@ -42,25 +42,21 @@ let config2 = """
 source "http://nuget.org/api/v2"
 
 github fsharp/fsharp:master foo.fs
-github fsharp/fsharp:fsharp4 bar.fs
+github fsharp/fsharp:fsharp4 foo.fs
 github fsprojects/FAKE:master test.fs
 github fsprojects/FAKE:vNext readme.md
 """
 
 let expectedError = """Found conflicting source file requirements:
-   - fsharp/fsharp
+   - fsharp/fsharpfoo.fs
      Versions:
      - master
      - fsharp4
-   - fsprojects/FAKE
-     Versions:
-     - master
-     - vNext
    Currently multiple versions for same source directory are not supported.
    Please adjust the dependencies file.""" |> normalizeLineEndings
 
 [<Test>]
-let ``should fail resolving source files from same repository but different versions``() =
+let ``should fail resolving same source files from same repository but different versions``() =
     try
         let cfg = DependenciesFile.FromCode(config2)
         ModuleResolver.Resolve(noGitHubConfigured, noGitHubConfigured, cfg.RemoteFiles) |> ignore