Github files (WIP #9)

src/Paket/BasicTypes.fs

@@ -84,6 +84,23 @@ type PackageSource =
         | true, uri -> if uri.Scheme = System.Uri.UriSchemeFile then LocalNuget(source) else Nuget(source)
         | _ -> failwith "unable to parse package source: %s" source
 
+// Represents details on a dependent source file.
+//TODO: As new sources e.g. fssnip etc. are added, this should probably become a DU or perhaps have an enum marker.
+type SourceFile =
+    { Owner : string
+      Project : string
+      Path : string
+      Commit : string option }
+    member this.FilePath =
+        this.Path
+            .TrimStart('/')
+            .Replace("/", "\\")
+    member this.CommitWithDefault = defaultArg this.Commit "master"
+    override this.ToString() = sprintf "(%s:%s:%s) %s" this.Owner this.Project this.CommitWithDefault this.Path
+
+//TODO: Perhaps Source File and Package should be merged into a DU?
+/// Represents a package.
+type Package = 
 
 /// Represents an unresolved package.
 type UnresolvedPackage =
@@ -131,4 +148,4 @@ type ResolvedDependency =
     | Conflict of Dependency * Dependency
 
 /// Represents a complete dependency resolution.
-type PackageResolution = Map<string,ResolvedDependency>
+type PackageResolution = Map<string,ResolvedDependency>

src/Paket/DependenciesFile.fs

@@ -33,24 +33,34 @@ module DependenciesFileParser =
         with
         | _ -> failwithf "could not parse version range \"%s\"" text
 
-    let private (|Remote|Package|Blank|) (line:string) =
+    let private (|Remote|Package|Blank|SourceFile|) (line:string) =
         match line.Trim() with
         | _ when String.IsNullOrWhiteSpace line -> Blank
         | trimmed when trimmed.StartsWith "source" -> 
             let fst = trimmed.IndexOf("\"")
             let snd = trimmed.IndexOf("\"",fst+1)
             Remote (trimmed.Substring(fst,snd-fst).Replace("\"",""))                
         | trimmed when trimmed.StartsWith "nuget" -> Package(trimmed.Replace("nuget","").Trim())
+        | trimmed when trimmed.StartsWith "github" ->
+            let parts = trimmed.Replace("\"", "").Split ' '
+            let getParts (projectSpec:string) =
+                match projectSpec.Split ':' with
+                | [| owner; project |] -> owner, project, None
+                | [| owner; project; commit |] -> owner, project, Some commit
+                | _ -> failwith "invalid github specification"
+            match parts with
+            | [| _; projectSpec; fileSpec |] -> SourceFile(getParts projectSpec, fileSpec)
+            | _ -> failwith "invalid github specification"
         | _ -> Blank
 
     let parseDependenciesFile (lines:string seq) = 
-        ((0,[], []), lines)
-        ||> Seq.fold(fun (lineNo, sources: PackageSource list, packages) line ->
+        ((0,[], [], []), lines)
+        ||> Seq.fold(fun (lineNo, sources: PackageSource list, packages, sourceFiles) line ->
             let lineNo = lineNo + 1
             try
                 match line with
-                | Remote newSource -> lineNo, (PackageSource.Parse(newSource.TrimEnd([|'/'|])) :: sources), packages
-                | Blank -> lineNo, sources, packages
+                | Remote newSource -> lineNo, (PackageSource.Parse(newSource.TrimEnd([|'/'|])) :: sources), packages, sourceFiles
+                | Blank -> lineNo, sources, packages, sourceFiles
                 | Package details ->
                     let parts = details.Split('"')
                     if parts.Length < 4 || String.IsNullOrWhiteSpace parts.[1] || String.IsNullOrWhiteSpace parts.[3] then
@@ -59,19 +69,30 @@ module DependenciesFileParser =
                     lineNo, sources, { Sources = sources
                                        Name = parts.[1]
                                        ResolverStrategy = if version.StartsWith "!" then ResolverStrategy.Min else ResolverStrategy.Max
-                                       VersionRange = parseVersionRange(version.Trim '!') } :: packages
+                                       VersionRange = parseVersionRange(version.Trim '!') } :: packages, sourceFiles
+                | SourceFile((owner,project, commit), path) ->
+                    let newSourceFile = { Owner = owner
+                                          Project = project
+                                          Commit = commit
+                                          Path = path }
+                    tracefn "  %O" newSourceFile
+                    lineNo, sources, packages, newSourceFile :: sourceFiles
+
             with
             | exn -> failwithf "Error in paket.dependencies line %d%s  %s" lineNo Environment.NewLine exn.Message)
-        |> fun (_,_,x) -> x
-        |> List.rev
+        |> fun (_,_,packages,remoteFiles) ->
+            packages |> List.rev,
+            remoteFiles |> List.rev
 
 /// Allows to parse and analyze Dependencies files.
+type DependenciesFile(packages : Package list, remoteFiles : SourceFile list) = 
 type DependenciesFile(packages : UnresolvedPackage seq) = 
     let packages = packages |> Seq.toList
     let dependencyMap = Map.ofSeq (packages |> Seq.map (fun p -> p.Name, p.VersionRange))
     member __.DirectDependencies = dependencyMap
     member __.Packages = packages
-    member __.Resolve(force, discovery : IDiscovery) = Resolver.Resolve(force, discovery, packages)
+    member __.RemoteFiles = remoteFiles
+    member __.Resolve(force, discovery : IDiscovery) = PackageResolver.Resolve(force, discovery, packages)
     static member FromCode(code:string) : DependenciesFile = 
         DependenciesFile(DependenciesFileParser.parseDependenciesFile <| code.Replace("\r\n","\n").Replace("\r","\n").Split('\n'))
     static member ReadFromFile fileName : DependenciesFile = 

src/Paket/GitHub.fs

@@ -0,0 +1,12 @@
+module Paket.GitHub
+
+open System.Net
+open System
+open Paket
+
+/// Gets a single file from github.
+let downloadFile remoteFile = 
+    let url = sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.CommitWithDefault remoteFile.Path
+    use wc = new WebClient()
+    Uri url |> wc.AsyncDownloadString
+

src/Paket/LockFile.fs

@@ -47,7 +47,7 @@ let extractErrors (resolved : PackageResolution) =
 
 
 /// [omit]
-let format (resolved : PackageResolution) = 
+let serializePackages (resolved : PackageResolution) = 
     let sources = 
         resolved
         |> Seq.map (fun x ->
@@ -74,6 +74,83 @@ let format (resolved : PackageResolution) =
 
     String.Join(Environment.NewLine, all)
 
+let serializeSourceFiles (files:SourceFile list) =
+    seq {
+        yield "GITHUB"
+        for (owner,project), files in files |> Seq.groupBy(fun f -> f.Owner,f.Project) do
+            yield sprintf "  remote: %s/%s" owner project
+            yield "  specs:"
+            for file in files do
+                let path = file.Path.TrimStart '/'
+                match file.Commit with
+                | Some commit -> yield sprintf "    %s (%s)" path commit
+                | None -> yield sprintf "    %s" path
+    }
+    |> fun all -> String.Join(Environment.NewLine, all)
+
+type private ParseState =
+    { RepositoryType : string option
+      Remote : string option
+      Packages : Package list
+      SourceFiles : SourceFile list }
+
+let private (|Remote|NugetPackage|NugetDependency|SourceFile|Spec|RepositoryType|Blank|) (state, line:string) =
+    match (state.RepositoryType, line.Trim()) with
+    | _, "NUGET" -> RepositoryType "NUGET"
+    | _, "GITHUB" -> RepositoryType "GITHUB"
+    | _, _ when String.IsNullOrWhiteSpace line -> Blank
+    | _, trimmed when trimmed.StartsWith "remote:" -> Remote (trimmed.Substring(trimmed.IndexOf(": ") + 2))
+    | _, trimmed when trimmed.StartsWith "specs:" -> Spec
+    | _, trimmed when line.StartsWith "      " -> NugetDependency (trimmed.Split ' ' |> Seq.head)
+    | Some "NUGET", trimmed -> NugetPackage trimmed
+    | Some "GITHUB", trimmed -> SourceFile trimmed
+    | Some _, _ -> failwith "unknown Repository Type."
+    | _ -> failwith "unknown lock file format."
+
+/// Parses a Lock file from lines
+let Parse(lines : string seq) =
+    let remove textToRemove (source:string) = source.Replace(textToRemove, "")
+    let removeBrackets = remove "(" >> remove ")"
+    ({ RepositoryType = None; Remote = None; Packages = []; SourceFiles = [] }, lines)
+    ||> Seq.fold(fun state line ->
+        match (state, line) with
+        | Remote remoteSource -> { state with Remote = Some remoteSource }
+        | Spec | Blank -> state
+        | RepositoryType repoType -> { state with RepositoryType = Some repoType }
+        | NugetPackage details ->
+            match state.Remote with
+            | Some remote ->
+                let parts = details.Split ' '
+                let version = parts.[1] |> removeBrackets
+                { state with Packages = { Sources = [PackageSource.Parse remote]
+                                          Name = parts.[0]
+                                          DirectDependencies = []
+                                          ResolverStrategy = Max
+                                          VersionRange = VersionRange.Exactly version } :: state.Packages }
+            | None -> failwith "no source has been specified."
+        | NugetDependency details ->
+            match state.Packages with
+            | currentPackage :: otherPackages -> 
+                { state with
+                    Packages = { currentPackage with
+                                    DirectDependencies = [details] |> List.append currentPackage.DirectDependencies
+                                } :: otherPackages }
+            | [] -> failwith "cannot set a dependency - no package has been specified."
+        | SourceFile details ->
+            match state.Remote |> Option.map(fun s -> s.Split '/') with
+            | Some [| owner; project |] ->
+                let path, commit = match details.Split ' ' with
+                                   | [| filePath; commit |] -> filePath, Some (commit |> removeBrackets)
+                                   | [| filePath |] -> filePath, None
+                                   | _ -> failwith "invalid file source details."
+                { state with
+                    SourceFiles = { Commit = commit
+                                    Owner = owner
+                                    Project = project
+                                    Path = path } :: state.SourceFiles }
+            | _ -> failwith "invalid remote details."
+        )
+    |> fun state -> List.rev state.Packages, List.rev state.SourceFiles
 let private (|Remote|Package|Dependency|Spec|Header|Blank|) (line:string) =
     match line.Trim() with
     | "NUGET" -> Header
@@ -111,17 +188,18 @@ let Parse(lines : string seq) : ResolvedPackage list =
     |> List.rev
 
 /// Analyzes the dependencies from the Dependencies file.
-let Create(force,dependenciesFile) =     
-    let cfg = DependenciesFile.ReadFromFile dependenciesFile
-    tracefn "Analyzing %s" dependenciesFile
-    cfg.Resolve(force,Nuget.NugetDiscovery)
+let Create(force, dependenciesFilename) =     
+    tracefn "Parsing %s" dependenciesFilename
+    let dependenciesFile = DependenciesFile.ReadFromFile dependenciesFilename
+    dependenciesFile.Resolve(force, Nuget.NugetDiscovery), dependenciesFile.RemoteFiles
 
 /// Updates the Lock file with the analyzed dependencies from the Dependencies file.
-let Update(force, packageFile, lockFile) = 
-    let resolution = Create(force,packageFile)
-    let errors = extractErrors resolution
+let Update(force, dependenciesFilename, lockFile) = 
+    let packageResolution, remoteFiles = Create(force, dependenciesFilename)
+    let errors = extractErrors packageResolution
     if errors = "" then
-        File.WriteAllText(lockFile, format resolution)
+        let output = String.Join(Environment.NewLine, serializePackages (packageResolution), serializeSourceFiles remoteFiles)
+        File.WriteAllText(lockFile, output)
         tracefn "Locked version resolutions written to %s" lockFile
     else
         failwith <| "Could not resolve dependencies." + Environment.NewLine + errors

src/Paket/Paket.fsproj

@@ -25,9 +25,11 @@
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <WarningLevel>3</WarningLevel>
     <DocumentationFile>..\..\bin\Paket.xml</DocumentationFile>
-    <StartArguments>update</StartArguments>
+    <StartArguments>install --dependencies-file "C:\Users\Isaac\Source\Repos\Paket\tests\ManualCSTest\paket.dependencies"</StartArguments>
     <StartAction>Project</StartAction>
     <StartProgram>paket.exe</StartProgram>
+    <StartWorkingDirectory>
+    </StartWorkingDirectory>
     <StartWorkingDirectory>c:\code\paket</StartWorkingDirectory>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
@@ -71,6 +73,7 @@
     <Compile Include="Resolver.fs" />
     <Compile Include="FrameworkHandling.fs" />
     <Compile Include="ProjectFile.fs" />
+    <Compile Include="GitHub.fs" />
     <Compile Include="Nuget.fs" />
     <Compile Include="DependenciesFile.fs" />
     <Compile Include="LockFile.fs" />

src/Paket/Process.fs

@@ -41,24 +41,45 @@ let extractReferencesFromListFile projectFile =
 let private findAllProjects(folder) = DirectoryInfo(folder).EnumerateFiles("*.*proj", SearchOption.AllDirectories)
 
 /// Installs the given packageFile.
-let Install(regenerate, force, dependenciesFile) = 
-    let lockfile = findLockfile dependenciesFile
-
-    if regenerate || (not lockfile.Exists) then 
-        LockFile.Update(force, dependenciesFile, lockfile.FullName)
-
-    let extracted = 
-        ExtractPackages(force, File.ReadAllLines lockfile.FullName |> LockFile.Parse)
+let Install(regenerate, force, dependenciesFilename) = 
+    let packages, sourceFiles =
+        let lockfile = findLockfile dependenciesFilename   
+
+        if regenerate || (not lockfile.Exists) then 
+            LockFile.Update(force, dependenciesFilename, lockfile.FullName)
+
+        File.ReadAllLines lockfile.FullName |> LockFile.Parse
+
+    let extractedPackages = 
+        ExtractPackages(force, packages)
+        |> Async.Parallel
+        |> Async.RunSynchronously    
+
+    let extractedSourceFiles =
+        let rootPath = dependenciesFilename |> Path.GetDirectoryName
+        sourceFiles
+        |> List.map(fun source ->
+                async {
+                    let destination = Path.Combine(rootPath, "paket-files", source.Owner, source.Project, source.CommitWithDefault, source.FilePath)
+
+                    if File.Exists destination then tracefn "%s already exists locally" (source.ToString())
+                    else
+                        tracefn "Downloading %s..." (source.ToString())
+                        let! file = GitHub.downloadFile source
+                        Directory.CreateDirectory(destination |> Path.GetDirectoryName) |> ignore
+                        File.WriteAllText(destination, file)
+                    return destination })
         |> Async.Parallel
         |> Async.RunSynchronously
+
     for proj in findAllProjects(".") do
         let directPackages = extractReferencesFromListFile proj.FullName
         let project = ProjectFile.Load proj.FullName
 
         let usedPackages = new HashSet<_>()
 
         let allPackages =
-            extracted
+            extractedPackages
             |> Array.map (fun (p,_) -> p.Name,p)
             |> Map.ofArray
 
@@ -73,16 +94,20 @@ let Install(regenerate, force, dependenciesFile) =
         directPackages
         |> Array.iter addPackage
 
-        project.UpdateReferences(extracted,usedPackages)
+        project.UpdateReferences(extractedPackages,usedPackages)
 
 
 /// Finds all outdated packages.
-let FindOutdated(packageFile) = 
-    let lockFile = findLockfile packageFile
-
-    let newPackages = LockFile.Create(true,packageFile)
-    let installed = if lockFile.Exists then LockFile.Parse(File.ReadAllLines lockFile.FullName) else []
+let FindOutdated(dependenciesFile) = 
+    let lockFile = findLockfile dependenciesFile
+
+    //TODO: Anything we need to do for source files here?    
+    let newPackages, _ = LockFile.Create(true, dependenciesFile)
+    let installedPackages, _ =
+        if lockFile.Exists then LockFile.Parse(File.ReadAllLines lockFile.FullName) else [], []
 
+    [for p in installedPackages do
+        match newPackages.ResolvedVersionMap.[p.Name] with
     [for p in installed do
         match newPackages.[p.Name] with
         | Resolved newVersion -> 

src/Paket/Program.fs

@@ -3,13 +3,12 @@ module Paket.Program
 
 open System
 open Nessos.UnionArgParser
-open System.IO
 
 type Command =
     | Install
     | Update
     | Outdated
-    | Unkown
+    | Unknown
 
 type CLIArguments =
     | [<First>][<NoAppSettings>][<CustomCommandLine("install")>] Install
@@ -39,7 +38,7 @@ let results,verbose =
             if results.Contains <@ CLIArguments.Install @> then Command.Install
             elif results.Contains <@ CLIArguments.Update @> then Command.Update
             elif results.Contains <@ CLIArguments.Outdated @> then Command.Outdated
-            else Command.Unkown
+            else Command.Unknown
         Some(command,results),results.Contains <@ CLIArguments.Verbose @>
     with
     | _ ->

src/Paket/Resolver.fs

@@ -1,5 +1,5 @@
 /// Contains logic which helps to resolve the dependency graph.
-module Paket.Resolver
+module Paket.PackageResolver
 
 open Paket
 open System

tests/ManualCSTest/paket.dependencies

@@ -1,3 +1,6 @@
 source "http://nuget.org/api/v2"
 
-nuget "Newtonsoft.Json" "~> 6.0"
+nuget "Newtonsoft.Json" "~> 6.0"
+github "fsharp:FAKE" "src/app/FAKE/Cli.fs"
+github "fsharp:FAKE:Globbing" "/src/app/Fake.Deploy.Lib/FakeDeployAgentHelper.fs"
+github "fsharp:FAKE:b019e6db78b88386447a1efd1bd70826f5008344" "src/app/FAKE/CommandlineParams.fs"