From 8199771a83be93e8dcf5593da96f50e0565c950a Mon Sep 17 00:00:00 2001 From: Tuomas Hietanen Date: Sat, 8 Nov 2014 15:34:53 +0200 Subject: [PATCH 1/2] Support downloading file from other remotes (fssnip, gist, etc) --- src/Paket.Core/DependenciesFile.fs | 17 ++++++ src/Paket.Core/GitHub.fs | 12 ++-- src/Paket.Core/LockFile.fs | 36 ++++++++---- src/Paket.Core/ModuleResolver.fs | 7 ++- .../DependenciesFile/ParserSpecs.fs | 55 +++++++++++++++++++ tests/Paket.Tests/Lockfile/GeneratorSpecs.fs | 26 ++++++++- tests/Paket.Tests/Lockfile/ParserSpecs.fs | 2 + 7 files changed, 138 insertions(+), 17 deletions(-) diff --git a/src/Paket.Core/DependenciesFile.fs b/src/Paket.Core/DependenciesFile.fs index 4cfe159da3..cd7ed388d4 100644 --- a/src/Paket.Core/DependenciesFile.fs +++ b/src/Paket.Core/DependenciesFile.fs @@ -127,6 +127,23 @@ module DependenciesFileParser = | [| _; projectSpec; fileSpec |] -> SourceFile(getParts projectSpec, fileSpec) | [| _; projectSpec; |] -> SourceFile(getParts projectSpec, "FULLPROJECT") | _ -> failwithf "invalid github specification:%s %s" Environment.NewLine trimmed + | trimmed when trimmed.StartsWith "http" -> + let parts = parseDependencyLine trimmed + let getParts (projectSpec:string) = + let ``project spec`` = + match projectSpec.EndsWith("/") with + | false -> projectSpec + | true -> projectSpec.Substring(0, projectSpec.Length-1) + let splitted = ``project spec``.Split [|':'; '/'|] + match splitted |> Seq.truncate 6 |> Seq.toArray with + //SourceFile((owner,project, commit), path) + | [| protocol; x; y; domain |] -> SourceFile((domain, domain, None), ``project spec``) + | [| protocol; x; y; domain; project |] -> SourceFile((domain,project, None), ``project spec``) + | [| protocol; x; y; owner; project; details |] -> SourceFile((owner+"/"+project,project+"/"+details, None), ``project spec``) + | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed + match parts with + | [| _; projectSpec; |] -> getParts projectSpec + | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed | _ -> Blank let parseDependenciesFile fileName (lines:string seq) = diff --git a/src/Paket.Core/GitHub.fs b/src/Paket.Core/GitHub.fs index 0a5c996bcd..45ab2fac72 100644 --- a/src/Paket.Core/GitHub.fs +++ b/src/Paket.Core/GitHub.fs @@ -57,9 +57,9 @@ let rec DirectoryCopy(sourceDirName, destDirName, copySubDirs) = DirectoryCopy(subdir.FullName, Path.Combine(destDirName, subdir.Name), copySubDirs) /// Gets a single file from github. -let downloadGithubFiles(remoteFile:ModuleResolver.ResolvedSourceFile,destitnation) = async { - match remoteFile.Name with - | "FULLPROJECT" -> +let downloadRemoteFiles(remoteFile:ModuleResolver.ResolvedSourceFile,destitnation) = async { + match remoteFile.Origin, remoteFile.Name with + | ModuleResolver.SourceFileOrigin.GitHubLink, "FULLPROJECT" -> let fi = FileInfo(destitnation) let projectPath = fi.Directory.FullName let zipFile = Path.Combine(projectPath,sprintf "%s.zip" remoteFile.Commit) @@ -72,7 +72,9 @@ let downloadGithubFiles(remoteFile:ModuleResolver.ResolvedSourceFile,destitnatio Directory.Delete(source,true) - | _ -> return! downloadFromUrl(None,sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destitnation + | ModuleResolver.SourceFileOrigin.GitHubLink, _ -> return! downloadFromUrl(None,sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destitnation + | ModuleResolver.SourceFileOrigin.HttpLink, _ -> return! downloadFromUrl(None,sprintf "%s" remoteFile.Name) destitnation + | ModuleResolver.SourceFileOrigin.NuGetPackage, _ -> failwith("Downloading single file from NuGet is not supported.") } let DownloadSourceFile(rootPath, source:ModuleResolver.ResolvedSourceFile) = @@ -91,6 +93,6 @@ let DownloadSourceFile(rootPath, source:ModuleResolver.ResolvedSourceFile) = tracefn "Downloading %s to %s" (source.ToString()) destination Directory.CreateDirectory(destination |> Path.GetDirectoryName) |> ignore - do! downloadGithubFiles(source,destination) + do! downloadRemoteFiles(source,destination) File.WriteAllText(versionFile.FullName, source.Commit) } \ No newline at end of file diff --git a/src/Paket.Core/LockFile.fs b/src/Paket.Core/LockFile.fs index 9f697f48ec..a05939c023 100644 --- a/src/Paket.Core/LockFile.fs +++ b/src/Paket.Core/LockFile.fs @@ -45,22 +45,35 @@ module LockFileSerializer = let serializeSourceFiles (files:ResolvedSourceFile list) = let all = let hasReported = ref false - [ for (owner,project), files in files |> Seq.groupBy(fun f -> f.Owner, f.Project) do - if not !hasReported then - yield "GITHUB" - hasReported := true + [ for (owner,project,origin), files in files |> Seq.groupBy(fun f -> f.Owner, f.Project, f.Origin) do + match origin with + | GitHubLink -> + if not !hasReported then + yield "GITHUB" + hasReported := true + yield sprintf " remote: %s/%s" owner project + yield " specs:" + | HttpLink -> + if not !hasReported then + yield "HTTP" + hasReported := true + yield sprintf " remote: LINK" + yield " specs:" + | NuGetPackage -> failwith("Referencing sigle file from NuGet is not supported.") - yield sprintf " remote: %s/%s" owner project - yield " specs:" for file in files |> Seq.sortBy (fun f -> f.Owner.ToLower(),f.Project.ToLower(),f.Name.ToLower()) do + let path = file.Name.TrimStart '/' - yield sprintf " %s (%s)" path file.Commit + match String.IsNullOrEmpty(file.Commit) with + | false -> yield sprintf " %s (%s)" path file.Commit + | true -> yield sprintf " %s" path for (name,v) in file.Dependencies do yield sprintf " %s (%s)" name (v.ToString())] String.Join(Environment.NewLine, all) module LockFileParser = + type ParseState = { RepositoryType : string option RemoteUrl :string option @@ -73,6 +86,7 @@ module LockFileParser = let private (|Remote|NugetPackage|NugetDependency|SourceFile|RepositoryType|Blank|InstallOption|) (state, line:string) = match (state.RepositoryType, line.Trim()) with + | _, "HTTP" -> RepositoryType "HTTP" | _, "NUGET" -> RepositoryType "NUGET" | _, "GITHUB" -> RepositoryType "GITHUB" | _, _ when String.IsNullOrWhiteSpace line -> Blank @@ -84,7 +98,8 @@ module LockFileParser = let parts = trimmed.Split '(' NugetDependency (parts.[0].Trim(),parts.[1].Replace("(", "").Replace(")", "").Trim()) | Some "NUGET", trimmed -> NugetPackage trimmed - | Some "GITHUB", trimmed -> SourceFile trimmed + | Some "HTTP", trimmed -> SourceFile(HttpLink, trimmed) + | Some "GITHUB", trimmed -> SourceFile(GitHubLink, trimmed) | Some _, _ -> failwith "unknown Repository Type." | _ -> failwith "unknown lock file format." @@ -130,16 +145,17 @@ module LockFileParser = Dependencies = Set.add (name, VersionRequirement.AllReleases) currentFile.Dependencies } :: rest } | [] -> failwith "cannot set a dependency - no remote file has been specified." - | SourceFile details -> + | SourceFile(origin, details) -> match state.RemoteUrl |> Option.map(fun s -> s.Split '/') with | Some [| owner; project |] -> let path, commit = match details.Split ' ' with | [| filePath; commit |] -> filePath, commit |> removeBrackets | _ -> failwith "invalid file source details." { state with - LastWasPackage = false + LastWasPackage = false SourceFiles = { Commit = commit Owner = owner + Origin = origin Project = project Dependencies = Set.empty Name = path } :: state.SourceFiles } diff --git a/src/Paket.Core/ModuleResolver.fs b/src/Paket.Core/ModuleResolver.fs index f8ee96919f..24b286ae86 100644 --- a/src/Paket.Core/ModuleResolver.fs +++ b/src/Paket.Core/ModuleResolver.fs @@ -25,12 +25,16 @@ type UnresolvedSourceFile = | Some commit -> sprintf "%s/%s:%s %s" this.Owner this.Project commit this.Name | None -> sprintf "%s/%s %s" this.Owner this.Project this.Name +type SourceFileOrigin = NuGetPackage | GitHubLink | HttpLink + type ResolvedSourceFile = { Owner : string Project : string Name : string Commit : string - Dependencies : Set } + Dependencies : Set + Origin : SourceFileOrigin + } member this.FilePath = this.ComputeFilePath(this.Name) member this.ComputeFilePath(name:string) = @@ -54,6 +58,7 @@ let Resolve(getPackages, getSha1, remoteFiles : UnresolvedSourceFile list) : Res let naked = { Commit = sha Owner = file.Owner + Origin = GitHubLink Project = file.Project Dependencies = Set.empty Name = file.Name } diff --git a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs index b99fa251d0..36b17d6b5e 100644 --- a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs +++ b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs @@ -218,6 +218,61 @@ let ``should read github source files withou sha1``() = Name = "src/app/FAKE/FileWithCommit.fs" Commit = Some "bla123zxc" } ] +[] +let ``should read http source file from config without quotes, simple real-life``() = + let config = """http http://www.fssnip.net/raw/1M + http http://www.fssnip.net/raw/1M/1 + http https://gist.githubusercontent.com/Thorium/1972308/raw/629ed3119e18ec0629142fb30351e50ac688e7fd/gistfile1.fs """ + let dependencies = DependenciesFile.FromCode(config) + dependencies.RemoteFiles + |> shouldEqual + [ { Owner = "www.fssnip.net/raw" + Project = "raw/1M" + Name = "http://www.fssnip.net/raw/1M" + Commit = None } + { Owner = "www.fssnip.net/raw" + Project = "raw/1M" + Name = "http://www.fssnip.net/raw/1M/1" + Commit = None } + { Owner = "gist.githubusercontent.com/Thorium" + Project = "Thorium/1972308" + Name = "https://gist.githubusercontent.com/Thorium/1972308/raw/629ed3119e18ec0629142fb30351e50ac688e7fd/gistfile1.fs" + Commit = None } ] + +[] +let ``should read http source file from config without quotes, parsing rules``() = + // The empty "/" should be ommited. After that, parsing amount of "/"-marks: + let config = """ + http http://example/ + http http://example/item + http http://example/item/ + http http://example/item/3 + http http://example/item/3/1""" + let dependencies = DependenciesFile.FromCode(config) + dependencies.RemoteFiles + |> shouldEqual + [ { Owner = "example" + Project = "example" + Name = "http://example" + Commit = None } + { Owner = "example" + Project = "item" + Name = "http://example/item" + Commit = None } + { Owner = "example" + Project = "item" + Name = "http://example/item" + Commit = None } + { Owner = "example/item" + Project = "item/3" + Name = "http://example/item/3" + Commit = None } + { Owner = "example/item" + Project = "item/3" + Name = "http://example/item/3/1" + Commit = None } ] + + let configWithoutVersions = """ source "http://nuget.org/api/v2" diff --git a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs index 09247134ba..fd0c9bab22 100644 --- a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs +++ b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs @@ -69,6 +69,7 @@ github "owner:project2:commit2" "folder/file.fs" """ match f.Commit with | Some commit -> { Commit = commit Owner = f.Owner + Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink Project = f.Project Dependencies = Set.empty Name = f.Name } : ModuleResolver.ResolvedSourceFile @@ -127,4 +128,27 @@ let ``should generate other version ranges for packages``() = let cfg = DependenciesFile.FromCode(config3) cfg.Resolve(noSha1,VersionsFromGraph graph3, PackageDetailsFromGraph graph3).ResolvedPackages.GetModelOrFail() |> LockFileSerializer.serializePackages cfg.Options - |> shouldEqual (normalizeLineEndings expected3) \ No newline at end of file + |> shouldEqual (normalizeLineEndings expected3) + +let expectedWithHttp = """HTTP + remote: LINK + specs: + http://www.fssnip.net/raw/1M""" + +[] +let ``should generate lock file for http source files``() = + let config = """http "http://www.fssnip.net/raw/1M" """ + + let cfg = DependenciesFile.FromCode(config) + + cfg.RemoteFiles + |> List.map (fun f -> + { Commit = "" + Owner = f.Owner + Origin = Paket.ModuleResolver.SourceFileOrigin.HttpLink + Project = f.Project + Dependencies = Set.empty + Name = f.Name } : ModuleResolver.ResolvedSourceFile) + |> LockFileSerializer.serializeSourceFiles + |> shouldEqual (normalizeLineEndings expectedWithHttp) + diff --git a/tests/Paket.Tests/Lockfile/ParserSpecs.fs b/tests/Paket.Tests/Lockfile/ParserSpecs.fs index 16b57e559c..d4d5059199 100644 --- a/tests/Paket.Tests/Lockfile/ParserSpecs.fs +++ b/tests/Paket.Tests/Lockfile/ParserSpecs.fs @@ -52,12 +52,14 @@ let ``should parse lock file``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" + Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink Dependencies = Set.empty Commit = "7699e40e335f3cc54ab382a8969253fecc1e08a9" } { Owner = "fsharp" Project = "FAKE" Dependencies = Set.empty Name = "src/app/Fake.Deploy.Lib/FakeDeployAgentHelper.fs" + Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink Commit = "Globbing" } ] sourceFiles.[0].Commit |> shouldEqual "7699e40e335f3cc54ab382a8969253fecc1e08a9" From 79ed799b9d20080e0275e90db20309ebd1647540 Mon Sep 17 00:00:00 2001 From: Tuomas Hietanen Date: Mon, 10 Nov 2014 20:49:31 +0200 Subject: [PATCH 2/2] Support downloading file from other remotes, part 2 --- README.md | 2 +- docs/content/dependencies-file.md | 7 +- docs/content/github-dependencies.md | 21 ++++- src/Paket.Core/DependenciesFile.fs | 73 ++++++++------ src/Paket.Core/InstallProcess.fs | 14 +-- src/Paket.Core/LockFile.fs | 94 ++++++++++++++----- src/Paket.Core/ModuleResolver.fs | 22 +++-- src/Paket.Core/NugetConvert.fs | 2 +- src/Paket.Core/Paket.Core.fsproj | 2 +- src/Paket.Core/ReferencesFile.fs | 14 +-- .../{GitHub.fs => RemoteDownload.fs} | 64 ++++++++++--- src/Paket.Core/RestoreProcess.fs | 2 +- .../DependenciesFile/ParserSpecs.fs | 61 ++++++++---- tests/Paket.Tests/Lockfile/GeneratorSpecs.fs | 54 ++++++++++- tests/Paket.Tests/Lockfile/ParserSpecs.fs | 4 +- .../ReferencesFile/ReferencesFileSpecs.fs | 16 ++-- 16 files changed, 326 insertions(+), 126 deletions(-) rename src/Paket.Core/{GitHub.fs => RemoteDownload.fs} (50%) diff --git a/README.md b/README.md index dee1f40686..f76a821915 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Even more importantly: If two packages reference conflicting versions of a packa Paket on the other hand maintains this information on a consistent and stable basis within the [`paket.lock` file][7] in the solution root. This file, together with the [`paket.dependencies` file][8] enables you to determine exactly what's happening with your dependencies. -Paket also enables you to [reference files directly from GitHub][9] repositories. +Paket also enables you to [reference files directly from GitHub (and Gist)][9] repositories or any http-resource. For more reasons see the [FAQ][10]. diff --git a/docs/content/dependencies-file.md b/docs/content/dependencies-file.md index 2ad156619b..10e0831b06 100644 --- a/docs/content/dependencies-file.md +++ b/docs/content/dependencies-file.md @@ -11,6 +11,8 @@ To give you an overview, consider the following `paket.dependencies` file: nuget DotNetZip >= 1.9 nuget SourceLink.Fake github forki/FsUnit FsUnit.fs + gist Thorium/1972349 timestamp.fs + http http://www.fssnip.net/1n decrypt.fs The file specifies that Paket's NuGet dependencies should be downloaded from [nuget.org](http://www.nuget.org) and that we need: @@ -19,6 +21,8 @@ The file specifies that Paket's NuGet dependencies should be downloaded from [nu * [DotNetZip](http://dotnetzip.codeplex.com/) with version which is at [least 1.9](http://fsprojects.github.io/Paket/nuget-dependencies.html#Greater-than-or-equal-version-constraint) * [SourceLink.Fake](https://github.com/ctaggart/SourceLink) in the latest version * [FSUnit.fs](https://github.com/forki/FsUnit) from GitHub. + * Gist number [1972349](https://gist.github.com/Thorium/1972349) from GitHub Gist. + * External HTTP-resource, e.g. [1n](http://www.fssnip.net/1n) from [FSSnip](http://www.fssnip.net/) -site. Paket uses this definition to compute a concrete dependency resolution, which also includes indirect dependencies. The resulting dependency graph is then persisted to the [`paket.lock` file](lock-file.html). @@ -29,7 +33,8 @@ Only direct dependencies should be listed and you can use the [`paket simplify` Paket supports the following source types: * [NuGet](nuget-dependencies.html) -* [GitHub](github-dependencies.html) +* [GitHub and Gist](github-dependencies.html) +* HTTP (any single file from any site without version control) ## Strict references diff --git a/docs/content/github-dependencies.md b/docs/content/github-dependencies.md index 0da4c500d8..a5653c43af 100644 --- a/docs/content/github-dependencies.md +++ b/docs/content/github-dependencies.md @@ -1,6 +1,6 @@ # GitHub dependencies -Paket allows one to automatically manage the linking of files from [github.com](http://www.github.com) into your projects. +Paket allows one to automatically manage the linking of files from [github.com](http://www.github.com) or [gist.github.com](https://gist.github.com/) into your projects. ## Referencing a single file @@ -89,4 +89,21 @@ This generates the following [`paket.lock` file](lock-file.html): modules/Octokit/Octokit.fsx (a25c2f256a99242c1106b5a3478aae6bb68c7a93) Octokit (>= 0) -As you can see Paket also resolved the Octokit dependency. \ No newline at end of file +As you can see Paket also resolved the Octokit dependency. + +## Gist + +Gist works the same way. You can fetch single files or multi-file-gists as well: + + gist Thorium/1972308 gistfile1.fs + gist Thorium/6088882 + +If you run the [`paket update` command](paket-update.html), it will add a new section to your [`paket.lock` file](lock-file.html): + + GIST + remote: Thorium/1972308 + specs: + gistfile1.fs + remote: Thorium/6088882 + specs: + FULLPROJECT \ No newline at end of file diff --git a/src/Paket.Core/DependenciesFile.fs b/src/Paket.Core/DependenciesFile.fs index cd7ed388d4..c2ac5b1131 100644 --- a/src/Paket.Core/DependenciesFile.fs +++ b/src/Paket.Core/DependenciesFile.fs @@ -89,6 +89,41 @@ module DependenciesFileParser = |> List.rev |> List.toArray + + let private ``parse git source`` trimmed origin originTxt = + let parts = parseDependencyLine trimmed + let getParts (projectSpec:string) = + match projectSpec.Split [|':'; '/'|] with + | [| owner; project |] -> owner, project, None + | [| owner; project; commit |] -> owner, project, Some commit + | _ -> failwithf "invalid %s specification:%s %s" originTxt Environment.NewLine trimmed + match parts with + | [| _; projectSpec; fileSpec |] -> origin, getParts projectSpec, fileSpec + | [| _; projectSpec; |] -> origin, getParts projectSpec, "FULLPROJECT" + | _ -> failwithf "invalid %s specification:%s %s" originTxt Environment.NewLine trimmed + + let private ``parse http source`` trimmed = + let parts = parseDependencyLine trimmed + let getParts (projectSpec:string) fileSpec = + let ``project spec`` = + match projectSpec.EndsWith("/") with + | false -> projectSpec + | true -> projectSpec.Substring(0, projectSpec.Length-1) + let splitted = ``project spec``.Split [|':'; '/'|] + let fileName = match String.IsNullOrEmpty(fileSpec) with + | true -> (splitted |> Seq.last) + ".fs" + | false -> fileSpec + match splitted |> Seq.truncate 6 |> Seq.toArray with + //SourceFile(origin(url), (owner,project, commit), path) + | [| protocol; x; y; domain |] -> HttpLink(``project spec``), (domain, domain, None), fileName + | [| protocol; x; y; domain; project |] -> HttpLink(``project spec``), (domain,project, None), fileName + | [| protocol; x; y; owner; project; details |] -> HttpLink(``project spec``), (owner,project+"/"+details, None), fileName + | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed + match parts with + | [| _; projectSpec; |] -> getParts projectSpec String.Empty + | [| _; projectSpec; fileSpec |] -> getParts projectSpec fileSpec + | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed + let private (|Remote|Package|Blank|ReferencesMode|OmitContent|SourceFile|) (line:string) = match line.Trim() with | _ when String.IsNullOrWhiteSpace line -> Blank @@ -116,34 +151,12 @@ module DependenciesFileParser = | _ -> failwithf "could not retrieve nuget package from %s" trimmed | trimmed when trimmed.StartsWith "references" -> ReferencesMode(trimmed.Replace("references","").Trim() = "strict") | trimmed when trimmed.StartsWith "content" -> OmitContent(trimmed.Replace("content","").Trim() = "none") + | trimmed when trimmed.StartsWith "gist" -> + SourceFile(``parse git source`` trimmed SingleSourceFileOrigin.GistLink "gist") | trimmed when trimmed.StartsWith "github" -> - let parts = parseDependencyLine trimmed - let getParts (projectSpec:string) = - match projectSpec.Split [|':'; '/'|] with - | [| owner; project |] -> owner, project, None - | [| owner; project; commit |] -> owner, project, Some commit - | _ -> failwithf "invalid github specification:%s %s" Environment.NewLine trimmed - match parts with - | [| _; projectSpec; fileSpec |] -> SourceFile(getParts projectSpec, fileSpec) - | [| _; projectSpec; |] -> SourceFile(getParts projectSpec, "FULLPROJECT") - | _ -> failwithf "invalid github specification:%s %s" Environment.NewLine trimmed + SourceFile(``parse git source`` trimmed SingleSourceFileOrigin.GitHubLink "github") | trimmed when trimmed.StartsWith "http" -> - let parts = parseDependencyLine trimmed - let getParts (projectSpec:string) = - let ``project spec`` = - match projectSpec.EndsWith("/") with - | false -> projectSpec - | true -> projectSpec.Substring(0, projectSpec.Length-1) - let splitted = ``project spec``.Split [|':'; '/'|] - match splitted |> Seq.truncate 6 |> Seq.toArray with - //SourceFile((owner,project, commit), path) - | [| protocol; x; y; domain |] -> SourceFile((domain, domain, None), ``project spec``) - | [| protocol; x; y; domain; project |] -> SourceFile((domain,project, None), ``project spec``) - | [| protocol; x; y; owner; project; details |] -> SourceFile((owner+"/"+project,project+"/"+details, None), ``project spec``) - | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed - match parts with - | [| _; projectSpec; |] -> getParts projectSpec - | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed + SourceFile(``parse http source`` trimmed) | _ -> Blank let parseDependenciesFile fileName (lines:string seq) = @@ -163,8 +176,8 @@ module DependenciesFileParser = ResolverStrategy = parseResolverStrategy version Parent = DependenciesFile fileName VersionRequirement = parseVersionRequirement(version.Trim '!') } :: packages, sourceFiles - | SourceFile((owner,project, commit), path) -> - lineNo, options, sources, packages, { Owner = owner; Project = project; Commit = commit; Name = path } :: sourceFiles + | SourceFile(origin, (owner,project, commit), path) -> + lineNo, options, sources, packages, { Owner = owner; Project = project; Commit = commit; Name = path; Origin = origin} :: sourceFiles with | exn -> failwithf "Error in paket.dependencies line %d%s %s" lineNo Environment.NewLine exn.Message) @@ -220,12 +233,12 @@ type DependenciesFile(fileName,options,packages : PackageRequirement list, remot member __.FileName = fileName member __.Sources = sources member this.Resolve(force) = - let getSha1 owner repo branch = GitHub.getSHA1OfBranch owner repo branch |> Async.RunSynchronously + let getSha1 origin owner repo branch = RemoteDownload.getSHA1OfBranch origin owner repo branch |> Async.RunSynchronously this.Resolve(getSha1,Nuget.GetVersions,Nuget.GetPackageDetails force) member __.Resolve(getSha1,getVersionF, getPackageDetailsF) = let resolveSourceFile(file:ResolvedSourceFile) : PackageRequirement list = - GitHub.downloadDependenciesFile(Path.GetDirectoryName fileName, file) + RemoteDownload.downloadDependenciesFile(Path.GetDirectoryName fileName, file) |> Async.RunSynchronously |> DependenciesFile.FromCode |> fun df -> df.Packages diff --git a/src/Paket.Core/InstallProcess.fs b/src/Paket.Core/InstallProcess.fs index c876cd6fc5..006392391f 100644 --- a/src/Paket.Core/InstallProcess.fs +++ b/src/Paket.Core/InstallProcess.fs @@ -77,7 +77,7 @@ let CreateInstallModel(sources, force, package) = let createModel(sources,force, lockFile:LockFile) = let sourceFileDownloads = lockFile.SourceFiles - |> Seq.map (fun file -> GitHub.DownloadSourceFile(Path.GetDirectoryName lockFile.FileName, file)) + |> Seq.map (fun file -> RemoteDownload.DownloadSourceFile(Path.GetDirectoryName lockFile.FileName, file)) |> Async.Parallel let packageDownloads = @@ -114,14 +114,16 @@ let Install(sources,force, hard, lockFile:LockFile) = removeCopiedFiles project - let getGitHubFilePath name = + let getSingleRemoteFilePath name = + printf "\nFilename %s " name + lockFile.SourceFiles |> List.iter (fun i -> printf "\n %s %s " i.Name i.FilePath) (lockFile.SourceFiles |> List.find (fun f -> Path.GetFileName(f.Name) = name)).FilePath - let gitHubFileItems = - referenceFile.GitHubFiles + let gitRemoteItems = + referenceFile.RemoteFiles |> List.map (fun file -> { BuildAction = project.DetermineBuildAction file.Name - Include = createRelativePath project.FileName (getGitHubFilePath file.Name) + Include = createRelativePath project.FileName (getSingleRemoteFilePath file.Name) Link = Some(if file.Link = "." then Path.GetFileName(file.Name) else Path.Combine(file.Link, Path.GetFileName(file.Name))) }) @@ -133,6 +135,6 @@ let Install(sources,force, hard, lockFile:LockFile) = Include = createRelativePath project.FileName file.FullName Link = None }) - project.UpdateFileItems(gitHubFileItems @ nuGetFileItems, hard) + project.UpdateFileItems(gitRemoteItems @ nuGetFileItems, hard) project.Save() diff --git a/src/Paket.Core/LockFile.fs b/src/Paket.Core/LockFile.fs index a05939c023..40c522eb0d 100644 --- a/src/Paket.Core/LockFile.fs +++ b/src/Paket.Core/LockFile.fs @@ -44,22 +44,34 @@ module LockFileSerializer = let serializeSourceFiles (files:ResolvedSourceFile list) = let all = - let hasReported = ref false + let updateHasReported = new List() + [ for (owner,project,origin), files in files |> Seq.groupBy(fun f -> f.Owner, f.Project, f.Origin) do match origin with | GitHubLink -> - if not !hasReported then + if not (updateHasReported.Contains(GitHubLink)) then yield "GITHUB" - hasReported := true + updateHasReported.Remove (HttpLink "") |> ignore + updateHasReported.Remove GistLink |> ignore + updateHasReported.Add GitHubLink + yield sprintf " remote: %s/%s" owner project + yield " specs:" + | GistLink -> + if not (updateHasReported.Contains(GistLink)) then + yield "GIST" + updateHasReported.Remove GitHubLink |> ignore + updateHasReported.Remove (HttpLink "") |> ignore + updateHasReported.Add GistLink yield sprintf " remote: %s/%s" owner project yield " specs:" - | HttpLink -> - if not !hasReported then + | HttpLink url -> + if not (updateHasReported.Contains(HttpLink(""))) then yield "HTTP" - hasReported := true - yield sprintf " remote: LINK" + updateHasReported.Remove GitHubLink |> ignore + updateHasReported.Remove GistLink |> ignore + updateHasReported.Add (HttpLink "") + yield sprintf " remote: " + url yield " specs:" - | NuGetPackage -> failwith("Referencing sigle file from NuGet is not supported.") for file in files |> Seq.sortBy (fun f -> f.Owner.ToLower(),f.Project.ToLower(),f.Name.ToLower()) do @@ -87,6 +99,7 @@ module LockFileParser = let private (|Remote|NugetPackage|NugetDependency|SourceFile|RepositoryType|Blank|InstallOption|) (state, line:string) = match (state.RepositoryType, line.Trim()) with | _, "HTTP" -> RepositoryType "HTTP" + | _, "GIST" -> RepositoryType "GIST" | _, "NUGET" -> RepositoryType "NUGET" | _, "GITHUB" -> RepositoryType "GITHUB" | _, _ when String.IsNullOrWhiteSpace line -> Blank @@ -98,8 +111,9 @@ module LockFileParser = let parts = trimmed.Split '(' NugetDependency (parts.[0].Trim(),parts.[1].Replace("(", "").Replace(")", "").Trim()) | Some "NUGET", trimmed -> NugetPackage trimmed - | Some "HTTP", trimmed -> SourceFile(HttpLink, trimmed) | Some "GITHUB", trimmed -> SourceFile(GitHubLink, trimmed) + | Some "GIST", trimmed -> SourceFile(GistLink, trimmed) + | Some "HTTP", trimmed -> SourceFile(HttpLink(String.Empty), trimmed) | Some _, _ -> failwith "unknown Repository Type." | _ -> failwith "unknown lock file format." @@ -146,20 +160,54 @@ module LockFileParser = } :: rest } | [] -> failwith "cannot set a dependency - no remote file has been specified." | SourceFile(origin, details) -> - match state.RemoteUrl |> Option.map(fun s -> s.Split '/') with - | Some [| owner; project |] -> - let path, commit = match details.Split ' ' with - | [| filePath; commit |] -> filePath, commit |> removeBrackets - | _ -> failwith "invalid file source details." - { state with - LastWasPackage = false - SourceFiles = { Commit = commit - Owner = owner - Origin = origin - Project = project - Dependencies = Set.empty - Name = path } :: state.SourceFiles } - | _ -> failwith "invalid remote details.") + match origin with + | GitHubLink | GistLink -> + match state.RemoteUrl |> Option.map(fun s -> s.Split '/') with + | Some [| owner; project |] -> + let path, commit = match details.Split ' ' with + | [| filePath; commit |] -> filePath, commit |> removeBrackets + | _ -> failwith "invalid file source details." + { state with + LastWasPackage = false + SourceFiles = { Commit = commit + Owner = owner + Origin = origin + Project = project + Dependencies = Set.empty + Name = path } :: state.SourceFiles } + | _ -> failwith "invalid remote details." + | HttpLink x -> + match state.RemoteUrl |> Option.map(fun s -> s.Split '/') with + | Some [| protocol; _; domain; |] -> + { state with + LastWasPackage = false + SourceFiles = { Commit = String.Empty + Owner = domain + Origin = origin + Project = domain + Dependencies = Set.empty + Name = details } :: state.SourceFiles } + | Some [| protocol; _; domain; project |] -> + { state with + LastWasPackage = false + SourceFiles = { Commit = String.Empty + Owner = domain + Origin = origin + Project = project + Dependencies = Set.empty + Name = details } :: state.SourceFiles } + | Some [| protocol; _; domain; project; moredetails |] -> + { state with + LastWasPackage = false + SourceFiles = { Commit = String.Empty + Owner = domain + Origin = origin + Project = project+"/"+moredetails + Dependencies = Set.empty + Name = details } :: state.SourceFiles } + | _ -> failwith "invalid remote details." + ) + /// Allows to parse and analyze paket.lock files. diff --git a/src/Paket.Core/ModuleResolver.fs b/src/Paket.Core/ModuleResolver.fs index 24b286ae86..084b6aa4bd 100644 --- a/src/Paket.Core/ModuleResolver.fs +++ b/src/Paket.Core/ModuleResolver.fs @@ -4,12 +4,17 @@ module Paket.ModuleResolver open System.IO open Paket.Requirements +type SingleSourceFileOrigin = +| GitHubLink +| GistLink +| HttpLink of string + // Represents details on a dependent source file. -//TODO: As new sources e.g. fssnip etc. are added, this should probably become a DU or perhaps have an enum marker. type UnresolvedSourceFile = { Owner : string Project : string Name : string + Origin : SingleSourceFileOrigin Commit : string option } member this.FilePath = let path = this.Name @@ -25,15 +30,13 @@ type UnresolvedSourceFile = | Some commit -> sprintf "%s/%s:%s %s" this.Owner this.Project commit this.Name | None -> sprintf "%s/%s %s" this.Owner this.Project this.Name -type SourceFileOrigin = NuGetPackage | GitHubLink | HttpLink - type ResolvedSourceFile = { Owner : string Project : string Name : string Commit : string Dependencies : Set - Origin : SourceFileOrigin + Origin : SingleSourceFileOrigin } member this.FilePath = this.ComputeFilePath(this.Name) @@ -51,14 +54,15 @@ type ResolvedSourceFile = // TODO: github has a rate limit - try to convince them to whitelist Paket let Resolve(getPackages, getSha1, remoteFiles : UnresolvedSourceFile list) : ResolvedSourceFile list = remoteFiles |> List.map (fun file -> - let sha = - file.Commit - |> defaultArg <| "master" - |> getSha1 file.Owner file.Project + let sha = + file.Commit + |> defaultArg <| "master" + |> getSha1 file.Origin file.Owner file.Project + let naked = { Commit = sha Owner = file.Owner - Origin = GitHubLink + Origin = file.Origin Project = file.Project Dependencies = Set.empty Name = file.Name } diff --git a/src/Paket.Core/NugetConvert.fs b/src/Paket.Core/NugetConvert.fs index 26b5057afb..f9a90c70f2 100644 --- a/src/Paket.Core/NugetConvert.fs +++ b/src/Paket.Core/NugetConvert.fs @@ -145,7 +145,7 @@ let private convertNugetToRefFile(nugetPackagesConfig) = for (name,_) in confictingRefs do traceWarnfn "Reference %s is already defined in %s" name refFilePath match existingRefFile with - | None -> {ReferencesFile.FileName = refFilePath; NugetPackages = refsToAdd |> List.map fst; GitHubFiles = []}.Save() + | None -> {ReferencesFile.FileName = refFilePath; NugetPackages = refsToAdd |> List.map fst; RemoteFiles = []}.Save() | Some refFile -> if not (refsToAdd |> List.isEmpty) then (refsToAdd |> List.fold (fun (refFile : ReferencesFile) (name,_) -> refFile.AddNugetRef(name)) refFile).Save() diff --git a/src/Paket.Core/Paket.Core.fsproj b/src/Paket.Core/Paket.Core.fsproj index d5d5dc4cef..07f9d6b5bb 100644 --- a/src/Paket.Core/Paket.Core.fsproj +++ b/src/Paket.Core/Paket.Core.fsproj @@ -78,7 +78,7 @@ - + diff --git a/src/Paket.Core/ReferencesFile.fs b/src/Paket.Core/ReferencesFile.fs index 3ba861a1c9..6582cf4975 100644 --- a/src/Paket.Core/ReferencesFile.fs +++ b/src/Paket.Core/ReferencesFile.fs @@ -4,27 +4,27 @@ open System open System.IO open Logging -type GitHubReference = +type RemoteFileReference = { Name : string Link : string } type ReferencesFile = { FileName: string NugetPackages: list - GitHubFiles: list } + RemoteFiles: list } static member DefaultLink = "paket-files" static member FromLines(lines : string[]) = - let isGitHubFile (line: string) = line.StartsWith "File:" + let isSingleFile (line: string) = line.StartsWith "File:" let notEmpty (line: string) = not <| String.IsNullOrWhiteSpace line { FileName = "" - NugetPackages = lines |> Array.filter notEmpty |> Array.filter (isGitHubFile >> not) |> Array.toList - GitHubFiles = + NugetPackages = lines |> Array.filter notEmpty |> Array.filter (isSingleFile >> not) |> Array.toList + RemoteFiles = lines |> Array.filter notEmpty - |> Array.filter isGitHubFile + |> Array.filter isSingleFile |> Array.map (fun s -> s.Replace("File:","").Split([|' '|], StringSplitOptions.RemoveEmptyEntries)) |> Array.map (fun segments -> { Name = segments.[0]; Link = if segments.Length = 2 then segments.[1] @@ -46,5 +46,5 @@ type ReferencesFile = override this.ToString() = List.append this.NugetPackages - (this.GitHubFiles |> List.map (fun s -> "File:" + s.Name + if s.Link <> ReferencesFile.DefaultLink then " " + s.Link else "")) + (this.RemoteFiles |> List.map (fun s -> "File:" + s.Name + if s.Link <> ReferencesFile.DefaultLink then " " + s.Link else "")) |> String.concat Environment.NewLine \ No newline at end of file diff --git a/src/Paket.Core/GitHub.fs b/src/Paket.Core/RemoteDownload.fs similarity index 50% rename from src/Paket.Core/GitHub.fs rename to src/Paket.Core/RemoteDownload.fs index 45ab2fac72..660658c8bb 100644 --- a/src/Paket.Core/GitHub.fs +++ b/src/Paket.Core/RemoteDownload.fs @@ -1,4 +1,4 @@ -module Paket.GitHub +module Paket.RemoteDownload open Paket open Newtonsoft.Json.Linq @@ -7,21 +7,35 @@ open Ionic.Zip open Paket.Logging // Gets the sha1 of a branch -let getSHA1OfBranch owner project branch = +let getSHA1OfBranch origin owner project branch = async { - let! document = getFromUrl(None,sprintf "https://api.github.com/repos/%s/%s/commits/%s" owner project branch) - let json = JObject.Parse(document) - return json.["sha"].ToString() + match origin with + | ModuleResolver.SingleSourceFileOrigin.GitHubLink -> + let url = sprintf "https://api.github.com/repos/%s/%s/commits/%s" owner project branch + let! document = getFromUrl(None, url) + let json = JObject.Parse(document) + return json.["sha"].ToString() + | ModuleResolver.SingleSourceFileOrigin.GistLink -> + let url = sprintf "https://api.github.com/gists/%s/%s" project branch + let! document = getFromUrl(None, url) + let json = JObject.Parse(document) + return json.["id"].ToString() + | ModuleResolver.SingleSourceFileOrigin.HttpLink _ -> return "" } - /// Gets a dependencies file from github. let downloadDependenciesFile(rootPath,remoteFile:ModuleResolver.ResolvedSourceFile) = async { let fi = FileInfo(remoteFile.Name) let dependenciesFileName = remoteFile.Name.Replace(fi.Name,"paket.dependencies") - let url = sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName + let url = + match remoteFile.Origin with + | ModuleResolver.GitHubLink -> + sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName + | ModuleResolver.GistLink -> + sprintf "https://gist.githubusercontent.com/%s/%s/raw/%s" remoteFile.Owner remoteFile.Project dependenciesFileName + | ModuleResolver.HttpLink url -> sprintf "%s" url let! result = safeGetFromUrl(None,url) match result with @@ -55,15 +69,39 @@ let rec DirectoryCopy(sourceDirName, destDirName, copySubDirs) = if copySubDirs then for subdir in dirs do DirectoryCopy(subdir.FullName, Path.Combine(destDirName, subdir.Name), copySubDirs) +open ModuleResolver /// Gets a single file from github. -let downloadRemoteFiles(remoteFile:ModuleResolver.ResolvedSourceFile,destitnation) = async { +let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destitnation) = async { match remoteFile.Origin, remoteFile.Name with - | ModuleResolver.SourceFileOrigin.GitHubLink, "FULLPROJECT" -> + | SingleSourceFileOrigin.GistLink, "FULLPROJECT" -> + let fi = FileInfo(destitnation) + let projectPath = fi.Directory.FullName + + let url = sprintf "https://api.github.com/gists/%s" remoteFile.Project + let! document = getFromUrl(None, url) + let json = JObject.Parse(document) + let files = json.["files"] |> Seq.map (fun i -> i.First.["filename"].ToString(), i.First.["raw_url"].ToString()) + + let task = + files |> Seq.map (fun (filename, url) -> + async { + let path = Path.Combine(projectPath,filename) + do! downloadFromUrl(None, url) path + } + ) |> Async.Parallel + task |> Async.RunSynchronously |> ignore + + // GIST currently does not support zip-packages, so now this fetches all files separately. + // let downloadUrl = sprintf "https://gist.github.com/%s/%s/download" remoteFile.Owner remoteFile.Project //is a tar.gz + + | SingleSourceFileOrigin.GitHubLink, "FULLPROJECT" -> let fi = FileInfo(destitnation) let projectPath = fi.Directory.FullName let zipFile = Path.Combine(projectPath,sprintf "%s.zip" remoteFile.Commit) - do! downloadFromUrl(None,sprintf "https://github.com/%s/%s/archive/%s.zip" remoteFile.Owner remoteFile.Project remoteFile.Commit) zipFile + let downloadUrl = sprintf "https://github.com/%s/%s/archive/%s.zip" remoteFile.Owner remoteFile.Project remoteFile.Commit + + do! downloadFromUrl(None, downloadUrl) zipFile ExtractZip(zipFile,projectPath) @@ -72,9 +110,9 @@ let downloadRemoteFiles(remoteFile:ModuleResolver.ResolvedSourceFile,destitnatio Directory.Delete(source,true) - | ModuleResolver.SourceFileOrigin.GitHubLink, _ -> return! downloadFromUrl(None,sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destitnation - | ModuleResolver.SourceFileOrigin.HttpLink, _ -> return! downloadFromUrl(None,sprintf "%s" remoteFile.Name) destitnation - | ModuleResolver.SourceFileOrigin.NuGetPackage, _ -> failwith("Downloading single file from NuGet is not supported.") + | SingleSourceFileOrigin.GistLink, _ -> return! downloadFromUrl(None,sprintf "https://gist.githubusercontent.com/%s/%s/raw/%s" remoteFile.Owner remoteFile.Project remoteFile.Name) destitnation + | SingleSourceFileOrigin.GitHubLink, _ -> return! downloadFromUrl(None,sprintf "https://github.com/%s/%s/raw/%s/%s" remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destitnation + | SingleSourceFileOrigin.HttpLink(url), _ -> return! downloadFromUrl(None,sprintf "%s" url) destitnation } let DownloadSourceFile(rootPath, source:ModuleResolver.ResolvedSourceFile) = diff --git a/src/Paket.Core/RestoreProcess.fs b/src/Paket.Core/RestoreProcess.fs index afab420065..21c8b16bd2 100644 --- a/src/Paket.Core/RestoreProcess.fs +++ b/src/Paket.Core/RestoreProcess.fs @@ -36,7 +36,7 @@ let ExtractPackage(sources, force, package : ResolvedPackage) = let internal restore(sources, force, lockFile:LockFile, packages:Set) = let sourceFileDownloads = lockFile.SourceFiles - |> Seq.map (fun file -> GitHub.DownloadSourceFile(Path.GetDirectoryName lockFile.FileName, file)) + |> Seq.map (fun file -> RemoteDownload.DownloadSourceFile(Path.GetDirectoryName lockFile.FileName, file)) |> Async.Parallel let packageDownloads = diff --git a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs index 36b17d6b5e..6cbb06877c 100644 --- a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs +++ b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs @@ -95,9 +95,11 @@ let ``should read source file from config``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = Some "master" } { Owner = "fsharp" Project = "FAKE" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Name = "src/app/FAKE/FileWithCommit.fs" Commit = Some "bla123zxc" } ] @@ -180,9 +182,11 @@ let ``should read github source file from config without quotes``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = Some "master" } { Owner = "fsharp" Project = "FAKE" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Name = "src/app/FAKE/FileWithCommit.fs" Commit = Some "bla123zxc" } ] @@ -196,10 +200,12 @@ let ``should read github source file from config with quotes``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = Some "master" } { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/FileWith Space.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = Some "bla123zxc" } ] [] @@ -212,33 +218,51 @@ let ``should read github source files withou sha1``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = None } { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/FileWithCommit.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = Some "bla123zxc" } ] [] -let ``should read http source file from config without quotes, simple real-life``() = - let config = """http http://www.fssnip.net/raw/1M - http http://www.fssnip.net/raw/1M/1 - http https://gist.githubusercontent.com/Thorium/1972308/raw/629ed3119e18ec0629142fb30351e50ac688e7fd/gistfile1.fs """ +let ``should read http source file from config without quotes with file specs``() = + let config = """http http://www.fssnip.net/raw/1M test1.fs + http http://www.fssnip.net/raw/1M/1 src/test2.fs """ let dependencies = DependenciesFile.FromCode(config) dependencies.RemoteFiles |> shouldEqual [ { Owner = "www.fssnip.net/raw" Project = "raw/1M" - Name = "http://www.fssnip.net/raw/1M" + Name = "test1.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M" Commit = None } { Owner = "www.fssnip.net/raw" Project = "raw/1M" - Name = "http://www.fssnip.net/raw/1M/1" + Name = "src/test2.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M/1" + Commit = None } ] + +[] +let ``should read gist source file from config without quotes with file specs``() = + let config = """gist Thorium/1972308 gistfile1.fs + gist Thorium/6088882 """ //Gist supports multiple files also + let dependencies = DependenciesFile.FromCode(config) + dependencies.RemoteFiles + |> shouldEqual + [ { Owner = "Thorium" + Project = "1972308" + Name = "gistfile1.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.GistLink Commit = None } - { Owner = "gist.githubusercontent.com/Thorium" - Project = "Thorium/1972308" - Name = "https://gist.githubusercontent.com/Thorium/1972308/raw/629ed3119e18ec0629142fb30351e50ac688e7fd/gistfile1.fs" + { Owner = "Thorium" + Project = "6088882" + Name = "FULLPROJECT" + Origin = ModuleResolver.SingleSourceFileOrigin.GistLink Commit = None } ] + [] let ``should read http source file from config without quotes, parsing rules``() = // The empty "/" should be ommited. After that, parsing amount of "/"-marks: @@ -253,23 +277,28 @@ let ``should read http source file from config without quotes, parsing rules``() |> shouldEqual [ { Owner = "example" Project = "example" - Name = "http://example" + Name = "example.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" Commit = None } { Owner = "example" Project = "item" - Name = "http://example/item" + Name = "item.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item" Commit = None } { Owner = "example" Project = "item" - Name = "http://example/item" + Name = "item.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item" Commit = None } - { Owner = "example/item" + { Owner = "example" Project = "item/3" - Name = "http://example/item/3" + Name = "3.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3" Commit = None } - { Owner = "example/item" + { Owner = "example" Project = "item/3" - Name = "http://example/item/3/1" + Name = "1.fs" + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3/1" Commit = None } ] diff --git a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs index fd0c9bab22..4e7acb89c2 100644 --- a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs +++ b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs @@ -69,7 +69,7 @@ github "owner:project2:commit2" "folder/file.fs" """ match f.Commit with | Some commit -> { Commit = commit Owner = f.Owner - Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Project = f.Project Dependencies = Set.empty Name = f.Name } : ModuleResolver.ResolvedSourceFile @@ -131,13 +131,13 @@ let ``should generate other version ranges for packages``() = |> shouldEqual (normalizeLineEndings expected3) let expectedWithHttp = """HTTP - remote: LINK + remote: http://www.fssnip.net/raw/1M specs: - http://www.fssnip.net/raw/1M""" + test.fs""" [] let ``should generate lock file for http source files``() = - let config = """http "http://www.fssnip.net/raw/1M" """ + let config = """http "http://www.fssnip.net/raw/1M" "test.fs" """ let cfg = DependenciesFile.FromCode(config) @@ -145,10 +145,54 @@ let ``should generate lock file for http source files``() = |> List.map (fun f -> { Commit = "" Owner = f.Owner - Origin = Paket.ModuleResolver.SourceFileOrigin.HttpLink + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M" Project = f.Project Dependencies = Set.empty Name = f.Name } : ModuleResolver.ResolvedSourceFile) |> LockFileSerializer.serializeSourceFiles |> shouldEqual (normalizeLineEndings expectedWithHttp) +let expectedMultiple = """HTTP + remote: http://www.fssnip.net/raw/32 + specs: + myFile2.fs +GIST + remote: Thorium/1972308 + specs: + gistfile1.fs + remote: Thorium/6088882 + specs: + FULLPROJECT +HTTP + remote: http://www.fssnip.net/raw/1M + specs: + myFile.fs + remote: http://www.fssnip.net/raw/15 + specs: + myFile3.fs""" + +[] +let ``should generate lock file for http and gist source files``() = + let config = """source "http://nuget.org/api/v2 + +http http://www.fssnip.net/raw/32 myFile2.fs + +gist Thorium/1972308 gistfile1.fs +gist Thorium/6088882 + +http http://www.fssnip.net/raw/1M myFile.fs +http http://www.fssnip.net/raw/15 myFile3.fs """ + + let cfg = DependenciesFile.FromCode(config) + + cfg.RemoteFiles + |> List.map (fun f -> + { Commit = "" + Owner = f.Owner + Origin = f.Origin + Project = f.Project + Dependencies = Set.empty + Name = f.Name } : ModuleResolver.ResolvedSourceFile) + |> LockFileSerializer.serializeSourceFiles + |> shouldEqual (normalizeLineEndings expectedMultiple) + diff --git a/tests/Paket.Tests/Lockfile/ParserSpecs.fs b/tests/Paket.Tests/Lockfile/ParserSpecs.fs index d4d5059199..c0cc7edeb7 100644 --- a/tests/Paket.Tests/Lockfile/ParserSpecs.fs +++ b/tests/Paket.Tests/Lockfile/ParserSpecs.fs @@ -52,14 +52,14 @@ let ``should parse lock file``() = [ { Owner = "fsharp" Project = "FAKE" Name = "src/app/FAKE/Cli.fs" - Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Dependencies = Set.empty Commit = "7699e40e335f3cc54ab382a8969253fecc1e08a9" } { Owner = "fsharp" Project = "FAKE" Dependencies = Set.empty Name = "src/app/Fake.Deploy.Lib/FakeDeployAgentHelper.fs" - Origin = Paket.ModuleResolver.SourceFileOrigin.GitHubLink + Origin = ModuleResolver.SingleSourceFileOrigin.GitHubLink Commit = "Globbing" } ] sourceFiles.[0].Commit |> shouldEqual "7699e40e335f3cc54ab382a8969253fecc1e08a9" diff --git a/tests/Paket.Tests/ReferencesFile/ReferencesFileSpecs.fs b/tests/Paket.Tests/ReferencesFile/ReferencesFileSpecs.fs index 06bd2c3bcf..a7be351a9e 100644 --- a/tests/Paket.Tests/ReferencesFile/ReferencesFileSpecs.fs +++ b/tests/Paket.Tests/ReferencesFile/ReferencesFileSpecs.fs @@ -18,13 +18,13 @@ let ``should parse lines correctly``() = refFile.NugetPackages.Length |> shouldEqual 3 refFile.NugetPackages.Head |> shouldEqual "Castle.Windsor" refFile.NugetPackages.Tail.Tail.Head |> shouldEqual "jQuery" - refFile.GitHubFiles.Length |> shouldEqual 1 - refFile.GitHubFiles.Head.Name |> shouldEqual "FsUnit.fs" - refFile.GitHubFiles.Head.Link |> shouldEqual "paket-files" + refFile.RemoteFiles.Length |> shouldEqual 1 + refFile.RemoteFiles.Head.Name |> shouldEqual "FsUnit.fs" + refFile.RemoteFiles.Head.Link |> shouldEqual "paket-files" [] let ``should serialize itself correctly``() = - let refFile = {FileName = ""; NugetPackages = ["A"; "B"]; GitHubFiles = [{Name = "FromGithub.fs"; Link = ReferencesFile.DefaultLink}]} + let refFile = {FileName = ""; NugetPackages = ["A"; "B"]; RemoteFiles = [{Name = "FromGithub.fs"; Link = ReferencesFile.DefaultLink}]} let expected = [|"A"; "B"; "File:FromGithub.fs"|] refFile.ToString() |> toLines |> shouldEqual expected @@ -37,13 +37,13 @@ File:FsUnit.fs Tests\Common let ``should parse custom path correctly``() = let refFile = ReferencesFile.FromLines(toLines refFileWithCustomPath) refFile.NugetPackages.Length |> shouldEqual 0 - refFile.GitHubFiles.Length |> shouldEqual 1 - refFile.GitHubFiles.Head.Name |> shouldEqual "FsUnit.fs" - refFile.GitHubFiles.Head.Link |> shouldEqual "Tests\Common" + refFile.RemoteFiles.Length |> shouldEqual 1 + refFile.RemoteFiles.Head.Name |> shouldEqual "FsUnit.fs" + refFile.RemoteFiles.Head.Link |> shouldEqual "Tests\Common" [] let ``should serialize customPath correctly``() = - let refFile = {FileName = ""; NugetPackages = []; GitHubFiles = [{Name = "FromGithub.fs"; Link = "CustomPath\Dir"}]} + let refFile = {FileName = ""; NugetPackages = []; RemoteFiles = [{Name = "FromGithub.fs"; Link = "CustomPath\Dir"}]} let expected = [|"File:FromGithub.fs CustomPath\Dir"|] refFile.ToString() |> toLines |> shouldEqual expected \ No newline at end of file