From 765f10c5ffaf6901a53cc536d142f680a3625abc Mon Sep 17 00:00:00 2001 From: mavnn Date: Thu, 26 Feb 2015 14:18:13 +0000 Subject: [PATCH] Rewrite template file parsing --- src/Paket.Core/PackageProcess.fs | 2 +- src/Paket.Core/TemplateFile.fs | 221 +++++++++++++---------- src/Paket/Paket.fsproj | 4 +- tests/Paket.Tests/TemplateFileParsing.fs | 32 ++-- 4 files changed, 150 insertions(+), 109 deletions(-) diff --git a/src/Paket.Core/PackageProcess.fs b/src/Paket.Core/PackageProcess.fs index f3aa01757a..492b368715 100644 --- a/src/Paket.Core/PackageProcess.fs +++ b/src/Paket.Core/PackageProcess.fs @@ -67,7 +67,7 @@ let Pack(dependencies : DependenciesFile, packageOutputPath, buildConfig, versio let missing = [ if merged.Id = None then yield "Id" if merged.Version = None then yield "Version" - if merged.Authors = None then yield "Authors" + if merged.Authors = None || merged.Authors = Some [] then yield "Authors" if merged.Description = None then yield "Description" ] |> fun xs -> String.Join(", ",xs) diff --git a/src/Paket.Core/TemplateFile.fs b/src/Paket.Core/TemplateFile.fs index 839423d47c..c58970f501 100644 --- a/src/Paket.Core/TemplateFile.fs +++ b/src/Paket.Core/TemplateFile.fs @@ -6,6 +6,80 @@ open System.IO open System.Text.RegularExpressions open Paket.Rop open Paket.Domain + +module private TemplateParser = + type private ParserState = + { + Remaining : string list + Map : Map + Line : int + } + + let private single = Regex("^(\S+)\s*$", RegexOptions.Compiled) + let private multi = Regex("^(\S+)\s+(\S.*)", RegexOptions.Compiled) + + let private (!!) (i : int) (m : Match) = + m.Groups.[i].Value.Trim().ToLowerInvariant() + + let private (|SingleToken|_|) line = + let m = single.Match line + match m.Success with + | true -> Some (!! 1 m) + | false -> None + + let private (|MultiToken|_|) line = + let m = multi.Match line + match m.Success with + | true -> + Some (!! 1 m, m.Groups.[2].Value.Trim()) + | false -> None + + let private indented = Regex("^\s+(.*)", RegexOptions.Compiled) + let private (|Indented|_|) line = + let i = indented.Match line + match i.Success with + | true -> i.Groups.[1].Value.Trim() |> Some + | false -> None + + let rec private indentedBlock acc i lines = + match lines with + | (Indented h)::t -> + indentedBlock (h::acc) (i + 1) t + | _ -> acc |> List.rev |> String.concat "\n", i, lines + + let rec private inner state = + match state with + | { Remaining = [] } -> Choice1Of2 state.Map + | { Remaining = h::t } -> + match h with + | Indented _ -> Choice2Of2 <| sprintf "Indented block with no name line %d" state.Line + | MultiToken (key, value) -> + inner { state with + Remaining = t + Map = Map.add key value state.Map + Line = state.Line + 1 } + | SingleToken key -> + let value, line, remaining = indentedBlock [] state.Line t + if value = "" then + Choice2Of2 <| sprintf "No indented block following name '%s' line %d" key line + else + inner { state with + Remaining = remaining + Map = Map.add key value state.Map + Line = line } + | "" -> + inner { state with Line = state.Line + 1; Remaining = t } + | _ -> + Choice2Of2 <| sprintf "Invalid syntax line %d" state.Line + + let parse (contents : string) = + inner { + Remaining = + contents.Split('\n') + |> Array.toList + Line = 1 + Map = Map.empty + } type internal CompleteCoreInfo = { Id : string @@ -86,60 +160,29 @@ module internal TemplateFile = | ProjectInfo(core, optional) -> ProjectInfo(core, { optional with ReleaseNotes = Some releaseNotes }) { templateFile with Contents = contents } - let private (!<) prefix lines = - let singleLine str = - let regex = sprintf "^%s (?<%s>.*)" prefix prefix - let reg = Regex(regex, RegexOptions.Compiled ||| RegexOptions.CultureInvariant ||| RegexOptions.IgnoreCase) - if reg.IsMatch str then Some <| (reg.Match str).Groups.[prefix].Value - else None - - let multiLine lines = - let rec findBody acc (lines : string list) = - match lines with - | h :: t when h.StartsWith " " -> findBody (h.Trim() :: acc) t - | _ -> - Some(acc - |> List.rev - |> String.concat Environment.NewLine) - - let rec findStart lines = - match (lines : String list) with - | h :: t when h.ToLowerInvariant() = prefix.ToLowerInvariant() -> findBody [] t - | h :: t -> findStart t - | [] -> None - - findStart lines - - [ lines |> List.tryPick singleLine - multiLine lines ] - |> List.tryPick id - let private failP str = fail <| PackagingConfigParseError str type private PackageConfigType = | FileType | ProjectType + + let private parsePackageConfigType map = + let t' = Map.tryFind "type" map + t' |> function + | Some s -> + match s with + | "file" -> succeed FileType + | "project" -> succeed ProjectType + | s -> failP (sprintf "Unknown package config type.") + | None -> failP (sprintf "First line of paket.package file had no 'type' declaration.") - let private parsePackageConfigType contents = - match contents with - | firstLine :: _ -> - let t' = (!<) "type" [ firstLine ] - t' |> function - | Some s -> - match s with - | "file" -> succeed FileType - | "project" -> succeed ProjectType - | s -> failP (sprintf "Unknown package config type.") - | None -> failP (sprintf "First line of paket.package file had no 'type' declaration.") - | [] -> failP "Empty paket.template file." - - let private getId lines = - (!<) "id" lines |> function + let private getId map = + Map.tryFind "id" map |> function | Some m -> succeed <| m | None -> failP "No id line in paket.template file." - let private getAuthors lines = - (!<) "authors" lines |> function + let private getAuthors (map : Map) = + Map.tryFind "authors" map |> function | Some m -> m.Split ',' |> Array.map (fun s -> s.Trim()) @@ -147,13 +190,13 @@ module internal TemplateFile = |> succeed | None -> failP "No authors line in paket.template file." - let private getDescription lines = - (!<) "description" lines |> function + let private getDescription map = + Map.tryFind "description" map |> function | Some m -> succeed m | None -> failP "No description line in paket.template file." - let private getDependencies lines = - (!<) "dependencies" lines + let private getDependencies (map : Map) = + Map.tryFind "dependencies" map |> Option.map (fun d -> d.Split '\n') |> Option.map (Array.map (fun d -> let reg = Regex(@"(?\S+)(?.*)").Match d @@ -167,8 +210,8 @@ module internal TemplateFile = let private fromReg = Regex("from (?.*)", RegexOptions.Compiled) let private toReg = Regex("to (?.*)", RegexOptions.Compiled) - let private getFiles lines = - (!<) "files" lines + let private getFiles (map : Map) = + Map.tryFind "files" map |> Option.map (fun d -> d.Split '\n') |> Option.map (Seq.map @@ -181,31 +224,31 @@ module internal TemplateFile = |> Option.map List.ofSeq |> fun x -> defaultArg x [] - let private getOptionalInfo configLines = - let title = (!<) "title" configLines + let private getOptionalInfo (map : Map) = + let title = Map.tryFind "title" map let owners = - (!<) "owners" configLines + Map.tryFind "owners" map |> Option.map (fun o -> o.Split(',') |> Array.map (fun o -> o.Trim()) |> Array.toList) |> fun x -> defaultArg x [] - let releaseNotes = (!<) "releaseNotes" configLines - let summary = (!<) "summary" configLines - let language = (!<) "language" configLines - let projectUrl = (!<) "projectUrl" configLines - let iconUrl = (!<) "iconUrl" configLines - let licenseUrl = (!<) "licenseUrl" configLines - let copyright = (!<) "copyright" configLines + let releaseNotes = Map.tryFind "releaseNotes" map + let summary = Map.tryFind "summary" map + let language = Map.tryFind "language" map + let projectUrl = Map.tryFind "projectUrl" map + let iconUrl = Map.tryFind "iconUrl" map + let licenseUrl = Map.tryFind "licenseUrl" map + let copyright = Map.tryFind "copyright" map let requireLicenseAcceptance = - match (!<) "requireLicenseAcceptance" configLines with + match Map.tryFind "requireLicenseAcceptance" map with | Some x when x.ToLower() = "true" -> true | _ -> false let tags = - (!<) "tags" configLines + Map.tryFind "tags" map |> Option.map (fun t -> t.Split ' ' |> Array.map (fun t -> t.Trim()) @@ -213,7 +256,7 @@ module internal TemplateFile = |> fun x -> defaultArg x [] let developmentDependency = - match (!<) "developmentDependency" configLines with + match Map.tryFind "developmentDependency" map with | Some x when x.ToLower() = "true" -> true | _ -> false @@ -229,48 +272,44 @@ module internal TemplateFile = RequireLicenseAcceptance = requireLicenseAcceptance Tags = tags DevelopmentDependency = developmentDependency - Dependencies = getDependencies configLines - Files = getFiles configLines } + Dependencies = getDependencies map + Files = getFiles map } let Parse(contentStream : Stream) = rop { - let configLines = - use sr = new StreamReader(contentStream, System.Text.Encoding.UTF8) - - let rec inner (s : StreamReader) = - seq { - let line = s.ReadLine() - if line <> null then - yield line - yield! inner s - } - inner sr |> Seq.toList - let! type' = parsePackageConfigType configLines + let sr = new StreamReader(contentStream) + let! map = + match TemplateParser.parse (sr.ReadToEnd()) with + | Choice1Of2 m -> succeed m + | Choice2Of2 f -> failP f + sr.Dispose() + let! type' = parsePackageConfigType map match type' with | ProjectType -> let core : ProjectCoreInfo = - { Id = (!<) "id" configLines - Version = (!<) "version" configLines |> Option.map SemVer.Parse + { Id = Map.tryFind "id" map + Version = Map.tryFind "version" map |> Option.map SemVer.Parse Authors = - (!<) "authors" configLines |> Option.map (fun s -> - s.Split(',') - |> Array.map (fun s -> s.Trim()) - |> Array.toList) - Description = (!<) "description" configLines } + Map.tryFind "authors" map + |> Option.map (fun s -> + s.Split(',') + |> Array.map (fun s -> s.Trim()) + |> Array.toList) + Description = Map.tryFind "description" map } - let optionalInfo = getOptionalInfo configLines + let optionalInfo = getOptionalInfo map return ProjectInfo(core, optionalInfo) | FileType -> - let! id' = getId configLines - let! authors = getAuthors configLines - let! description = getDescription configLines + let! id' = getId map + let! authors = getAuthors map + let! description = getDescription map let core : CompleteCoreInfo = { Id = id' - Version = (!<) "version" configLines |> Option.map SemVer.Parse + Version = Map.tryFind "version" map |> Option.map SemVer.Parse Authors = authors Description = description } - let optionalInfo = getOptionalInfo configLines + let optionalInfo = getOptionalInfo map return CompleteInfo(core, optionalInfo) } diff --git a/src/Paket/Paket.fsproj b/src/Paket/Paket.fsproj index 5e3d9cd08e..7ed55de4f2 100644 --- a/src/Paket/Paket.fsproj +++ b/src/Paket/Paket.fsproj @@ -27,10 +27,10 @@ 3 - update + pack output temp version 1.0.0.0 Project paket.exe - d:\code\Paket09x + c:\rip\FSharpx.Async pdbonly diff --git a/tests/Paket.Tests/TemplateFileParsing.fs b/tests/Paket.Tests/TemplateFileParsing.fs index ef104d247e..523e9107b5 100644 --- a/tests/Paket.Tests/TemplateFileParsing.fs +++ b/tests/Paket.Tests/TemplateFileParsing.fs @@ -20,7 +20,7 @@ let FileBasedLongDesc = """type file id My.Thing version 1.0 authors Bob McBob -description +description A longer description on two lines. """ @@ -115,24 +115,26 @@ description A short description [] let DescriptionTest = """type project -id - 15below.TravelStatus.CommonMessages -title - 15below.TravelStatus.CommonMessages -authors - 15below owners - 15below -requireLicenseAcceptance - false -description - Common messages for Travel Status + Thomas Petricek, David Thomas, Ryan Riley, Steffen Forkmann +authors + Thomas Petricek, David Thomas, Ryan Riley, Steffen Forkmann projectUrl - https://github.com/15below/Pasngr.TravelStatus/tree/master/src/15below.TravelStatus.CommonMessages + http://fsprojects.github.io/FSharpx.Async/ iconUrl - https://si0.twimg.com/profile_images/3046082295/a10bd2175096bd5faebbd8285e319d54_bigger.png + http://fsprojects.github.io/FSharpx.Async/img/logo.png +licenseUrl + http://fsprojects.github.io/FSharpx.Async/license.html +requireLicenseAcceptance + false copyright - Copyright 2013 + Copyright 2015 +tags + F#, async, fsharpx +summary + Async extensions for F# +description + Async extensions for F# """