Skip to content

Commit

Permalink
Optimize tokens to LineCommentOnSingleLine and LineCommentAfterSource…
Browse files Browse the repository at this point in the history
…Code trivia. (fsprojects#1505)
  • Loading branch information
nojaf authored Mar 5, 2021
1 parent c86b187 commit 69fd7ea
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 34 deletions.
40 changes: 40 additions & 0 deletions src/Fantomas.Tests/CommentTests.fs
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,27 @@ type IExports =
// U2<MomentConstructor1, MomentConstructor2>
"""

[<Test>]
let ``comment newline comment`` () =
formatSourceString
false
"""
// foo

// bar
let x = 8
"""
config
|> prepend newline
|> should
equal
"""
// foo

// bar
let x = 8
"""

[<Test>]
let ``comments before no warn should not be removed, 1220`` () =
formatSourceString
Expand Down Expand Up @@ -1142,3 +1163,22 @@ if stateSub.Value |> State.hasChanges then
|> Option.bind Dto.changesAsImport
|> Option.iter (fun changes -> update (Import changes) |> ignore)
"""

[<Test>]
let ``very long comment on single line`` () =
formatSourceString
false
"""
// Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec ipsum nulla, pellentesque eget maximus et, facilisis eu nibh. Suspendisse convallis scelerisque urna, id fringilla dolor mollis id. Etiam dictum pellentesque nisl, vel ullamcorper neque accumsan eget. Pellentesque at mattis magna. Cras varius nisl nisi, sed iaculis tortor auctor quis. Sed luctus eget ante in dapibus. Cras ac leo nibh. Sed commodo, ex vel interdum egestas, risus lorem volutpat sapien, at pellentesque lectus ipsum non libero. Pellentesque malesuada scelerisque augue at blandit. Fusce in nisl sapien. In hac habitasse platea dictumst. Aenean tristique nibh ac tortor laoreet, rutrum aliquam elit rutrum. In vitae dignissim neque.
// Dollar
let meh = 7
"""
config
|> prepend newline
|> should
equal
"""
// Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec ipsum nulla, pellentesque eget maximus et, facilisis eu nibh. Suspendisse convallis scelerisque urna, id fringilla dolor mollis id. Etiam dictum pellentesque nisl, vel ullamcorper neque accumsan eget. Pellentesque at mattis magna. Cras varius nisl nisi, sed iaculis tortor auctor quis. Sed luctus eget ante in dapibus. Cras ac leo nibh. Sed commodo, ex vel interdum egestas, risus lorem volutpat sapien, at pellentesque lectus ipsum non libero. Pellentesque malesuada scelerisque augue at blandit. Fusce in nisl sapien. In hac habitasse platea dictumst. Aenean tristique nibh ac tortor laoreet, rutrum aliquam elit rutrum. In vitae dignissim neque.
// Dollar
let meh = 7
"""
119 changes: 85 additions & 34 deletions src/Fantomas/TokenParser.fs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ open FSharp.Compiler.SourceCodeServices

let private whiteSpaceTag = 4
let private lineCommentTag = 8
let private commentTag = 3
let private greaterTag = 160

// workaround for cases where tokenizer dont output "delayed" part of operator after ">."
Expand Down Expand Up @@ -599,57 +600,107 @@ let ``only whitespaces were found in the remainder of the line`` lineNumber toke
&& t.TokenInfo.Tag <> whiteSpaceTag)
|> not

let private (|LineComment|_|) (token: Token) =
if token.TokenInfo.Tag = lineCommentTag then
Some token
else
None

let private (|NoCommentToken|_|) (token: Token) =
if token.TokenInfo.Tag <> lineCommentTag
&& token.TokenInfo.Tag <> commentTag then
Some token
else
None

let private (|WhiteSpaceToken|_|) (token: Token) =
if token.TokenInfo.Tag = whiteSpaceTag then
Some token
else
None

let private (|LineComments|_|) (tokens: Token list) =
let rec collect (tokens: Token list) (lastLineNumber: int) (commentTokens: Token list) =
match tokens with
| LineComment lc :: rest when (lc.LineNumber <= lastLineNumber + 1) ->
collect rest lc.LineNumber (lc :: commentTokens)
| _ -> commentTokens

match tokens with
| LineComment h :: _ ->
collect tokens (h.LineNumber) []
|> fun commentTokens -> Some(List.rev commentTokens, List.skip commentTokens.Length tokens)
| _ -> None

let private collectComment (commentTokens: Token list) =
commentTokens
|> List.groupBy (fun t -> t.LineNumber)
|> List.map (snd >> getContentFromTokens)
|> String.concat "\n"

let rec private getTriviaFromTokensThemSelves
(mkRange: MkRange)
(allTokens: Token list)
(tokens: Token list)
foundTrivia
=
match tokens with
| headToken :: rest when (headToken.TokenInfo.Tag = lineCommentTag) ->
let lineCommentTokens =
Seq.zip
rest
(headToken :: rest
|> List.map (fun x -> x.LineNumber))
|> Seq.takeWhile
(fun (t, currentLineNumber) ->
t.TokenInfo.Tag = lineCommentTag
&& t.LineNumber <= (currentLineNumber + 1))
|> Seq.map fst
|> Seq.toList
| (NoCommentToken nct) :: WhiteSpaceToken _ :: (LineComments (commentTokens, nextTokens)) ->
let comment =
let commentText = collectComment commentTokens
let firstCommentToken = List.head commentTokens

(if nct.LineNumber < firstCommentToken.LineNumber then
LineCommentOnSingleLine commentText
else
LineCommentAfterSourceCode commentText)
|> Comment

let range =
let headToken = List.head commentTokens
let lastToken = List.tryLast commentTokens
getRangeBetween mkRange headToken (Option.defaultValue headToken lastToken)

let info =
Trivia.Create comment range
|> List.appendItem foundTrivia

getTriviaFromTokensThemSelves mkRange allTokens (nct :: nextTokens) info
| (NoCommentToken nct) :: (LineComments (commentTokens, nextTokens)) ->
let comment =
headToken
|> List.prependItem lineCommentTokens
|> List.groupBy (fun t -> t.LineNumber)
|> List.map (snd >> getContentFromTokens)
|> String.concat "\n"
let commentText = collectComment commentTokens
let firstCommentToken = List.head commentTokens

let nextTokens =
List.length lineCommentTokens
|> fun length -> List.skip length rest
(if nct.LineNumber < firstCommentToken.LineNumber
|| (nct.TokenInfo.Tag = whiteSpaceTag
&& firstCommentToken.LineNumber = nct.LineNumber) then
LineCommentOnSingleLine commentText
else
LineCommentAfterSourceCode commentText)
|> Comment

let range =
let lastToken = List.tryLast lineCommentTokens
let headToken = List.head commentTokens
let lastToken = List.tryLast commentTokens
getRangeBetween mkRange headToken (Option.defaultValue headToken lastToken)

let info =
let toLineComment =
allTokens
|> List.exists
(fun t ->
t.LineNumber = headToken.LineNumber
&& t.TokenInfo.Tag <> whiteSpaceTag
&& t.TokenInfo.RightColumn < headToken.TokenInfo.LeftColumn)
|> fun e ->
if e then
LineCommentAfterSourceCode
else
LineCommentOnSingleLine
Trivia.Create comment range
|> List.appendItem foundTrivia

let comment = toLineComment comment |> Comment
getTriviaFromTokensThemSelves mkRange allTokens (nct :: nextTokens) info
| LineComments (commentTokens, nextTokens) ->
let comment =
collectComment commentTokens
|> LineCommentOnSingleLine
|> Comment

let range =
let headToken = List.head commentTokens
let lastToken = List.tryLast commentTokens
getRangeBetween mkRange headToken (Option.defaultValue headToken lastToken)

let info =
Trivia.Create comment range
|> List.appendItem foundTrivia

Expand Down

0 comments on commit 69fd7ea

Please sign in to comment.