Skip to content

Commit

Permalink
Parse Ruby comments as CommonMark
Browse files Browse the repository at this point in the history 
Updates the formatting of Ruby comments to expect comments in the format 
of [CommonMark](https://commonmark.org/). Proto comments generally 
follow the CommonMark spec.

Replaces ad hoc comment transformations with a visitor for the Markdown 
elements. Adds tests for the RubyCommentReformatter.

Cleans up comment regex patterns.

Updates googleapis#722
  • Loading branch information
@evaogbe
evaogbe committed Feb 12, 2019
1 parent bbec4bb commit 3981768
Show file tree
Hide file tree
Showing 7 changed files with 622 additions and 193 deletions.
35 changes: 0 additions & 35 deletions src/main/java/com/google/api/codegen/util/CommentPatterns.java
View file

This file was deleted.

14 changes: 11 additions & 3 deletions src/main/java/com/google/api/codegen/util/LinkPattern.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,17 @@
* </pre>
*/
public class LinkPattern {
public static LinkPattern ABSOLUTE = new LinkPattern(CommentPatterns.ABSOLUTE_LINK_PATTERN, "");
public static LinkPattern RELATIVE = new LinkPattern(CommentPatterns.RELATIVE_LINK_PATTERN, "");
public static LinkPattern PROTO = new LinkPattern(CommentPatterns.PROTO_LINK_PATTERN, "");

private static final Pattern ABSOLUTE_LINK_PATTERN =
Pattern.compile("\\[([^\\]]+)\\]\\((\\p{Alpha}+:[^\\)]+)\\)");
private static final Pattern RELATIVE_LINK_PATTERN =
Pattern.compile("\\[([^\\]]+)\\]\\(((?!\\p{Alpha}+:)[^\\)]+)\\)");
private static final Pattern PROTO_LINK_PATTERN =
Pattern.compile("\\[([^\\]]+)\\]\\[([A-Za-z_][A-Za-z_.0-9]*)?\\]");

public static LinkPattern ABSOLUTE = new LinkPattern(ABSOLUTE_LINK_PATTERN, "");
public static LinkPattern RELATIVE = new LinkPattern(RELATIVE_LINK_PATTERN, "");
public static LinkPattern PROTO = new LinkPattern(PROTO_LINK_PATTERN, "");

private Pattern pattern;
private String urlPrefix;
Expand Down
316 changes: 233 additions & 83 deletions src/main/java/com/google/api/codegen/util/ruby/RubyCommentReformatter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,106 +14,256 @@
*/
package com.google.api.codegen.util.ruby;

import com.google.api.codegen.util.CommentPatterns;
import com.google.api.codegen.util.CommentReformatter;
import com.google.api.codegen.util.CommentTransformer;
import com.google.api.codegen.util.CommentTransformer.Transformation;
import com.google.api.codegen.util.LinkPattern;
import com.google.common.base.Function;
import com.google.api.codegen.util.ErrorMarkdownVisitor;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.commonmark.node.BulletList;
import org.commonmark.node.Code;
import org.commonmark.node.Document;
import org.commonmark.node.Emphasis;
import org.commonmark.node.HardLineBreak;
import org.commonmark.node.Heading;
import org.commonmark.node.HtmlBlock;
import org.commonmark.node.HtmlInline;
import org.commonmark.node.IndentedCodeBlock;
import org.commonmark.node.Link;
import org.commonmark.node.ListBlock;
import org.commonmark.node.ListItem;
import org.commonmark.node.Node;
import org.commonmark.node.Paragraph;
import org.commonmark.node.SoftLineBreak;
import org.commonmark.node.Text;
import org.commonmark.parser.Parser;

public class RubyCommentReformatter implements CommentReformatter {
private static final String BULLET = "* ";

private static Transformation PROTO_TO_RUBY_DOC_TRANSFORMATION =
new Transformation(
CommentPatterns.PROTO_LINK_PATTERN,
new Function<String, String>() {
@Override
public String apply(String matchedString) {
Matcher matcher = CommentPatterns.PROTO_LINK_PATTERN.matcher(matchedString);
matcher.find();
String title = matcher.group(1);
String ref = matcher.group(2);
if (ref == null || ref.equals(title)) {
return String.format("{%s}", Matcher.quoteReplacement(protoToRubyDoc(title)));
}
return String.format(
"{%s %s}",
Matcher.quoteReplacement(protoToRubyDoc(ref)),
Matcher.quoteReplacement(protoToRubyDoc(title, false)));
}
});

private CommentTransformer transformer =
CommentTransformer.newBuilder()
.transform(PROTO_TO_RUBY_DOC_TRANSFORMATION)
.transform(
LinkPattern.RELATIVE
.withUrlPrefix(CommentTransformer.CLOUD_URL_PREFIX)
.toFormat("[$TITLE]($URL)"))
.transform(LinkPattern.ABSOLUTE.toFormat("[$TITLE]($URL)"))
.scopedReplace(CommentPatterns.HEADLINE_PATTERN, "#", "=")
.build();

private static final Logger LOGGER = Logger.getLogger(RubyCommentReformatter.class.getName());

// Might as well create only one. Parser is thread-safe.
private static final Parser PARSER = Parser.builder().build();

private static String CLOUD_URL_PREFIX = "https://cloud.google.com";

private static final Pattern LINE_SEPARATOR = Pattern.compile("\\v");

private static final Pattern RELATIVE_LINK_DEST_PATTERN = Pattern.compile("(?!\\p{Alpha}+:).+");

private static final Pattern PROTO_LINK_PATTERN =
Pattern.compile("\\[(?<title>[^\\]]+)\\]\\[(?<element>[A-Za-z_][A-Za-z_.0-9]*)?\\]");

@Override
public String reformat(String comment) {
Node root = PARSER.parse(comment);
RubyVisitor visitor = new RubyVisitor();
try {
root.accept(visitor);
return visitor.toString();
} catch (ErrorMarkdownVisitor.UnimplementedRenderException e) {
LOGGER.log(
Level.WARNING, "markdown contains elements we don't handle; copying doc verbatim", e);
return comment;
}
}

private static class RubyVisitor extends ErrorMarkdownVisitor {
StringBuffer sb = new StringBuffer();
int listIndent = 0;
boolean followsListItem = false;
boolean followsBlankLine = false;
for (String line : Splitter.on("\n").split(comment)) {
Matcher listMatcher = CommentPatterns.UNORDERED_LIST_PATTERN.matcher(line);
boolean matchesList = listMatcher.lookingAt();
Matcher indentMatcher = CommentPatterns.INDENT_PATTERN.matcher(line);
indentMatcher.lookingAt();
int indent = indentMatcher.group().length();
if (matchesList) {
line = listMatcher.replaceFirst(BULLET);
int indentLevel = 0;

@Override
public String toString() {
return sb.toString().trim();
}

@Override
public void visit(BulletList bulletList) {
++indentLevel;
visitChildren(bulletList);
--indentLevel;
}

@Override
public void visit(Code code) {
sb.append("`").append(code.getLiteral()).append("`");
visitChildren(code);
}

@Override
public void visit(Document document) {
visitChildren(document);
}

@Override
public void visit(Emphasis emphasis) {
sb.append('*');
visitChildren(emphasis);
sb.append('*');
}

@Override
public void visit(HtmlInline htmlInline) {
sb.append(htmlInline.getLiteral());
visitChildren(htmlInline);
}

@Override
public void visit(HtmlBlock htmlBlock) {
sb.append(htmlBlock.getLiteral());
visitChildren(htmlBlock);
}

@Override
public void visit(Link link) {
sb.append('[');

// The text is in the child.
visitChildren(link);

sb.append("](");

if (RELATIVE_LINK_DEST_PATTERN.matcher(link.getDestination()).matches()) {
sb.append(CLOUD_URL_PREFIX);
}
if (indent < listIndent && (matchesList || followsBlankLine)) {
listIndent -= BULLET.length();
} else if (followsListItem && (!matchesList || indent > listIndent)) {
listIndent += BULLET.length();

sb.append(link.getDestination()).append(')');
}

@Override
public void visit(ListItem listItem) {
sb.append('\n');

// Adjust loose list.
ListBlock parent = (ListBlock) listItem.getParent();
if (!parent.isTight()) {
sb.append('\n');
}
if (listIndent > 0) {
line = line.trim();
sb.append(Strings.repeat(" ", listIndent));

// Reduce indent for bullet.
--indentLevel;
printIndent();
sb.append("* ");
++indentLevel;

Node child = listItem.getFirstChild();
if (child != null) {
// Skip formatting first Paragraph child.
visitChildren(child);
child = child.getNext();
}

while (child != null) {
child.accept(this);
child = child.getNext();
}
sb.append(transformer.transform(line)).append("\n");
followsListItem = matchesList;
followsBlankLine = line.isEmpty();
}
return sb.toString().trim();
}

private static String protoToRubyDoc(String comment) {
return protoToRubyDoc(comment, true);
}
@Override
public void visit(Paragraph paragraph) {
sb.append("\n\n");
printIndent();
visitChildren(paragraph);
}

private static String protoToRubyDoc(String comment, boolean changeCase) {
boolean messageFound = false;
boolean isFirstSegment = true;
StringBuilder builder = new StringBuilder();
for (String name : Splitter.on(".").split(comment)) {
char firstChar = name.charAt(0);
if (Character.isUpperCase(firstChar)) {
builder.append(isFirstSegment ? "" : "::").append(name);
messageFound = true;
} else if (messageFound) {
// Lowercase segment after message is found is field.
// In Ruby, it is referred as "Message#field" format.
builder.append("#").append(name);
} else {
builder
.append(isFirstSegment ? "" : "::")
.append(changeCase ? Character.toUpperCase(firstChar) : firstChar)
.append(name.substring(1));
@Override
public void visit(SoftLineBreak softLineBreak) {
sb.append('\n');
printIndent();
visitChildren(softLineBreak);
}

@Override
public void visit(Text text) {
Matcher matcher = PROTO_LINK_PATTERN.matcher(text.getLiteral());
while (matcher.find()) {
String protoLink =
getProtoLink(matcher.group("title"), matcher.group("element")).replace("$", "\\$");
matcher.appendReplacement(sb, protoLink);
}
isFirstSegment = false;

matcher.appendTail(sb);

visitChildren(text);
}

@Override
public void visit(HardLineBreak hardLineBreak) {
sb.append('\n');
printIndent();
visitChildren(hardLineBreak);
}

@Override
public void visit(Heading heading) {
sb.append("\n\n");
IntStream.range(0, heading.getLevel()).forEach(i -> sb.append('='));
sb.append(' ');
visitChildren(heading);
}

@Override
public void visit(IndentedCodeBlock indentedCodeBlock) {
sb.append('\n');
indentLevel += 2;
LINE_SEPARATOR
.splitAsStream(indentedCodeBlock.getLiteral())
.peek(line -> sb.append('\n'))
.peek(line -> printIndent())
.forEach(sb::append);
visitChildren(indentedCodeBlock);
indentLevel -= 2;
}

private void printIndent() {
IntStream.range(0, indentLevel).forEach(i -> sb.append(" "));
}

private String getProtoLink(String title, String element) {
if (Strings.isNullOrEmpty(element) || title.equals(element)) {
return String.format("{%s}", protoToRubyDoc(title));
}

return String.format("{%s %s}", protoToRubyDoc(element), protoToRubyDoc(title, false));
}

private static String protoToRubyDoc(String protoElement) {
return protoToRubyDoc(protoElement, true);
}

private static String protoToRubyDoc(String protoElement, boolean changeCase) {
List<String> nameSegments = Splitter.on('.').splitToList(protoElement);
if (nameSegments.isEmpty()) {
return protoElement;
}

String message =
nameSegments
.stream()
.filter(n -> !n.isEmpty())
.limit(nameSegments.size() - 1)
.map(n -> changeCase ? capitalize(n) : n)
.collect(Collectors.joining("::"));
String field = nameSegments.get(nameSegments.size() - 1);

if (field.isEmpty() || field.equals(protoElement)) {
return protoElement;
}

if (Character.isUpperCase(field.charAt(0))) {
return String.format("%s::%s", message, field);
}

return String.format("%s#%s", message, field);
}

private static String capitalize(String str) {
return Character.toUpperCase(str.charAt(0)) + str.substring(1, str.length());
}
return builder.toString();
}
}
Loading

0 comments on commit 3981768

Please sign in to comment.