From 691b79c46ab2caf6cc9e9f07a638654b6ed79661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Fri, 25 Aug 2023 14:04:54 +0200 Subject: [PATCH 01/12] Re-write/Fix antlr-utils --- .../de/jplag/antlr/AbstractAntlrListener.java | 198 +++++------------- .../antlr/AbstractAntlrParserAdapter.java | 33 ++- .../java/de/jplag/antlr/AbstractVisitor.java | 122 +++++++++++ .../de/jplag/antlr/ContextTokenBuilder.java | 53 ----- .../jplag/antlr/ContextTokenBuilderType.java | 10 - .../java/de/jplag/antlr/ContextVisitor.java | 98 +++++++++ .../antlr/InternalListenerException.java | 15 -- .../java/de/jplag/antlr/RangeBuilder.java | 112 ---------- .../de/jplag/antlr/TerminalTokenBuilder.java | 29 --- .../java/de/jplag/antlr/TerminalVisitor.java | 21 ++ .../java/de/jplag/antlr/TokenBuilder.java | 140 ------------- .../java/de/jplag/antlr/TokenCollector.java | 56 ++++- .../antlr/testLanguage/TestListener.java | 19 +- .../main/java/de/jplag/cpp2/CPPListener.java | 80 +++---- .../java/de/jplag/kotlin/KotlinListener.java | 86 ++++---- 15 files changed, 452 insertions(+), 620 deletions(-) create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilder.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilderType.java create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/InternalListenerException.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/RangeBuilder.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/TerminalTokenBuilder.java create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java delete mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/TokenBuilder.java diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index b8a38cd28..19c1b00ad 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -13,7 +13,6 @@ import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.TerminalNode; -import de.jplag.TokenType; import de.jplag.semantics.VariableRegistry; /** @@ -23,15 +22,11 @@ */ @SuppressWarnings("unused") public class AbstractAntlrListener implements ParseTreeListener { - private final List> startMappings; - private final List> endMappings; - - private final List terminalMapping; - private final TokenCollector collector; private final File currentFile; - - private VariableRegistry variableRegistry; + private final List> contextVisitors; + private final List terminalVisitors; + protected final VariableRegistry variableRegistry; /** * New instance @@ -42,15 +37,9 @@ public class AbstractAntlrListener implements ParseTreeListener { public AbstractAntlrListener(TokenCollector collector, File currentFile, boolean extractsSemantics) { this.collector = collector; this.currentFile = currentFile; - - this.startMappings = new ArrayList<>(); - this.endMappings = new ArrayList<>(); - - this.terminalMapping = new ArrayList<>(); - - if (extractsSemantics) { - this.variableRegistry = new VariableRegistry(); - } + this.contextVisitors = new ArrayList<>(); + this.terminalVisitors = new ArrayList<>(); + this.variableRegistry = new VariableRegistry(); } /** @@ -62,157 +51,70 @@ public AbstractAntlrListener(TokenCollector collector, File currentFile) { this(collector, currentFile, false); } - @Override - public void visitTerminal(TerminalNode terminalNode) { - this.terminalMapping.stream().filter(mapping -> mapping.matches(terminalNode.getSymbol())) - .forEach(mapping -> mapping.createToken(terminalNode.getSymbol(), variableRegistry)); - } - - @Override - public void visitErrorNode(ErrorNode errorNode) { - // does nothing, because we do not handle error nodes right now. - } - - @Override - public void enterEveryRule(ParserRuleContext rule) { - this.startMappings.stream().filter(mapping -> mapping.matches(rule)).forEach(mapping -> mapping.createToken(rule, variableRegistry)); - } - - @Override - public void exitEveryRule(ParserRuleContext rule) { - this.endMappings.stream().filter(mapping -> mapping.matches(rule)).forEach(mapping -> mapping.createToken(rule, variableRegistry)); - } - /** - * Creates a mapping using the start token from antlr as the location - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param The type of {@link ParserRuleContext} - * @return The builder for the token - */ - protected ContextTokenBuilder mapEnter(Class antlrType, TokenType jplagType) { - return this.mapEnter(antlrType, jplagType, it -> true); - } - - /** - * Creates a mapping using the start token from antlr as the location - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param condition The condition under which the mapping applies - * @param The type of {@link ParserRuleContext} - * @return The builder for the token + * Visit the given node. + * @param antlrType The antlr type of the node. + * @param condition An additional condition for the visit. + * @return A visitor for the node. + * @param The class of the node. */ @SuppressWarnings("unchecked") - protected ContextTokenBuilder mapEnter(Class antlrType, TokenType jplagType, Predicate condition) { - ContextTokenBuilder builder = initTypeBuilder(antlrType, jplagType, condition, ContextTokenBuilderType.START); - this.startMappings.add((ContextTokenBuilder) builder); - return builder; + public ContextVisitor visit(Class antlrType, Predicate condition) { + ContextVisitor visitor = new ContextVisitor<>(condition.and(rule -> rule.getClass() == antlrType), collector, variableRegistry); + contextVisitors.add((ContextVisitor) visitor); + return visitor; } /** - * Creates a mapping using the stop token from antlr as the location - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param The type of {@link ParserRuleContext} - * @return The builder for the token + * Visit the given node. + * @param antlrType The antlr type of the node. + * @return A visitor for the node. + * @param The class of the node. */ - protected ContextTokenBuilder mapExit(Class antlrType, TokenType jplagType) { - return this.mapExit(antlrType, jplagType, it -> true); + public ContextVisitor visit(Class antlrType) { + return visit(antlrType, ignore -> true); } /** - * Creates a mapping using the stop token from antlr as the location - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param condition The condition under which the mapping applies - * @param The type of {@link ParserRuleContext} - * @return The builder for the token + * Visit the given terminal. + * @param terminalType The type of the terminal. + * @param condition An additional condition for the visit. + * @return A visitor for the node. */ - @SuppressWarnings("unchecked") - protected ContextTokenBuilder mapExit(Class antlrType, TokenType jplagType, Predicate condition) { - ContextTokenBuilder builder = initTypeBuilder(antlrType, jplagType, condition, ContextTokenBuilderType.STOP); - this.endMappings.add((ContextTokenBuilder) builder); - return builder; + public TerminalVisitor visit(int terminalType, Predicate condition) { + TerminalVisitor visitor = new TerminalVisitor(condition.and(rule -> rule.getType() == terminalType), collector, variableRegistry); + terminalVisitors.add(visitor); + return visitor; } /** - * Creates a mapping using the beginning of the start token as the start location and the distance from the start to the - * stop token as the length - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param The type of {@link ParserRuleContext} - * @return The builder for the token + * Visit the given terminal. + * @param terminalType The type of the terminal. + * @return A visitor for the node. */ - protected ContextTokenBuilder mapRange(Class antlrType, TokenType jplagType) { - return this.mapRange(antlrType, jplagType, it -> true); - } - - /** - * Creates a mapping using the beginning of the start token as the start location and the distance from the start to the - * stop token as the length - * @param antlrType The antlr context type - * @param jplagType The Jplag token type - * @param condition The condition under which the mapping applies - * @param The type of {@link ParserRuleContext} - * @return The builder for the token - */ - @SuppressWarnings("unchecked") - protected ContextTokenBuilder mapRange(Class antlrType, TokenType jplagType, Predicate condition) { - ContextTokenBuilder builder = initTypeBuilder(antlrType, jplagType, condition, ContextTokenBuilderType.RANGE); - this.startMappings.add((ContextTokenBuilder) builder); - return builder; + public TerminalVisitor visit(int terminalType) { + return visit(terminalType, ignore -> true); } - /** - * Creates a start mapping from antlrType to startType and a stop mapping from antlrType to stopType. - * @param antlrType The antlr token type - * @param startType The token type for the start mapping - * @param stopType The token type for the stop mapping - * @param The type of {@link ParserRuleContext} - * @return The builder for the token - */ - protected RangeBuilder mapEnterExit(Class antlrType, TokenType startType, TokenType stopType) { - return mapEnterExit(antlrType, startType, stopType, it -> true); + @Override + public void visitTerminal(TerminalNode terminalNode) { + this.terminalVisitors.stream().filter(visitor -> visitor.matches(terminalNode.getSymbol())) + .forEach(visitor -> visitor.enter(terminalNode.getSymbol())); } - /** - * Creates a start mapping from antlrType to startType and a stop mapping from antlrType to stopType. - * @param antlrType The antlr token type - * @param startType The token type for the start mapping - * @param stopType The token type for the stop mapping - * @param condition The condition under which the mapping applies - * @param The type of {@link ParserRuleContext} - * @return The builder for the token - */ - protected RangeBuilder mapEnterExit(Class antlrType, TokenType startType, TokenType stopType, - Predicate condition) { - ContextTokenBuilder start = this.mapEnter(antlrType, startType, condition); - ContextTokenBuilder end = this.mapExit(antlrType, stopType, condition); - return new RangeBuilder<>(start, end); + @Override + public void visitErrorNode(ErrorNode errorNode) { + // does nothing, because we do not handle error nodes right now. } - /** - * Creates a mapping for terminal tokens - * @param terminalType The type of the terminal node - * @param jplagType The jplag token type - * @return The builder for the token - */ - protected TerminalTokenBuilder mapTerminal(int terminalType, TokenType jplagType) { - return this.mapTerminal(terminalType, jplagType, it -> true); + @Override + public void enterEveryRule(ParserRuleContext rule) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(rule)).forEach(visitor -> visitor.enter(rule)); } - /** - * Creates a mapping for terminal tokens - * @param terminalType The type of the terminal node - * @param jplagType The jplag token type - * @param condition The condition under which the mapping applies - * @return The builder for the token - */ - protected TerminalTokenBuilder mapTerminal(int terminalType, TokenType jplagType, Predicate condition) { - TerminalTokenBuilder builder = new TerminalTokenBuilder(jplagType, token -> token.getType() == terminalType && condition.test(token), - this.collector, this.currentFile); - this.terminalMapping.add(builder); - return builder; + @Override + public void exitEveryRule(ParserRuleContext rule) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(rule)).forEach(visitor -> visitor.exit(rule)); } /** @@ -280,10 +182,4 @@ protected final T getDescendant(ParserRuleContext } return null; } - - private ContextTokenBuilder initTypeBuilder(Class antlrType, TokenType jplagType, Predicate condition, - ContextTokenBuilderType type) { - return new ContextTokenBuilder<>(jplagType, rule -> rule.getClass() == antlrType && condition.test(antlrType.cast(rule)), this.collector, - this.currentFile, type); - } } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index c5cdc478f..c2a09fc8e 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -25,6 +26,25 @@ * @param The type of the antlr parser */ public abstract class AbstractAntlrParserAdapter extends AbstractParser { + + private final boolean extractsSemantics; + + /** + * New instance + * @param extractsSemantics If true, the listener will extract semantics along with every token + */ + public AbstractAntlrParserAdapter(boolean extractsSemantics) { + super(); + this.extractsSemantics = extractsSemantics; + } + + /** + * New instance + */ + public AbstractAntlrParserAdapter() { + this(false); + } + /** * Parsers the set of files * @param files The files @@ -32,12 +52,15 @@ public abstract class AbstractAntlrParserAdapter extends Abstr * @throws ParsingException If anything goes wrong */ public List parse(Set files) throws ParsingException { - TokenCollector collector = new TokenCollector(); - - for (File file : files) { + List filesList = new ArrayList<>(files); + File firstFile = filesList.remove(0); + TokenCollector collector = new TokenCollector(extractsSemantics, firstFile); + parseFile(firstFile, collector); + for (File file : filesList) { + collector.addFileEndToken(file); // takes the NEXT file parseFile(file, collector); } - + collector.addFileEndToken(null); return collector.getTokens(); } @@ -54,8 +77,6 @@ private void parseFile(File file, TokenCollector collector) throws ParsingExcept for (ParseTree child : entryContext.children) { treeWalker.walk(listener, child); } - - collector.addToken(Token.fileEnd(file)); } catch (IOException exception) { throw new ParsingException(file, exception.getMessage(), exception); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java new file mode 100644 index 000000000..66621452a --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -0,0 +1,122 @@ +package de.jplag.antlr; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import org.antlr.v4.runtime.Token; + +import de.jplag.TokenType; +import de.jplag.semantics.CodeSemantics; +import de.jplag.semantics.VariableRegistry; + +/** + * The abstract visitor. + * @param The type of the visited entity. + */ +public abstract class AbstractVisitor { + private final Predicate condition; + private final List> entryHandlers; + private final TokenCollector tokenCollector; + private Function semanticsSupplier; + VariableRegistry variableRegistry; // used in ContextVisitor + + /** + * @param condition The condition for the visit. + * @param tokenCollector The used token collector. + * @param variableRegistry The used variable registry. + */ + AbstractVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { + this.condition = condition; + this.tokenCollector = tokenCollector; + this.entryHandlers = new ArrayList<>(); + this.variableRegistry = variableRegistry; + } + + /** + * Add an action the visitor runs upon entering the entity. + * @param handler The action, takes the entity as parameter. + * @return Self + */ + public AbstractVisitor onEnter(Consumer handler) { + this.entryHandlers.add(handler); + return this; + } + + /** + * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. + * @param tokenType The type of the token. + * @return Self + */ + public AbstractVisitor mapEnter(TokenType tokenType) { + map(entryHandlers, tokenType, this::extractEnterToken); + return this; + } + + /** + * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. + * Alias for {@link #mapEnter(TokenType)}. + * @param tokenType The type of the token. + * @return Self + */ + public AbstractVisitor map(TokenType tokenType) { + mapEnter(tokenType); + return this; + } + + /** + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. If it doesn't + * generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * @param semanticsSupplier A function that takes the entity and returns the semantics. + * @return Self + */ + public AbstractVisitor withSemantics(Function semanticsSupplier) { + this.semanticsSupplier = semanticsSupplier; + return this; + } + + /** + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. If it doesn't + * generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * @param semanticsSupplier A function that returns the semantics. + * @return Self + */ + public AbstractVisitor withSemantics(Supplier semanticsSupplier) { + this.semanticsSupplier = ignore -> semanticsSupplier.get(); + return this; + } + + /** + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type control. If + * it doesn't generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * @return Self + */ + public AbstractVisitor withControlSemantics() { + this.semanticsSupplier = ignore -> CodeSemantics.createControl(); + return this; + } + + /** + * @param entity The entity to check. + * @return Whether the visitor should be visited. + */ + boolean matches(T entity) { + return this.condition.test(entity); + } + + void enter(T entity) { + entryHandlers.forEach(handler -> handler.accept(entity)); + } + + void exit(T entity) { + } + + void map(List> handlers, TokenType tokenType, Function extractToken) { + handlers.add(0, value -> tokenCollector.addToken(tokenType, semanticsSupplier, value, extractToken, variableRegistry)); + } + + abstract Token extractEnterToken(T entity); +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilder.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilder.java deleted file mode 100644 index c48ffc678..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -package de.jplag.antlr; - -import java.io.File; -import java.util.function.Function; -import java.util.function.Predicate; - -import org.antlr.v4.runtime.ParserRuleContext; - -import de.jplag.TokenType; -import de.jplag.semantics.VariableScope; - -/** - * Builds tokens for {@link ParserRuleContext}s. - * @param The type of context - */ -public class ContextTokenBuilder extends TokenBuilder { - private final ContextTokenBuilderType type; - - ContextTokenBuilder(TokenType tokenType, Predicate condition, TokenCollector collector, File file, ContextTokenBuilderType type) { - super(tokenType, condition, collector, file); - this.type = type; - } - - /** - * Adds this builder as a variable to the variable registry - * @param scope The scope of the variable - * @param mutable true, if the variable is mutable - * @param nameGetter The getter for the name, from the current {@link ParserRuleContext} - * @return Self - */ - public ContextTokenBuilder addAsVariable(VariableScope scope, boolean mutable, Function nameGetter) { - addSemanticsHandler((registry, rule) -> registry.registerVariable(nameGetter.apply(rule), scope, mutable)); - return this; - } - - @Override - protected org.antlr.v4.runtime.Token getAntlrToken(T antlrContent) { - if (this.type != ContextTokenBuilderType.STOP) { - return antlrContent.getStart(); - } else { - return antlrContent.getStop(); - } - } - - @Override - protected int getLength(T antlrContent) { - if (this.type != ContextTokenBuilderType.RANGE) { - return super.getLength(antlrContent); - } else { - return antlrContent.getStop().getStopIndex() - antlrContent.getStart().getStartIndex() + 1; - } - } -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilderType.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilderType.java deleted file mode 100644 index ab7d9231c..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextTokenBuilderType.java +++ /dev/null @@ -1,10 +0,0 @@ -package de.jplag.antlr; - -/** - * The types of context token builder. Either start, stop or range. Should only be used internally. - */ -enum ContextTokenBuilderType { - START, - STOP, - RANGE -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java new file mode 100644 index 000000000..1d2350f28 --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -0,0 +1,98 @@ +package de.jplag.antlr; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; + +import de.jplag.TokenType; +import de.jplag.semantics.VariableRegistry; + +/** + * The visitor for nodes, or contexts. + * @param The antlr type of the node. + */ +public class ContextVisitor extends AbstractVisitor { + private final List> exitHandlers; + + ContextVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { + super(condition, tokenCollector, variableRegistry); + this.exitHandlers = new ArrayList<>(); + } + + /** + * Add an action the visitor runs upon exiting the entity. + * @param handler The action, takes the entity as parameter. + * @return Self + */ + public AbstractVisitor onExit(Consumer handler) { + this.exitHandlers.add(handler); + return this; + } + + /** + * Tell the visitor that it should generate a token upon exiting the entity. Should only be invoked once per visitor. + * @param tokenType The type of the token. + * @return Self + */ + public ContextVisitor mapExit(TokenType tokenType) { + map(exitHandlers, tokenType, ParserRuleContext::getStop); + return this; + } + + /** + * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be + * invoked once per visitor. + * @param enterTokenType The type of the token generated on enter. + * @param exitTokenType The type of the token generated on exit. + * @return Self + */ + public ContextVisitor mapEnterExit(TokenType enterTokenType, TokenType exitTokenType) { + mapEnter(enterTokenType); + mapExit(exitTokenType); + return this; + } + + /** + * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be + * invoked once per visitor. Alias for {@link #mapEnterExit(TokenType, TokenType)}. + * @param enterTokenType The type of the token generated on enter. + * @param exitTokenType The type of the token generated on exit. + * @return Self + */ + public ContextVisitor map(TokenType enterTokenType, TokenType exitTokenType) { + mapEnterExit(enterTokenType, exitTokenType); + return this; + } + + /** + * Tell the visitor that the entity represents a local scope. + * @return Self + */ + public ContextVisitor addLocalScope() { + onEnter(ignore -> variableRegistry.enterLocalScope()); + onExit(ignore -> variableRegistry.exitLocalScope()); + return this; + } + + /** + * Tell the visitor that the entity represents a class scope. + * @return Self + */ + public ContextVisitor addClassScope() { + onEnter(ignore -> variableRegistry.enterClass()); + onExit(ignore -> variableRegistry.exitClass()); + return this; + } + + void exit(T entity) { + exitHandlers.forEach(handler -> handler.accept(entity)); + } + + Token extractEnterToken(T entity) { + return entity.getStart(); + } +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListenerException.java b/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListenerException.java deleted file mode 100644 index 70422ee29..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListenerException.java +++ /dev/null @@ -1,15 +0,0 @@ -package de.jplag.antlr; - -/** - * Exception type used internally within the antlr utils. Has to be a {@link RuntimeException}, because it is thrown - * within the antlr listener methods. Should not be thrown outside the antlr utils. - */ -public class InternalListenerException extends RuntimeException { - /** - * New instance - * @param message The message of the exception - */ - public InternalListenerException(String message) { - super(message); - } -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/RangeBuilder.java b/language-antlr-utils/src/main/java/de/jplag/antlr/RangeBuilder.java deleted file mode 100644 index ea84cf3bd..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/RangeBuilder.java +++ /dev/null @@ -1,112 +0,0 @@ -package de.jplag.antlr; - -import java.util.function.Consumer; -import java.util.function.Function; - -import org.antlr.v4.runtime.ParserRuleContext; - -import de.jplag.semantics.CodeSemantics; -import de.jplag.semantics.VariableRegistry; -import de.jplag.semantics.VariableScope; - -/** - * Builder for semantics on range mappings - * @param The type of rule - */ -@SuppressWarnings("unused") -public class RangeBuilder { - private final ContextTokenBuilder start; - private final ContextTokenBuilder end; - - /** - * New instance - * @param start The builder for the start token - * @param end The builder for the end token - */ - RangeBuilder(ContextTokenBuilder start, ContextTokenBuilder end) { - this.start = start; - this.end = end; - } - - /** - * Adds a class context to the variable registry - * @return Self - */ - public RangeBuilder addClassContext() { - this.start.addSemanticsHandler(VariableRegistry::enterClass); - this.end.addSemanticsHandler(VariableRegistry::exitClass); - return this; - } - - /** - * Adds a local scope to the variable registry - * @return Self - */ - public RangeBuilder addLocalScope() { - this.start.addSemanticsHandler(VariableRegistry::enterLocalScope); - this.end.addSemanticsHandler(VariableRegistry::exitLocalScope); - return this; - } - - /** - * Adds a semantics handler to the start token builder - * @param handler The handler - * @return Self - */ - public RangeBuilder addStartSemanticHandler(Consumer handler) { - this.start.addSemanticsHandler(handler); - return this; - } - - /** - * Adds a semantic handler to the end token builder - * @param handler The handler - * @return Self - */ - public RangeBuilder addEndSemanticHandler(Consumer handler) { - this.end.addSemanticsHandler(handler); - return this; - } - - /** - * Adds the start token as a variable when it is extracted - * @param scope The scope for the variable - * @param mutable true if the variable is mutable - * @param nameGetter The getter for the name - * @return Self - */ - public RangeBuilder addAsVariableOnStart(VariableScope scope, boolean mutable, Function nameGetter) { - this.start.addAsVariable(scope, mutable, nameGetter); - return this; - } - - /** - * Sets the given semantic for the start token - * @param semantics The semantic - * @return Self - */ - public RangeBuilder withStartSemantics(CodeSemantics semantics) { - this.start.withSemantics(semantics); - return this; - } - - /** - * Sets the given semantic for the end token - * @param semantics The semantic - * @return Self - */ - public RangeBuilder withEndSemantics(CodeSemantics semantics) { - this.end.withSemantics(semantics); - return this; - } - - /** - * Sets a control semantics for both tokens - * @return Self - */ - public RangeBuilder withControlSemantics() { - this.start.withControlSemantics(); - this.end.withControlSemantics(); - return this; - } -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalTokenBuilder.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalTokenBuilder.java deleted file mode 100644 index 3f074e6ad..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalTokenBuilder.java +++ /dev/null @@ -1,29 +0,0 @@ -package de.jplag.antlr; - -import java.io.File; -import java.util.function.Predicate; - -import org.antlr.v4.runtime.Token; - -import de.jplag.TokenType; - -/** - * Builds tokens from terminal antlr nodes - */ -public class TerminalTokenBuilder extends TokenBuilder { - /** - * New instance - * @param tokenType The token type - * @param condition The condition - * @param collector The token collector for the listener - * @param file The file the listener is for - */ - TerminalTokenBuilder(TokenType tokenType, Predicate condition, TokenCollector collector, File file) { - super(tokenType, condition, collector, file); - } - - @Override - protected Token getAntlrToken(Token antlrContent) { - return antlrContent; - } -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java new file mode 100644 index 000000000..e63cbd9b8 --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java @@ -0,0 +1,21 @@ +package de.jplag.antlr; + +import java.util.function.Predicate; + +import org.antlr.v4.runtime.Token; + +import de.jplag.semantics.VariableRegistry; + +/** + * The visitor for terminals. + */ +public class TerminalVisitor extends AbstractVisitor { + + TerminalVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { + super(condition, tokenCollector, variableRegistry); + } + + Token extractEnterToken(Token token) { + return token; + } +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenBuilder.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenBuilder.java deleted file mode 100644 index 08e393ef7..000000000 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenBuilder.java +++ /dev/null @@ -1,140 +0,0 @@ -package de.jplag.antlr; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.function.*; -import java.util.logging.Logger; - -import de.jplag.Token; -import de.jplag.TokenType; -import de.jplag.semantics.CodeSemantics; -import de.jplag.semantics.VariableRegistry; - -/** - * Handles the extraction of tokens. Contains information on the appropriate antlr types, the conditions under which the - * token should be extracted and semantics information. - * @param The antlr type being mapped - */ -public abstract class TokenBuilder { - private static final Logger logger = Logger.getLogger(TokenBuilder.class.getName()); - private static final String UNEXPECTED_SEMANTICS = "The listener %s indicates, it does not extract semantics. But the token (%s) has semantics information"; - private static final String MISSING_SEMANTICS = "Tokens should contain semantics, but none were supplied"; - - private final Predicate condition; - protected final TokenType tokenType; - - private Function semanticsSupplier = null; - private final List> semanticsHandler; - - protected final TokenCollector tokenCollector; - protected final File file; - - /** - * New instance - * @param tokenType The token type - * @param condition The condition - * @param collector The token collector for the listener - * @param file The file the listener is for - */ - TokenBuilder(TokenType tokenType, Predicate condition, TokenCollector collector, File file) { - this.condition = condition; - this.tokenType = tokenType; - this.tokenCollector = collector; - this.file = file; - - this.semanticsHandler = new ArrayList<>(); - } - - /** - * Checks if the token should be extracted for this node. - * @param value The node to check - * @return true, if the token should be extracted - */ - boolean matches(T value) { - return this.condition.test(value); - } - - /** - * Sets the given semantics for the token - * @param semantics The semantics - * @return Self - */ - public TokenBuilder withSemantics(CodeSemantics semantics) { - this.semanticsSupplier = ignore -> semantics; - return this; - } - - /** - * Uses the given function to build the token semantics from the antlr node - * @param function The function - * @return Self - */ - public TokenBuilder withSemantics(Function function) { - this.semanticsSupplier = function; - return this; - } - - /** - * Sets control semantics for the token - * @return Self - */ - public TokenBuilder withControlSemantics() { - withSemantics(CodeSemantics.createControl()); - return this; - } - - /** - * Adds a semantics handler to this builder. This can be used to perform additional operation like calling methods in - * the {@link de.jplag.semantics.VariableRegistry}. - * @param handler The handler function - * @return Self - */ - public TokenBuilder addSemanticsHandler(Consumer handler) { - this.semanticsHandler.add((semantics, rule) -> handler.accept(semantics)); - return this; - } - - /** - * Adds a semantics handler, that can perform additional operations required for semantics using the Semantics context - * objects and the antlr node. - * @param handler The handler - * @return Self - */ - public TokenBuilder addSemanticsHandler(BiConsumer handler) { - this.semanticsHandler.add(handler); - return this; - } - - void createToken(T antlrContent, VariableRegistry semantics) { - org.antlr.v4.runtime.Token antlrToken = getAntlrToken(antlrContent); - - int line = antlrToken.getLine(); - int column = antlrToken.getCharPositionInLine() + 1; - int length = getLength(antlrContent); - - Token token; - if (semantics != null) { - if (semanticsSupplier == null) { - throw new IllegalStateException(MISSING_SEMANTICS); - } - - this.semanticsHandler.forEach(it -> it.accept(semantics, antlrContent)); - token = new Token(this.tokenType, this.file, line, column, length, semanticsSupplier.apply(antlrContent)); - } else { - if (semanticsSupplier != null) { - logger.warning(() -> String.format(UNEXPECTED_SEMANTICS, this.getClass().getName(), this.tokenType.getDescription())); - } - - token = new Token(this.tokenType, this.file, line, column, length); - } - - this.tokenCollector.addToken(token); - } - - protected abstract org.antlr.v4.runtime.Token getAntlrToken(T antlrContent); - - protected int getLength(T antlrContent) { - return getAntlrToken(antlrContent).getText().length(); - } -} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index 30ab0ce78..a56b8f35a 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -1,36 +1,74 @@ package de.jplag.antlr; +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.function.Function; +import java.util.logging.Logger; import de.jplag.Token; +import de.jplag.TokenType; +import de.jplag.semantics.CodeSemantics; +import de.jplag.semantics.VariableRegistry; /** * Collects the tokens during parsing. */ public class TokenCollector { + private static final Logger logger = Logger.getLogger(TokenCollector.class.getName()); private final List collected; + private final boolean extractsSemantics; + private File file; /** - * New instance + * @param extractsSemantics If semantics are extracted */ - public TokenCollector() { + TokenCollector(boolean extractsSemantics, File initialFile) { this.collected = new ArrayList<>(); + this.extractsSemantics = extractsSemantics; + this.file = initialFile; } /** - * Adds a token to the collector - * @param token The token to add + * @return All collected tokens */ - public void addToken(Token token) { - this.collected.add(token); + List getTokens() { + return Collections.unmodifiableList(this.collected); + } + + void addToken(TokenType jplagType, Function semanticsSupplier, T entity, + Function extractToken, VariableRegistry variableRegistry) { + org.antlr.v4.runtime.Token antlrToken = extractToken.apply(entity); + int line = antlrToken.getLine(); + int column = antlrToken.getCharPositionInLine() + 1; + int length = antlrToken.getText().length(); + Token token; + if (extractsSemantics) { + if (semanticsSupplier == null) + throw new IllegalStateException(String.format("Expected semantics bud did not receive any for token %s", jplagType.getDescription())); + CodeSemantics semantics = semanticsSupplier.apply(entity); + token = new Token(jplagType, this.file, line, column, length, semantics); + variableRegistry.updateSemantics(semantics); + } else { + if (semanticsSupplier != null) + logger.warning(() -> String.format("Received semantics for token %s despite not expecting any", jplagType.getDescription())); + token = new Token(jplagType, this.file, line, column, length); + } + addToken(token); } /** - * @return All collected tokens + * Add a file end token. + * @param newFile The next file, null if there isn't one. */ - public List getTokens() { - return Collections.unmodifiableList(this.collected); + void addFileEndToken(File newFile) { + addToken(extractsSemantics ? Token.semanticFileEnd(file) : Token.fileEnd(file)); + // don't need to update semantics because variable registry is new for every file + this.file = newFile; + } + + private void addToken(Token token) { + this.collected.add(token); } } diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java index 49d30b023..51afe2896 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java @@ -19,17 +19,12 @@ public class TestListener extends AbstractAntlrListener { */ public TestListener(TokenCollector collector, File currentFile) { super(collector, currentFile, true); - - mapEnter(VarDefContext.class, VARDEF).addAsVariable(VariableScope.FILE, false, rule -> rule.VAR_NAME().getText()) - .withSemantics(CodeSemantics.createKeep()); - - mapRange(CalcExpressionContext.class, ADDITION, rule -> rule.operator() != null && rule.operator().PLUS() != null).withControlSemantics(); - mapRange(OperatorContext.class, SUBTRACTION, rule -> rule.MINUS() != null).withControlSemantics(); - mapEnterExit(SubExpressionContext.class, SUB_EXPRESSION_BEGIN, SUB_EXPRESSION_END) - // .addEndSemanticHandler(registry -> registry.addAllNonLocalVariablesAsReads()) - // does not work here, because there is no class context. Is still here as an example. - .addLocalScope().withControlSemantics(); - mapTerminal(TestParser.NUMBER, NUMBER).withSemantics(CodeSemantics.createKeep()); - mapEnter(VarRefContext.class, VARREF).withSemantics(CodeSemantics.createKeep()); + visit(VarDefContext.class).map(VARDEF).withSemantics(CodeSemantics::createKeep) + .onEnter(rule -> variableRegistry.registerVariable(rule.VAR_NAME().getText(), VariableScope.FILE, false)); + visit(CalcExpressionContext.class, rule -> rule.operator() != null && rule.operator().PLUS() != null).map(ADDITION).withControlSemantics(); + visit(OperatorContext.class, rule -> rule.MINUS() != null).map(SUBTRACTION).withControlSemantics(); + visit(SubExpressionContext.class).map(SUB_EXPRESSION_BEGIN, SUB_EXPRESSION_END).addLocalScope().withControlSemantics(); + visit(TestParser.NUMBER).map(NUMBER).withSemantics(CodeSemantics::createKeep); + visit(VarDefContext.class).map(VARDEF).withSemantics(CodeSemantics::createKeep); } } diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java index 90ef46b3b..bc2438f7d 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java +++ b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java @@ -24,50 +24,50 @@ public class CPPListener extends AbstractAntlrListener { public CPPListener(TokenCollector collector, File currentFile) { super(collector, currentFile); - mapEnterExit(ClassSpecifierContext.class, UNION_BEGIN, UNION_END, rule -> rule.classHead().Union() != null); - mapEnterExit(ClassSpecifierContext.class, CLASS_BEGIN, CLASS_END, - rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Class() != null); - mapEnterExit(ClassSpecifierContext.class, STRUCT_BEGIN, STRUCT_END, - rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Struct() != null); - mapEnterExit(EnumSpecifierContext.class, ENUM_BEGIN, ENUM_END); + visit(ClassSpecifierContext.class, rule -> rule.classHead().Union() != null).map(UNION_BEGIN, UNION_END); + visit(ClassSpecifierContext.class, rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Class() != null) + .map(CLASS_BEGIN, CLASS_END); + visit(ClassSpecifierContext.class, rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Struct() != null) + .map(STRUCT_BEGIN, STRUCT_END); + visit(EnumSpecifierContext.class).map(ENUM_BEGIN, ENUM_END); - mapEnterExit(FunctionDefinitionContext.class, FUNCTION_BEGIN, FUNCTION_END); + visit(FunctionDefinitionContext.class).map(FUNCTION_BEGIN, FUNCTION_END); - mapEnterExit(IterationStatementContext.class, DO_BEGIN, DO_END, rule -> rule.Do() != null); - mapEnterExit(IterationStatementContext.class, FOR_BEGIN, FOR_END, rule -> rule.For() != null); - mapEnterExit(IterationStatementContext.class, WHILE_BEGIN, WHILE_END, rule -> rule.While() != null && rule.Do() == null); + visit(IterationStatementContext.class, rule -> rule.Do() != null).map(DO_BEGIN, DO_END); + visit(IterationStatementContext.class, rule -> rule.For() != null).map(FOR_BEGIN, FOR_END); + visit(IterationStatementContext.class, rule -> rule.While() != null && rule.Do() == null).map(WHILE_BEGIN, WHILE_END); - mapEnterExit(SelectionStatementContext.class, SWITCH_BEGIN, SWITCH_END, rule -> rule.Switch() != null); - mapEnterExit(SelectionStatementContext.class, IF_BEGIN, IF_END, rule -> rule.If() != null); - mapTerminal(CPP14Parser.Else, ELSE); + visit(SelectionStatementContext.class, rule -> rule.Switch() != null).map(SWITCH_BEGIN, SWITCH_END); + visit(SelectionStatementContext.class, rule -> rule.If() != null).map(IF_BEGIN, IF_END); + visit(CPP14Parser.Else).map(ELSE); - mapEnter(LabeledStatementContext.class, CASE, rule -> rule.Case() != null); - mapEnter(LabeledStatementContext.class, DEFAULT, rule -> rule.Default() != null); + visit(LabeledStatementContext.class, rule -> rule.Case() != null).map(CASE); + visit(LabeledStatementContext.class, rule -> rule.Default() != null).map(DEFAULT); - mapEnter(TryBlockContext.class, TRY); - mapEnterExit(HandlerContext.class, CATCH_BEGIN, CATCH_END); + visit(TryBlockContext.class).map(TRY); + visit(HandlerContext.class).map(CATCH_BEGIN, CATCH_END); - mapEnter(JumpStatementContext.class, BREAK, rule -> rule.Break() != null); - mapEnter(JumpStatementContext.class, CONTINUE, rule -> rule.Continue() != null); - mapEnter(JumpStatementContext.class, GOTO, rule -> rule.Goto() != null); - mapEnter(JumpStatementContext.class, RETURN, rule -> rule.Return() != null); + visit(JumpStatementContext.class, rule -> rule.Break() != null).map(BREAK); + visit(JumpStatementContext.class, rule -> rule.Continue() != null).map(CONTINUE); + visit(JumpStatementContext.class, rule -> rule.Goto() != null).map(GOTO); + visit(JumpStatementContext.class, rule -> rule.Return() != null).map(RETURN); - mapEnter(ThrowExpressionContext.class, THROW); + visit(ThrowExpressionContext.class).map(THROW); - mapEnter(NewExpressionContext.class, NEWCLASS, rule -> rule.newInitializer() != null); - mapEnter(NewExpressionContext.class, NEWARRAY, rule -> rule.newInitializer() == null); + visit(NewExpressionContext.class, rule -> rule.newInitializer() != null).map(NEWCLASS); + visit(NewExpressionContext.class, rule -> rule.newInitializer() == null).map(NEWARRAY); - mapEnter(TemplateDeclarationContext.class, GENERIC); + visit(TemplateDeclarationContext.class).map(GENERIC); - mapEnter(AssignmentOperatorContext.class, ASSIGN); - mapEnter(BraceOrEqualInitializerContext.class, ASSIGN, rule -> rule.Assign() != null); - mapEnter(UnaryExpressionContext.class, ASSIGN, rule -> rule.PlusPlus() != null || rule.MinusMinus() != null); + visit(AssignmentOperatorContext.class).map(ASSIGN); + visit(BraceOrEqualInitializerContext.class, rule -> rule.Assign() != null).map(ASSIGN); + visit(UnaryExpressionContext.class, rule -> rule.PlusPlus() != null || rule.MinusMinus() != null).map(ASSIGN); - mapEnter(StaticAssertDeclarationContext.class, STATIC_ASSERT); - mapEnter(EnumeratorDefinitionContext.class, VARDEF); - mapEnterExit(BracedInitListContext.class, BRACED_INIT_BEGIN, BRACED_INIT_END); + visit(StaticAssertDeclarationContext.class).map(STATIC_ASSERT); + visit(EnumeratorDefinitionContext.class).map(VARDEF); + visit(BracedInitListContext.class).map(BRACED_INIT_BEGIN, BRACED_INIT_END); - mapEnter(SimpleTypeSpecifierContext.class, VARDEF, rule -> { + visit(SimpleTypeSpecifierContext.class, rule -> { if (hasAncestor(rule, MemberdeclarationContext.class, FunctionDefinitionContext.class)) { return true; } @@ -80,23 +80,23 @@ public CPPListener(TokenCollector collector, File currentFile) { } return false; - }); + }).map(VARDEF); - mapEnter(SimpleDeclarationContext.class, APPLY, rule -> { + visit(SimpleDeclarationContext.class, rule -> { if (!hasAncestor(rule, FunctionBodyContext.class)) { return false; } NoPointerDeclaratorContext noPointerDecl = getDescendant(rule, NoPointerDeclaratorContext.class); return noPointerInFunctionCallContext(noPointerDecl); - }); + }).map(APPLY); - mapEnter(InitDeclaratorContext.class, APPLY, rule -> rule.initializer() != null && rule.initializer().LeftParen() != null); - mapEnter(ParameterDeclarationContext.class, VARDEF); - mapEnter(ConditionalExpressionContext.class, QUESTIONMARK, rule -> rule.Question() != null); + visit(InitDeclaratorContext.class, rule -> rule.initializer() != null && rule.initializer().LeftParen() != null).map(APPLY); + visit(ParameterDeclarationContext.class).map(VARDEF); + visit(ConditionalExpressionContext.class, rule -> rule.Question() != null).map(QUESTIONMARK); - mapEnter(PostfixExpressionContext.class, APPLY, rule -> rule.LeftParen() != null); - mapEnter(PostfixExpressionContext.class, ASSIGN, rule -> rule.PlusPlus() != null || rule.MinusMinus() != null); + visit(PostfixExpressionContext.class, rule -> rule.LeftParen() != null).map(APPLY); + visit(PostfixExpressionContext.class, rule -> rule.PlusPlus() != null || rule.MinusMinus() != null).map(ASSIGN); } /** diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java index bb308df61..9728ec889 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java @@ -103,49 +103,49 @@ public class KotlinListener extends AbstractAntlrListener { public KotlinListener(TokenCollector collector, File currentFile) { super(collector, currentFile); - this.mapRange(PackageHeaderContext.class, PACKAGE); - this.mapRange(ImportHeaderContext.class, IMPORT); - this.mapEnter(ClassDeclarationContext.class, CLASS_DECLARATION); - this.mapRange(ObjectDeclarationContext.class, OBJECT_DECLARATION); - this.mapRange(CompanionObjectContext.class, COMPANION_DECLARATION); - this.mapRange(TypeParameterContext.class, TYPE_PARAMETER); - this.mapRange(PrimaryConstructorContext.class, CONSTRUCTOR); - this.mapRange(ClassParameterContext.class, PROPERTY_DECLARATION); - this.mapEnterExit(ClassBodyContext.class, CLASS_BODY_BEGIN, CLASS_BODY_END); - this.mapEnterExit(EnumClassBodyContext.class, ENUM_CLASS_BODY_BEGIN, ENUM_CLASS_BODY_END); - this.mapEnter(EnumEntryContext.class, ENUM_ENTRY); - this.mapRange(SecondaryConstructorContext.class, CONSTRUCTOR); - this.mapEnter(PropertyDeclarationContext.class, PROPERTY_DECLARATION); - this.mapEnter(AnonymousInitializerContext.class, INITIALIZER); - this.mapEnterExit(InitBlockContext.class, INITIALIZER_BODY_START, INITIALIZER_BODY_END); - this.mapEnter(FunctionDeclarationContext.class, FUNCTION); - this.mapEnter(GetterContext.class, GETTER); - this.mapEnter(SetterContext.class, SETTER); - this.mapRange(FunctionValueParameterContext.class, FUNCTION_PARAMETER); - this.mapEnterExit(FunctionBodyContext.class, FUNCTION_BODY_BEGIN, FUNCTION_BODY_END); - this.mapEnterExit(FunctionLiteralContext.class, FUNCTION_LITERAL_BEGIN, FUNCTION_LITERAL_END); - this.mapEnterExit(ForExpressionContext.class, FOR_EXPRESSION_BEGIN, FOR_EXPRESSION_END); - this.mapEnterExit(IfExpressionContext.class, IF_EXPRESSION_BEGIN, IF_EXPRESSION_END); - this.mapEnterExit(WhileExpressionContext.class, WHILE_EXPRESSION_START, WHILE_EXPRESSION_END); - this.mapEnterExit(DoWhileExpressionContext.class, DO_WHILE_EXPRESSION_START, DO_WHILE_EXPRESSION_END); - this.mapEnter(TryExpressionContext.class, TRY_EXPRESSION); - this.mapEnterExit(TryBodyContext.class, TRY_BODY_START, TRY_BODY_END); - this.mapEnter(CatchStatementContext.class, CATCH); - this.mapEnterExit(CatchBodyContext.class, CATCH_BODY_START, CATCH_BODY_END); - this.mapEnter(FinallyStatementContext.class, FINALLY); - this.mapEnterExit(FinallyBodyContext.class, FINALLY_BODY_START, FINALLY_BODY_END); - this.mapEnterExit(WhenExpressionContext.class, WHEN_EXPRESSION_START, WHEN_EXPRESSION_END); - this.mapEnter(WhenConditionContext.class, WHEN_CONDITION); - this.mapEnterExit(ControlStructureBodyContext.class, CONTROL_STRUCTURE_BODY_START, CONTROL_STRUCTURE_BODY_END); - this.mapEnter(VariableDeclarationContext.class, VARIABLE_DECLARATION); - this.mapRange(ConstructorInvocationContext.class, CREATE_OBJECT); - this.mapRange(CallSuffixContext.class, FUNCTION_INVOCATION); - this.mapEnter(AssignmentOperatorContext.class, ASSIGNMENT); + visit(PackageHeaderContext.class).map(PACKAGE); + visit(ImportHeaderContext.class).map(IMPORT); + visit(ClassDeclarationContext.class).map(CLASS_DECLARATION); + visit(ObjectDeclarationContext.class).map(OBJECT_DECLARATION); + visit(CompanionObjectContext.class).map(COMPANION_DECLARATION); + visit(TypeParameterContext.class).map(TYPE_PARAMETER); + visit(PrimaryConstructorContext.class).map(CONSTRUCTOR); + visit(ClassParameterContext.class).map(PROPERTY_DECLARATION); + visit(ClassBodyContext.class).map(CLASS_BODY_BEGIN, CLASS_BODY_END); + visit(EnumClassBodyContext.class).map(ENUM_CLASS_BODY_BEGIN, ENUM_CLASS_BODY_END); + visit(EnumEntryContext.class).map(ENUM_ENTRY); + visit(SecondaryConstructorContext.class).map(CONSTRUCTOR); + visit(PropertyDeclarationContext.class).map(PROPERTY_DECLARATION); + visit(AnonymousInitializerContext.class).map(INITIALIZER); + visit(InitBlockContext.class).map(INITIALIZER_BODY_START, INITIALIZER_BODY_END); + visit(FunctionDeclarationContext.class).map(FUNCTION); + visit(GetterContext.class).map(GETTER); + visit(SetterContext.class).map(SETTER); + visit(FunctionValueParameterContext.class).map(FUNCTION_PARAMETER); + visit(FunctionBodyContext.class).map(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END); + visit(FunctionLiteralContext.class).map(FUNCTION_LITERAL_BEGIN, FUNCTION_LITERAL_END); + visit(ForExpressionContext.class).map(FOR_EXPRESSION_BEGIN, FOR_EXPRESSION_END); + visit(IfExpressionContext.class).map(IF_EXPRESSION_BEGIN, IF_EXPRESSION_END); + visit(WhileExpressionContext.class).map(WHILE_EXPRESSION_START, WHILE_EXPRESSION_END); + visit(DoWhileExpressionContext.class).map(DO_WHILE_EXPRESSION_START, DO_WHILE_EXPRESSION_END); + visit(TryExpressionContext.class).map(TRY_EXPRESSION); + visit(TryBodyContext.class).map(TRY_BODY_START, TRY_BODY_END); + visit(CatchStatementContext.class).map(CATCH); + visit(CatchBodyContext.class).map(CATCH_BODY_START, CATCH_BODY_END); + visit(FinallyStatementContext.class).map(FINALLY); + visit(FinallyBodyContext.class).map(FINALLY_BODY_START, FINALLY_BODY_END); + visit(WhenExpressionContext.class).map(WHEN_EXPRESSION_START, WHEN_EXPRESSION_END); + visit(WhenConditionContext.class).map(WHEN_CONDITION); + visit(ControlStructureBodyContext.class).map(CONTROL_STRUCTURE_BODY_START, CONTROL_STRUCTURE_BODY_END); + visit(VariableDeclarationContext.class).map(VARIABLE_DECLARATION); + visit(ConstructorInvocationContext.class).map(CREATE_OBJECT); + visit(CallSuffixContext.class).map(FUNCTION_INVOCATION); + visit(AssignmentOperatorContext.class).map(ASSIGNMENT); - this.mapTerminal(KotlinParser.THROW, THROW); - this.mapTerminal(KotlinParser.RETURN, RETURN); - this.mapTerminal(KotlinParser.CONTINUE, CONTINUE); - this.mapTerminal(KotlinParser.BREAK, BREAK); - this.mapTerminal(KotlinParser.BREAK_AT, BREAK); + visit(KotlinParser.THROW).map(THROW); + visit(KotlinParser.RETURN).map(RETURN); + visit(KotlinParser.CONTINUE).map(CONTINUE); + visit(KotlinParser.BREAK).map(BREAK); + visit(KotlinParser.BREAK_AT).map(BREAK); } } From ef9dce36bbe943a897c057952cd5f1a198538488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Fri, 25 Aug 2023 14:37:22 +0200 Subject: [PATCH 02/12] Fix errors --- .../main/java/de/jplag/antlr/AbstractAntlrListener.java | 9 ++++++--- .../java/de/jplag/antlr/AbstractAntlrParserAdapter.java | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index 19c1b00ad..e0e136f57 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -8,6 +8,7 @@ import java.util.function.Predicate; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -60,7 +61,8 @@ public AbstractAntlrListener(TokenCollector collector, File currentFile) { */ @SuppressWarnings("unchecked") public ContextVisitor visit(Class antlrType, Predicate condition) { - ContextVisitor visitor = new ContextVisitor<>(condition.and(rule -> rule.getClass() == antlrType), collector, variableRegistry); + Predicate typeCheck = rule -> rule.getClass() == antlrType; + ContextVisitor visitor = new ContextVisitor<>(typeCheck.and(condition), collector, variableRegistry); contextVisitors.add((ContextVisitor) visitor); return visitor; } @@ -81,8 +83,9 @@ public ContextVisitor visit(Class antlrType) * @param condition An additional condition for the visit. * @return A visitor for the node. */ - public TerminalVisitor visit(int terminalType, Predicate condition) { - TerminalVisitor visitor = new TerminalVisitor(condition.and(rule -> rule.getType() == terminalType), collector, variableRegistry); + public TerminalVisitor visit(int terminalType, Predicate condition) { + Predicate typeCheck = rule -> rule.getType() == terminalType; + TerminalVisitor visitor = new TerminalVisitor(typeCheck.and(condition), collector, variableRegistry); terminalVisitors.add(visitor); return visitor; } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index c2a09fc8e..8bd9c56ff 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -53,6 +53,8 @@ public AbstractAntlrParserAdapter() { */ public List parse(Set files) throws ParsingException { List filesList = new ArrayList<>(files); + if (files.isEmpty()) + return new ArrayList<>(); File firstFile = filesList.remove(0); TokenCollector collector = new TokenCollector(extractsSemantics, firstFile); parseFile(firstFile, collector); From bb8a13930b7c3a715e933f68b22c27a2e5133da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Fri, 25 Aug 2023 15:09:21 +0200 Subject: [PATCH 03/12] Adjust llvmir listener --- .../java/de/jplag/llvmir/LLVMIRListener.java | 216 +++++++++--------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java index ed0efc65f..3a6dfad43 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java @@ -130,113 +130,113 @@ public class LLVMIRListener extends AbstractAntlrListener { public LLVMIRListener(TokenCollector collector, File currentFile) { super(collector, currentFile); - this.mapEnter(SourceFilenameContext.class, FILENAME); - this.mapRange(ModuleAsmContext.class, ASSEMBLY); - this.mapEnter(TypeDefContext.class, TYPE_DEFINITION); - this.mapRange(GlobalDeclContext.class, GLOBAL_VARIABLE); - this.mapRange(GlobalDefContext.class, GLOBAL_VARIABLE); - this.mapRange(FuncDeclContext.class, FUNCTION_DECLARATION); - this.mapRange(FuncDefContext.class, FUNCTION_DEFINITION); - this.mapEnterExit(FuncBodyContext.class, FUNCTION_BODY_BEGIN, FUNCTION_BODY_END); - this.mapEnterExit(BasicBlockContext.class, BASIC_BLOCK_BEGIN, BASIC_BLOCK_END); - this.mapRange(RetTermContext.class, RETURN); - this.mapRange(BrTermContext.class, BRANCH); - this.mapRange(CondBrTermContext.class, CONDITIONAL_BRANCH); - this.mapRange(SwitchTermContext.class, SWITCH); - this.mapRange(IndirectBrTermContext.class, BRANCH); - this.mapRange(ResumeTermContext.class, RESUME); - this.mapRange(CatchRetTermContext.class, CATCH_RETURN); - this.mapRange(CleanupRetTermContext.class, CLEAN_UP_RETURN); - this.mapRange(InvokeTermContext.class, INVOKE); - this.mapRange(CallBrTermContext.class, CALL_BRANCH); - this.mapRange(CatchSwitchTermContext.class, CATCH_SWITCH); - this.mapRange(Case_Context.class, CASE); - this.mapRange(StructConstContext.class, STRUCTURE); - this.mapRange(ArrayConstContext.class, ARRAY); - this.mapRange(VectorConstContext.class, VECTOR); - this.mapRange(InlineAsmContext.class, ASSEMBLY); - this.mapRange(BitCastExprContext.class, BITCAST); - this.mapRange(GetElementPtrExprContext.class, GET_ELEMENT_POINTER); - this.mapEnter(AddrSpaceCastExprContext.class, CONVERSION); - this.mapEnter(IntToPtrExprContext.class, CONVERSION); - this.mapRange(ICmpExprContext.class, COMPARISON); - this.mapRange(FCmpExprContext.class, COMPARISON); - this.mapRange(SelectExprContext.class, SELECT); - this.mapEnter(TruncExprContext.class, CONVERSION); - this.mapEnter(ZExtExprContext.class, CONVERSION); - this.mapEnter(SExtExprContext.class, CONVERSION); - this.mapEnter(FpTruncExprContext.class, CONVERSION); - this.mapEnter(FpExtExprContext.class, CONVERSION); - this.mapEnter(FpToUiExprContext.class, CONVERSION); - this.mapEnter(FpToSiExprContext.class, CONVERSION); - this.mapEnter(UiToFpExprContext.class, CONVERSION); - this.mapEnter(SiToFpExprContext.class, CONVERSION); - this.mapEnter(PtrToIntExprContext.class, CONVERSION); - this.mapEnter(ExtractElementExprContext.class, EXTRACT_ELEMENT); - this.mapEnter(InsertElementExprContext.class, INSERT_ELEMENT); - this.mapEnter(ShuffleVectorExprContext.class, SHUFFLE_VECTOR); - this.mapRange(ShlExprContext.class, SHIFT); - this.mapRange(LShrExprContext.class, SHIFT); - this.mapRange(AShrExprContext.class, SHIFT); - this.mapRange(AndExprContext.class, AND); - this.mapRange(OrExprContext.class, OR); - this.mapRange(XorExprContext.class, XOR); - this.mapRange(AddExprContext.class, ADDITION); - this.mapRange(SubExprContext.class, SUBTRACTION); - this.mapRange(MulExprContext.class, MULTIPLICATION); - this.mapRange(StoreInstContext.class, STORE); - this.mapRange(FenceInstContext.class, FENCE); - this.mapRange(AddInstContext.class, ADDITION); - this.mapRange(FAddInstContext.class, ADDITION); - this.mapRange(SubInstContext.class, SUBTRACTION); - this.mapRange(FSubInstContext.class, SUBTRACTION); - this.mapRange(MulInstContext.class, MULTIPLICATION); - this.mapRange(FMulInstContext.class, MULTIPLICATION); - this.mapRange(UDivInstContext.class, DIVISION); - this.mapRange(SDivInstContext.class, DIVISION); - this.mapRange(FDivInstContext.class, DIVISION); - this.mapRange(URemInstContext.class, REMAINDER); - this.mapRange(SRemInstContext.class, REMAINDER); - this.mapRange(FRemInstContext.class, REMAINDER); - this.mapRange(ShlInstContext.class, SHIFT); - this.mapRange(LShrInstContext.class, SHIFT); - this.mapRange(AShrInstContext.class, SHIFT); - this.mapRange(AndInstContext.class, AND); - this.mapRange(OrInstContext.class, OR); - this.mapRange(XorInstContext.class, XOR); - this.mapEnter(ExtractElementInstContext.class, EXTRACT_ELEMENT); - this.mapEnter(InsertElementInstContext.class, INSERT_ELEMENT); - this.mapEnter(ShuffleVectorInstContext.class, SHUFFLE_VECTOR); - this.mapRange(ExtractValueInstContext.class, EXTRACT_VALUE); - this.mapRange(InsertValueInstContext.class, INSERT_VALUE); - this.mapRange(AllocaInstContext.class, ALLOCATION); - this.mapRange(LoadInstContext.class, LOAD); - this.mapRange(CmpXchgInstContext.class, COMPARE_EXCHANGE); - this.mapRange(AtomicRMWInstContext.class, ATOMIC_READ_MODIFY_WRITE); - this.mapRange(GetElementPtrInstContext.class, GET_ELEMENT_POINTER); - this.mapEnter(TruncInstContext.class, CONVERSION); - this.mapEnter(ZExtInstContext.class, CONVERSION); - this.mapEnter(SExtInstContext.class, CONVERSION); - this.mapEnter(FpTruncInstContext.class, CONVERSION); - this.mapEnter(FpExtInstContext.class, CONVERSION); - this.mapEnter(FpToUiInstContext.class, CONVERSION); - this.mapEnter(FpToSiInstContext.class, CONVERSION); - this.mapEnter(UiToFpInstContext.class, CONVERSION); - this.mapEnter(SiToFpInstContext.class, CONVERSION); - this.mapEnter(PtrToIntInstContext.class, CONVERSION); - this.mapEnter(IntToPtrInstContext.class, CONVERSION); - this.mapRange(BitCastInstContext.class, BITCAST); - this.mapEnter(AddrSpaceCastInstContext.class, CONVERSION); - this.mapRange(ICmpInstContext.class, COMPARISON); - this.mapRange(FCmpInstContext.class, COMPARISON); - this.mapRange(PhiInstContext.class, PHI); - this.mapRange(SelectInstContext.class, SELECT); - this.mapRange(CallInstContext.class, CALL); - this.mapRange(VaargInstContext.class, VARIABLE_ARGUMENT); - this.mapRange(LandingPadInstContext.class, LANDING_PAD); - this.mapRange(CatchPadInstContext.class, CATCH_PAD); - this.mapRange(CleanupPadInstContext.class, CLEAN_UP_PAD); - this.mapRange(ClauseContext.class, CLAUSE); - this.mapRange(AtomicOrderingContext.class, ATOMIC_ORDERING); + visit(SourceFilenameContext.class).map(FILENAME); + visit(ModuleAsmContext.class).map(ASSEMBLY); + visit(TypeDefContext.class).map(TYPE_DEFINITION); + visit(GlobalDeclContext.class).map(GLOBAL_VARIABLE); + visit(GlobalDefContext.class).map(GLOBAL_VARIABLE); + visit(FuncDeclContext.class).map(FUNCTION_DECLARATION); + visit(FuncDefContext.class).map(FUNCTION_DEFINITION); + visit(FuncBodyContext.class).map(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END); + visit(BasicBlockContext.class).map(BASIC_BLOCK_BEGIN, BASIC_BLOCK_END); + visit(RetTermContext.class).map(RETURN); + visit(BrTermContext.class).map(BRANCH); + visit(CondBrTermContext.class).map(CONDITIONAL_BRANCH); + visit(SwitchTermContext.class).map(SWITCH); + visit(IndirectBrTermContext.class).map(BRANCH); + visit(ResumeTermContext.class).map(RESUME); + visit(CatchRetTermContext.class).map(CATCH_RETURN); + visit(CleanupRetTermContext.class).map(CLEAN_UP_RETURN); + visit(InvokeTermContext.class).map(INVOKE); + visit(CallBrTermContext.class).map(CALL_BRANCH); + visit(CatchSwitchTermContext.class).map(CATCH_SWITCH); + visit(Case_Context.class).map(CASE); + visit(StructConstContext.class).map(STRUCTURE); + visit(ArrayConstContext.class).map(ARRAY); + visit(VectorConstContext.class).map(VECTOR); + visit(InlineAsmContext.class).map(ASSEMBLY); + visit(BitCastExprContext.class).map(BITCAST); + visit(GetElementPtrExprContext.class).map(GET_ELEMENT_POINTER); + visit(AddrSpaceCastExprContext.class).map(CONVERSION); + visit(IntToPtrExprContext.class).map(CONVERSION); + visit(ICmpExprContext.class).map(COMPARISON); + visit(FCmpExprContext.class).map(COMPARISON); + visit(SelectExprContext.class).map(SELECT); + visit(TruncExprContext.class).map(CONVERSION); + visit(ZExtExprContext.class).map(CONVERSION); + visit(SExtExprContext.class).map(CONVERSION); + visit(FpTruncExprContext.class).map(CONVERSION); + visit(FpExtExprContext.class).map(CONVERSION); + visit(FpToUiExprContext.class).map(CONVERSION); + visit(FpToSiExprContext.class).map(CONVERSION); + visit(UiToFpExprContext.class).map(CONVERSION); + visit(SiToFpExprContext.class).map(CONVERSION); + visit(PtrToIntExprContext.class).map(CONVERSION); + visit(ExtractElementExprContext.class).map(EXTRACT_ELEMENT); + visit(InsertElementExprContext.class).map(INSERT_ELEMENT); + visit(ShuffleVectorExprContext.class).map(SHUFFLE_VECTOR); + visit(ShlExprContext.class).map(SHIFT); + visit(LShrExprContext.class).map(SHIFT); + visit(AShrExprContext.class).map(SHIFT); + visit(AndExprContext.class).map(AND); + visit(OrExprContext.class).map(OR); + visit(XorExprContext.class).map(XOR); + visit(AddExprContext.class).map(ADDITION); + visit(SubExprContext.class).map(SUBTRACTION); + visit(MulExprContext.class).map(MULTIPLICATION); + visit(StoreInstContext.class).map(STORE); + visit(FenceInstContext.class).map(FENCE); + visit(AddInstContext.class).map(ADDITION); + visit(FAddInstContext.class).map(ADDITION); + visit(SubInstContext.class).map(SUBTRACTION); + visit(FSubInstContext.class).map(SUBTRACTION); + visit(MulInstContext.class).map(MULTIPLICATION); + visit(FMulInstContext.class).map(MULTIPLICATION); + visit(UDivInstContext.class).map(DIVISION); + visit(SDivInstContext.class).map(DIVISION); + visit(FDivInstContext.class).map(DIVISION); + visit(URemInstContext.class).map(REMAINDER); + visit(SRemInstContext.class).map(REMAINDER); + visit(FRemInstContext.class).map(REMAINDER); + visit(ShlInstContext.class).map(SHIFT); + visit(LShrInstContext.class).map(SHIFT); + visit(AShrInstContext.class).map(SHIFT); + visit(AndInstContext.class).map(AND); + visit(OrInstContext.class).map(OR); + visit(XorInstContext.class).map(XOR); + visit(ExtractElementInstContext.class).map(EXTRACT_ELEMENT); + visit(InsertElementInstContext.class).map(INSERT_ELEMENT); + visit(ShuffleVectorInstContext.class).map(SHUFFLE_VECTOR); + visit(ExtractValueInstContext.class).map(EXTRACT_VALUE); + visit(InsertValueInstContext.class).map(INSERT_VALUE); + visit(AllocaInstContext.class).map(ALLOCATION); + visit(LoadInstContext.class).map(LOAD); + visit(CmpXchgInstContext.class).map(COMPARE_EXCHANGE); + visit(AtomicRMWInstContext.class).map(ATOMIC_READ_MODIFY_WRITE); + visit(GetElementPtrInstContext.class).map(GET_ELEMENT_POINTER); + visit(TruncInstContext.class).map(CONVERSION); + visit(ZExtInstContext.class).map(CONVERSION); + visit(SExtInstContext.class).map(CONVERSION); + visit(FpTruncInstContext.class).map(CONVERSION); + visit(FpExtInstContext.class).map(CONVERSION); + visit(FpToUiInstContext.class).map(CONVERSION); + visit(FpToSiInstContext.class).map(CONVERSION); + visit(UiToFpInstContext.class).map(CONVERSION); + visit(SiToFpInstContext.class).map(CONVERSION); + visit(PtrToIntInstContext.class).map(CONVERSION); + visit(IntToPtrInstContext.class).map(CONVERSION); + visit(BitCastInstContext.class).map(BITCAST); + visit(AddrSpaceCastInstContext.class).map(CONVERSION); + visit(ICmpInstContext.class).map(COMPARISON); + visit(FCmpInstContext.class).map(COMPARISON); + visit(PhiInstContext.class).map(PHI); + visit(SelectInstContext.class).map(SELECT); + visit(CallInstContext.class).map(CALL); + visit(VaargInstContext.class).map(VARIABLE_ARGUMENT); + visit(LandingPadInstContext.class).map(LANDING_PAD); + visit(CatchPadInstContext.class).map(CATCH_PAD); + visit(CleanupPadInstContext.class).map(CLEAN_UP_PAD); + visit(ClauseContext.class).map(CLAUSE); + visit(AtomicOrderingContext.class).map(ATOMIC_ORDERING); } } From 6b7758843c7e565c439fcfe97fe649f9875086f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Fri, 25 Aug 2023 16:41:54 +0200 Subject: [PATCH 04/12] Fix code smells --- .../main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java | 4 ++-- .../src/main/java/de/jplag/antlr/ContextVisitor.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index 8bd9c56ff..172aa45fd 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -33,7 +33,7 @@ public abstract class AbstractAntlrParserAdapter extends Abstr * New instance * @param extractsSemantics If true, the listener will extract semantics along with every token */ - public AbstractAntlrParserAdapter(boolean extractsSemantics) { + protected AbstractAntlrParserAdapter(boolean extractsSemantics) { super(); this.extractsSemantics = extractsSemantics; } @@ -41,7 +41,7 @@ public AbstractAntlrParserAdapter(boolean extractsSemantics) { /** * New instance */ - public AbstractAntlrParserAdapter() { + protected AbstractAntlrParserAdapter() { this(false); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 1d2350f28..baf8f971d 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -88,6 +88,7 @@ public ContextVisitor addClassScope() { return this; } + @Override void exit(T entity) { exitHandlers.forEach(handler -> handler.accept(entity)); } From 2337898700ef4368eac4e18dc4ea1726ee4ce42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Sat, 26 Aug 2023 13:41:39 +0200 Subject: [PATCH 05/12] Create language listeners statically --- .../de/jplag/antlr/AbstractAntlrListener.java | 72 +++++-------------- .../antlr/AbstractAntlrParserAdapter.java | 11 +-- .../java/de/jplag/antlr/AbstractVisitor.java | 44 ++++++------ .../java/de/jplag/antlr/ContextVisitor.java | 35 ++++++--- .../main/java/de/jplag/antlr/HandlerData.java | 9 +++ .../java/de/jplag/antlr/InternalListener.java | 48 +++++++++++++ .../java/de/jplag/antlr/TerminalVisitor.java | 6 +- .../antlr/testLanguage/TestListener.java | 18 ++--- .../antlr/testLanguage/TestParserAdapter.java | 8 +-- .../main/java/de/jplag/cpp2/CPPListener.java | 9 +-- .../java/de/jplag/cpp2/CPPParserAdapter.java | 9 ++- .../java/de/jplag/kotlin/KotlinListener.java | 7 +- .../de/jplag/kotlin/KotlinParserAdapter.java | 9 ++- .../java/de/jplag/llvmir/LLVMIRListener.java | 9 +-- .../de/jplag/llvmir/LLVMIRParserAdapter.java | 9 ++- 15 files changed, 153 insertions(+), 150 deletions(-) create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index e0e136f57..db254c61a 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -1,6 +1,5 @@ package de.jplag.antlr; -import java.io.File; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; @@ -9,47 +8,19 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; - -import de.jplag.semantics.VariableRegistry; /** - * Base class for Antlr listeners. You can use the create*Mapping functions to map antlr tokens to jplag tokens. - *

- * You should create a constructor matching one of the constructors and create your mapping after calling super. + * Base class for Antlr listeners. This is a quasi-static class that is only created once per language. Use by + * overwriting the constructor, calling super(), and then calling the visit methods. */ -@SuppressWarnings("unused") -public class AbstractAntlrListener implements ParseTreeListener { - private final TokenCollector collector; - private final File currentFile; +public abstract class AbstractAntlrListener { private final List> contextVisitors; private final List terminalVisitors; - protected final VariableRegistry variableRegistry; - /** - * New instance - * @param collector The token collector - * @param currentFile The currently processed file - * @param extractsSemantics If true, the listener will extract semantics along with every token - */ - public AbstractAntlrListener(TokenCollector collector, File currentFile, boolean extractsSemantics) { - this.collector = collector; - this.currentFile = currentFile; - this.contextVisitors = new ArrayList<>(); - this.terminalVisitors = new ArrayList<>(); - this.variableRegistry = new VariableRegistry(); - } - - /** - * Creates a new AbstractAntlrListener, that does not collect semantics information - * @param collector The collector, obtained by the parser - * @param currentFile The current file, obtained by the parser - */ - public AbstractAntlrListener(TokenCollector collector, File currentFile) { - this(collector, currentFile, false); + public AbstractAntlrListener() { + contextVisitors = new ArrayList<>(); + terminalVisitors = new ArrayList<>(); } /** @@ -62,7 +33,7 @@ public AbstractAntlrListener(TokenCollector collector, File currentFile) { @SuppressWarnings("unchecked") public ContextVisitor visit(Class antlrType, Predicate condition) { Predicate typeCheck = rule -> rule.getClass() == antlrType; - ContextVisitor visitor = new ContextVisitor<>(typeCheck.and(condition), collector, variableRegistry); + ContextVisitor visitor = new ContextVisitor<>(typeCheck.and(condition)); contextVisitors.add((ContextVisitor) visitor); return visitor; } @@ -85,7 +56,7 @@ public ContextVisitor visit(Class antlrType) */ public TerminalVisitor visit(int terminalType, Predicate condition) { Predicate typeCheck = rule -> rule.getType() == terminalType; - TerminalVisitor visitor = new TerminalVisitor(typeCheck.and(condition), collector, variableRegistry); + TerminalVisitor visitor = new TerminalVisitor(typeCheck.and(condition)); terminalVisitors.add(visitor); return visitor; } @@ -99,25 +70,16 @@ public TerminalVisitor visit(int terminalType) { return visit(terminalType, ignore -> true); } - @Override - public void visitTerminal(TerminalNode terminalNode) { - this.terminalVisitors.stream().filter(visitor -> visitor.matches(terminalNode.getSymbol())) - .forEach(visitor -> visitor.enter(terminalNode.getSymbol())); - } - - @Override - public void visitErrorNode(ErrorNode errorNode) { - // does nothing, because we do not handle error nodes right now. + void visitTerminal(HandlerData handlerData) { + this.terminalVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.enter(handlerData)); } - @Override - public void enterEveryRule(ParserRuleContext rule) { - this.contextVisitors.stream().filter(visitor -> visitor.matches(rule)).forEach(visitor -> visitor.enter(rule)); + void enterEveryRule(HandlerData handlerData) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.enter(handlerData)); } - @Override - public void exitEveryRule(ParserRuleContext rule) { - this.contextVisitors.stream().filter(visitor -> visitor.matches(rule)).forEach(visitor -> visitor.exit(rule)); + void exitEveryRule(HandlerData handlerData) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.exit(handlerData)); } /** @@ -129,7 +91,7 @@ public void exitEveryRule(ParserRuleContext rule) { * @return an ancestor of the specified type, or null if not found. */ @SafeVarargs - protected final T getAncestor(ParserRuleContext context, Class ancestor, + protected static T getAncestor(ParserRuleContext context, Class ancestor, Class... stops) { ParserRuleContext currentContext = context; Set> forbidden = Set.of(stops); @@ -156,7 +118,7 @@ protected final T getAncestor(ParserRuleContext co * @see #getAncestor(ParserRuleContext, Class, Class[]) */ @SafeVarargs - protected final boolean hasAncestor(ParserRuleContext context, Class parent, + protected static boolean hasAncestor(ParserRuleContext context, Class parent, Class... stops) { return getAncestor(context, parent, stops) != null; } @@ -168,7 +130,7 @@ protected final boolean hasAncestor(ParserRuleContext context, Class the type to search for. * @return the first appearance of an element of the given type in the subtree, or null if no such element exists. */ - protected final T getDescendant(ParserRuleContext context, Class descendant) { + protected static T getDescendant(ParserRuleContext context, Class descendant) { // simple iterative bfs ArrayDeque queue = new ArrayDeque<>(); queue.add(context); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index 172aa45fd..21f0d516f 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -71,11 +71,9 @@ private void parseFile(File file, TokenCollector collector) throws ParsingExcept Lexer lexer = this.createLexer(CharStreams.fromReader(reader)); CommonTokenStream tokenStream = new CommonTokenStream(lexer); T parser = this.createParser(tokenStream); - ParserRuleContext entryContext = this.getEntryContext(parser); ParseTreeWalker treeWalker = new ParseTreeWalker(); - - AbstractAntlrListener listener = this.createListener(collector, file); + InternalListener listener = new InternalListener(this.getListener(), collector); for (ParseTree child : entryContext.children) { treeWalker.walk(listener, child); } @@ -106,10 +104,7 @@ private void parseFile(File file, TokenCollector collector) throws ParsingExcept protected abstract ParserRuleContext getEntryContext(T parser); /** - * Creates the listener - * @param collector The token collector - * @param currentFile The current file - * @return The parser + * @return The listener. Should be created once statically since it never changes. */ - protected abstract AbstractAntlrListener createListener(TokenCollector collector, File currentFile); + protected abstract AbstractAntlrListener getListener(); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 66621452a..32e318ebe 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -2,10 +2,7 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; +import java.util.function.*; import org.antlr.v4.runtime.Token; @@ -19,21 +16,25 @@ */ public abstract class AbstractVisitor { private final Predicate condition; - private final List> entryHandlers; - private final TokenCollector tokenCollector; + private final List>> entryHandlers; private Function semanticsSupplier; - VariableRegistry variableRegistry; // used in ContextVisitor /** * @param condition The condition for the visit. - * @param tokenCollector The used token collector. - * @param variableRegistry The used variable registry. */ - AbstractVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { + AbstractVisitor(Predicate condition) { this.condition = condition; - this.tokenCollector = tokenCollector; this.entryHandlers = new ArrayList<>(); - this.variableRegistry = variableRegistry; + } + + /** + * Add an action the visitor runs upon entering the entity. + * @param handler The action, takes the entity and the variable registry as parameter. + * @return Self + */ + public AbstractVisitor onEnter(BiConsumer handler) { + entryHandlers.add(handlerData -> handler.accept(handlerData.entity(), handlerData.variableRegistry())); + return this; } /** @@ -42,7 +43,7 @@ public abstract class AbstractVisitor { * @return Self */ public AbstractVisitor onEnter(Consumer handler) { - this.entryHandlers.add(handler); + entryHandlers.add(handlerData -> handler.accept(handlerData.entity())); return this; } @@ -101,21 +102,22 @@ public AbstractVisitor withControlSemantics() { /** * @param entity The entity to check. - * @return Whether the visitor should be visited. + * @return Whether to visit the entity. */ boolean matches(T entity) { return this.condition.test(entity); } - void enter(T entity) { - entryHandlers.forEach(handler -> handler.accept(entity)); - } - - void exit(T entity) { + /** + * Enter a given entity, injecting the needed dependencies. + */ + void enter(HandlerData handlerData) { + entryHandlers.forEach(handler -> handler.accept(handlerData)); } - void map(List> handlers, TokenType tokenType, Function extractToken) { - handlers.add(0, value -> tokenCollector.addToken(tokenType, semanticsSupplier, value, extractToken, variableRegistry)); + void map(List>> handlers, TokenType tokenType, Function extractToken) { + handlers.add(0, handlerData -> handlerData.collector().addToken(tokenType, semanticsSupplier, handlerData.entity(), extractToken, + handlerData.variableRegistry())); } abstract Token extractEnterToken(T entity); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index baf8f971d..e860684e8 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; @@ -16,20 +17,30 @@ * @param The antlr type of the node. */ public class ContextVisitor extends AbstractVisitor { - private final List> exitHandlers; + private final List>> exitHandlers; - ContextVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { - super(condition, tokenCollector, variableRegistry); + ContextVisitor(Predicate condition) { + super(condition); this.exitHandlers = new ArrayList<>(); } + /** + * Add an action the visitor runs upon exiting the entity. + * @param handler The action, takes the entity and the variable registry as parameter. + * @return Self + */ + public AbstractVisitor onExit(BiConsumer handler) { + exitHandlers.add(handlerData -> handler.accept(handlerData.entity(), handlerData.variableRegistry())); + return this; + } + /** * Add an action the visitor runs upon exiting the entity. * @param handler The action, takes the entity as parameter. * @return Self */ public AbstractVisitor onExit(Consumer handler) { - this.exitHandlers.add(handler); + exitHandlers.add(handlerData -> handler.accept(handlerData.entity())); return this; } @@ -73,8 +84,8 @@ public ContextVisitor map(TokenType enterTokenType, TokenType exitTokenType) * @return Self */ public ContextVisitor addLocalScope() { - onEnter(ignore -> variableRegistry.enterLocalScope()); - onExit(ignore -> variableRegistry.exitLocalScope()); + onEnter((ignore, variableRegistry) -> variableRegistry.enterLocalScope()); + onExit((ignore, variableRegistry) -> variableRegistry.exitLocalScope()); return this; } @@ -83,14 +94,16 @@ public ContextVisitor addLocalScope() { * @return Self */ public ContextVisitor addClassScope() { - onEnter(ignore -> variableRegistry.enterClass()); - onExit(ignore -> variableRegistry.exitClass()); + onEnter((ignore, variableRegistry) -> variableRegistry.enterClass()); + onExit((ignore, variableRegistry) -> variableRegistry.exitClass()); return this; } - @Override - void exit(T entity) { - exitHandlers.forEach(handler -> handler.accept(entity)); + /** + * Exit a given entity, injecting the needed dependencies. + */ + void exit(HandlerData handlerData) { + exitHandlers.forEach(handler -> handler.accept(handlerData)); } Token extractEnterToken(T entity) { diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java new file mode 100644 index 000000000..6c1bb4095 --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java @@ -0,0 +1,9 @@ +package de.jplag.antlr; + +import de.jplag.semantics.VariableRegistry; + +/** + * Holds the data passed to the (quasi-static) listeners. + */ +record HandlerData(T entity, VariableRegistry variableRegistry, TokenCollector collector) { +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java new file mode 100644 index 000000000..39178e0fb --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java @@ -0,0 +1,48 @@ +package de.jplag.antlr; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; + +import de.jplag.semantics.VariableRegistry; + +/** + * Internal listener that implements pre-existing antlr methods that are called automatically. This listener is created + * for every file. + */ +class InternalListener implements ParseTreeListener { + private final AbstractAntlrListener listener; + private final TokenCollector collector; + protected final VariableRegistry variableRegistry; + + InternalListener(AbstractAntlrListener listener, TokenCollector collector) { + this.listener = listener; + this.collector = collector; + this.variableRegistry = new VariableRegistry(); + } + + @Override + public void visitTerminal(TerminalNode terminalNode) { + listener.visitTerminal(getHandlerData(terminalNode.getSymbol())); + } + + @Override + public void enterEveryRule(ParserRuleContext rule) { + listener.enterEveryRule(getHandlerData(rule)); + } + + @Override + public void exitEveryRule(ParserRuleContext rule) { + listener.exitEveryRule(getHandlerData(rule)); + } + + @Override + public void visitErrorNode(ErrorNode errorNode) { + // does nothing, because we do not handle error nodes right now. + } + + private HandlerData getHandlerData(T entity) { + return new HandlerData<>(entity, variableRegistry, collector); + } +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java index e63cbd9b8..170f5d627 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java @@ -4,15 +4,13 @@ import org.antlr.v4.runtime.Token; -import de.jplag.semantics.VariableRegistry; - /** * The visitor for terminals. */ public class TerminalVisitor extends AbstractVisitor { - TerminalVisitor(Predicate condition, TokenCollector tokenCollector, VariableRegistry variableRegistry) { - super(condition, tokenCollector, variableRegistry); + TerminalVisitor(Predicate condition) { + super(condition); } Token extractEnterToken(Token token) { diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java index 51afe2896..6a949d4ab 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java @@ -2,25 +2,17 @@ import static de.jplag.antlr.testLanguage.TestTokenType.*; -import java.io.File; - -import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.TestParser; +import de.jplag.antlr.*; import de.jplag.antlr.TestParser.*; -import de.jplag.antlr.TokenCollector; import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableScope; public class TestListener extends AbstractAntlrListener { - /** - * New instance - * @param collector The token collector - * @param currentFile The currently processed file - */ - public TestListener(TokenCollector collector, File currentFile) { - super(collector, currentFile, true); + + public TestListener() { + super(); visit(VarDefContext.class).map(VARDEF).withSemantics(CodeSemantics::createKeep) - .onEnter(rule -> variableRegistry.registerVariable(rule.VAR_NAME().getText(), VariableScope.FILE, false)); + .onEnter((rule, variableRegistry) -> variableRegistry.registerVariable(rule.VAR_NAME().getText(), VariableScope.FILE, false)); visit(CalcExpressionContext.class, rule -> rule.operator() != null && rule.operator().PLUS() != null).map(ADDITION).withControlSemantics(); visit(OperatorContext.class, rule -> rule.MINUS() != null).map(SUBTRACTION).withControlSemantics(); visit(SubExpressionContext.class).map(SUB_EXPRESSION_BEGIN, SUB_EXPRESSION_END).addLocalScope().withControlSemantics(); diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java index 5b292e18d..2f6eb7aca 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java @@ -1,7 +1,5 @@ package de.jplag.antlr.testLanguage; -import java.io.File; - import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.Lexer; @@ -10,6 +8,8 @@ import de.jplag.antlr.*; public class TestParserAdapter extends AbstractAntlrParserAdapter { + private static final TestListener testListener = new TestListener(); + @Override protected Lexer createLexer(CharStream input) { return new TestLexer(input); @@ -26,7 +26,7 @@ protected ParserRuleContext getEntryContext(TestParser parser) { } @Override - protected AbstractAntlrListener createListener(TokenCollector collector, File currentFile) { - return new TestListener(collector, currentFile); + protected AbstractAntlrListener getListener() { + return testListener; } } diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java index bc2438f7d..acef7b2f1 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java +++ b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java @@ -2,10 +2,7 @@ import static de.jplag.cpp2.CPPTokenType.*; -import java.io.File; - import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.TokenCollector; import de.jplag.cpp2.grammar.CPP14Parser; import de.jplag.cpp2.grammar.CPP14Parser.*; @@ -18,11 +15,9 @@ public class CPPListener extends AbstractAntlrListener { /** * New instance - * @param collector The token collector the token will be added to - * @param currentFile The currently processed file */ - public CPPListener(TokenCollector collector, File currentFile) { - super(collector, currentFile); + public CPPListener() { + super(); visit(ClassSpecifierContext.class, rule -> rule.classHead().Union() != null).map(UNION_BEGIN, UNION_END); visit(ClassSpecifierContext.class, rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Class() != null) diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java index fc73e2b11..c876023ee 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java +++ b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java @@ -1,7 +1,5 @@ package de.jplag.cpp2; -import java.io.File; - import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.Lexer; @@ -10,7 +8,6 @@ import de.jplag.AbstractParser; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.AbstractAntlrParserAdapter; -import de.jplag.antlr.TokenCollector; import de.jplag.cpp2.grammar.CPP14Lexer; import de.jplag.cpp2.grammar.CPP14Parser; @@ -18,6 +15,8 @@ * The adapter between {@link AbstractParser} and the ANTLR based parser of this language module. */ public class CPPParserAdapter extends AbstractAntlrParserAdapter { + private static final CPPListener listener = new CPPListener(); + @Override protected Lexer createLexer(CharStream input) { return new CPP14Lexer(input); @@ -34,7 +33,7 @@ protected ParserRuleContext getEntryContext(CPP14Parser parser) { } @Override - protected AbstractAntlrListener createListener(TokenCollector collector, File currentFile) { - return new CPPListener(collector, currentFile); + protected AbstractAntlrListener getListener() { + return listener; } } diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java index 9728ec889..b6231b446 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java @@ -93,15 +93,12 @@ import static de.jplag.kotlin.grammar.KotlinParser.WhenExpressionContext; import static de.jplag.kotlin.grammar.KotlinParser.WhileExpressionContext; -import java.io.File; - import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.TokenCollector; import de.jplag.kotlin.grammar.KotlinParser; public class KotlinListener extends AbstractAntlrListener { - public KotlinListener(TokenCollector collector, File currentFile) { - super(collector, currentFile); + public KotlinListener() { + super(); visit(PackageHeaderContext.class).map(PACKAGE); visit(ImportHeaderContext.class).map(IMPORT); diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java index 49537918d..0fed85032 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java @@ -1,16 +1,15 @@ package de.jplag.kotlin; -import java.io.File; - import org.antlr.v4.runtime.*; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.AbstractAntlrParserAdapter; -import de.jplag.antlr.TokenCollector; import de.jplag.kotlin.grammar.KotlinLexer; import de.jplag.kotlin.grammar.KotlinParser; public class KotlinParserAdapter extends AbstractAntlrParserAdapter { + private static final KotlinListener listener = new KotlinListener(); + @Override protected Lexer createLexer(CharStream input) { return new KotlinLexer(input); @@ -27,7 +26,7 @@ protected ParserRuleContext getEntryContext(KotlinParser parser) { } @Override - protected AbstractAntlrListener createListener(TokenCollector collector, File currentFile) { - return new KotlinListener(collector, currentFile); + protected AbstractAntlrListener getListener() { + return listener; } } diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java index 3a6dfad43..8b247be02 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java @@ -110,10 +110,7 @@ import static de.jplag.llvmir.grammar.LLVMIRParser.ZExtExprContext; import static de.jplag.llvmir.grammar.LLVMIRParser.ZExtInstContext; -import java.io.File; - import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.TokenCollector; /** * Extracts tokens from the ANTLR parse tree. The token abstraction includes nesting tokens for functions and basic @@ -124,11 +121,9 @@ public class LLVMIRListener extends AbstractAntlrListener { /** * New instance - * @param collector The token collector the token will be added to - * @param currentFile The currently processed file */ - public LLVMIRListener(TokenCollector collector, File currentFile) { - super(collector, currentFile); + public LLVMIRListener() { + super(); visit(SourceFilenameContext.class).map(FILENAME); visit(ModuleAsmContext.class).map(ASSEMBLY); diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java index 0d96e301e..edbe94148 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java @@ -1,13 +1,10 @@ package de.jplag.llvmir; -import java.io.File; - import org.antlr.v4.runtime.*; import de.jplag.AbstractParser; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.AbstractAntlrParserAdapter; -import de.jplag.antlr.TokenCollector; import de.jplag.llvmir.grammar.LLVMIRLexer; import de.jplag.llvmir.grammar.LLVMIRParser; @@ -15,6 +12,8 @@ * The adapter between {@link AbstractParser} and the ANTLR based parser of this language module. */ public class LLVMIRParserAdapter extends AbstractAntlrParserAdapter { + private static final LLVMIRListener listener = new LLVMIRListener(); + @Override protected Lexer createLexer(CharStream input) { return new LLVMIRLexer(input); @@ -31,7 +30,7 @@ protected ParserRuleContext getEntryContext(LLVMIRParser parser) { } @Override - protected AbstractAntlrListener createListener(TokenCollector collector, File currentFile) { - return new LLVMIRListener(collector, currentFile); + protected AbstractAntlrListener getListener() { + return listener; } } From 889f178f83939561cd03dfc8f3a88a4f425e203b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Sat, 26 Aug 2023 14:07:20 +0200 Subject: [PATCH 06/12] Fix code smell --- .../src/main/java/de/jplag/antlr/AbstractAntlrListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index db254c61a..c8a2f2931 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -18,7 +18,7 @@ public abstract class AbstractAntlrListener { private final List> contextVisitors; private final List terminalVisitors; - public AbstractAntlrListener() { + protected AbstractAntlrListener() { contextVisitors = new ArrayList<>(); terminalVisitors = new ArrayList<>(); } From 5bcd1edd271d2081ff45a7f53e6d3f133500d6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Sat, 26 Aug 2023 14:18:38 +0200 Subject: [PATCH 07/12] Make language listeners package-private --- .../java/de/jplag/antlr/testLanguage/TestListener.java | 5 +++-- .../cpp2/src/main/java/de/jplag/cpp2/CPPListener.java | 8 +++----- .../src/main/java/de/jplag/kotlin/KotlinListener.java | 5 +++-- .../src/main/java/de/jplag/llvmir/LLVMIRListener.java | 7 ++----- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java index 6a949d4ab..c87e50128 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java @@ -7,10 +7,11 @@ import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableScope; -public class TestListener extends AbstractAntlrListener { +class TestListener extends AbstractAntlrListener { - public TestListener() { + TestListener() { super(); + visit(VarDefContext.class).map(VARDEF).withSemantics(CodeSemantics::createKeep) .onEnter((rule, variableRegistry) -> variableRegistry.registerVariable(rule.VAR_NAME().getText(), VariableScope.FILE, false)); visit(CalcExpressionContext.class, rule -> rule.operator() != null && rule.operator().PLUS() != null).map(ADDITION).withControlSemantics(); diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java index acef7b2f1..c9710635f 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java +++ b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java @@ -12,11 +12,9 @@ *

* Those cases are covered by {@link SimpleTypeSpecifierContext} and {@link SimpleDeclarationContext} */ -public class CPPListener extends AbstractAntlrListener { - /** - * New instance - */ - public CPPListener() { +class CPPListener extends AbstractAntlrListener { + + CPPListener() { super(); visit(ClassSpecifierContext.class, rule -> rule.classHead().Union() != null).map(UNION_BEGIN, UNION_END); diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java index b6231b446..eaa8c43e2 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java @@ -96,8 +96,9 @@ import de.jplag.antlr.AbstractAntlrListener; import de.jplag.kotlin.grammar.KotlinParser; -public class KotlinListener extends AbstractAntlrListener { - public KotlinListener() { +class KotlinListener extends AbstractAntlrListener { + + KotlinListener() { super(); visit(PackageHeaderContext.class).map(PACKAGE); diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java index 8b247be02..6c4988db6 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java @@ -117,12 +117,9 @@ * blocks and separate tokens for different elements. These include binary and bitwise instructions, memory operations, * terminator instructions, conversions, global variables, type definitions, constants, and others. */ -public class LLVMIRListener extends AbstractAntlrListener { +class LLVMIRListener extends AbstractAntlrListener { - /** - * New instance - */ - public LLVMIRListener() { + LLVMIRListener() { super(); visit(SourceFilenameContext.class).map(FILENAME); From a1f34b25c2414937e45ced8096f24ba9d3a92c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Sat, 26 Aug 2023 14:50:15 +0200 Subject: [PATCH 08/12] Add withLoopSemantics --- .../java/de/jplag/antlr/AbstractVisitor.java | 14 ++++---- .../java/de/jplag/antlr/ContextVisitor.java | 34 ++++++++++++++++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 32e318ebe..88edba2eb 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -17,7 +17,7 @@ public abstract class AbstractVisitor { private final Predicate condition; private final List>> entryHandlers; - private Function semanticsSupplier; + private Function entrySemantics; /** * @param condition The condition for the visit. @@ -53,7 +53,7 @@ public AbstractVisitor onEnter(Consumer handler) { * @return Self */ public AbstractVisitor mapEnter(TokenType tokenType) { - map(entryHandlers, tokenType, this::extractEnterToken); + map(entryHandlers, tokenType, entrySemantics, this::extractEnterToken); return this; } @@ -75,7 +75,7 @@ public AbstractVisitor map(TokenType tokenType) { * @return Self */ public AbstractVisitor withSemantics(Function semanticsSupplier) { - this.semanticsSupplier = semanticsSupplier; + this.entrySemantics = semanticsSupplier; return this; } @@ -86,7 +86,7 @@ public AbstractVisitor withSemantics(Function semanticsSupp * @return Self */ public AbstractVisitor withSemantics(Supplier semanticsSupplier) { - this.semanticsSupplier = ignore -> semanticsSupplier.get(); + this.entrySemantics = ignore -> semanticsSupplier.get(); return this; } @@ -96,7 +96,7 @@ public AbstractVisitor withSemantics(Supplier semanticsSupplie * @return Self */ public AbstractVisitor withControlSemantics() { - this.semanticsSupplier = ignore -> CodeSemantics.createControl(); + withSemantics(CodeSemantics::createControl); return this; } @@ -115,8 +115,8 @@ void enter(HandlerData handlerData) { entryHandlers.forEach(handler -> handler.accept(handlerData)); } - void map(List>> handlers, TokenType tokenType, Function extractToken) { - handlers.add(0, handlerData -> handlerData.collector().addToken(tokenType, semanticsSupplier, handlerData.entity(), extractToken, + void map(List>> handlers, TokenType tokenType, Function semantics, Function extractToken) { + handlers.add(0, handlerData -> handlerData.collector().addToken(tokenType, semantics, handlerData.entity(), extractToken, handlerData.variableRegistry())); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index e860684e8..2f8a7d1a0 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -2,14 +2,13 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Predicate; +import java.util.function.*; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import de.jplag.TokenType; +import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableRegistry; /** @@ -18,6 +17,7 @@ */ public class ContextVisitor extends AbstractVisitor { private final List>> exitHandlers; + private Function exitSemantics; ContextVisitor(Predicate condition) { super(condition); @@ -50,7 +50,7 @@ public AbstractVisitor onExit(Consumer handler) { * @return Self */ public ContextVisitor mapExit(TokenType tokenType) { - map(exitHandlers, tokenType, ParserRuleContext::getStop); + map(exitHandlers, tokenType, exitSemantics, ParserRuleContext::getStop); return this; } @@ -79,6 +79,32 @@ public ContextVisitor map(TokenType enterTokenType, TokenType exitTokenType) return this; } + @Override + public ContextVisitor withSemantics(Function semantics) { + super.withSemantics(semantics); + this.exitSemantics = semantics; + return this; + } + + @Override + public ContextVisitor withSemantics(Supplier semantics) { + super.withSemantics(semantics); + this.exitSemantics = ignore -> semantics.get(); + return this; + } + + /** + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type loop begin, + * same for the exit and loop end. If it doesn't generate a token, the semantics are discarded. This is not checked and + * does not lead to a warning. + * @return Self + */ + public ContextVisitor withLoopSemantics() { + super.withSemantics(CodeSemantics::createLoopBegin); + this.exitSemantics = ignore -> CodeSemantics.createLoopEnd(); + return this; + } + /** * Tell the visitor that the entity represents a local scope. * @return Self From 468dfd6eb430c830fad0000c25bb51ac2978b9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Sat, 26 Aug 2023 15:01:44 +0200 Subject: [PATCH 09/12] Add tokens directly instead of using a handler --- .../java/de/jplag/antlr/AbstractAntlrListener.java | 12 ++++++------ .../main/java/de/jplag/antlr/AbstractVisitor.java | 13 +++++++------ .../main/java/de/jplag/antlr/ContextVisitor.java | 8 +++++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index c8a2f2931..fc6a2d11f 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -70,16 +70,16 @@ public TerminalVisitor visit(int terminalType) { return visit(terminalType, ignore -> true); } - void visitTerminal(HandlerData handlerData) { - this.terminalVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.enter(handlerData)); + void visitTerminal(HandlerData data) { + this.terminalVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.enter(data)); } - void enterEveryRule(HandlerData handlerData) { - this.contextVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.enter(handlerData)); + void enterEveryRule(HandlerData data) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.enter(data)); } - void exitEveryRule(HandlerData handlerData) { - this.contextVisitors.stream().filter(visitor -> visitor.matches(handlerData.entity())).forEach(visitor -> visitor.exit(handlerData)); + void exitEveryRule(HandlerData data) { + this.contextVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.exit(data)); } /** diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 88edba2eb..839180227 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -17,6 +17,7 @@ public abstract class AbstractVisitor { private final Predicate condition; private final List>> entryHandlers; + private TokenType entryTokenType; private Function entrySemantics; /** @@ -53,7 +54,7 @@ public AbstractVisitor onEnter(Consumer handler) { * @return Self */ public AbstractVisitor mapEnter(TokenType tokenType) { - map(entryHandlers, tokenType, entrySemantics, this::extractEnterToken); + entryTokenType = tokenType; return this; } @@ -111,13 +112,13 @@ boolean matches(T entity) { /** * Enter a given entity, injecting the needed dependencies. */ - void enter(HandlerData handlerData) { - entryHandlers.forEach(handler -> handler.accept(handlerData)); + void enter(HandlerData data) { + addToken(data, entryTokenType, entrySemantics, this::extractEnterToken); + entryHandlers.forEach(handler -> handler.accept(data)); } - void map(List>> handlers, TokenType tokenType, Function semantics, Function extractToken) { - handlers.add(0, handlerData -> handlerData.collector().addToken(tokenType, semantics, handlerData.entity(), extractToken, - handlerData.variableRegistry())); + void addToken(HandlerData data, TokenType tokenType, Function semantics, Function extractToken) { + data.collector().addToken(tokenType, semantics, data.entity(), extractToken, data.variableRegistry()); } abstract Token extractEnterToken(T entity); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 2f8a7d1a0..5a0060aa7 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -17,6 +17,7 @@ */ public class ContextVisitor extends AbstractVisitor { private final List>> exitHandlers; + private TokenType exitToken; private Function exitSemantics; ContextVisitor(Predicate condition) { @@ -50,7 +51,7 @@ public AbstractVisitor onExit(Consumer handler) { * @return Self */ public ContextVisitor mapExit(TokenType tokenType) { - map(exitHandlers, tokenType, exitSemantics, ParserRuleContext::getStop); + exitToken = tokenType; return this; } @@ -128,8 +129,9 @@ public ContextVisitor addClassScope() { /** * Exit a given entity, injecting the needed dependencies. */ - void exit(HandlerData handlerData) { - exitHandlers.forEach(handler -> handler.accept(handlerData)); + void exit(HandlerData data) { + addToken(data, exitToken, exitSemantics, ParserRuleContext::getStop); + exitHandlers.forEach(handler -> handler.accept(data)); } Token extractEnterToken(T entity) { From 3dad69d3ed6a0b9cac9d4a68f8dbba893c98e783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Mon, 28 Aug 2023 11:46:26 +0200 Subject: [PATCH 10/12] Fix bug, add warning --- .../java/de/jplag/antlr/AbstractAntlrListener.java | 10 ++++++++++ .../src/main/java/de/jplag/antlr/AbstractVisitor.java | 9 +++------ .../src/main/java/de/jplag/antlr/ContextVisitor.java | 3 +-- .../src/main/java/de/jplag/antlr/TokenCollector.java | 5 +++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index fc6a2d11f..a1ea6ebdb 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -9,6 +9,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; /** * Base class for Antlr listeners. This is a quasi-static class that is only created once per language. Use by @@ -70,14 +71,23 @@ public TerminalVisitor visit(int terminalType) { return visit(terminalType, ignore -> true); } + /** + * Called by {@link InternalListener#visitTerminal(TerminalNode)} as part of antlr framework. + */ void visitTerminal(HandlerData data) { this.terminalVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.enter(data)); } + /** + * Called by {@link InternalListener#enterEveryRule(ParserRuleContext)} as part of antlr framework. + */ void enterEveryRule(HandlerData data) { this.contextVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.enter(data)); } + /** + * Called by {@link InternalListener#exitEveryRule(ParserRuleContext)} as part of antlr framework. + */ void exitEveryRule(HandlerData data) { this.contextVisitors.stream().filter(visitor -> visitor.matches(data.entity())).forEach(visitor -> visitor.exit(data)); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 839180227..cad061e19 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -70,8 +70,7 @@ public AbstractVisitor map(TokenType tokenType) { } /** - * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. If it doesn't - * generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. * @param semanticsSupplier A function that takes the entity and returns the semantics. * @return Self */ @@ -81,8 +80,7 @@ public AbstractVisitor withSemantics(Function semanticsSupp } /** - * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. If it doesn't - * generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. * @param semanticsSupplier A function that returns the semantics. * @return Self */ @@ -92,8 +90,7 @@ public AbstractVisitor withSemantics(Supplier semanticsSupplie } /** - * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type control. If - * it doesn't generate a token, the semantics are discarded. This is not checked and does not lead to a warning. + * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type control. * @return Self */ public AbstractVisitor withControlSemantics() { diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 5a0060aa7..92dd4ae79 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -96,8 +96,7 @@ public ContextVisitor withSemantics(Supplier semantics) { /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type loop begin, - * same for the exit and loop end. If it doesn't generate a token, the semantics are discarded. This is not checked and - * does not lead to a warning. + * same for the exit and loop end. * @return Self */ public ContextVisitor withLoopSemantics() { diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index a56b8f35a..d9b063f1c 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -39,6 +39,11 @@ List getTokens() { void addToken(TokenType jplagType, Function semanticsSupplier, T entity, Function extractToken, VariableRegistry variableRegistry) { + if (jplagType == null) { + if (semanticsSupplier != null) + logger.warning("Received semantics, but no token type, so no token was generated and the semantics discarded"); + return; + } org.antlr.v4.runtime.Token antlrToken = extractToken.apply(entity); int line = antlrToken.getLine(); int column = antlrToken.getCharPositionInLine() + 1; From cc156ec676febeb388ce981271135f2ba79ae427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Mon, 28 Aug 2023 17:19:35 +0200 Subject: [PATCH 11/12] Simplify token collector file update --- .../de/jplag/antlr/AbstractAntlrParserAdapter.java | 14 ++++---------- .../main/java/de/jplag/antlr/TokenCollector.java | 14 ++++++-------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index 21f0d516f..d6582b793 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -3,7 +3,6 @@ import java.io.File; import java.io.IOException; import java.io.Reader; -import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -52,21 +51,15 @@ protected AbstractAntlrParserAdapter() { * @throws ParsingException If anything goes wrong */ public List parse(Set files) throws ParsingException { - List filesList = new ArrayList<>(files); - if (files.isEmpty()) - return new ArrayList<>(); - File firstFile = filesList.remove(0); - TokenCollector collector = new TokenCollector(extractsSemantics, firstFile); - parseFile(firstFile, collector); - for (File file : filesList) { - collector.addFileEndToken(file); // takes the NEXT file + TokenCollector collector = new TokenCollector(extractsSemantics); + for (File file : files) { parseFile(file, collector); } - collector.addFileEndToken(null); return collector.getTokens(); } private void parseFile(File file, TokenCollector collector) throws ParsingException { + collector.enterFile(file); try (Reader reader = FileUtils.openFileReader(file)) { Lexer lexer = this.createLexer(CharStreams.fromReader(reader)); CommonTokenStream tokenStream = new CommonTokenStream(lexer); @@ -80,6 +73,7 @@ private void parseFile(File file, TokenCollector collector) throws ParsingExcept } catch (IOException exception) { throw new ParsingException(file, exception.getMessage(), exception); } + collector.addFileEndToken(); } /** diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index d9b063f1c..d82e30155 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -24,10 +24,9 @@ public class TokenCollector { /** * @param extractsSemantics If semantics are extracted */ - TokenCollector(boolean extractsSemantics, File initialFile) { + TokenCollector(boolean extractsSemantics) { this.collected = new ArrayList<>(); this.extractsSemantics = extractsSemantics; - this.file = initialFile; } /** @@ -63,14 +62,13 @@ void addToken(TokenType jplagType, Function semanticsSuppl addToken(token); } - /** - * Add a file end token. - * @param newFile The next file, null if there isn't one. - */ - void addFileEndToken(File newFile) { + void enterFile(File newFile) { + this.file = newFile; + } + + void addFileEndToken() { addToken(extractsSemantics ? Token.semanticFileEnd(file) : Token.fileEnd(file)); // don't need to update semantics because variable registry is new for every file - this.file = newFile; } private void addToken(Token token) { From 3efd49600a7e4c890e27e38ca6183f65aaeb252e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Br=C3=B6del?= Date: Wed, 13 Sep 2023 13:58:10 +0200 Subject: [PATCH 12/12] Remove explicit super() calls --- .../main/java/de/jplag/antlr/AbstractAntlrListener.java | 7 +++++-- .../java/de/jplag/antlr/AbstractAntlrParserAdapter.java | 1 - .../src/main/java/de/jplag/antlr/TokenCollector.java | 9 ++++++--- .../java/de/jplag/antlr/testLanguage/TestListener.java | 2 -- .../cpp2/src/main/java/de/jplag/cpp2/CPPListener.java | 2 -- .../src/main/java/de/jplag/kotlin/KotlinListener.java | 2 -- .../src/main/java/de/jplag/llvmir/LLVMIRListener.java | 2 -- 7 files changed, 11 insertions(+), 14 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java index a1ea6ebdb..f4144b72e 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrListener.java @@ -12,13 +12,16 @@ import org.antlr.v4.runtime.tree.TerminalNode; /** - * Base class for Antlr listeners. This is a quasi-static class that is only created once per language. Use by - * overwriting the constructor, calling super(), and then calling the visit methods. + * Base class for Antlr listeners. This is a quasi-static class that is only created once per language. Use by calling + * the visit methods in the overwritten constructor. */ public abstract class AbstractAntlrListener { private final List> contextVisitors; private final List terminalVisitors; + /** + * New instance + */ protected AbstractAntlrListener() { contextVisitors = new ArrayList<>(); terminalVisitors = new ArrayList<>(); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java index d6582b793..1644ccf7b 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrParserAdapter.java @@ -33,7 +33,6 @@ public abstract class AbstractAntlrParserAdapter extends Abstr * @param extractsSemantics If true, the listener will extract semantics along with every token */ protected AbstractAntlrParserAdapter(boolean extractsSemantics) { - super(); this.extractsSemantics = extractsSemantics; } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index d82e30155..7a284d015 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -39,8 +39,9 @@ List getTokens() { void addToken(TokenType jplagType, Function semanticsSupplier, T entity, Function extractToken, VariableRegistry variableRegistry) { if (jplagType == null) { - if (semanticsSupplier != null) + if (semanticsSupplier != null) { logger.warning("Received semantics, but no token type, so no token was generated and the semantics discarded"); + } return; } org.antlr.v4.runtime.Token antlrToken = extractToken.apply(entity); @@ -49,14 +50,16 @@ void addToken(TokenType jplagType, Function semanticsSuppl int length = antlrToken.getText().length(); Token token; if (extractsSemantics) { - if (semanticsSupplier == null) + if (semanticsSupplier == null) { throw new IllegalStateException(String.format("Expected semantics bud did not receive any for token %s", jplagType.getDescription())); + } CodeSemantics semantics = semanticsSupplier.apply(entity); token = new Token(jplagType, this.file, line, column, length, semantics); variableRegistry.updateSemantics(semantics); } else { - if (semanticsSupplier != null) + if (semanticsSupplier != null) { logger.warning(() -> String.format("Received semantics for token %s despite not expecting any", jplagType.getDescription())); + } token = new Token(jplagType, this.file, line, column, length); } addToken(token); diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java index c87e50128..9073ad954 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java @@ -10,8 +10,6 @@ class TestListener extends AbstractAntlrListener { TestListener() { - super(); - visit(VarDefContext.class).map(VARDEF).withSemantics(CodeSemantics::createKeep) .onEnter((rule, variableRegistry) -> variableRegistry.registerVariable(rule.VAR_NAME().getText(), VariableScope.FILE, false)); visit(CalcExpressionContext.class, rule -> rule.operator() != null && rule.operator().PLUS() != null).map(ADDITION).withControlSemantics(); diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java index c9710635f..6b2a68c49 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java +++ b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java @@ -15,8 +15,6 @@ class CPPListener extends AbstractAntlrListener { CPPListener() { - super(); - visit(ClassSpecifierContext.class, rule -> rule.classHead().Union() != null).map(UNION_BEGIN, UNION_END); visit(ClassSpecifierContext.class, rule -> rule.classHead().classKey() != null && rule.classHead().classKey().Class() != null) .map(CLASS_BEGIN, CLASS_END); diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java index eaa8c43e2..19a475ffa 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java @@ -99,8 +99,6 @@ class KotlinListener extends AbstractAntlrListener { KotlinListener() { - super(); - visit(PackageHeaderContext.class).map(PACKAGE); visit(ImportHeaderContext.class).map(IMPORT); visit(ClassDeclarationContext.class).map(CLASS_DECLARATION); diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java index 6c4988db6..24e70a9ca 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java @@ -120,8 +120,6 @@ class LLVMIRListener extends AbstractAntlrListener { LLVMIRListener() { - super(); - visit(SourceFilenameContext.class).map(FILENAME); visit(ModuleAsmContext.class).map(ASSEMBLY); visit(TypeDefContext.class).map(TYPE_DEFINITION);