Merge pull request #2381 from guwirth/pp-refactoring-2

preprocessor refactoring

cxx-sensors/src/main/java/org/sonar/cxx/sensors/tests/dotnet/VisualStudioTestResultsFileParser.java

@@ -122,7 +122,7 @@ private Date getRequiredDateAttribute(XmlParserHelper xmlParserHelper, String na
     }
 
     private String keepOnlyMilliseconds(String value) {
-      var sb = new StringBuffer(256);
+      var sb = new StringBuilder(256);
 
       var matcher = millisecondsPattern.matcher(value);
       var trailingZeros = new StringBuilder(128);

cxx-squid-bridge/src/main/java/org/sonar/cxx/squidbridge/api/SourceCode.java

@@ -45,11 +45,11 @@ public abstract class SourceCode implements Measurable, Comparable<SourceCode> {
   private SourceCodeIndexer indexer;
   private Set<CheckMessage> messages;
 
-  public SourceCode(String key) {
+  protected SourceCode(String key) {
     this(key, null);
   }
 
-  public SourceCode(String key, @Nullable String name) {
+  protected SourceCode(String key, @Nullable String name) {
     this.key = key;
     this.name = name;
   }

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java

@@ -394,12 +394,18 @@ private boolean addUnitForcedIncludes(String level) {
 
   private void parseIncludeLine(String includeLine) {
     AstNode astNode = lineParser(includeLine);
-    handleIncludeLine(astNode, astNode.getFirstDescendant(PPGrammarImpl.includeBodyQuoted).getToken());
+    if (astNode != null) {
+      handleIncludeLine(astNode, astNode.getFirstDescendant(PPGrammarImpl.includeBodyQuoted).getToken());
+    }
   }
 
+  @CheckForNull
   PPMacro parseMacroDefinition(String macroDef) {
     AstNode astNode = lineParser(macroDef);
-    return PPMacro.create(astNode.getFirstDescendant(PPGrammarImpl.defineLine));
+    if (astNode != null) {
+      return PPMacro.create(astNode.getFirstDescendant(PPGrammarImpl.defineLine));
+    }
+    return null;
   }
 
   /**
@@ -573,42 +579,33 @@ private PreprocessorAction handleModuleLine(AstNode ast, Token token) {
     return oneConsumedToken(token);
   }
 
-  private static List<Token> createChildrenTokenList(AstNode ast) {
-    var children = ast.getChildren();
-    var list = new ArrayList<Token>(children.size());
-    for (var child : children) {
-      list.add(child.getToken());
-    }
-    return list;
-  }
-
   private PreprocessorAction mapFromPPToCxx(AstNode ast, Token token) {
-    List<Token> replTokens = new ArrayList<>();
-    for (Token ppToken : TokenUtils.removeLastTokenIfEof(createChildrenTokenList(ast))) { // TODO removeLastTokenIfEof?
-      String value = ppToken.getValue();
-      if (!value.isBlank()) {
-        // call CXX lexer to create a CXX token
-        var lexer = lineLexerwithoutPP.borrowLexer();
-        List<Token> cxxTokens = lexer.lex(value);
-        lineLexerwithoutPP.returnLexer(lexer);
-        var cxxToken = cxxTokens.get(0);
-        var cxxType = cxxToken.getType();
-
-        if (!cxxType.equals(GenericTokenType.EOF)) {
-          cxxToken = Token.builder()
+    var ppTokens = ast.getTokens();
+    List<Token> result = new ArrayList<>(ppTokens.size());
+    var lexer = lineLexerwithoutPP.borrowLexer();
+    try {
+      for (var ppToken : ppTokens) {
+        String value = ppToken.getValue();
+        if (!"EOF".equals(value) && !value.isBlank()) {
+
+          // call CXX lexer to create a CXX token
+          List<Token> cxxTokens = lexer.lex(value);
+
+          var cxxToken = Token.builder()
             .setLine(token.getLine() + ppToken.getLine() - 1)
             .setColumn(token.getColumn() + ppToken.getColumn())
             .setURI(ppToken.getURI())
             .setValueAndOriginalValue(ppToken.getValue())
-            .setType(cxxType)
+            .setType(cxxTokens.get(0).getType())
             .build();
 
-          replTokens.add(cxxToken);
+          result.add(cxxToken);
         }
       }
+    } finally {
+      lineLexerwithoutPP.returnLexer(lexer);
     }
-
-    return new PreprocessorAction(1, Collections.singletonList(Trivia.createPreprocessingToken(token)), replTokens);
+    return new PreprocessorAction(1, Collections.singletonList(Trivia.createPreprocessingToken(token)), result);
   }
 
   static private List<Token> adjustPosition(List<Token> tokens, Token position) {

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/IncludeDirectiveLexer.java

@@ -27,16 +27,16 @@
 import org.sonar.cxx.channels.PreprocessorChannel;
 import org.sonar.cxx.config.CxxSquidConfiguration;
 
-public final class IncludeDirectiveLexer {
+final class IncludeDirectiveLexer {
 
   private IncludeDirectiveLexer() {
   }
 
-  public static Lexer create(Preprocessor... preprocessors) {
+  static Lexer create(Preprocessor... preprocessors) {
     return create(new CxxSquidConfiguration(), preprocessors);
   }
 
-  public static Lexer create(CxxSquidConfiguration squidConfig, Preprocessor... preprocessors) {
+  static Lexer create(CxxSquidConfiguration squidConfig, Preprocessor... preprocessors) {
     var builder = Lexer.builder()
       .withCharset(squidConfig.getCharset())
       .withFailIfNoChannelToConsumeOneCharacter(true)

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/MacroContainer.java

@@ -19,9 +19,10 @@
  */
 package org.sonar.cxx.preprocessor;
 
+import java.util.ArrayDeque;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Stack;
 import java.util.stream.Collectors;
 
 /**
@@ -35,7 +36,7 @@
 public class MacroContainer<K, V> {
 
   private final Map<K, V> values = new HashMap<>();
-  private final Stack<K> disabled = new Stack<>();
+  private final Deque<K> disabled = new ArrayDeque<>();
 
   /**
    * get value for key.
@@ -44,11 +45,10 @@ public class MacroContainer<K, V> {
    */
   public V get(K key) {
     V v = values.get(key);
-    if (v != null) {
-      if (disabled.isEmpty() || disabled.search(key) == -1) {
-        return v;
-      }
+    if ((v != null) && (disabled.isEmpty() || !disabled.contains(key))) {
+      return v;
     }
+
     return null;
   }
 
@@ -109,10 +109,9 @@ public void popDisable() {
    */
   @Override
   public String toString() {
-    String mapAsString = values.values().stream()
-      .map(value -> value.toString())
+    return values.values().stream()
+      .map(Object::toString)
       .collect(Collectors.joining(", ", "[", "]"));
-    return mapAsString;
   }
 
 }

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPConcatenation.java

@@ -35,7 +35,7 @@
  * a longer identifier, digits that form a number, or operators + and = that form a +=. A comment cannot be created by
  * pasting / and * because comments are removed from text before macro substitution is considered.
  */
-class PPConcatenation {
+final class PPConcatenation {
 
   private static final Logger LOG = Loggers.get(PPConcatenation.class);
 

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPGeneratedToken.java

@@ -26,7 +26,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-class PPGeneratedToken {
+final class PPGeneratedToken {
 
   private PPGeneratedToken() {
 

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPInclude.java

@@ -28,7 +28,7 @@ class PPInclude {
 
   private CxxPreprocessor pp;
 
-  public PPInclude(CxxPreprocessor pp) {
+  PPInclude(CxxPreprocessor pp) {
     this.pp = pp;
   }
 

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java

@@ -73,7 +73,7 @@ private PPMacro(String name, String body) {
     this.isVariadic = false;
   }
 
-  public static PPMacro create(AstNode defineLineAst) {
+  static PPMacro create(AstNode defineLineAst) {
     var ast = defineLineAst.getFirstChild();
     var nameNode = ast.getFirstDescendant(PPGrammarImpl.ppToken);
     String macroName = nameNode.getTokenValue();
@@ -131,7 +131,7 @@ public String toString() {
     return ab.toString();
   }
 
-  public boolean checkArgumentsCount(int count) {
+  boolean checkArgumentsCount(int count) {
     if (params != null) {
       return isVariadic ? count >= params.size() - 1 : count == params.size();
     }

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPPredefinedMacros.java

@@ -24,9 +24,9 @@
  *
  * The macro names of this class are predefined in every translation unit.
  */
-class PPPredefinedMacros {
+final class PPPredefinedMacros {
 
-  static String[] predefinedMacros = {
+  private static final String[] predefinedMacros = {
     "__FILE__ \"file\"",
     "__LINE__ 1",
     // indicates 'date unknown'. should suffice
@@ -41,6 +41,10 @@ class PPPredefinedMacros {
     "__has_include 1"
   };
 
+  private PPPredefinedMacros() {
+
+  }
+
   static String[] predefinedMacroValues() {
     return predefinedMacros.clone();
   }

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPReplace.java

@@ -38,7 +38,7 @@ class PPReplace {
   private static final Logger LOG = Loggers.get(PPReplace.class);
   private CxxPreprocessor pp;
 
-  public PPReplace(CxxPreprocessor pp) {
+  PPReplace(CxxPreprocessor pp) {
     this.pp = pp;
   }
 

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPSpecialIdentifier.java

@@ -38,7 +38,7 @@ public enum PPSpecialIdentifier implements TokenType {
     this.value = value;
   }
 
-  public static String[] keywordValues() {
+  static String[] keywordValues() {
     PPSpecialIdentifier[] keywordsEnum = PPSpecialIdentifier.values();
     var keywords = new String[keywordsEnum.length];
     for (var i = 0; i < keywords.length; i++) {

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPStringification.java

@@ -28,7 +28,7 @@
  * operation is called "stringification". If the result of stringification is not a valid string literal, the behavior
  * is undefined.
  */
-class PPStringification {
+final class PPStringification {
 
   private PPStringification() {
 

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/JoinStringsPreprocessorTest.java

@@ -27,19 +27,19 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-public class JoinStringsPreprocessorTest {
+class JoinStringsPreprocessorTest {
 
   private JoinStringsPreprocessor pp;
   private Lexer lexer;
 
   @BeforeEach
-  public void setUp() {
+  void setUp() {
     pp = new JoinStringsPreprocessor();
     lexer = PPLexer.create();
   }
 
   @Test
-  public void testProcess() {
+  void testProcess() {
     List<Token> tokens = lexer.lex("\"A\"\"B\"");
     PreprocessorAction result = pp.process(tokens);
     assertThat(result.getNumberOfConsumedTokens()).isEqualTo(2);

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/MacroContainerTest.java

@@ -26,7 +26,7 @@ class MacroContainerTest {
 
   private final MacroContainer<String, String> mc;
 
-  public MacroContainerTest() {
+  MacroContainerTest() {
     mc = new MacroContainer<>();
   }
 

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPConcatenationTest.java

@@ -26,17 +26,17 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-public class PPConcatenationTest {
+class PPConcatenationTest {
 
   private Lexer lexer;
 
   @BeforeEach
-  public void setUp() {
+  void setUp() {
     lexer = PPLexer.create();
   }
 
   @Test
-  public void testConcatenate() {
+  void testConcatenate() {
     List<Token> tokens = lexer.lex("x ## y");
     List<Token> result = PPConcatenation.concatenate(tokens);
     assertThat(result)

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPGeneratedTokenTest.java

@@ -28,17 +28,17 @@
 import static org.assertj.core.api.Assertions.*;
 import org.junit.jupiter.api.Test;
 
-public class PPGeneratedTokenTest {
+class PPGeneratedTokenTest {
 
   @Test
-  public void testNewGeneratedTokenFromExistingToken() {
+  void testNewGeneratedTokenFromExistingToken() {
     Token token = TokenUtils.tokenBuilder(IDENTIFIER, "Token", 1, 2);
     Token result = PPGeneratedToken.build(token);
     assertThat(result.isGeneratedCode()).isTrue();
   }
 
   @Test
-  public void testNewGeneratedTokenWithNewValueAndType() {
+  void testNewGeneratedTokenWithNewValueAndType() {
     Token token = TokenUtils.tokenBuilder(IDENTIFIER, "Token", 1, 2);
     Token result = PPGeneratedToken.build(token, CONSTANT, "3");
 
@@ -50,7 +50,7 @@ public void testNewGeneratedTokenWithNewValueAndType() {
   }
 
   @Test
-  public void testNewGeneratedTokenWithNewPosition() throws URISyntaxException {
+  void testNewGeneratedTokenWithNewPosition() throws URISyntaxException {
     Token token = TokenUtils.tokenBuilder(IDENTIFIER, "Token", 1, 2);
     var uri = new URI("tests://sample");
     Token result = PPGeneratedToken.build(token, uri, 3, 4);
@@ -64,7 +64,7 @@ public void testNewGeneratedTokenWithNewPosition() throws URISyntaxException {
   }
 
   @Test
-  public void testNewGeneratedToken() {
+  void testNewGeneratedToken() {
     Token result = PPGeneratedToken.build(CONSTANT, "3", 1, 2);
 
     assertThat(result.isGeneratedCode()).isTrue();
@@ -75,7 +75,7 @@ public void testNewGeneratedToken() {
   }
 
   @Test
-  public void testMarkAllAsGenerated() {
+  void testMarkAllAsGenerated() {
     List<Token> tokens = PPLexer.create().lex("A B");
     List<Token> result = PPGeneratedToken.markAllAsGenerated(tokens);