Merge pull request #967 from guwirth/i64-support

improve preprocessor number decoding
SonarOpenCommunity · Oct 6, 2016 · 2b5cc69 · 2b5cc69
2 parents 1f8d00e + f668ec2
commit 2b5cc69
Show file tree

Hide file tree

Showing 7 changed files with 117 additions and 25 deletions.
diff --git a/cxx-squid/src/main/java/org/sonar/cxx/lexer/CxxLexer.java b/cxx-squid/src/main/java/org/sonar/cxx/lexer/CxxLexer.java
@@ -51,7 +51,7 @@ public static Lexer create(Preprocessor... preprocessors) { //@todo deprecated P
   }
 
   private static final String EXP = "([Ee][+-]?+[0-9_]([']?+[0-9_]++)*+)";
-  //private static final String INTEGER_SUFFIX = "(((U|u)(LL|ll|L|l)?)|((LL|ll|L|l)(u|U)?))";  
+  //private static final String INTEGER_SUFFIX = "(((U|u)(i64|LL|ll|L|l)?)|((i64|LL|ll|L|l)(u|U)?))";  
   //private static final String FLOAT_SUFFIX = "(f|l|F|L)";
   private static final String UD_SUFFIX = "([_a-zA-Z]([_a-zA-Z0-9]*+))"; // ud-suffix: identifier (including INTEGER_SUFFIX, FLOAT_SUFFIX)
 

diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/ExpressionEvaluator.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/ExpressionEvaluator.java
@@ -21,8 +21,6 @@
 
 import java.math.BigInteger;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
@@ -40,7 +38,6 @@
 public final class ExpressionEvaluator {
 
   private static final BigInteger UINT64_MAX = new BigInteger("FFFFFFFFFFFFFFFF", 16);
-  private static final String HEX_REGEX = "0[xX]([0-9A-Fa-f]+)(ui64)?";
   public static final Logger LOG = Loggers.get(ExpressionEvaluator.class);
 
   private final Parser<Grammar> parser;
@@ -459,28 +456,74 @@ private BigInteger evalFunctionlikeMacro(AstNode exprAst) {
     return evalToInt(value, exprAst);
   }
 
-  public static String stripSuffix(String number) {
-    return number.replaceAll("[LlUu]", "");
-  }
+  public static BigInteger decode(String number) {
 
-  private static BigInteger decode(String number) {
+    // This fuction is only responsible for providing a string and a radix to BigInteger.
+    // The lexer ensures that the number has a valid format.
+
     int radix = 10;
+    int begin = 0;
     if (number.length() > 2) {
       if (number.charAt(0) == '0') {
-        radix = 8; // 0...
-        try {
-          Pattern p = Pattern.compile(HEX_REGEX);
-          Matcher m = p.matcher(number);
-          if (m.find()) {
-            radix = 16;
-            number = m.group(1);
-          }
-        } catch (java.lang.IllegalStateException e) {
-          // ignore "No match found"
+        switch (number.charAt(1)) {
+          case 'x':
+          case 'X':
+            radix = 16; // 0x...
+            begin = 2;
+            break;
+          case 'b':
+          case 'B':
+            radix = 2; // 0b...
+            begin = 2;
+            break;
+          default:
+            radix = 8; // 0...
+            break;
         }
       }
     }
 
-    return new BigInteger(stripSuffix(number), radix);
+    StringBuilder sb = new StringBuilder(number.length());
+    Boolean suffix = false;
+    for (int index = begin; index < number.length() && !suffix; index++) {
+      char c = number.charAt(index);
+      switch (c) {
+        case '0':
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9':
+
+        case 'a':
+        case 'b':
+        case 'c':
+        case 'd':
+        case 'e':
+        case 'f':
+
+        case 'A':
+        case 'B':
+        case 'C':
+        case 'D':
+        case 'E':
+        case 'F':
+          sb.append(c);
+          break;
+
+        case '\'': // ignore digit separator
+          break;
+
+        default: // suffix
+          suffix = true;
+          break;
+      }
+    }
+
+    return new BigInteger(sb.toString(), radix);
   }
 }
diff --git a/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerTest.java b/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerTest.java
@@ -114,6 +114,10 @@ public void decimal_integer_literals() {
     assertThat(lexer.lex("7llU"), hasToken("7llU", CxxTokenType.NUMBER));
     assertThat(lexer.lex("7LLu"), hasToken("7LLu", CxxTokenType.NUMBER));
     assertThat(lexer.lex("7LLU"), hasToken("7LLU", CxxTokenType.NUMBER));
+
+    // With Micosoft specific 64-bit integer-suffix: i64
+    assertThat(lexer.lex("7i64"), hasToken("7i64", CxxTokenType.NUMBER));
+    assertThat(lexer.lex("7ui64"), hasToken("7ui64", CxxTokenType.NUMBER));
   }
 
   /**
@@ -159,6 +163,10 @@ public void octal_integer_literals() {
     assertThat(lexer.lex("07llU"), hasToken("07llU", CxxTokenType.NUMBER));
     assertThat(lexer.lex("07LLu"), hasToken("07LLu", CxxTokenType.NUMBER));
     assertThat(lexer.lex("07LLU"), hasToken("07LLU", CxxTokenType.NUMBER));
+
+    // With Micosoft specific 64-bit integer-suffix: i64
+    assertThat(lexer.lex("07i64"), hasToken("07i64", CxxTokenType.NUMBER));
+    assertThat(lexer.lex("07ui64"), hasToken("07ui64", CxxTokenType.NUMBER));
   }
 
   /**
@@ -204,6 +212,10 @@ public void hex_integer_literals() {
     assertThat(lexer.lex("0x7llU"), hasToken("0x7llU", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0x7LLu"), hasToken("0x7LLu", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0x7LLU"), hasToken("0x7LLU", CxxTokenType.NUMBER));
+
+    // With Micosoft specific 64-bit integer-suffix: i64
+    assertThat(lexer.lex("0x7i64"), hasToken("0x7i64", CxxTokenType.NUMBER));
+    assertThat(lexer.lex("0x7ui64"), hasToken("0x7ui64", CxxTokenType.NUMBER));
   }
 
   /**
@@ -260,6 +272,10 @@ public void hex_integer_literals_bigX() {
     assertThat(lexer.lex("0X7llU"), hasToken("0X7llU", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0X7LLu"), hasToken("0X7LLu", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0X7LLU"), hasToken("0X7LLU", CxxTokenType.NUMBER));
+
+    // With Micosoft specific 64-bit integer-suffix: i64
+    assertThat(lexer.lex("0X7i64"), hasToken("0X7i64", CxxTokenType.NUMBER));
+    assertThat(lexer.lex("0X7ui64"), hasToken("0X7ui64", CxxTokenType.NUMBER));
   }
 
   /**
@@ -305,6 +321,10 @@ public void bin_integer_literals_bigX() {
     assertThat(lexer.lex("0B1llU"), hasToken("0B1llU", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0B1LLu"), hasToken("0B1LLu", CxxTokenType.NUMBER));
     assertThat(lexer.lex("0B1LLU"), hasToken("0B1LLU", CxxTokenType.NUMBER));
+
+    // With Micosoft specific 64-bit integer-suffix: i64
+    assertThat(lexer.lex("0B1i64"), hasToken("0B1i64", CxxTokenType.NUMBER));
+    assertThat(lexer.lex("0B1ui64"), hasToken("0B1ui64", CxxTokenType.NUMBER));
   }
 
   /**

diff --git a/cxx-squid/src/test/java/org/sonar/cxx/parser/CxxParserTest.java b/cxx-squid/src/test/java/org/sonar/cxx/parser/CxxParserTest.java
@@ -37,7 +37,7 @@
 public class CxxParserTest extends ParserBaseTest {
 
   String errSources = "/parser/bad/error_recovery_declaration.cc";
-  String[] goodFiles = {"own", "examples", "cli", "cuda"};
+  String[] goodFiles = {"own", "VC", "cli", "cuda", "examples"};
   String[] preprocessorFiles = {"preprocessor"};
   String[] cCompatibilityFiles = {"C", "C99"};
   String rootDir = "src/test/resources/parser";

diff --git a/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/ExpressionEvaluatorTest.java b/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/ExpressionEvaluatorTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.cxx.preprocessor;
 
+import java.math.BigInteger;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -317,11 +318,29 @@ public void defined_false_with_parantheses() {
   }
 
   @Test
-  public void stripping_suffix_from_numbers() {
-    assertEquals(evaluator.stripSuffix("1L"), "1");
-    assertEquals(evaluator.stripSuffix("1l"), "1");
-    assertEquals(evaluator.stripSuffix("1U"), "1");
-    assertEquals(evaluator.stripSuffix("1u"), "1");
+  public void decode_numbers() {
+    assertEquals(ExpressionEvaluator.decode("1"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("067"), new BigInteger("67", 8));
+    assertEquals(ExpressionEvaluator.decode("0b11"), new BigInteger("11", 2));
+    assertEquals(ExpressionEvaluator.decode("0xab"), new BigInteger("ab", 16));
+
+    assertEquals(ExpressionEvaluator.decode("1L"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1l"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1U"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1u"), new BigInteger("1", 10));
+
+    assertEquals(ExpressionEvaluator.decode("1ul"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1ll"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1i64"), new BigInteger("1", 10));
+    assertEquals(ExpressionEvaluator.decode("1ui64"), new BigInteger("1", 10));
+
+    assertEquals(ExpressionEvaluator.decode("067ll"), new BigInteger("67", 8));
+    assertEquals(ExpressionEvaluator.decode("0b11ul"), new BigInteger("11", 2));
+    assertEquals(ExpressionEvaluator.decode("0xabui64"), new BigInteger("ab", 16));
+
+    assertEquals(ExpressionEvaluator.decode("1'234"), new BigInteger("1234", 10));
+    assertEquals(ExpressionEvaluator.decode("0b1111'0000'1111"), new BigInteger("111100001111", 2));
+    assertEquals(ExpressionEvaluator.decode("0xAAAA'bbbb"), new BigInteger("AAAAbbbb", 16));
   }
 
   @Test(expected = EvaluationException.class)

diff --git a/cxx-squid/src/test/resources/parser/VC/64-bit-integer-suffix.cc b/cxx-squid/src/test/resources/parser/VC/64-bit-integer-suffix.cc
@@ -0,0 +1,10 @@
+long long i1 = 7i64;
+long long i2 = 7ll;
+long long i3 = 7LL;
+
+unsigned long long ui1 = 7ui64;
+unsigned long long ui2 = 7ull;
+unsigned long long ui3 = 7ULL;
+
+// real life samples
+long long r1 = 9223372036854775807i64;
diff --git a/.../parser/own/VC/exception-specification.cc → ...rces/parser/VC/exception-specification.cc b/.../parser/own/VC/exception-specification.cc → ...rces/parser/VC/exception-specification.cc