Skip to content

Commit

Permalink
Support for long number declaration
Browse files Browse the repository at this point in the history
  • Loading branch information
@aNNiMON
aNNiMON committed Feb 10, 2024
1 parent 3194547 commit 7acad44
Show file tree
Hide file tree
Showing 9 changed files with 102 additions and 53 deletions.
1 change: 1 addition & 0 deletions docs/docs/en/changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

### Changes
- Introducing Constants. Constant can be imported only when using a module.
- Support for long number declaration: `700L`, `0xABL`
- Fixed variables scope in shadowing.
- Better error visualizing. Parse errors shows exact line in which an error occurs. Same for Linter and Runtime errors.
- Semantic linter as a required stage.
Expand Down
1 change: 1 addition & 0 deletions docs/docs/ru/changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

### Изменения
- Добавлены константы. Константа может быть импортирована только при подключении модуля.
- Возможность задать числа типа long: `700L`, `0xABL`
- Исправлена область видимости переменных при шедоуинге.
- Улучшена визуализация ошибок. Ошибки парсинга показывают конкретное место, где возникла ошибка. То же самое с линтером и ошибками времени исполнения.
- Семантический линтер как обязательный этап работы интерпретатора.
Expand Down
13 changes: 11 additions & 2 deletions ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,8 @@ private void tokenizeNumber() {
while (true) {
if (current == '.') {
decimal = true;
if (hasDot) throw error("Invalid float number " + buffer, startPos);
if (hasDot)
throw error("Invalid float number " + buffer, startPos);
hasDot = true;
} else if (current == 'e' || current == 'E') {
decimal = true;
Expand All @@ -182,6 +183,9 @@ private void tokenizeNumber() {
}
if (decimal) {
addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos);
} else if (current == 'L') {
next();
addToken(TokenType.LONG_NUMBER, buffer.toString(), startPos);
} else {
addToken(TokenType.NUMBER, buffer.toString(), startPos);
}
Expand Down Expand Up @@ -232,7 +236,12 @@ private void tokenizeHexNumber(int skipChars) {

if (buffer.isEmpty()) throw error("Empty HEX value", startPos);
if (peek(-1) == '_') throw error("HEX value cannot end with _", startPos, markEndPos());
addToken(TokenType.HEX_NUMBER, buffer.toString(), startPos);
if (current == 'L') {
next();
addToken(TokenType.HEX_LONG_NUMBER, buffer.toString(), startPos);
} else {
addToken(TokenType.HEX_NUMBER, buffer.toString(), startPos);
}
}

private static boolean isNumber(char current) {
Expand Down
99 changes: 61 additions & 38 deletions ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ public static Node parse(List<Token> tokens) {
ASSIGN_OPERATORS.put(TokenType.ATEQ, BinaryExpression.Operator.AT);
}

private static final EnumSet<TokenType> NUMBER_TOKEN_TYPES = EnumSet.of(
TokenType.NUMBER,
TokenType.LONG_NUMBER,
TokenType.DECIMAL_NUMBER,
TokenType.HEX_NUMBER,
TokenType.HEX_LONG_NUMBER
);

private final List<Token> tokens;
private final int size;
private final ParseErrors parseErrors;
Expand Down Expand Up @@ -347,7 +355,7 @@ private Node functionChain(Node qualifiedNameExpr) {
}
if (lookMatch(0, TokenType.DOT)) {
final List<Node> indices = variableSuffix();
if (indices == null || indices.isEmpty()) {
if (indices.isEmpty()) {
return expr;
}

Expand Down Expand Up @@ -411,20 +419,10 @@ private MatchExpression match() {
consume(TokenType.CASE);
MatchExpression.Pattern pattern = null;
final Token current = get(0);
if (match(TokenType.NUMBER)) {
// case 20:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createNumber(current.text(), 10))
);
} else if (match(TokenType.DECIMAL_NUMBER)) {
// case 0.5:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createDecimalNumber(current.text()))
);
} else if (match(TokenType.HEX_NUMBER)) {
// case #FF:
if (isNumberToken(current.type())) {
// case 20: / case 0.5: / case #FF:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createNumber(current.text(), 16))
NumberValue.of(getAsNumber(current))
);
} else if (match(TokenType.TEXT)) {
// case "text":
Expand Down Expand Up @@ -859,7 +857,7 @@ private Node qualifiedName() {
final List<Node> indices = variableSuffix();
final var variable = new VariableExpression(current.text());
variable.setRange(getRange(startTokenIndex, index - 1));
if (indices == null || indices.isEmpty()) {
if (indices.isEmpty()) {
return variable;
} else {
return new ContainerAccessExpression(variable, indices, variable.getRange());
Expand All @@ -869,7 +867,7 @@ private Node qualifiedName() {
private List<Node> variableSuffix() {
// .key1.arr1[expr1][expr2].key2
if (!lookMatch(0, TokenType.DOT) && !lookMatch(0, TokenType.LBRACKET)) {
return null;
return Collections.emptyList();
}
final List<Node> indices = new ArrayList<>();
while (lookMatch(0, TokenType.DOT) || lookMatch(0, TokenType.LBRACKET)) {
Expand All @@ -888,47 +886,72 @@ private List<Node> variableSuffix() {

private Node value() {
final Token current = get(0);
if (match(TokenType.NUMBER)) {
return new ValueExpression(createNumber(current.text(), 10));
}
if (match(TokenType.DECIMAL_NUMBER)) {
return new ValueExpression(createDecimalNumber(current.text()));
}
if (match(TokenType.HEX_NUMBER)) {
return new ValueExpression(createNumber(current.text(), 16));
if (isNumberToken(current.type())) {
return new ValueExpression(getAsNumber(current));
}
if (match(TokenType.TEXT)) {
final ValueExpression strExpr = new ValueExpression(current.text());
// "text".property || "text".func()
if (lookMatch(0, TokenType.DOT)) {
if (lookMatch(1, TokenType.WORD) && lookMatch(2, TokenType.LPAREN)) {
match(TokenType.DOT);
return functionChain(new ContainerAccessExpression(
strExpr,
Collections.singletonList(new ValueExpression(consume(TokenType.WORD).text())),
getRange()
));
}
final List<Node> indices = variableSuffix();
if (indices == null || indices.isEmpty()) {
return strExpr;
}
return new ContainerAccessExpression(strExpr, indices, getRange());
return stringProperty(strExpr);
}
return strExpr;
}
throw error("Unknown expression: " + current);
}

private Node stringProperty(ValueExpression strExpr) {
if (lookMatch(1, TokenType.WORD) && lookMatch(2, TokenType.LPAREN)) {
match(TokenType.DOT);
return functionChain(new ContainerAccessExpression(
strExpr,
Collections.singletonList(new ValueExpression(consume(TokenType.WORD).text())),
getRange()
));
}
final List<Node> indices = variableSuffix();
if (indices.isEmpty()) {
return strExpr;
}
return new ContainerAccessExpression(strExpr, indices, getRange());
}

private boolean isNumberToken(TokenType type) {
return NUMBER_TOKEN_TYPES.contains(type);
}

private Number getAsNumber(Token current) {
if (match(TokenType.NUMBER)) {
return createNumber(current.text(), 10);
}
if (match(TokenType.LONG_NUMBER)) {
return createLongNumber(current.text(), 10);
}
if (match(TokenType.DECIMAL_NUMBER)) {
return createDecimalNumber(current.text());
}
if (match(TokenType.HEX_NUMBER)) {
return createNumber(current.text(), 16);
}
if (match(TokenType.HEX_LONG_NUMBER)) {
return createLongNumber(current.text(), 16);
}
throw error("Unknown number expression: " + current);
}

private Number createNumber(String text, int radix) {
// Integer
try {
return Integer.parseInt(text, radix);
} catch (NumberFormatException nfe) {
return Long.parseLong(text, radix);
return createLongNumber(text, radix);
}
}

private Number createLongNumber(String text, int radix) {
return Long.parseLong(text, radix);
}

private Number createDecimalNumber(String text) {
// Double
return Double.parseDouble(text);
Expand Down
2 changes: 2 additions & 0 deletions ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@
public enum TokenType {

NUMBER,
LONG_NUMBER,
DECIMAL_NUMBER,
HEX_NUMBER,
HEX_LONG_NUMBER,
WORD,
TEXT,

Expand Down
22 changes: 11 additions & 11 deletions ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,17 @@ public static Stream<Arguments> invalidData() {
}

@Test
public void testNumbers() {
String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF";
void testNumbers() {
String input = "0 800L 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF 0x7FL";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
assertTokens(result, NUMBER, LONG_NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_LONG_NUMBER);
assertThat(result)
.extracting(Token::text)
.containsExactly("0", "3.1415", "CAFEBABE", "f7d6c5", "FFFF");
.containsExactly("0", "800", "3.1415", "CAFEBABE", "f7d6c5", "FFFF", "7F");
}

@Test
public void testDecimalNumbersExponent() {
void testDecimalNumbersExponent() {
String input = "4e+7 0.3E-19 2e0 5e0000000000000200 5E-000000089";
List<Token> result = Lexer.tokenize(input);
assertThat(result)
Expand All @@ -61,15 +61,15 @@ public void testDecimalNumbersExponent() {
}

@Test
public void testString() {
void testString() {
String input = "\"1\\\"2\"";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, TEXT);
assertEquals("1\"2", result.get(0).text());
}

@Test
public void testEscapeString() {
void testEscapeString() {
String input = """
"\\\\/\\\\"
""".stripIndent();
Expand All @@ -79,15 +79,15 @@ public void testEscapeString() {
}

@Test
public void testEmptyString() {
void testEmptyString() {
String input = "\"\"";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, TEXT);
assertEquals("", result.get(0).text());
}

@Test
public void testComments() {
void testComments() {
String input = "// 1234 \n /* */ 123 /* \n 12345 \n\n\n */";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, NUMBER);
Expand All @@ -96,7 +96,7 @@ public void testComments() {

@ParameterizedTest
@MethodSource("validData")
public void testValidInput(String name, String input, List<TokenType> tokenTypes) throws IOException {
void testValidInput(String name, String input, List<TokenType> tokenTypes) throws IOException {
List<Token> result = Lexer.tokenize(input);
assertThat(result)
.hasSize(tokenTypes.size())
Expand All @@ -106,7 +106,7 @@ public void testValidInput(String name, String input, List<TokenType> tokenTypes

@ParameterizedTest
@MethodSource("invalidData")
public void testInvalidInput(String name, String input) throws IOException {
void testInvalidInput(String name, String input) throws IOException {
assertThatThrownBy(() -> Lexer.tokenize(input))
.isInstanceOf(OwnLangParserException.class);
}
Expand Down
5 changes: 4 additions & 1 deletion ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ private static List<Arguments> numbers() {
List.of(DECIMAL_NUMBER)),
Arguments.of("Hex numbers",
"#FF 0xCA 0x12fb 0xFF",
List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER))
List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER)),
Arguments.of("Long numbers",
"680L #80L 0x700L",
List.of(LONG_NUMBER, HEX_LONG_NUMBER, HEX_LONG_NUMBER))
);
}

Expand Down
2 changes: 1 addition & 1 deletion ownlang-parser/src/test/java/com/annimon/ownlang/parser/ProgramsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public static void createStage() {

@ParameterizedTest
@MethodSource("data")
public void testProgram(InputSource inputSource) {
void testProgram(InputSource inputSource) {
final StagesDataMap stagesData = new StagesDataMap();
try {
testPipeline.perform(stagesData, inputSource);
Expand Down
10 changes: 10 additions & 0 deletions ownlang-parser/src/test/resources/expressions/binaryExpressionOnNumbers.own
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@ def testMultiplicationOnNumbers() {
assertEquals(30, 5 * (-2 * -3))
}

def testMultiplicationOverflowOnNumbers() {
assertNotEquals(1234567890000L, 1234567890 * 1000)
assertNotEquals(0xFFFFFF00, 0x100 * 0xFFFFFF)
}

def testMultiplicationOnLongNumbers() {
assertEquals(1234567890000L, 1234567890 * 1000L)
assertEquals(0xFFFFFF00L, 0x100L * 0xFFFFFF)
}

def testDivisionOnNumbers() {
assertEquals(3, 6 / 2)
assertEquals(30, -900 / (60 / -2))
Expand Down

0 comments on commit 7acad44

Please sign in to comment.